From 56c67132f3d5c4f596f86ef5f9d66221058dc0a5 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Sat, 13 Jun 2026 09:49:29 +0200 Subject: [PATCH] claude: fold in old sow-tools --- AGENTS.md | 31 +- README.md | 19 +- docker/Dockerfile | 19 +- docs/command-surface.md | 12 +- docs/migration-from-nwn-tool.md | 15 +- go.mod | 5 + go.sum | 6 + internal/app/app.go | 2113 +++ internal/app/app_test.go | 574 + internal/changelog/changelog.go | 455 + internal/changelog/changelog_test.go | 213 + internal/dispatch/dispatch.go | 107 +- internal/dispatch/dispatch_test.go | 52 +- internal/erf/erf.go | 424 + internal/erf/erf_test.go | 130 + internal/gff/binary.go | 675 + internal/gff/binary_test.go | 55 + internal/gff/json.go | 317 + internal/gff/types.go | 140 + internal/music/config.go | 47 + internal/music/credits.go | 292 + internal/music/metadata.go | 62 + internal/music/music.go | 226 + internal/music/music_test.go | 404 + internal/music/naming.go | 197 + internal/pipeline/apply_manifest.go | 70 + internal/pipeline/build.go | 1627 ++ internal/pipeline/chunk_validation.go | 27 + internal/pipeline/compare.go | 359 + internal/pipeline/extract.go | 621 + internal/pipeline/music.go | 524 + internal/pipeline/pipeline_test.go | 4057 +++++ internal/project/effective.go | 738 + internal/project/project.go | 2158 +++ internal/project/project_test.go | 1830 ++ ..._PART_MODELS_IN_2DA_GENERATION_CONTRACT.md | 155 + .../CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md | 76 + internal/topdata/FAMILY_EXPANSION_CONTRACT.md | 119 + .../FEAT_GENERATED_FAMILIES_CONTRACT.md | 174 + internal/topdata/INHERITANCE_CONTRACT.md | 78 + internal/topdata/MASTERFEATS_CONTRACT.md | 82 + internal/topdata/appearance_migrate.go | 96 + internal/topdata/armor_migrate.go | 160 + internal/topdata/assets_repo.go | 46 + internal/topdata/autogen.go | 1132 ++ internal/topdata/base_dialog_migrate.go | 41 + internal/topdata/baseitems_migrate.go | 105 + internal/topdata/class_spellbooks.go | 373 + internal/topdata/classes_migrate.go | 226 + internal/topdata/cloakmodel_migrate.go | 6 + internal/topdata/convert.go | 888 + internal/topdata/convert_test.go | 611 + internal/topdata/creaturespeed_migrate.go | 145 + internal/topdata/damagetypes_migrate.go | 288 + internal/topdata/damagetypes_registry.go | 243 + internal/topdata/doortypes_migrate.go | 99 + internal/topdata/editorconfig_format.go | 246 + internal/topdata/expansion_native.go | 284 + internal/topdata/family_expansion.go | 345 + internal/topdata/feat_migrate.go | 64 + internal/topdata/generated_assets.go | 205 + internal/topdata/genericdoors_migrate.go | 6 + internal/topdata/itemprops_registry.go | 1214 ++ internal/topdata/legacy_dataset_migrate.go | 176 + internal/topdata/loadscreens_migrate.go | 6 + internal/topdata/masterfeats_migrate.go | 237 + internal/topdata/metadata_wiki.go | 190 + internal/topdata/migrate.go | 415 + internal/topdata/native.go | 6936 +++++++ internal/topdata/parts_discovery.go | 692 + internal/topdata/parts_manifest.go | 287 + internal/topdata/parts_manifest_test.go | 26 + internal/topdata/placeables_migrate.go | 6 + internal/topdata/portraits_migrate.go | 8 + internal/topdata/progfx_migrate.go | 6 + internal/topdata/racialtypes_migrate.go | 282 + internal/topdata/racialtypes_registry.go | 270 + internal/topdata/repadjust_migrate.go | 63 + internal/topdata/ruleset_migrate.go | 171 + internal/topdata/skills_migrate.go | 140 + internal/topdata/skyboxes_migrate.go | 6 + internal/topdata/spells_migrate.go | 103 + internal/topdata/tailmodel_migrate.go | 86 + internal/topdata/tlk_native.go | 729 + internal/topdata/top_package.go | 713 + internal/topdata/topdata.go | 3009 +++ internal/topdata/topdata_test.go | 15115 ++++++++++++++++ internal/topdata/vfx_persistent_migrate.go | 198 + internal/topdata/visualeffects_migrate.go | 6 + internal/topdata/wiki_data_providers.go | 518 + internal/topdata/wiki_deploy.go | 1861 ++ internal/topdata/wiki_deploy_test.go | 2387 +++ internal/topdata/wiki_formatters.go | 286 + internal/topdata/wiki_native.go | 2159 +++ internal/topdata/wiki_native_test.go | 3008 +++ internal/topdata/wiki_page_paths.go | 77 + internal/topdata/wiki_preserve.go | 54 + internal/topdata/wiki_slug.go | 210 + internal/topdata/wiki_tables.go | 1749 ++ internal/topdata/wiki_template.go | 433 + internal/topdata/wiki_template_expr.go | 629 + internal/topdata/wiki_templates/classes.txt | 17 + internal/topdata/wiki_templates/feat.txt | 13 + internal/topdata/wiki_templates/items.txt | 13 + internal/topdata/wiki_templates/races.txt | 15 + internal/topdata/wiki_templates/skills.txt | 13 + internal/topdata/wiki_templates/spells.txt | 13 + internal/topdata/wiki_visibility.go | 407 + internal/topdata/wingmodel_migrate.go | 86 + internal/validator/validator.go | 546 + internal/validator/validator_test.go | 356 + scripts/crucible-smoke.sh | 52 +- 112 files changed, 70812 insertions(+), 74 deletions(-) create mode 100644 go.sum create mode 100644 internal/app/app.go create mode 100644 internal/app/app_test.go create mode 100644 internal/changelog/changelog.go create mode 100644 internal/changelog/changelog_test.go create mode 100644 internal/erf/erf.go create mode 100644 internal/erf/erf_test.go create mode 100644 internal/gff/binary.go create mode 100644 internal/gff/binary_test.go create mode 100644 internal/gff/json.go create mode 100644 internal/gff/types.go create mode 100644 internal/music/config.go create mode 100644 internal/music/credits.go create mode 100644 internal/music/metadata.go create mode 100644 internal/music/music.go create mode 100644 internal/music/music_test.go create mode 100644 internal/music/naming.go create mode 100644 internal/pipeline/apply_manifest.go create mode 100644 internal/pipeline/build.go create mode 100644 internal/pipeline/chunk_validation.go create mode 100644 internal/pipeline/compare.go create mode 100644 internal/pipeline/extract.go create mode 100644 internal/pipeline/music.go create mode 100644 internal/pipeline/pipeline_test.go create mode 100644 internal/project/effective.go create mode 100644 internal/project/project.go create mode 100644 internal/project/project_test.go create mode 100644 internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md create mode 100644 internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md create mode 100644 internal/topdata/FAMILY_EXPANSION_CONTRACT.md create mode 100644 internal/topdata/FEAT_GENERATED_FAMILIES_CONTRACT.md create mode 100644 internal/topdata/INHERITANCE_CONTRACT.md create mode 100644 internal/topdata/MASTERFEATS_CONTRACT.md create mode 100644 internal/topdata/appearance_migrate.go create mode 100644 internal/topdata/armor_migrate.go create mode 100644 internal/topdata/assets_repo.go create mode 100644 internal/topdata/autogen.go create mode 100644 internal/topdata/base_dialog_migrate.go create mode 100644 internal/topdata/baseitems_migrate.go create mode 100644 internal/topdata/class_spellbooks.go create mode 100644 internal/topdata/classes_migrate.go create mode 100644 internal/topdata/cloakmodel_migrate.go create mode 100644 internal/topdata/convert.go create mode 100644 internal/topdata/convert_test.go create mode 100644 internal/topdata/creaturespeed_migrate.go create mode 100644 internal/topdata/damagetypes_migrate.go create mode 100644 internal/topdata/damagetypes_registry.go create mode 100644 internal/topdata/doortypes_migrate.go create mode 100644 internal/topdata/editorconfig_format.go create mode 100644 internal/topdata/expansion_native.go create mode 100644 internal/topdata/family_expansion.go create mode 100644 internal/topdata/feat_migrate.go create mode 100644 internal/topdata/generated_assets.go create mode 100644 internal/topdata/genericdoors_migrate.go create mode 100644 internal/topdata/itemprops_registry.go create mode 100644 internal/topdata/legacy_dataset_migrate.go create mode 100644 internal/topdata/loadscreens_migrate.go create mode 100644 internal/topdata/masterfeats_migrate.go create mode 100644 internal/topdata/metadata_wiki.go create mode 100644 internal/topdata/migrate.go create mode 100644 internal/topdata/native.go create mode 100644 internal/topdata/parts_discovery.go create mode 100644 internal/topdata/parts_manifest.go create mode 100644 internal/topdata/parts_manifest_test.go create mode 100644 internal/topdata/placeables_migrate.go create mode 100644 internal/topdata/portraits_migrate.go create mode 100644 internal/topdata/progfx_migrate.go create mode 100644 internal/topdata/racialtypes_migrate.go create mode 100644 internal/topdata/racialtypes_registry.go create mode 100644 internal/topdata/repadjust_migrate.go create mode 100644 internal/topdata/ruleset_migrate.go create mode 100644 internal/topdata/skills_migrate.go create mode 100644 internal/topdata/skyboxes_migrate.go create mode 100644 internal/topdata/spells_migrate.go create mode 100644 internal/topdata/tailmodel_migrate.go create mode 100644 internal/topdata/tlk_native.go create mode 100644 internal/topdata/top_package.go create mode 100644 internal/topdata/topdata.go create mode 100644 internal/topdata/topdata_test.go create mode 100644 internal/topdata/vfx_persistent_migrate.go create mode 100644 internal/topdata/visualeffects_migrate.go create mode 100644 internal/topdata/wiki_data_providers.go create mode 100644 internal/topdata/wiki_deploy.go create mode 100644 internal/topdata/wiki_deploy_test.go create mode 100644 internal/topdata/wiki_formatters.go create mode 100644 internal/topdata/wiki_native.go create mode 100644 internal/topdata/wiki_native_test.go create mode 100644 internal/topdata/wiki_page_paths.go create mode 100644 internal/topdata/wiki_preserve.go create mode 100644 internal/topdata/wiki_slug.go create mode 100644 internal/topdata/wiki_tables.go create mode 100644 internal/topdata/wiki_template.go create mode 100644 internal/topdata/wiki_template_expr.go create mode 100644 internal/topdata/wiki_templates/classes.txt create mode 100644 internal/topdata/wiki_templates/feat.txt create mode 100644 internal/topdata/wiki_templates/items.txt create mode 100644 internal/topdata/wiki_templates/races.txt create mode 100644 internal/topdata/wiki_templates/skills.txt create mode 100644 internal/topdata/wiki_templates/spells.txt create mode 100644 internal/topdata/wiki_visibility.go create mode 100644 internal/topdata/wingmodel_migrate.go create mode 100644 internal/validator/validator.go create mode 100644 internal/validator/validator_test.go diff --git a/AGENTS.md b/AGENTS.md index 5f545bb..aa18984 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,8 +8,7 @@ alwaysApply: true This repo owns the **builder logic** for the migration: one Go module (`gitea.westgate.pw/ShadowsOverWestgate/sow-tools`) producing the `crucible` dispatcher and the `crucible-` binaries (D11). Read -[`../AGENTS.md`](../AGENTS.md) (migration hub) and -[`../../KICKOFF_PROMPT.md`](../../KICKOFF_PROMPT.md) first. +`../AGENTS.md` (migration hub) and `../../KICKOFF_PROMPT.md` first. ## What this repo owns / does not own @@ -19,29 +18,33 @@ changelog. Does **not** own authored game content (that is `sow-module` / `sow-topdata` / `sow-assets-manifest`) or any production deploy authority (that is `sow-platform`). -## Scaffold rules (Phase 5) +## Rules -1. **Source is not transplanted.** The migration hard rules forbid copying the - `internal/` packages from `gitea/sow-tools` automatically. The operator - migrates them at cutover; see [`docs/migration-from-nwn-tool.md`](docs/migration-from-nwn-tool.md). -2. **Fail closed, never fake.** Unwired builders exit `70`. Do not stub a - builder to emit a placeholder artifact. +1. **Migrated logic, not a fresh rewrite.** The `internal/` packages (`app`, + `pipeline`, `project`, `erf`, `gff`, `topdata`, `music`, `changelog`, + `validator`) were folded in from `gitea/sow-tools` at cutover; wired builders + delegate to `internal/app`'s command surface. Keep them in step with upstream + fixes rather than diverging silently. +2. **Fail closed, never fake.** A builder with no migrated logic yet (`depot`) + exits `70`. Do not stub a builder to emit a placeholder artifact. 3. **Binaries are not committed.** They are CI artifacts / image layers (D19). `/bin/`, `*.exe`, `nwn-tool`, `sow-toolkit` are gitignored. 4. **No home-dir / `NWN_ROOT` guessing.** Builders take roots explicitly via - flag or env. See [`docs/consumer-contract.md`](docs/consumer-contract.md). + flag or env (project resolution is CWD-based, never `$HOME`). See + [`docs/consumer-contract.md`](docs/consumer-contract.md). 5. **The registry is the command surface.** `internal/dispatch.Registry` is the single source of truth; keep it in sync with `cmd/` and [`docs/command-surface.md`](docs/command-surface.md). Adding a builder = a `cmd/crucible-/main.go` shim + a `Registry` entry + a doc row. -## Wiring a builder (operator cutover) +## Wiring a builder -1. Migrate the relevant `internal/` package(s) from `gitea/sow-tools`. -2. Register a handler and flip the builder off the unwired path in - `internal/dispatch`. +1. Ensure the relevant `internal/` package(s) cover the work. +2. Add the legacy command(s) to the builder's `Legacy`/`Extra` set and set + `Wired: true` in `internal/dispatch`; the dispatcher delegates to + `app.Run`. `depot` is the remaining unwired builder. 3. Add tests; keep outputs deterministic (same input → same bytes). -4. `make check` must stay green; `make smoke` is updated to expect a wired exit. +4. `make check` must stay green; update `make smoke` to expect the wired exit. ## Commands diff --git a/README.md b/README.md index 4ddd5f9..d80b35f 100644 --- a/README.md +++ b/README.md @@ -27,16 +27,19 @@ The dispatcher and the standalone shims share one registry single-token command. The full legacy `nwn-tool` command surface and where each command lands is mapped in [`docs/command-surface.md`](docs/command-surface.md). -## Status (Phase 5 scaffold) +## Status (cutover performed) -The suite **builds, vets, tests, and runs**, but every builder is **unwired**: -running one fails closed with exit `70` and never fakes an artifact. The internal -pipeline/topdata/erf/wiki/music packages from `gitea/sow-tools` are migrated by -the operator at cutover — the workspace hard rules forbid transplanting that -source automatically. See [`docs/migration-from-nwn-tool.md`](docs/migration-from-nwn-tool.md). +The internal `app`/`pipeline`/`project`/`erf`/`gff`/`topdata`/`music`/`changelog`/ +`validator` packages from `gitea/sow-tools` have been migrated into this tree, and +the `module`, `topdata`, `hak`, and `wiki` builders now **delegate to the migrated +`nwn-tool` command surface** (mapped in +[`docs/command-surface.md`](docs/command-surface.md)). `config` and `changelog` +are global commands on the dispatcher. `depot` has no migrated logic yet, so it +keeps the fail-closed path: exit `70`, never a faked artifact. -This matches the downstream skeletons: `sow-module` / `sow-topdata` package -scripts already fail closed until they can resolve a Crucible binary. +See [`docs/migration-from-nwn-tool.md`](docs/migration-from-nwn-tool.md) for what +was done and what remains (the consumer `--manifest/--source/--out` flag contract +is the open Phase-6 item). ## Develop diff --git a/docker/Dockerfile b/docker/Dockerfile index f7e2f54..27472cc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,15 +6,15 @@ # and ships them on a static base. The `crucible` dispatcher is the entrypoint; # consumer CI can also call the standalone crucible- binaries by path. # -# NOTE (Phase 5 scaffold): the binaries are pure stdlib and fail closed until -# the operator migrates the internal pipeline. When the music pipeline is wired -# it needs ffmpeg at runtime — switch the final stage to a debian-slim base with -# ffmpeg then (see docs/migration-from-nwn-tool.md). +# NOTE: the internal pipeline is migrated and the music conversion path is wired, +# so the runtime stage is debian-slim with ffmpeg on PATH (BMU encode/decode). +# See docs/migration-from-nwn-tool.md. FROM golang:1.26-alpine AS build WORKDIR /src RUN apk add --no-cache git -# go.sum is optional while the scaffold has no external deps. +# go.sum is committed now that the migrated packages pull golang.org/x/text and +# gopkg.in/yaml.v3; the glob keeps the build working if it is ever absent. COPY go.mod go.sum* ./ RUN go mod download COPY . . @@ -29,7 +29,14 @@ RUN set -eux; \ -o "/out/${name}" "${dir}"; \ done -FROM gcr.io/distroless/static-debian12:nonroot +FROM debian:12-slim +# ffmpeg: the migrated music pipeline shells out to it for BMU conversion. +# ca-certificates: builders fetch published manifests over HTTPS. +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends ffmpeg ca-certificates; \ + rm -rf /var/lib/apt/lists/*; \ + useradd --system --create-home --uid 65532 nonroot COPY --from=build /out/ /usr/local/bin/ USER nonroot ENTRYPOINT ["/usr/local/bin/crucible"] diff --git a/docs/command-surface.md b/docs/command-surface.md index c7570e8..3b5ef68 100644 --- a/docs/command-surface.md +++ b/docs/command-surface.md @@ -25,19 +25,21 @@ These legacy commands are **not** top-level builders: - `music *` (`list-datasets`, `scan`, `build`, `credits`, `validate`, `manifest`, `normalize`) → folds into the **module/hak build pipeline** (music BMUs are packed into HAKs at build time). Exposed as `crucible-module music ...` / - `crucible-hak music ...` when wired. Needs `ffmpeg` at runtime. + `crucible-hak music ...`. Needs `ffmpeg` at runtime (in the image). - `config *` (`validate`, `effective`, `inspect`, `explain`, `sources`) → a - **global** concern shared by every builder; exposed as a global subcommand - group, not its own binary. -- `build-changelog` → release tooling; lands as a global `crucible changelog` + **global** concern shared by every builder; exposed as the global + `crucible config ...`, not its own binary. +- `build-changelog` → release tooling; exposed as the global `crucible changelog` rather than a content builder. -## Global commands (implemented in the scaffold) +## Global commands (implemented) ```text crucible help | -h usage + builder list crucible version | -V build version (git sha via -ldflags) crucible list builders, one per line: namebinsummary +crucible config [args] -> legacy `config` command group +crucible changelog [args] -> legacy `build-changelog` ``` `crucible list` is machine-readable so CI can enumerate builders without parsing diff --git a/docs/migration-from-nwn-tool.md b/docs/migration-from-nwn-tool.md index 389b33c..5c3ed34 100644 --- a/docs/migration-from-nwn-tool.md +++ b/docs/migration-from-nwn-tool.md @@ -1,9 +1,18 @@ -# Migrating from `nwn-tool` to Crucible (operator cutover) +# Migrating from `nwn-tool` to Crucible (cutover — DONE) The legacy toolchain lives at `gitea/sow-tools` as a single `nwn-tool` binary (entry `cmd/nwn-tool`, logic under `internal/`). This repo is the Crucible -rewrite. Per the migration hard rules, the `internal/` source is **not** -transplanted automatically — the operator migrates it. This is the checklist. +rewrite. The cutover below has been **performed**: the `internal/` packages are +migrated and the `module`/`topdata`/`hak`/`wiki` builders delegate to them. The +checklist is kept as the record of what was done. + +> **Status.** Steps 1–8 are complete. `depot` is the one remaining unwired +> builder (no legacy source — it was shell `mc`/S3). The open follow-up is the +> consumer flag contract: `sow-module` / `sow-topdata` / `sow-assets-manifest` +> wrappers invoke `crucible-` with `--manifest/--source/--out` style flags +> (see [`consumer-contract.md`](consumer-contract.md)), whereas the migrated +> commands use the legacy CWD/project model. Reconciling the two is the Phase-6 +> consumer task and is **not** part of this cutover. ## Why a clean scaffold first diff --git a/go.mod b/go.mod index ee2a72d..23f6c1a 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,8 @@ module git.westgate.pw/ShadowsOverWestgate/sow-tools go 1.26.0 + +require ( + golang.org/x/text v0.35.0 + gopkg.in/yaml.v3 v3.0.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..fece86f --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 0000000..d4f0c3d --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,2113 @@ +package app + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/changelog" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/topdata" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/validator" + "gopkg.in/yaml.v3" +) + +type command struct { + name string + description string + run func(context) error +} + +type context struct { + stdout io.Writer + stderr io.Writer + cwd string + args []string + logLevel logLevel +} + +type spinner struct { + mu sync.Mutex + on bool + text string + msg []string + painted bool + writer io.Writer + enabled bool +} + +func newSpinner() *spinner { + return &spinner{ + msg: []string{"-", "\\", "|", "/"}, + } +} + +var spin = newSpinner() + +func (s *spinner) configure(writer io.Writer, enabled bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.writer = writer + s.enabled = enabled +} + +func (s *spinner) start(text string) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.enabled || s.writer == nil { + s.on = false + s.painted = false + s.text = text + return + } + if s.on { + s.text = text + return + } + s.on = true + s.text = text + go s.run() +} + +func (s *spinner) stop() { + s.mu.Lock() + defer s.mu.Unlock() + if !s.enabled || s.writer == nil { + s.on = false + s.painted = false + return + } + s.on = false + if s.painted { + fmt.Fprint(s.writer, "\r\033[K") + s.painted = false + } +} + +func (s *spinner) linebreak() { + s.mu.Lock() + defer s.mu.Unlock() + if s.enabled && s.on && s.painted { + fmt.Fprint(s.writer, "\r\033[K") + s.painted = false + } +} + +func (s *spinner) update(text string) { + s.mu.Lock() + defer s.mu.Unlock() + s.text = text +} + +func (s *spinner) run() { + i := 0 + for { + s.mu.Lock() + if !s.on { + s.mu.Unlock() + return + } + msg := s.msg[i%len(s.msg)] + fmt.Fprintf(s.writer, "\r[%s] %s", msg, s.text) + s.painted = true + s.mu.Unlock() + i++ + time.Sleep(100 * time.Millisecond) + } +} + +func isInteractiveTTY(w io.Writer) bool { + file, ok := w.(*os.File) + if !ok { + return false + } + info, err := file.Stat() + if err != nil { + return false + } + return (info.Mode() & os.ModeCharDevice) != 0 +} + +func spinnerEnabledFor(w io.Writer, level logLevel) bool { + mode := strings.TrimSpace(strings.ToLower(os.Getenv("SOW_TOOLS_TTY_MODE"))) + switch mode { + case "plain", "off", "0", "false", "no": + return false + case "spinner", "on", "1", "true", "yes": + return level == logLevelNormal && isInteractiveTTY(w) + case "", "auto": + return isInteractiveTTY(w) && strings.TrimSpace(os.Getenv("CI")) == "" && level == logLevelNormal + default: + return isInteractiveTTY(w) && strings.TrimSpace(os.Getenv("CI")) == "" && level == logLevelNormal + } +} + +var commands = []command{ + { + name: "build", + description: "Build module, HAK, and configured top package outputs.", + run: runBuild, + }, + { + name: "build-module", + description: "Build only the configured module archive.", + run: runBuildModule, + }, + { + name: "build-haks", + description: "Build only configured HAK archives and manifest.", + run: runBuildHAKs, + }, + { + name: "extract", + description: "Extract built archives back into the configured source and asset trees.", + run: runExtract, + }, + { + name: "validate", + description: "Validate project layout, config, and source inventory.", + run: runValidate, + }, + { + name: "config", + description: "Inspect, explain, and validate the resolved project configuration.", + run: runConfig, + }, + { + name: "music", + description: "List, scan, build, and validate configured music datasets.", + run: runMusic, + }, + { + name: "compare", + description: "Compare current source content against the built archives.", + run: runCompare, + }, + { + name: "apply-hak-manifest", + description: "Apply a generated HAK manifest to the configured module source.", + run: runApplyHAKManifest, + }, + { + name: "validate-topdata", + description: "Validate topdata source layout and canonical JSON compatibility.", + run: runValidateTopData, + }, + { + name: "build-topdata", + description: "Build configured topdata outputs, then refresh the configured top package.", + run: runBuildTopData, + }, + { + name: "build-top-package", + description: "Build the configured top package from cached native topdata output and authored topdata assets.", + run: runBuildTopPackage, + }, + { + name: "compare-topdata", + description: "Rebuild topdata natively and compare the current build output against that fresh native result.", + run: runCompareTopData, + }, + { + name: "convert-topdata", + description: "Convert between 2DA and native topdata JSON/module formats.", + run: runConvertTopData, + }, + { + name: "build-wiki", + description: "Build wiki pages from the current topdata state.", + run: runBuildWiki, + }, + { + name: "deploy-wiki", + description: "Deploy generated wiki pages to NodeBB.", + run: runDeployWiki, + }, + { + name: "build-changelog", + description: "Build a release changelog from tagged pushes and write it to stdout or a file.", + run: runBuildChangelog, + }, +} + +func Run(args []string) (int, error) { + ctx, err := newContext() + if err != nil { + return 1, err + } + + if len(args) == 0 { + printUsage(ctx.stdout) + return 0, nil + } + + switch args[0] { + case "-h", "--help", "help": + printUsage(ctx.stdout) + return 0, nil + default: + cmdArgs, level := parseGlobalFlags(args[1:]) + ctx.logLevel = level + ctx.args = append([]string{args[0]}, cmdArgs...) + for _, cmd := range commands { + if cmd.name == args[0] { + if err := cmd.run(ctx); err != nil { + return 1, err + } + return 0, nil + } + } + } + + return 1, fmt.Errorf("unknown command %q", args[0]) +} + +func parseGlobalFlags(args []string) ([]string, logLevel) { + level := logLevelNormal + var filtered []string + for _, arg := range args { + switch arg { + case "--quiet": + level = logLevelQuiet + case "--verbose": + level = logLevelVerbose + case "--debug": + level = logLevelDebug + default: + filtered = append(filtered, arg) + } + } + return filtered, level +} + +func newContext() (context, error) { + cwd, err := os.Getwd() + if err != nil { + return context{}, fmt.Errorf("resolve working directory: %w", err) + } + + return context{ + stdout: os.Stdout, + stderr: os.Stderr, + cwd: cwd, + args: os.Args[1:], + }, nil +} + +func runBuild(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + preflight, err := runBuildModulePreflight(ctx, p, nil) + if err != nil { + return err + } + console := newProjectConsole(ctx, p, "build") + + result, err := pipeline.Build(p) + if err != nil { + return err + } + + console.emitBuildResult(result, preflight) + return nil +} + +func runBuildModule(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + console := newProjectConsole(ctx, p, "build-module") + spin.configure(ctx.stderr, console.spinnerEnabled) + spin.start("Build Module: starting") + defer spin.stop() + + progress := func(message string) { + console.progress(message) + } + preflight, err := runBuildModulePreflight(ctx, p, progress) + if err != nil { + return err + } + + result, err := pipeline.BuildModuleWithProgress(p, func(message string) { + progress(message) + }) + if err != nil { + return err + } + + console.emitBuildModuleResult(result, preflight) + return nil +} + +type buildModulePreflightResult struct { + ManifestStatus string + ManifestPath string +} + +func runBuildModulePreflight(ctx context, p *project.Project, progress func(string)) (buildModulePreflightResult, error) { + result := buildModulePreflightResult{} + if progress == nil { + progress = func(string) {} + } + manifestStatus, manifestPath, err := refreshBuildModuleManifest(ctx, p, progress) + if err != nil { + return result, err + } + result.ManifestStatus = manifestStatus + result.ManifestPath = manifestPath + return result, nil +} + +func refreshBuildModuleManifest(ctx context, p *project.Project, progress func(string)) (string, string, error) { + manifestPath := p.HAKManifestPath() + if override := strings.TrimSpace(os.Getenv("SOW_HAK_MANIFEST_PATH")); override != "" { + manifestPath = override + if !filepath.IsAbs(manifestPath) { + manifestPath = filepath.Join(p.Root, manifestPath) + } + } + + if envEnabled("SOW_MODULE_SKIP_HAK_MANIFEST_REFRESH") { + if _, err := os.Stat(manifestPath); err != nil { + return "", "", fmt.Errorf("SOW_MODULE_SKIP_HAK_MANIFEST_REFRESH is set, but %s does not exist", manifestPath) + } + progress(fmt.Sprintf("Using existing hak manifest at %s; skipping published sow-assets refresh.", relPathFromRoot(p.Root, manifestPath))) + return "reused", manifestPath, nil + } + + progress("Refreshing hak list from the latest published sow-assets manifest...") + if err := runProjectScript(ctx, p, []string{"scripts", "fetch-hak-manifest"}, manifestPath); err != nil { + return "", "", err + } + if _, err := pipeline.ApplyHAKManifest(p, manifestPath); err != nil { + return "", "", err + } + return "refreshed", manifestPath, nil +} + +func runProjectScript(ctx context, p *project.Project, scriptParts []string, args ...string) error { + if len(scriptParts) == 0 { + return errors.New("project script path is required") + } + scriptPath := projectScriptPath(p.Root, scriptParts...) + if !fileExists(scriptPath) { + return fmt.Errorf("required project script is missing: %s", scriptPath) + } + + var cmd *exec.Cmd + switch filepath.Ext(scriptPath) { + case ".ps1": + powershell := "powershell" + if runtime.GOOS != "windows" { + powershell = "pwsh" + } + commandArgs := []string{"-NoProfile", "-File", scriptPath} + commandArgs = append(commandArgs, args...) + cmd = exec.Command(powershell, commandArgs...) + default: + cmd = exec.Command(scriptPath, args...) + } + cmd.Dir = p.Root + cmd.Stdout = ctx.stderr + cmd.Stderr = ctx.stderr + return cmd.Run() +} + +func projectScriptPath(root string, pathParts ...string) string { + parts := append([]string{root}, pathParts...) + base := filepath.Join(parts...) + if runtime.GOOS == "windows" { + if strings.HasSuffix(base, ".ps1") || strings.HasSuffix(base, ".sh") { + return base + } + return base + ".ps1" + } + if strings.HasSuffix(base, ".ps1") || strings.HasSuffix(base, ".sh") { + return base + } + return base + ".sh" +} + +func envEnabled(name string) bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(name))) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} + +func fileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} + +func relPathFromRoot(root, path string) string { + rel, err := filepath.Rel(root, path) + if err != nil { + return filepath.ToSlash(path) + } + return filepath.ToSlash(rel) +} + +type buildHAKOptions struct { + filteredHAKs []string + filteredArchives []string + sourceManifest string + musicDatasets []string + skipMusic bool + planOnly bool + logLevel logLevel +} + +type logLevel int + +const ( + logLevelNormal logLevel = iota + logLevelQuiet + logLevelVerbose + logLevelDebug +) + +type projectConsole struct { + stdout io.Writer + projectRoot string + projectName string + commandName string + commandLabel string + level logLevel + spinnerEnabled bool +} + +func newProjectConsole(ctx context, p *project.Project, commandName string) *projectConsole { + return &projectConsole{ + stdout: ctx.stdout, + projectRoot: p.Root, + projectName: p.Config.Module.Name, + commandName: commandName, + commandLabel: projectCommandLabel(commandName), + level: ctx.logLevel, + spinnerEnabled: spinnerEnabledFor(ctx.stderr, ctx.logLevel), + } +} + +func projectCommandLabel(commandName string) string { + switch commandName { + case "build": + return "Build" + case "build-module": + return "Build Module" + case "extract": + return "Extract" + case "validate": + return "Validate" + case "compare": + return "Compare" + case "apply-hak-manifest": + return "Apply HAK Manifest" + default: + return commandName + } +} + +func (c *projectConsole) progress(message string) { + if phase := c.phaseLabel(message); phase != "" { + spin.update(c.commandLabel + ": " + phase) + } + if c.level != logLevelDebug { + return + } + spin.linebreak() + fmt.Fprintf(c.stdout, "[debug] %s\n", message) +} + +func (c *projectConsole) phaseLabel(message string) string { + switch { + case strings.HasPrefix(message, "Refreshing hak list from the latest published sow-assets manifest"): + return "refreshing HAK manifest" + case strings.HasPrefix(message, "Using existing hak manifest at "): + return "reusing HAK manifest" + case strings.HasPrefix(message, "Validating project"): + return "validating project" + case strings.HasPrefix(message, "Resolving module HAK order"): + return "resolving HAK order" + case strings.HasPrefix(message, "Collecting module resources"): + return "collecting module resources" + case strings.HasPrefix(message, "Writing module archive"): + return "writing module archive" + default: + return "" + } +} + +func (c *projectConsole) emitBuildResult(result pipeline.BuildResult, preflight buildModulePreflightResult) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Build ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "module: %s\n", c.relPath(result.ModulePath)) + fmt.Fprintf(c.stdout, "resources: %d\n", result.Resources) + if preflight.ManifestStatus != "" { + fmt.Fprintf(c.stdout, "hak manifest: %s", preflight.ManifestStatus) + if preflight.ManifestPath != "" { + fmt.Fprintf(c.stdout, " (%s)", c.relPath(preflight.ManifestPath)) + } + fmt.Fprintln(c.stdout) + } + if result.TopPackageHAK != "" { + fmt.Fprintf(c.stdout, "top package hak: %s\n", c.relPath(result.TopPackageHAK)) + fmt.Fprintf(c.stdout, "top package tlk: %s\n", c.relPath(result.TopPackageTLK)) + } + if len(result.HAKPaths) > 0 { + fmt.Fprintf(c.stdout, "hak archives: %d\n", len(result.HAKPaths)) + fmt.Fprintf(c.stdout, "hak assets: %d\n", result.HAKAssets) + fmt.Fprintf(c.stdout, "hak manifest: %s\n", c.relPath(result.Manifest)) + } +} + +func (c *projectConsole) emitBuildModuleResult(result pipeline.BuildResult, preflight buildModulePreflightResult) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Build Module ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "module: %s\n", c.relPath(result.ModulePath)) + fmt.Fprintf(c.stdout, "resources: %d\n", result.Resources) + if preflight.ManifestStatus != "" { + fmt.Fprintf(c.stdout, "hak manifest: %s", preflight.ManifestStatus) + if preflight.ManifestPath != "" { + fmt.Fprintf(c.stdout, " (%s)", c.relPath(preflight.ManifestPath)) + } + fmt.Fprintln(c.stdout) + } +} + +func (c *projectConsole) emitExtractResult(result pipeline.ExtractResult) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Extract ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "module: %s\n", c.relPath(result.ModulePath)) + if len(result.HAKPaths) > 0 { + fmt.Fprintf(c.stdout, "hak archives: %d\n", len(result.HAKPaths)) + } + if len(result.DeletedArchivePaths) > 0 { + fmt.Fprintf(c.stdout, "deleted archives: %d\n", len(result.DeletedArchivePaths)) + } + fmt.Fprintf(c.stdout, "written: %d\n", result.Written) + fmt.Fprintf(c.stdout, "overwritten: %d\n", result.Overwritten) + fmt.Fprintf(c.stdout, "removed: %d\n", result.Removed) + fmt.Fprintf(c.stdout, "skipped: %d\n", result.Skipped) +} + +func (c *projectConsole) emitValidationResult(report validator.Report, root string, inventoryReport project.InventoryReport) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Validate ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "root: %s\n", c.relPath(root)) + fmt.Fprintf(c.stdout, "source files: %d\n", inventoryReport.SourceFiles) + fmt.Fprintf(c.stdout, "script files: %d\n", inventoryReport.ScriptFiles) + fmt.Fprintf(c.stdout, "asset files: %d\n", inventoryReport.AssetFiles) + fmt.Fprintf(c.stdout, "module file types: %s\n", strings.Join(inventoryReport.Extensions, ", ")) + if warnings := report.WarningCount(); warnings > 0 { + fmt.Fprintf(c.stdout, "warnings: %d\n", warnings) + } + fmt.Fprintln(c.stdout, "validation: ok") +} + +func (c *projectConsole) emitCompareResult(result pipeline.CompareResult) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Compare ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "module: %s\n", c.relPath(result.ModulePath)) + if len(result.HAKPaths) > 0 { + fmt.Fprintf(c.stdout, "hak archives: %d\n", len(result.HAKPaths)) + } + fmt.Fprintf(c.stdout, "checked resources: %d\n", result.Checked) + fmt.Fprintln(c.stdout, "compare: ok") +} + +func (c *projectConsole) emitApplyManifestResult(result pipeline.ApplyManifestResult) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Apply HAK Manifest ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "manifest: %s\n", c.relPath(result.ManifestPath)) + fmt.Fprintf(c.stdout, "module source: %s\n", c.relPath(result.ModuleSource)) + fmt.Fprintf(c.stdout, "hak entries: %d\n", result.HAKCount) +} + +func (c *projectConsole) relPath(path string) string { + if path == "" { + return path + } + rel, err := filepath.Rel(c.projectRoot, path) + if err != nil { + return filepath.ToSlash(path) + } + return filepath.ToSlash(rel) +} + +type buildHAKConsole struct { + stdout io.Writer + projectRoot string + projectName string + planOnly bool + level logLevel + spinnerEnabled bool +} + +func newBuildHAKConsole(ctx context, p *project.Project, opts buildHAKOptions) *buildHAKConsole { + return &buildHAKConsole{ + stdout: ctx.stdout, + projectRoot: p.Root, + projectName: p.Config.Module.Name, + planOnly: opts.planOnly, + level: opts.logLevel, + spinnerEnabled: spinnerEnabledFor(ctx.stderr, opts.logLevel), + } +} + +func (c *buildHAKConsole) progress(message string) { + if phase := c.phaseLabel(message); phase != "" { + spin.update("Build HAKs: " + phase) + } + if c.level != logLevelDebug { + return + } + spin.linebreak() + fmt.Fprintf(c.stdout, "[debug] %s\n", message) +} + +func (c *buildHAKConsole) phaseLabel(message string) string { + switch { + case strings.HasPrefix(message, "Validating project"): + return "validating project" + case strings.HasPrefix(message, "Collecting asset resources"): + return "collecting asset resources" + case strings.HasPrefix(message, "Planning HAK chunks"): + return "planning HAK chunks" + case strings.HasPrefix(message, "Resolving module HAK order"): + return "resolving module HAK order" + case strings.HasPrefix(message, "Reusing unchanged generated HAKs"): + return "reusing unchanged HAKs" + case strings.HasPrefix(message, "Reusing HAK "), strings.HasPrefix(message, "Writing HAK "): + return "writing HAK archives" + case strings.HasPrefix(message, "Cleaning stale generated HAKs"): + return "cleaning stale generated HAKs" + case strings.HasPrefix(message, "Cleaning previous generated HAKs"): + return "cleaning previous generated HAKs" + case strings.HasPrefix(message, "Writing HAK manifest"): + return "writing HAK manifest" + case strings.HasPrefix(message, "Writing autogen manifest "): + return "writing autogen manifests" + default: + return "" + } +} + +func (c *buildHAKConsole) emitResult(result pipeline.BuildResult) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Build HAKs ----------") + if c.level != logLevelQuiet { + c.emitCreditsSummary(result) + c.emitCoreSteps(result) + } + c.emitFinalSummary(result) +} + +func (c *buildHAKConsole) emitCreditsSummary(result pipeline.BuildResult) { + summary := result.CreditsSummary + if len(summary.ArtifactPaths) == 0 { + return + } + fmt.Fprintln(c.stdout, "Refreshing credits cache") + if len(summary.ScannedDirs) > 0 { + fmt.Fprintf(c.stdout, " scanned: %s\n", strings.Join(summary.ScannedDirs, ", ")) + } + fmt.Fprintf(c.stdout, " tracks: %d\n", summary.TrackCount) + if len(summary.GeneratedPaths) > 0 { + for _, path := range summary.GeneratedPaths { + fmt.Fprintf(c.stdout, " output: %s\n", c.relPath(path)) + } + } + switch { + case summary.ChangedFiles == 0: + fmt.Fprintln(c.stdout, " status: unchanged") + default: + fmt.Fprintf(c.stdout, " updated: %d artifact(s)\n", summary.ChangedFiles) + } + if summary.TrackCount > 0 { + if c.level >= logLevelVerbose { + fmt.Fprintln(c.stdout, " mappings:") + for _, mapping := range summary.Mappings { + fmt.Fprintf(c.stdout, " %s -> %s\n", mapping.Source, mapping.Output) + } + } else { + fmt.Fprintf(c.stdout, " mapped: %d music file(s); use --verbose to list mappings\n", summary.TrackCount) + } + } + fmt.Fprintln(c.stdout) +} + +func (c *buildHAKConsole) emitCoreSteps(result pipeline.BuildResult) { + fmt.Fprintln(c.stdout, "Validating project") + fmt.Fprintln(c.stdout, "Collecting asset resources") + if len(result.AutogenManifestPaths) > 0 { + fmt.Fprintln(c.stdout, "Writing autogen manifests") + for _, path := range result.AutogenManifestPaths { + fmt.Fprintf(c.stdout, " %s\n", filepath.Base(path)) + } + fmt.Fprintln(c.stdout) + } + fmt.Fprintln(c.stdout, "Planning HAK chunks") + fmt.Fprintln(c.stdout, "Resolving module HAK order") + if !c.planOnly { + fmt.Fprintln(c.stdout, "Reusing unchanged HAKs") + fmt.Fprintf(c.stdout, " reused: %d / %d\n", result.HAKSummary.Reused, result.HAKSummary.Total) + fmt.Fprintf(c.stdout, " written: %d / %d\n", result.HAKSummary.Written, result.HAKSummary.Total) + fmt.Fprintf(c.stdout, " assets: %s\n", formatCount(result.HAKAssets)) + if c.level >= logLevelVerbose { + for _, action := range result.HAKSummary.Actions { + verb := "wrote" + if action.Reused { + verb = "reused" + } + fmt.Fprintf(c.stdout, " %s: %s (%s assets)\n", verb, action.Name, formatCount(action.AssetCount)) + } + } + fmt.Fprintln(c.stdout) + fmt.Fprintln(c.stdout, "Cleaning stale generated HAKs") + } else { + fmt.Fprintf(c.stdout, "Planned HAK archives: %d\n", result.HAKSummary.Total) + } + fmt.Fprintln(c.stdout, "Writing HAK manifest") + fmt.Fprintln(c.stdout) +} + +func (c *buildHAKConsole) emitFinalSummary(result pipeline.BuildResult) { + fmt.Fprintln(c.stdout, "Summary ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + if c.planOnly { + fmt.Fprintf(c.stdout, "planned archives: %d\n", result.HAKSummary.Total) + } else { + fmt.Fprintf(c.stdout, "hak archives: %d\n", len(result.HAKPaths)) + fmt.Fprintf(c.stdout, "hak assets: %s\n", formatCount(result.HAKAssets)) + } + if result.Manifest != "" { + fmt.Fprintf(c.stdout, "manifest: %s\n", c.relPath(result.Manifest)) + } + if len(result.CreditsArtifactPaths) > 0 { + fmt.Fprintf(c.stdout, "credits artifacts: %d\n", len(result.CreditsArtifactPaths)) + } + if len(result.AutogenManifestPaths) > 0 { + fmt.Fprintf(c.stdout, "autogen manifests: %d\n", len(result.AutogenManifestPaths)) + } + status := "complete" + if c.planOnly { + status = "planned" + } + fmt.Fprintf(c.stdout, "status: %s\n", status) +} + +func (c *buildHAKConsole) relPath(path string) string { + if path == "" { + return path + } + rel, err := filepath.Rel(c.projectRoot, path) + if err != nil { + return filepath.ToSlash(path) + } + return filepath.ToSlash(rel) +} + +type topdataConsole struct { + stdout io.Writer + projectRoot string + projectName string + commandName string + commandLabel string + level logLevel + spinnerEnabled bool +} + +func newTopdataConsole(ctx context, p *project.Project, commandName string) *topdataConsole { + return &topdataConsole{ + stdout: ctx.stdout, + projectRoot: p.Root, + projectName: p.Config.Module.Name, + commandName: commandName, + commandLabel: topdataCommandLabel(commandName), + level: ctx.logLevel, + spinnerEnabled: spinnerEnabledFor(ctx.stderr, ctx.logLevel), + } +} + +func topdataCommandLabel(commandName string) string { + switch commandName { + case "validate-topdata": + return "Validate Topdata" + case "build-topdata": + return "Build Topdata" + case "build-top-package": + return "Build Top Package" + case "compare-topdata": + return "Compare Topdata" + case "build-wiki": + return "Build Wiki" + case "deploy-wiki": + return "Deploy Wiki" + default: + return commandName + } +} + +func (c *topdataConsole) progress(message string) { + if phase := c.phaseLabel(message); phase != "" { + spin.update(c.commandLabel + ": " + phase) + } + if c.level != logLevelDebug { + return + } + spin.linebreak() + fmt.Fprintf(c.stdout, "[debug] %s\n", message) +} + +func (c *topdataConsole) phaseLabel(message string) string { + switch { + case strings.HasPrefix(message, "Building native topdata outputs"): + return "building native outputs" + case strings.HasPrefix(message, "Topdata outputs are current"): + return "checking cached outputs" + case strings.HasPrefix(message, "Compiled topdata outputs are current"): + return "refreshing top package" + case strings.HasPrefix(message, "Packaging compiled topdata resources into "): + return "packaging HAK" + case strings.HasPrefix(message, "Preparing compiled TLK release output "): + return "packaging TLK" + case strings.HasPrefix(message, "Building native wiki pages"): + return "building wiki pages" + case strings.HasPrefix(message, "NodeBB wiki plan: "): + return "planning wiki deploy" + case strings.HasPrefix(message, "Planning NodeBB wiki deploy"): + return "planning wiki deploy" + case strings.HasPrefix(message, "Planning NodeBB wiki page "): + return "planning wiki pages" + case strings.HasPrefix(message, "Executing NodeBB wiki actions"): + return "executing wiki deploy" + case strings.HasPrefix(message, "Executing NodeBB wiki action "): + return "executing wiki actions" + case strings.HasPrefix(message, "Listing NodeBB wiki namespace category "): + return "listing remote wiki pages" + case strings.HasPrefix(message, "Collecting "): + return "collecting pages" + default: + return "" + } +} + +func (c *topdataConsole) emitValidationResult(report topdata.ValidationReport, topdataRoot string) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Validate Topdata ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "topdata root: %s\n", c.relPath(topdataRoot)) + fmt.Fprintf(c.stdout, "topdata files: %d\n", report.Files) + fmt.Fprintf(c.stdout, "data files: %d\n", report.DataFiles) + fmt.Fprintf(c.stdout, "tlk files: %d\n", report.TLKFiles) + if warnings := report.WarningCount(); warnings > 0 { + fmt.Fprintf(c.stdout, "warnings: %d\n", warnings) + } + fmt.Fprintln(c.stdout, "topdata validation: ok") +} + +func (c *topdataConsole) emitPackageResult(result topdata.PackageResult) { + spin.linebreak() + fmt.Fprintf(c.stdout, "%s ----------\n", c.commandLabel) + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "mode: %s\n", result.Mode) + fmt.Fprintf(c.stdout, "topdata 2da output: %s\n", c.relPath(result.Output2DADir)) + fmt.Fprintf(c.stdout, "topdata tlk output: %s\n", c.relPath(result.OutputTLKDir)) + fmt.Fprintf(c.stdout, "top package hak: %s\n", c.relPath(result.OutputHAKPath)) + fmt.Fprintf(c.stdout, "top package tlk: %s\n", c.relPath(result.OutputTLKPath)) + fmt.Fprintf(c.stdout, "2da files: %d\n", result.Files2DA) + fmt.Fprintf(c.stdout, "tlk files: %d\n", result.FilesTLK) + fmt.Fprintf(c.stdout, "top package resources: %d\n", result.HAKResources) + fmt.Fprintf(c.stdout, "top package assets: %d\n", result.AssetFiles) + if result.WikiOutputDir != "" { + fmt.Fprintf(c.stdout, "wiki output: %s\n", c.relPath(result.WikiOutputDir)) + fmt.Fprintf(c.stdout, "wiki pages: %d\n", result.WikiPages) + fmt.Fprintf(c.stdout, "wiki status: %s\n", result.WikiStatus) + } +} + +func (c *topdataConsole) emitCompareResult(result topdata.CompareResult) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Compare Topdata ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "mode: %s\n", result.Mode) + fmt.Fprintf(c.stdout, "checked 2da files: %d\n", result.Compared2DA) + fmt.Fprintf(c.stdout, "checked tlk files: %d\n", result.ComparedTLK) + fmt.Fprintf(c.stdout, "native-pass: %d\n", result.NativePass) + fmt.Fprintf(c.stdout, "not-yet-canonical: %d\n", result.NotYetCanonical) + fmt.Fprintf(c.stdout, "native-mismatch: %d\n", result.NativeMismatch) + fmt.Fprintln(c.stdout, "topdata compare: ok") +} + +func (c *topdataConsole) emitWikiBuildResult(outputDir string, pageCount int, status string) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Build Wiki ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "wiki output: %s\n", c.relPath(outputDir)) + fmt.Fprintf(c.stdout, "wiki pages: %d\n", pageCount) + fmt.Fprintf(c.stdout, "wiki status: %s\n", status) +} + +func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skipped, stale, archived, purged, drifted int, manifest string) { + spin.linebreak() + fmt.Fprintln(c.stdout, "Deploy Wiki ----------") + fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) + fmt.Fprintf(c.stdout, "local pages: %d\n", localPages) + fmt.Fprintf(c.stdout, "created: %d\n", created) + fmt.Fprintf(c.stdout, "updated: %d\n", updated) + fmt.Fprintf(c.stdout, "skipped: %d\n", skipped) + fmt.Fprintf(c.stdout, "stale: %d\n", stale) + fmt.Fprintf(c.stdout, "archived: %d\n", archived) + fmt.Fprintf(c.stdout, "purged: %d\n", purged) + fmt.Fprintf(c.stdout, "drifted: %d\n", drifted) + fmt.Fprintf(c.stdout, "manifest: %s\n", c.relPath(manifest)) +} + +func (c *topdataConsole) relPath(path string) string { + if path == "" { + return path + } + rel, err := filepath.Rel(c.projectRoot, path) + if err != nil { + return filepath.ToSlash(path) + } + return filepath.ToSlash(rel) +} + +func formatCount(value int) string { + raw := strconv.Itoa(value) + if value < 1000 && value > -1000 { + return raw + } + sign := "" + if strings.HasPrefix(raw, "-") { + sign = "-" + raw = strings.TrimPrefix(raw, "-") + } + parts := make([]string, 0, (len(raw)+2)/3) + for len(raw) > 3 { + parts = append(parts, raw[len(raw)-3:]) + raw = raw[:len(raw)-3] + } + parts = append(parts, raw) + slices := make([]string, 0, len(parts)) + for index := len(parts) - 1; index >= 0; index-- { + slices = append(slices, parts[index]) + } + return sign + strings.Join(slices, ",") +} + +func runBuildHAKs(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + + opts, err := parseBuildHAKArgs(ctx.args[1:]) + if err != nil { + return err + } + if ctx.logLevel > opts.logLevel { + opts.logLevel = ctx.logLevel + } + p, err = p.CloneWithHAKNames(opts.filteredHAKs) + if err != nil { + return err + } + + console := newBuildHAKConsole(ctx, p, opts) + spin.configure(ctx.stderr, console.spinnerEnabled) + spin.start("Build HAKs: starting") + defer spin.stop() + var result pipeline.BuildResult + pipelineOpts := pipeline.BuildHAKOptions{ + Progress: console.progress, + ArchiveNames: opts.filteredArchives, + SourceManifestPath: opts.sourceManifest, + SkipMusic: opts.skipMusic, + MusicDatasetIDs: opts.musicDatasets, + } + if opts.planOnly { + result, err = pipeline.PlanHAKsWithOptions(p, pipelineOpts) + } else { + result, err = pipeline.BuildHAKsWithOptions(p, pipelineOpts) + } + if err != nil { + return err + } + + console.emitResult(result) + return nil +} + +func parseBuildHAKArgs(args []string) (buildHAKOptions, error) { + opts := buildHAKOptions{} + if len(args) == 0 { + return opts, nil + } + + for index := 0; index < len(args); index++ { + arg := args[index] + switch arg { + case "-h", "--help": + return opts, errors.New("usage: build-haks [--hak ...] [--archive ...] [--source-manifest ] [--plan-only] [--skip-music] [--music-dataset ...] [--quiet|--verbose|--debug]") + case "--hak": + index++ + if index >= len(args) { + return opts, errors.New("--hak requires a value") + } + opts.filteredHAKs = append(opts.filteredHAKs, args[index]) + case "--archive": + index++ + if index >= len(args) { + return opts, errors.New("--archive requires a value") + } + opts.filteredArchives = append(opts.filteredArchives, args[index]) + case "--plan-only": + opts.planOnly = true + case "--skip-music": + opts.skipMusic = true + case "--music-dataset": + index++ + if index >= len(args) { + return opts, errors.New("--music-dataset requires a value") + } + opts.musicDatasets = append(opts.musicDatasets, args[index]) + case "--quiet": + opts.logLevel = logLevelQuiet + case "--verbose": + opts.logLevel = logLevelVerbose + case "--debug": + opts.logLevel = logLevelDebug + case "--source-manifest": + index++ + if index >= len(args) { + return opts, errors.New("--source-manifest requires a value") + } + opts.sourceManifest = args[index] + default: + if value, ok, err := requireInlineFlagValue(arg, "--hak"); ok || err != nil { + if err != nil { + return opts, err + } + opts.filteredHAKs = append(opts.filteredHAKs, value) + continue + } + if value, ok, err := requireInlineFlagValue(arg, "--archive"); ok || err != nil { + if err != nil { + return opts, err + } + opts.filteredArchives = append(opts.filteredArchives, value) + continue + } + if value, ok, err := requireInlineFlagValue(arg, "--source-manifest"); ok || err != nil { + if err != nil { + return opts, err + } + opts.sourceManifest = value + continue + } + if value, ok, err := requireInlineFlagValue(arg, "--music-dataset"); ok || err != nil { + if err != nil { + return opts, err + } + opts.musicDatasets = append(opts.musicDatasets, value) + continue + } + if arg == "--quiet=true" { + opts.logLevel = logLevelQuiet + continue + } + return opts, fmt.Errorf("unknown build-haks argument %q", arg) + } + } + return opts, nil +} + +func parseInlineFlagValue(arg, name string) (string, bool) { + prefix := name + "=" + if !strings.HasPrefix(arg, prefix) { + return "", false + } + return strings.TrimSpace(strings.TrimPrefix(arg, prefix)), true +} + +func requireInlineFlagValue(arg, name string) (string, bool, error) { + value, ok := parseInlineFlagValue(arg, name) + if !ok { + return "", false, nil + } + if value == "" { + return "", true, fmt.Errorf("%s requires a value", name) + } + return value, true, nil +} + +type musicCommandOptions struct { + datasets []string + json bool + dryRun bool + check bool + force bool +} + +func runMusic(ctx context) error { + if len(ctx.args) < 2 { + return errors.New("usage: music [--dataset ] [--json] [--dry-run] [--check] [--force]") + } + p, err := loadProject(ctx) + if err != nil { + return err + } + if err := p.Scan(); err != nil { + return err + } + subcommand := ctx.args[1] + opts, err := parseMusicCommandArgs(ctx.args[2:]) + if err != nil { + return err + } + switch subcommand { + case "list-datasets": + return emitMusicDatasets(ctx, p, opts) + case "scan": + result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false}) + if err != nil { + return err + } + return emitMusicResult(ctx, "scan", result, opts) + case "build", "credits": + write := true + if subcommand == "build" && opts.dryRun { + write = false + } + if subcommand == "credits" && opts.check { + write = false + } + result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: write}) + if err != nil { + return err + } + return emitMusicResult(ctx, subcommand, result, opts) + case "manifest", "normalize": + result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false}) + if err != nil { + return err + } + return emitMusicResult(ctx, subcommand, result, opts) + case "validate": + if err := p.ValidateLayout(); err != nil { + return err + } + result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false}) + if err != nil { + return err + } + return emitMusicResult(ctx, "validate", result, opts) + default: + return fmt.Errorf("unknown music subcommand %q", subcommand) + } +} + +func parseMusicCommandArgs(args []string) (musicCommandOptions, error) { + opts := musicCommandOptions{} + for index := 0; index < len(args); index++ { + arg := args[index] + switch arg { + case "--dataset": + index++ + if index >= len(args) || strings.TrimSpace(args[index]) == "" { + return opts, errors.New("--dataset requires a value") + } + opts.datasets = append(opts.datasets, args[index]) + case "--json": + opts.json = true + case "--dry-run": + opts.dryRun = true + case "--check": + opts.check = true + case "--force": + opts.force = true + default: + if value, ok, err := requireInlineFlagValue(arg, "--dataset"); ok || err != nil { + if err != nil { + return opts, err + } + opts.datasets = append(opts.datasets, value) + continue + } + return opts, fmt.Errorf("unknown music argument %q", arg) + } + } + return opts, nil +} + +func emitMusicDatasets(ctx context, p *project.Project, opts musicCommandOptions) error { + effective := p.EffectiveConfig() + if opts.json { + return emitConfigValue(ctx.stdout, effective.Music.Datasets, "json") + } + if len(effective.Music.Datasets) == 0 { + fmt.Fprintln(ctx.stdout, "music datasets: none") + return nil + } + fmt.Fprintln(ctx.stdout, "music datasets:") + ids := make([]string, 0, len(effective.Music.Datasets)) + for id := range effective.Music.Datasets { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + ds := effective.Music.Datasets[id] + fmt.Fprintf(ctx.stdout, " %s: source=%s output=%s prefix=%s\n", id, ds.Source, ds.Output, ds.Prefix) + } + return nil +} + +func emitMusicResult(ctx context, action string, result pipeline.BuildResult, opts musicCommandOptions) error { + if opts.json { + return emitConfigValue(ctx.stdout, result.CreditsSummary, "json") + } + fmt.Fprintf(ctx.stdout, "music %s\n", action) + if len(result.CreditsSummary.ScannedDirs) > 0 { + fmt.Fprintf(ctx.stdout, "scanned: %s\n", strings.Join(result.CreditsSummary.ScannedDirs, ", ")) + } + fmt.Fprintf(ctx.stdout, "tracks: %d\n", result.CreditsSummary.TrackCount) + if action != "scan" && len(result.CreditsArtifactPaths) > 0 { + fmt.Fprintf(ctx.stdout, "credits artifacts: %d\n", len(result.CreditsArtifactPaths)) + } + if len(result.CreditsSummary.Mappings) > 0 && ctx.logLevel >= logLevelVerbose { + fmt.Fprintln(ctx.stdout, "mappings:") + for _, mapping := range result.CreditsSummary.Mappings { + fmt.Fprintf(ctx.stdout, " %s -> %s\n", mapping.Source, mapping.Output) + } + } + fmt.Fprintln(ctx.stdout, "status: ok") + return nil +} + +func runExtract(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + console := newProjectConsole(ctx, p, "extract") + + files := ctx.args[1:] + result, err := pipeline.Extract(p, files...) + if err != nil { + return err + } + + console.emitExtractResult(result) + return nil +} + +func runValidate(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + console := newProjectConsole(ctx, p, "validate") + + validationReport := validator.ValidateProject(p) + emitValidatorReport(ctx.stderr, validationReport) + if validationReport.HasErrors() { + return fmt.Errorf("validation failed with %d error(s)", validationReport.ErrorCount()) + } + + inventoryReport := p.Inventory.Report() + console.emitValidationResult(validationReport, p.Root, inventoryReport) + return nil +} + +func runConfig(ctx context) error { + if len(ctx.args) < 2 { + return errors.New("usage: config ...") + } + root, err := project.FindRoot(ctx.cwd) + if err != nil { + return err + } + p, err := project.Load(root) + if err != nil { + return err + } + + switch ctx.args[1] { + case "validate": + if len(ctx.args) > 2 { + return fmt.Errorf("usage: config validate") + } + if err := p.ValidateLayout(); err != nil { + return err + } + fmt.Fprintf(ctx.stdout, "config: ok\n") + fmt.Fprintf(ctx.stdout, "source: %s\n", p.ConfigSource.Path) + return nil + case "effective": + format, err := parseConfigFormatArgs("config effective", ctx.args[2:]) + if err != nil { + return err + } + return emitConfigValue(ctx.stdout, p.EffectiveConfig(), format) + case "inspect": + key := "" + format := "yaml" + for _, arg := range ctx.args[2:] { + switch arg { + case "--json": + format = "json" + case "--yaml": + format = "yaml" + default: + if key != "" { + return fmt.Errorf("usage: config inspect [] [--json|--yaml]") + } + key = arg + } + } + if key == "" { + return emitConfigValue(ctx.stdout, p.EffectiveConfig(), format) + } + value, _, ok := p.ExplainConfig(key) + if !ok { + return fmt.Errorf("unknown config key %q", key) + } + return emitConfigValue(ctx.stdout, value, format) + case "explain": + if len(ctx.args) != 3 { + return fmt.Errorf("usage: config explain ") + } + key := ctx.args[2] + value, provenance, ok := p.ExplainConfig(key) + if !ok { + return fmt.Errorf("unknown config key %q", key) + } + fmt.Fprintf(ctx.stdout, "key: %s\n", key) + fmt.Fprintf(ctx.stdout, "value: %s\n", formatConfigScalar(value)) + fmt.Fprintf(ctx.stdout, "source: %s\n", provenance.Source) + if provenance.Detail != "" { + fmt.Fprintf(ctx.stdout, "detail: %s\n", provenance.Detail) + } + if provenance.Deprecated { + fmt.Fprintf(ctx.stdout, "deprecated: true\n") + } + if rules := configValidationRules(key); len(rules) > 0 { + fmt.Fprintf(ctx.stdout, "validation: %s\n", strings.Join(rules, "; ")) + } + return nil + case "sources": + if len(ctx.args) > 2 { + return fmt.Errorf("usage: config sources") + } + fmt.Fprintf(ctx.stdout, "loaded: %s\n", p.ConfigSource.Path) + fmt.Fprintf(ctx.stdout, "name: %s\n", p.ConfigSource.Name) + fmt.Fprintf(ctx.stdout, "format: %s\n", p.ConfigSource.Format) + fmt.Fprintf(ctx.stdout, "legacy: %t\n", p.ConfigSource.Legacy) + fmt.Fprintf(ctx.stdout, "candidates: %s\n", strings.Join(project.ConfigFileCandidates, ", ")) + overrides := p.ActiveOverrides() + if len(overrides) == 0 { + fmt.Fprintln(ctx.stdout, "active overrides: none") + return nil + } + fmt.Fprintln(ctx.stdout, "active overrides:") + for _, override := range overrides { + fmt.Fprintf(ctx.stdout, " %s=%s (%s)\n", override.Key, override.Value, override.Source) + } + return nil + default: + return fmt.Errorf("unknown config subcommand %q", ctx.args[1]) + } +} + +func configValidationRules(key string) []string { + switch { + case strings.HasPrefix(key, "paths."), strings.Contains(key, ".root"), strings.Contains(key, ".dir"), strings.Contains(key, ".cache"): + return []string{"relative paths must not escape the repository root unless explicitly documented"} + case strings.HasPrefix(key, "outputs."): + return []string{"generated output names must use the configured extension and must not escape the repository root"} + case key == "extract.layout": + return []string{"supported value: nwn_canonical_json"} + case key == "extract.archives": + return []string{"build-relative .mod/.hak glob patterns; .erf is intentionally rejected"} + case key == "extract.consume_archives": + return []string{"when true, selected build archives are deleted only after successful extraction"} + case key == "extract.hak_discovery": + return []string{"deprecated; use extract.archives instead"} + case key == "autogen.cache.max_age": + return []string{"Go duration string, for example 1h or 30m"} + case strings.HasPrefix(key, "topdata.package_hak"): + return []string{"must be a .hak file name"} + case strings.HasPrefix(key, "topdata.package_tlk"), strings.HasPrefix(key, "topdata.compiled_tlk"): + return []string{"must be a .tlk file name"} + default: + return nil + } +} + +func parseConfigFormatArgs(commandName string, args []string) (string, error) { + format := "yaml" + for _, arg := range args { + switch arg { + case "--json": + format = "json" + case "--yaml": + format = "yaml" + default: + return "", fmt.Errorf("usage: %s [--json|--yaml]", commandName) + } + } + return format, nil +} + +func emitConfigValue(w io.Writer, value any, format string) error { + var ( + raw []byte + err error + ) + switch format { + case "json": + raw, err = json.MarshalIndent(value, "", " ") + case "yaml": + raw, err = yaml.Marshal(value) + default: + return fmt.Errorf("unsupported config output format %q", format) + } + if err != nil { + return err + } + if len(raw) == 0 || raw[len(raw)-1] != '\n' { + raw = append(raw, '\n') + } + _, err = w.Write(raw) + return err +} + +func formatConfigScalar(value any) string { + raw, err := json.Marshal(value) + if err != nil { + return fmt.Sprint(value) + } + return string(raw) +} + +func emitValidatorReport(w io.Writer, report validator.Report) { + lines := report.SummaryLines(5) + emitSummaryLines(w, lines, 20, "validation diagnostics") + if line := report.DecompiledSummaryLine(12); line != "" { + fmt.Fprintln(w, line) + } +} + +func emitTopdataValidationReport(w io.Writer, report topdata.ValidationReport) { + lines := report.SummaryLines(5) + emitSummaryLines(w, lines, 20, "topdata validation diagnostics") +} + +func emitSummaryLines(w io.Writer, lines []string, maxLines int, label string) { + if maxLines <= 0 || len(lines) <= maxLines { + for _, line := range lines { + fmt.Fprintln(w, line) + } + return + } + for _, line := range lines[:maxLines] { + fmt.Fprintln(w, line) + } + fmt.Fprintf(w, "info: %d additional %s omitted\n", len(lines)-maxLines, label) +} + +func runCompare(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + console := newProjectConsole(ctx, p, "compare") + + result, err := pipeline.Compare(p) + if err != nil { + return err + } + + console.emitCompareResult(result) + return nil +} + +func runApplyHAKManifest(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + console := newProjectConsole(ctx, p, "apply-hak-manifest") + + manifestPath := p.HAKManifestPath() + if len(ctx.args) > 1 { + manifestPath = ctx.args[1] + if !filepath.IsAbs(manifestPath) { + manifestPath = filepath.Join(ctx.cwd, manifestPath) + } + } + + result, err := pipeline.ApplyHAKManifest(p, manifestPath) + if err != nil { + return err + } + + console.emitApplyManifestResult(result) + return nil +} + +func runValidateTopData(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + + console := newTopdataConsole(ctx, p, "validate-topdata") + report := topdata.ValidateProject(p) + emitTopdataValidationReport(ctx.stderr, report) + if report.HasErrors() { + return fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount()) + } + + console.emitValidationResult(report, p.TopDataSourceDir()) + return nil +} + +func runBuildTopData(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + + opts, err := parseBuildTopDataArgs("build-topdata", ctx.args[1:]) + if err != nil { + return err + } + + console := newTopdataConsole(ctx, p, "build-topdata") + spin.configure(ctx.stderr, console.spinnerEnabled) + spin.start("Build Topdata: starting") + defer spin.stop() + result, err := topdata.BuildAndPackageWithOptions(p, opts, console.progress) + if err != nil { + return err + } + + console.emitPackageResult(result) + return nil +} + +func runBuildTopPackage(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + + opts, err := parseBuildTopDataArgs("build-top-package", ctx.args[1:]) + if err != nil { + return err + } + + if opts.BuildWiki { + return fmt.Errorf("--wiki is not supported with build-top-package; use build-topdata when wiki generation is required") + } + + console := newTopdataConsole(ctx, p, "build-top-package") + spin.configure(ctx.stderr, console.spinnerEnabled) + spin.start("Build Top Package: starting") + defer spin.stop() + result, err := topdata.BuildPackage(p, console.progress) + if err != nil { + return err + } + + console.emitPackageResult(result) + return nil +} + +func parseBuildTopDataArgs(commandName string, args []string) (topdata.BuildAndPackageOptions, error) { + opts := topdata.BuildAndPackageOptions{} + for _, arg := range args { + switch arg { + case "--force": + opts.Force = true + case "--wiki": + opts.BuildWiki = true + case "-h", "--help": + return opts, fmt.Errorf("usage: %s [--force] [--wiki]", commandName) + default: + return opts, fmt.Errorf("unknown %s argument %q", commandName, arg) + } + } + return opts, nil +} + +func runCompareTopData(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + + console := newTopdataConsole(ctx, p, "compare-topdata") + spin.configure(ctx.stderr, console.spinnerEnabled) + spin.start("Compare Topdata: starting") + defer spin.stop() + result, err := topdata.Compare(p, console.progress) + if err != nil { + return err + } + + console.emitCompareResult(result) + return nil +} + +func runConvertTopData(ctx context) error { + if len(ctx.args) < 2 { + return errors.New("usage: convert-topdata <2da-to-json|2da-to-module|json-to-2da> ...") + } + return topdata.RunConvertCommand(ctx.args[1:], ctx.stdout) +} + +func runBuildWiki(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + + opts, err := parseBuildWikiArgs("build-wiki", ctx.args[1:]) + if err != nil { + return err + } + + console := newTopdataConsole(ctx, p, "build-wiki") + spin.configure(ctx.stderr, console.spinnerEnabled) + spin.start("Build Wiki: starting") + defer spin.stop() + result, err := topdata.BuildWikiWithOptions(p, opts, console.progress) + if err != nil { + return err + } + + console.emitWikiBuildResult(result.OutputDir, result.PageCount, result.Status) + return nil +} + +func parseBuildWikiArgs(commandName string, args []string) (topdata.BuildWikiOptions, error) { + opts := topdata.BuildWikiOptions{} + for _, arg := range args { + switch arg { + case "--force": + opts.Force = true + case "-h", "--help": + return opts, fmt.Errorf("usage: %s [--force]", commandName) + default: + return opts, fmt.Errorf("unknown %s argument %q", commandName, arg) + } + } + return opts, nil +} + +func runDeployWiki(ctx context) error { + p, err := loadProject(ctx) + if err != nil { + return err + } + + opts, err := parseDeployWikiArgs("deploy-wiki", ctx.args[1:]) + if err != nil { + return err + } + + console := newTopdataConsole(ctx, p, "deploy-wiki") + spin.configure(ctx.stderr, console.spinnerEnabled) + spin.start("Deploy Wiki: starting") + defer spin.stop() + result, err := topdata.DeployWikiWithOptions(p, opts, console.progress) + if err != nil { + return err + } + + console.emitWikiDeployResult(result.LocalPages, result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Purged, result.Drifted, result.Manifest) + return nil +} + +func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiOptions, error) { + opts := topdata.DeployWikiOptions{} + for i := 0; i < len(args); i++ { + arg := args[i] + switch arg { + case "--source-dir": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--source-dir requires a value") + } + opts.SourceDir = args[i] + case "--endpoint": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--endpoint requires a value") + } + opts.Endpoint = args[i] + case "--username": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--username requires a value") + } + opts.Username = args[i] + case "--password": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--password requires a value") + } + opts.Password = args[i] + case "--token": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--token requires a value") + } + opts.Token = args[i] + case "--version": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--version requires a value") + } + opts.Version = args[i] + case "--namespace": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--namespace requires a value") + } + opts.Namespaces = append(opts.Namespaces, args[i]) + case "--category": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--category requires a namespace=cid value") + } + if opts.CategoryIDs == nil { + opts.CategoryIDs = map[string]int{} + } + if err := parseWikiCategoryArg(args[i], opts.CategoryIDs); err != nil { + return opts, err + } + case "--manifest": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--manifest requires a value") + } + opts.ManifestPath = args[i] + case "--stale-policy": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--stale-policy requires a value") + } + opts.StalePolicy = args[i] + case "--dry-run": + opts.DryRun = true + case "--force": + opts.Force = true + case "--create": + opts.AllowCreates = true + case "--reset-managed-namespaces": + opts.ResetManagedNamespaces = true + case "-h", "--help": + return opts, fmt.Errorf("usage: %s [--source-dir ] [--endpoint ] [--token ] [--version ] [--namespace ...] [--category ...] [--manifest ] [--stale-policy ] [--dry-run] [--create] [--force] [--reset-managed-namespaces]", commandName) + default: + if value, ok, err := requireInlineFlagValue(arg, "--source-dir"); ok || err != nil { + if err != nil { + return opts, err + } + opts.SourceDir = value + } else if value, ok, err := requireInlineFlagValue(arg, "--endpoint"); ok || err != nil { + if err != nil { + return opts, err + } + opts.Endpoint = value + } else if value, ok, err := requireInlineFlagValue(arg, "--username"); ok || err != nil { + if err != nil { + return opts, err + } + opts.Username = value + } else if value, ok, err := requireInlineFlagValue(arg, "--password"); ok || err != nil { + if err != nil { + return opts, err + } + opts.Password = value + } else if value, ok, err := requireInlineFlagValue(arg, "--token"); ok || err != nil { + if err != nil { + return opts, err + } + opts.Token = value + } else if value, ok, err := requireInlineFlagValue(arg, "--version"); ok || err != nil { + if err != nil { + return opts, err + } + opts.Version = value + } else if value, ok, err := requireInlineFlagValue(arg, "--namespace"); ok || err != nil { + if err != nil { + return opts, err + } + opts.Namespaces = append(opts.Namespaces, value) + } else if value, ok, err := requireInlineFlagValue(arg, "--category"); ok || err != nil { + if err != nil { + return opts, err + } + if opts.CategoryIDs == nil { + opts.CategoryIDs = map[string]int{} + } + if err := parseWikiCategoryArg(value, opts.CategoryIDs); err != nil { + return opts, err + } + } else if value, ok, err := requireInlineFlagValue(arg, "--manifest"); ok || err != nil { + if err != nil { + return opts, err + } + opts.ManifestPath = value + } else if value, ok, err := requireInlineFlagValue(arg, "--stale-policy"); ok || err != nil { + if err != nil { + return opts, err + } + opts.StalePolicy = value + } else if arg == "--dry-run" { + opts.DryRun = true + } else if arg == "--force" { + opts.Force = true + } else if arg == "--create" { + opts.AllowCreates = true + } else if arg == "--reset-managed-namespaces" { + opts.ResetManagedNamespaces = true + } else { + return opts, fmt.Errorf("unknown %s argument %q", commandName, arg) + } + } + } + + if opts.Endpoint == "" { + opts.Endpoint = os.Getenv("NODEBB_API_ENDPOINT") + } + if opts.Token == "" { + opts.Token = os.Getenv("NODEBB_API_TOKEN") + } + if opts.Version == "" { + opts.Version = os.Getenv("GITHUB_REF_NAME") + } + if opts.CategoryIDs == nil { + opts.CategoryIDs = map[string]int{} + } + if raw := os.Getenv("NODEBB_WIKI_CATEGORIES"); raw != "" { + if err := parseWikiCategoryList(raw, opts.CategoryIDs); err != nil { + return opts, err + } + } + + return opts, nil +} + +type buildChangelogOptions struct { + configPath string + outputPath string + currentTag string + previousTag string + apiBaseURL string + token string +} + +func runBuildChangelog(ctx context) error { + opts, err := parseBuildChangelogArgs("build-changelog", ctx.args[1:]) + if err != nil { + return err + } + + return changelog.Generate(changelog.Options{ + RepoRoot: ctx.cwd, + ConfigPath: opts.configPath, + OutputPath: opts.outputPath, + CurrentTag: opts.currentTag, + PreviousTag: opts.previousTag, + APIBaseURL: opts.apiBaseURL, + Token: opts.token, + Stdout: ctx.stdout, + }) +} + +func parseBuildChangelogArgs(commandName string, args []string) (buildChangelogOptions, error) { + opts := buildChangelogOptions{} + usage := fmt.Errorf("usage: %s [--config ] [--output ] [--current-tag ] [--previous-tag ] [--api-base-url ] [--token ]", commandName) + + for i := 0; i < len(args); i++ { + arg := args[i] + switch arg { + case "--config": + i++ + if i >= len(args) || strings.TrimSpace(args[i]) == "" { + return opts, usage + } + opts.configPath = args[i] + case "--output": + i++ + if i >= len(args) || strings.TrimSpace(args[i]) == "" { + return opts, usage + } + opts.outputPath = args[i] + case "--current-tag": + i++ + if i >= len(args) || strings.TrimSpace(args[i]) == "" { + return opts, usage + } + opts.currentTag = args[i] + case "--previous-tag": + i++ + if i >= len(args) || strings.TrimSpace(args[i]) == "" { + return opts, usage + } + opts.previousTag = args[i] + case "--api-base-url": + i++ + if i >= len(args) || strings.TrimSpace(args[i]) == "" { + return opts, usage + } + opts.apiBaseURL = args[i] + case "--token": + i++ + if i >= len(args) || strings.TrimSpace(args[i]) == "" { + return opts, usage + } + opts.token = args[i] + case "-h", "--help": + return opts, usage + default: + if value, ok, err := requireInlineFlagValue(arg, "--config"); ok || err != nil { + if err != nil { + return opts, err + } + opts.configPath = value + } else if value, ok, err := requireInlineFlagValue(arg, "--output"); ok || err != nil { + if err != nil { + return opts, err + } + opts.outputPath = value + } else if value, ok, err := requireInlineFlagValue(arg, "--current-tag"); ok || err != nil { + if err != nil { + return opts, err + } + opts.currentTag = value + } else if value, ok, err := requireInlineFlagValue(arg, "--previous-tag"); ok || err != nil { + if err != nil { + return opts, err + } + opts.previousTag = value + } else if value, ok, err := requireInlineFlagValue(arg, "--api-base-url"); ok || err != nil { + if err != nil { + return opts, err + } + opts.apiBaseURL = value + } else if value, ok, err := requireInlineFlagValue(arg, "--token"); ok || err != nil { + if err != nil { + return opts, err + } + opts.token = value + } else { + return opts, fmt.Errorf("unknown %s argument %q", commandName, arg) + } + } + } + + return opts, nil +} + +func parseWikiCategoryList(raw string, target map[string]int) error { + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if err := parseWikiCategoryArg(part, target); err != nil { + return err + } + } + return nil +} + +func parseWikiCategoryArg(raw string, target map[string]int) error { + namespace, rawCID, ok := strings.Cut(raw, "=") + if !ok { + return fmt.Errorf("wiki category mapping %q must be namespace=cid", raw) + } + namespace = strings.TrimSpace(namespace) + rawCID = strings.TrimSpace(rawCID) + if namespace == "" || rawCID == "" { + return fmt.Errorf("wiki category mapping %q must be namespace=cid", raw) + } + cid, err := strconv.Atoi(rawCID) + if err != nil || cid <= 0 { + return fmt.Errorf("wiki category mapping %q has invalid cid", raw) + } + target[namespace] = cid + return nil +} + +func loadProject(ctx context) (*project.Project, error) { + root, err := project.FindRoot(ctx.cwd) + if err != nil { + return nil, err + } + + p, err := project.Load(root) + if err != nil { + return nil, err + } + if ctx.stderr != nil && ctx.logLevel != logLevelQuiet { + fmt.Fprintf(ctx.stderr, "Loaded config: %s\n", p.ConfigSource.Name) + if p.ConfigSource.Legacy { + fmt.Fprintf(ctx.stderr, "Warning: %s is legacy repository configuration; migrate to %s.\n", project.LegacyConfigFile, project.ConfigFile) + } + } + + var failures []error + if err := p.ValidateLayout(); err != nil { + failures = append(failures, err) + } + if err := p.Scan(); err != nil { + failures = append(failures, err) + } + + if len(failures) > 0 { + return nil, errors.Join(failures...) + } + + return p, nil +} + +func printUsage(w io.Writer) { + exe := filepath.Base(os.Args[0]) + fmt.Fprintf(w, "%s \n\n", exe) + fmt.Fprintln(w, "Commands:") + for _, cmd := range commands { + fmt.Fprintf(w, " %-8s %s\n", cmd.name, cmd.description) + } +} diff --git a/internal/app/app_test.go b/internal/app/app_test.go new file mode 100644 index 0000000..89e3ffb --- /dev/null +++ b/internal/app/app_test.go @@ -0,0 +1,574 @@ +package app + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline" +) + +func TestRunBuildTopPackageUsesCachedCompiledOutputs(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "build")) + mkdirAll(t, filepath.Join(root, ".cache", "2da")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: test_module +topdata: + source: topdata + build: .cache + package_hak: sow_top.hak + package_tlk: sow_tlk.tlk +`) + writeFile(t, filepath.Join(root, ".cache", "2da", "repadjust.2da"), "2DA V2.0\n\n Label\n0 TEST_LABEL\n") + writeFile(t, filepath.Join(root, "build", "sow_tlk.tlk"), "compiled tlk") + writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.png"), "icon-data") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + + sourceTime := time.Now().Add(-2 * time.Hour) + outputTime := time.Now().Add(-1 * time.Hour) + setTreeTime(t, filepath.Join(root, "topdata"), sourceTime) + setFileTime(t, filepath.Join(root, ".cache", "2da", "repadjust.2da"), outputTime) + setFileTime(t, filepath.Join(root, "build", "sow_tlk.tlk"), outputTime) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"build-top-package"}, + } + + if err := runBuildTopPackage(ctx); err != nil { + t.Fatalf("runBuildTopPackage failed: %v", err) + } + output := stdout.String() + if !strings.Contains(output, "top package hak: build/sow_top.hak") { + t.Fatalf("expected build-top-package output, got %q", output) + } + if strings.Contains(output, "[build-top-package]") { + t.Fatalf("did not expect raw progress lines in normal output, got %q", output) + } + if _, err := os.Stat(filepath.Join(root, "build", "sow_top.hak")); err != nil { + t.Fatalf("expected packaged hak output: %v", err) + } +} + +func TestRunBuildHAKsEmitsCompactSummary(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets", "envi", "music", "westgate")) + mkdirAll(t, filepath.Join(root, "build")) + mkdirAll(t, filepath.Join(root, "src")) + + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +haks: + - name: envi + priority: 1 + max_bytes: 1048576 + split: false + include: + - envi/** +`) + writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "AleandAnecdotes.mp3"), "source-mp3") + writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "CREDITS.md"), "# NWN Music Pack Credits\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Darren Curtis | Ale and Anecdotes | `mus_wg_andnc.bmu` | `AleandAnecdotes.mp3` | Darren's Commercially Free Music | | Darren Curtis Music | |\n") + + ffprobePath := filepath.Join(root, "ffprobe") + writeFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Unknown\",\"title\":\"Broken Metadata\"}}}'\n") + if err := os.Chmod(ffprobePath, 0o755); err != nil { + t.Fatalf("chmod ffprobe: %v", err) + } + + ffmpegPath := filepath.Join(root, "ffmpeg") + writeFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'bmu-data' > \"$last\"\n") + if err := os.Chmod(ffmpegPath, 0o755); err != nil { + t.Fatalf("chmod ffmpeg: %v", err) + } + + t.Setenv("SOW_FFMPEG", ffmpegPath) + t.Setenv("SOW_FFPROBE", ffprobePath) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"build-haks"}, + } + + if err := runBuildHAKs(ctx); err != nil { + t.Fatalf("runBuildHAKs failed: %v", err) + } + + output := stdout.String() + if !strings.Contains(output, "Build HAKs ----------") { + t.Fatalf("expected build header, got %q", output) + } + if !strings.Contains(output, "mapped: 1 music file(s); use --verbose to list mappings") { + t.Fatalf("expected compact mapping summary, got %q", output) + } + if strings.Contains(output, "AleandAnecdotes.mp3 -> mus_wg_andnc.bmu") { + t.Fatalf("did not expect verbose mapping in normal mode, got %q", output) + } + if !strings.Contains(output, "manifest: build/haks.json") { + t.Fatalf("expected relative manifest path, got %q", output) + } +} + +func setTreeTime(t *testing.T, root string, modTime time.Time) { + t.Helper() + err := filepath.WalkDir(root, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + return os.Chtimes(path, modTime, modTime) + }) + if err != nil { + t.Fatalf("set tree time under %s: %v", root, err) + } +} + +func TestRunBuildHAKsVerboseListsMappings(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets", "envi", "music", "westgate")) + mkdirAll(t, filepath.Join(root, "build")) + mkdirAll(t, filepath.Join(root, "src")) + + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +haks: + - name: envi + priority: 1 + max_bytes: 1048576 + split: false + include: + - envi/** +`) + writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "AleandAnecdotes.mp3"), "source-mp3") + writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "CREDITS.md"), "# NWN Music Pack Credits\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Darren Curtis | Ale and Anecdotes | `mus_wg_andnc.bmu` | `AleandAnecdotes.mp3` | Darren's Commercially Free Music | | Darren Curtis Music | |\n") + + ffprobePath := filepath.Join(root, "ffprobe") + writeFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Unknown\",\"title\":\"Broken Metadata\"}}}'\n") + if err := os.Chmod(ffprobePath, 0o755); err != nil { + t.Fatalf("chmod ffprobe: %v", err) + } + + ffmpegPath := filepath.Join(root, "ffmpeg") + writeFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'bmu-data' > \"$last\"\n") + if err := os.Chmod(ffmpegPath, 0o755); err != nil { + t.Fatalf("chmod ffmpeg: %v", err) + } + + t.Setenv("SOW_FFMPEG", ffmpegPath) + t.Setenv("SOW_FFPROBE", ffprobePath) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"build-haks", "--verbose"}, + } + + if err := runBuildHAKs(ctx); err != nil { + t.Fatalf("runBuildHAKs failed: %v", err) + } + + output := stdout.String() + if !strings.Contains(output, "mappings:") { + t.Fatalf("expected verbose mappings header, got %q", output) + } + if !strings.Contains(output, "AleandAnecdotes.mp3 -> mus_wg_andnc.bmu") { + t.Fatalf("expected verbose mapping output, got %q", output) + } + if !strings.Contains(output, "wrote: envi (1 assets)") { + t.Fatalf("expected verbose archive action, got %q", output) + } +} + +func TestTopdataConsoleSuppressesProgressInNormalMode(t *testing.T) { + var stdout bytes.Buffer + console := &topdataConsole{ + stdout: &stdout, + projectRoot: "/workspace/project", + projectName: "Test Module", + commandName: "build-topdata", + commandLabel: "Build Topdata", + level: logLevelNormal, + } + + console.progress("Packaging compiled topdata resources into sow_top.hak...") + + if stdout.String() != "" { + t.Fatalf("expected no normal-mode progress output, got %q", stdout.String()) + } +} + +func TestSpinnerEnabledForHonorsPlainTTYMode(t *testing.T) { + t.Setenv("SOW_TOOLS_TTY_MODE", "plain") + if spinnerEnabledFor(&bytes.Buffer{}, logLevelNormal) { + t.Fatal("expected plain tty mode to disable spinner output") + } +} + +func TestTopdataConsoleDebugProgressAndRelativePaths(t *testing.T) { + var stdout bytes.Buffer + console := &topdataConsole{ + stdout: &stdout, + projectRoot: "/workspace/project", + projectName: "Test Module", + commandName: "deploy-wiki", + commandLabel: "Deploy Wiki", + level: logLevelDebug, + } + + console.progress("NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, purge 6, drift 0") + console.emitWikiDeployResult(10, 1, 2, 3, 4, 5, 6, 0, "/workspace/project/build/wiki/deploy-manifest.json") + + output := stdout.String() + if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, purge 6, drift 0") { + t.Fatalf("expected debug progress line, got %q", output) + } + if !strings.Contains(output, "archived: 5") { + t.Fatalf("expected archived deploy count, got %q", output) + } + if !strings.Contains(output, "purged: 6") { + t.Fatalf("expected purged deploy count, got %q", output) + } + if !strings.Contains(output, "manifest: build/wiki/deploy-manifest.json") { + t.Fatalf("expected relative deploy manifest path, got %q", output) + } +} + +func TestParseDeployWikiHelpListsPurgeStalePolicy(t *testing.T) { + _, err := parseDeployWikiArgs("deploy-wiki", []string{"--help"}) + if err == nil || !strings.Contains(err.Error(), "--stale-policy ") { + t.Fatalf("expected deploy-wiki help to list purge stale policy, got %v", err) + } +} + +func TestParseDeployWikiResetManagedNamespacesFlag(t *testing.T) { + opts, err := parseDeployWikiArgs("deploy-wiki", []string{"--reset-managed-namespaces"}) + if err != nil { + t.Fatalf("parse deploy wiki reset flag: %v", err) + } + if !opts.ResetManagedNamespaces { + t.Fatalf("expected --reset-managed-namespaces to enable namespace reset") + } +} + +func TestProjectConsoleSuppressesBuildModuleProgressInNormalMode(t *testing.T) { + var stdout bytes.Buffer + console := &projectConsole{ + stdout: &stdout, + projectRoot: "/workspace/project", + projectName: "Test Module", + commandName: "build-module", + commandLabel: "Build Module", + level: logLevelNormal, + } + + console.progress("Writing module archive...") + + if stdout.String() != "" { + t.Fatalf("expected no normal-mode progress output, got %q", stdout.String()) + } +} + +func TestProjectConsoleEmitsRelativePaths(t *testing.T) { + var stdout bytes.Buffer + console := &projectConsole{ + stdout: &stdout, + projectRoot: "/workspace/project", + projectName: "Test Module", + commandName: "compare", + commandLabel: "Compare", + level: logLevelNormal, + } + + console.emitCompareResult(pipeline.CompareResult{ + ModulePath: "/workspace/project/build/test.mod", + HAKPaths: []string{"/workspace/project/build/core.hak"}, + Checked: 42, + }) + + output := stdout.String() + if !strings.Contains(output, "module: build/test.mod") { + t.Fatalf("expected relative module path, got %q", output) + } +} + +func TestRunMusicScanUsesConfiguredDatasetsWithoutWriting(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets", "audio", "westgate")) + mkdirAll(t, filepath.Join(root, "build")) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + assets: assets + build: build +music: + datasets: + westgate_audio: + source: audio/westgate + output: generated/music + prefix: wg_ +`) + writeFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3") + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"music", "scan", "--dataset", "westgate_audio"}, + } + + if err := runMusic(ctx); err != nil { + t.Fatalf("runMusic failed: %v", err) + } + output := stdout.String() + if !strings.Contains(output, "music scan") || !strings.Contains(output, "tracks: 1") { + t.Fatalf("unexpected music scan output: %q", output) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "credits")); !os.IsNotExist(err) { + t.Fatalf("music scan should not write credits artifacts, stat err=%v", err) + } +} + +func TestRunMusicListDatasets(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + assets: assets +music: + datasets: + westgate_audio: + source: audio/westgate + output: generated/music + prefix: wg_ +`) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"music", "list-datasets"}, + } + + if err := runMusic(ctx); err != nil { + t.Fatalf("runMusic failed: %v", err) + } + if !strings.Contains(stdout.String(), "westgate_audio: source=audio/westgate output=generated/music prefix=wg_") { + t.Fatalf("unexpected dataset list: %q", stdout.String()) + } +} + +func TestRunConfigEffectiveJSONShowsDefaultsAndProvenance(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src +`) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"config", "effective", "--json"}, + } + + if err := runConfig(ctx); err != nil { + t.Fatalf("runConfig failed: %v", err) + } + output := stdout.String() + if !strings.Contains(output, `"hak_manifest": "haks.json"`) { + t.Fatalf("expected HAK manifest default in effective config, got %q", output) + } + if !strings.Contains(output, `"paths.build":`) || !strings.Contains(output, `"toolkit default"`) { + t.Fatalf("expected default provenance in effective config, got %q", output) + } +} + +func TestRunConfigExplainReportsYAMLSource(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + build: output +`) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"config", "explain", "paths.build"}, + } + + if err := runConfig(ctx); err != nil { + t.Fatalf("runConfig failed: %v", err) + } + output := stdout.String() + if !strings.Contains(output, "value: \"output\"") { + t.Fatalf("expected configured build value, got %q", output) + } + if !strings.Contains(output, "source: yaml") { + t.Fatalf("expected YAML source, got %q", output) + } +} + +func TestRunConfigValidateDoesNotScanSourceInventory(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "build")) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: missing-src + build: build +`) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"config", "validate"}, + } + + if err := runConfig(ctx); err != nil { + t.Fatalf("runConfig failed: %v", err) + } + if !strings.Contains(stdout.String(), "config: ok") { + t.Fatalf("expected config validation output, got %q", stdout.String()) + } +} + +func TestRunConfigSourcesListsActiveOverrides(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src +`) + t.Setenv("SOW_BUILD_HAKS_KEEP_EXISTING", "1") + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"config", "sources"}, + } + + if err := runConfig(ctx); err != nil { + t.Fatalf("runConfig failed: %v", err) + } + output := stdout.String() + if !strings.Contains(output, "active overrides:") { + t.Fatalf("expected active overrides section, got %q", output) + } + if !strings.Contains(output, "build.keep_existing_haks=1") { + t.Fatalf("expected keep existing override, got %q", output) + } +} + +func TestInlineFlagParsersRejectEmptyValues(t *testing.T) { + if _, err := parseBuildHAKArgs([]string{"--hak="}); err == nil || !strings.Contains(err.Error(), "--hak requires a value") { + t.Fatalf("expected empty --hak inline value error, got %v", err) + } + if _, err := parseMusicCommandArgs([]string{"--dataset="}); err == nil || !strings.Contains(err.Error(), "--dataset requires a value") { + t.Fatalf("expected empty --dataset inline value error, got %v", err) + } + if _, err := parseDeployWikiArgs("deploy-wiki", []string{"--endpoint="}); err == nil || !strings.Contains(err.Error(), "--endpoint requires a value") { + t.Fatalf("expected empty --endpoint inline value error, got %v", err) + } + if _, err := parseBuildChangelogArgs("build-changelog", []string{"--output="}); err == nil || !strings.Contains(err.Error(), "--output requires a value") { + t.Fatalf("expected empty --output inline value error, got %v", err) + } +} + +func TestParseBuildChangelogArgsAcceptsInlineValues(t *testing.T) { + opts, err := parseBuildChangelogArgs("build-changelog", []string{ + "--config=scripts/changelog.json", + "--output=CHANGELOG.md", + "--current-tag=v2", + "--previous-tag=v1", + "--api-base-url=https://gitea.example/api/v1", + "--token=secret", + }) + if err != nil { + t.Fatalf("parseBuildChangelogArgs failed: %v", err) + } + if opts.configPath != "scripts/changelog.json" || + opts.outputPath != "CHANGELOG.md" || + opts.currentTag != "v2" || + opts.previousTag != "v1" || + opts.apiBaseURL != "https://gitea.example/api/v1" || + opts.token != "secret" { + t.Fatalf("unexpected changelog opts: %#v", opts) + } +} + +func mkdirAll(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func setFileTime(t *testing.T, path string, modTime time.Time) { + t.Helper() + if err := os.Chtimes(path, modTime, modTime); err != nil { + t.Fatalf("set file time %s: %v", path, err) + } +} diff --git a/internal/changelog/changelog.go b/internal/changelog/changelog.go new file mode 100644 index 0000000..4df370a --- /dev/null +++ b/internal/changelog/changelog.go @@ -0,0 +1,455 @@ +package changelog + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" +) + +type Config struct { + RepoURL string `json:"repo_url"` + Categories []Category `json:"categories"` +} + +type Category struct { + Title string `json:"title"` + Sections []Section `json:"sections"` +} + +type Section struct { + Title string `json:"title"` + Paths []string `json:"paths"` +} + +type Options struct { + RepoRoot string + ConfigPath string + OutputPath string + CurrentTag string + PreviousTag string + APIBaseURL string + Token string + Stdout io.Writer +} + +type pullInfo struct { + AuthorName string +} + +type generator struct { + repoRoot string + config Config + currentTag string + previousTag string + apiBaseURL string + repoURL string + owner string + name string + token string + httpClient *http.Client + pullCache map[string]pullInfo +} + +type changelogEntry struct { + Title string + ReferenceLabel string + ReferenceURL string + ReferenceSort string + AuthorName string +} + +var pullNumberPattern = regexp.MustCompile(`\(#([0-9]+)\)`) + +func Generate(opts Options) error { + repoRoot := strings.TrimSpace(opts.RepoRoot) + if repoRoot == "" { + detectedRepoRoot, err := gitOutput("", "rev-parse", "--show-toplevel") + if err != nil { + return fmt.Errorf("detect repo root: %w", err) + } + repoRoot = strings.TrimSpace(detectedRepoRoot) + } + + configPath := opts.ConfigPath + if configPath == "" { + configPath = filepath.Join(repoRoot, "scripts", "changelog.json") + } + config, err := loadConfig(configPath) + if err != nil { + return err + } + + currentTag := strings.TrimSpace(opts.CurrentTag) + if currentTag == "" { + currentTag, err = gitOutput(repoRoot, "describe", "--tags", "--exact-match") + if err != nil { + return fmt.Errorf("detect current tag: %w", err) + } + currentTag = strings.TrimSpace(currentTag) + } + if currentTag == "" { + return fmt.Errorf("could not determine current tag") + } + + previousTag := strings.TrimSpace(opts.PreviousTag) + if previousTag == "" { + previousTag, err = previousTagFor(repoRoot, currentTag) + if err != nil { + return err + } + } + + repoURL := strings.TrimSpace(config.RepoURL) + if repoURL == "" { + return fmt.Errorf("config %s is missing repo_url", configPath) + } + apiBaseURL := strings.TrimSpace(opts.APIBaseURL) + if apiBaseURL == "" { + if apiBaseURL = strings.TrimSpace(os.Getenv("GITEA_SERVER_URL")); apiBaseURL == "" { + apiBaseURL = deriveAPIBaseURL(repoURL) + } + } + apiBaseURL = normalizeAPIBaseURL(apiBaseURL) + if apiBaseURL == "" { + return fmt.Errorf("could not determine Gitea API base URL") + } + + owner, name, err := ownerAndRepoFromURL(repoURL) + if err != nil { + return err + } + + g := generator{ + repoRoot: repoRoot, + config: config, + currentTag: currentTag, + previousTag: previousTag, + apiBaseURL: strings.TrimRight(apiBaseURL, "/"), + repoURL: strings.TrimRight(repoURL, "/"), + owner: owner, + name: name, + token: firstNonEmpty(opts.Token, os.Getenv("GITEA_API_TOKEN"), os.Getenv("GITEA_TOKEN"), os.Getenv("SOW_TOOLS_TOKEN")), + httpClient: http.DefaultClient, + pullCache: map[string]pullInfo{}, + } + + rendered, err := g.render() + if err != nil { + return err + } + + if opts.OutputPath != "" { + if err := os.MkdirAll(filepath.Dir(opts.OutputPath), 0o755); err != nil { + return fmt.Errorf("create output directory: %w", err) + } + if err := os.WriteFile(opts.OutputPath, rendered, 0o644); err != nil { + return fmt.Errorf("write changelog: %w", err) + } + return nil + } + + stdout := opts.Stdout + if stdout == nil { + stdout = os.Stdout + } + _, err = stdout.Write(rendered) + return err +} + +func loadConfig(path string) (Config, error) { + var config Config + data, err := os.ReadFile(path) + if err != nil { + return Config{}, fmt.Errorf("read changelog config %s: %w", path, err) + } + if err := json.Unmarshal(data, &config); err != nil { + return Config{}, fmt.Errorf("parse changelog config %s: %w", path, err) + } + if len(config.Categories) == 0 { + return Config{}, fmt.Errorf("changelog config %s does not define any categories", path) + } + return config, nil +} + +func previousTagFor(repoRoot string, currentTag string) (string, error) { + output, err := gitOutput(repoRoot, "for-each-ref", "--sort=-creatordate", "--format=%(refname:strip=2)", "refs/tags") + if err != nil { + return "", fmt.Errorf("list tags: %w", err) + } + + tags := strings.Fields(output) + for idx, tag := range tags { + if tag != currentTag { + continue + } + if idx+1 < len(tags) { + return tags[idx+1], nil + } + return "", nil + } + + return "", fmt.Errorf("current tag %s was not found in repository tag list", currentTag) +} + +func deriveAPIBaseURL(repoURL string) string { + parsed, err := url.Parse(repoURL) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "" + } + return fmt.Sprintf("%s://%s/api/v1", parsed.Scheme, parsed.Host) +} + +func normalizeAPIBaseURL(raw string) string { + raw = strings.TrimRight(strings.TrimSpace(raw), "/") + if raw == "" { + return "" + } + if strings.HasSuffix(raw, "/api/v1") { + return raw + } + return raw + "/api/v1" +} + +func ownerAndRepoFromURL(repoURL string) (string, string, error) { + parsed, err := url.Parse(repoURL) + if err != nil { + return "", "", fmt.Errorf("parse repo_url %s: %w", repoURL, err) + } + + parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") + if len(parts) < 2 { + return "", "", fmt.Errorf("repo_url %s does not include owner/repo", repoURL) + } + + owner := strings.TrimSpace(parts[0]) + name := strings.TrimSuffix(strings.TrimSpace(parts[1]), ".git") + if owner == "" || name == "" { + return "", "", fmt.Errorf("repo_url %s does not include owner/repo", repoURL) + } + return owner, name, nil +} + +func (g generator) render() ([]byte, error) { + publishDate, err := gitOutput(g.repoRoot, "log", "-1", "--format=%cs", g.currentTag) + if err != nil { + return nil, fmt.Errorf("resolve publish date: %w", err) + } + publishDate = strings.TrimSpace(publishDate) + + var buf bytes.Buffer + fmt.Fprintf(&buf, "# %s - %s\n\n", g.currentTag, publishDate) + if g.previousTag != "" { + fmt.Fprintf(&buf, "_Compared against %s_\n\n", g.previousTag) + } else { + buf.WriteString("_Initial tagged release_\n\n") + } + + for _, category := range g.config.Categories { + categoryText, err := g.renderCategory(category) + if err != nil { + return nil, err + } + if categoryText == "" { + continue + } + fmt.Fprintf(&buf, "## %s\n\n", category.Title) + buf.WriteString(categoryText) + } + + return buf.Bytes(), nil +} + +func (g generator) renderCategory(category Category) (string, error) { + var buf bytes.Buffer + for _, section := range category.Sections { + entries, err := g.sectionEntries(section) + if err != nil { + return "", err + } + if len(entries) == 0 { + continue + } + fmt.Fprintf(&buf, "### %s\n", section.Title) + for _, entry := range entries { + fmt.Fprintf(&buf, "- %s ([%s](%s)) - From %s\n", entry.Title, entry.ReferenceLabel, entry.ReferenceURL, entry.AuthorName) + } + buf.WriteString("\n") + } + return buf.String(), nil +} + +func (g generator) sectionEntries(section Section) ([]changelogEntry, error) { + if len(section.Paths) == 0 { + return nil, nil + } + + args := []string{"log", "--first-parent", "--pretty=format:%H%x1f%s%x1f%an"} + if g.previousTag != "" { + args = append(args, fmt.Sprintf("%s..%s", g.previousTag, g.currentTag)) + } else { + args = append(args, g.currentTag) + } + args = append(args, "--") + args = append(args, section.Paths...) + + output, err := gitOutput(g.repoRoot, args...) + if err != nil { + return nil, fmt.Errorf("list git log for section %s: %w", section.Title, err) + } + + seen := map[string]struct{}{} + var entries []changelogEntry + for _, line := range strings.Split(output, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + parts := strings.SplitN(line, "\x1f", 3) + commitHash := strings.TrimSpace(parts[0]) + subject := "" + if len(parts) > 1 { + subject = strings.TrimSpace(parts[1]) + } + fallbackAuthor := "" + if len(parts) > 2 { + fallbackAuthor = strings.TrimSpace(parts[2]) + } + matches := pullNumberPattern.FindStringSubmatch(subject) + entryKey := commitHash + if len(matches) == 2 { + entryKey = "pr:" + matches[1] + } + if _, exists := seen[entryKey]; exists { + continue + } + seen[entryKey] = struct{}{} + + entry := changelogEntry{ + Title: sanitizeSubject(subject), + AuthorName: fallbackAuthor, + } + + if len(matches) == 2 { + pullNumber := matches[1] + pullInfo, err := g.pullDetails(pullNumber) + if err != nil { + return nil, err + } + authorName := strings.TrimSpace(pullInfo.AuthorName) + if authorName == "" { + authorName = fallbackAuthor + } + entry.ReferenceLabel = "#" + pullNumber + entry.ReferenceURL = fmt.Sprintf("%s/pulls/%s", g.repoURL, pullNumber) + entry.ReferenceSort = "pr:" + pullNumber + entry.AuthorName = authorName + } else { + entry.ReferenceLabel = shortCommitHash(commitHash) + entry.ReferenceURL = fmt.Sprintf("%s/commit/%s", g.repoURL, commitHash) + entry.ReferenceSort = "commit:" + commitHash + } + + entries = append(entries, entry) + } + + sort.Slice(entries, func(i, j int) bool { + if entries[i].Title == entries[j].Title { + return entries[i].ReferenceSort < entries[j].ReferenceSort + } + return entries[i].Title < entries[j].Title + }) + + return entries, nil +} + +func sanitizeSubject(subject string) string { + title := strings.TrimSpace(subject) + title = pullNumberPattern.ReplaceAllString(title, "") + title = strings.TrimSpace(title) + title = strings.TrimPrefix(title, "Merge pull request") + title = strings.TrimSpace(title) + title = strings.Trim(title, `"'`) + title = strings.TrimSpace(title) + if title == "" { + return strings.TrimSpace(subject) + } + return title +} + +func shortCommitHash(commitHash string) string { + commitHash = strings.TrimSpace(commitHash) + if len(commitHash) <= 7 { + return commitHash + } + return commitHash[:7] +} + +func (g generator) pullDetails(pullNumber string) (pullInfo, error) { + if cached, ok := g.pullCache[pullNumber]; ok { + return cached, nil + } + + endpoint := fmt.Sprintf("%s/repos/%s/%s/pulls/%s", g.apiBaseURL, url.PathEscape(g.owner), url.PathEscape(g.name), url.PathEscape(pullNumber)) + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return pullInfo{}, fmt.Errorf("build Gitea request for pull #%s: %w", pullNumber, err) + } + req.Header.Set("Accept", "application/json") + if g.token != "" { + req.Header.Set("Authorization", "token "+g.token) + } + + resp, err := g.httpClient.Do(req) + if err != nil { + return pullInfo{}, fmt.Errorf("request Gitea pull #%s: %w", pullNumber, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return pullInfo{}, fmt.Errorf("request Gitea pull #%s: unexpected HTTP %d: %s", pullNumber, resp.StatusCode, strings.TrimSpace(string(body))) + } + + var payload struct { + User struct { + FullName string `json:"full_name"` + Login string `json:"login"` + } `json:"user"` + } + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return pullInfo{}, fmt.Errorf("decode Gitea pull #%s: %w", pullNumber, err) + } + + info := pullInfo{AuthorName: firstNonEmpty(strings.TrimSpace(payload.User.FullName), strings.TrimSpace(payload.User.Login))} + g.pullCache[pullNumber] = info + return info, nil +} + +func gitOutput(repoRoot string, args ...string) (string, error) { + cmd := exec.Command("git", args...) + if repoRoot != "" { + cmd.Dir = repoRoot + } + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("%s: %s", strings.Join(cmd.Args, " "), strings.TrimSpace(string(output))) + } + return string(output), nil +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} diff --git a/internal/changelog/changelog_test.go b/internal/changelog/changelog_test.go new file mode 100644 index 0000000..7f50ec3 --- /dev/null +++ b/internal/changelog/changelog_test.go @@ -0,0 +1,213 @@ +package changelog + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestSanitizeSubject(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + subject string + want string + }{ + { + name: "merge prefix with quotes", + subject: "Merge pull request 'Add classes overhaul (#123)'", + want: "Add classes overhaul", + }, + { + name: "plain subject", + subject: "Add classes overhaul (#123)", + want: "Add classes overhaul", + }, + { + name: "double quotes", + subject: `Merge pull request "Add classes overhaul (#123)"`, + want: "Add classes overhaul", + }, + { + name: "direct push", + subject: "Fix direct push handling", + want: "Fix direct push handling", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := sanitizeSubject(tt.subject); got != tt.want { + t.Fatalf("sanitizeSubject(%q) = %q, want %q", tt.subject, got, tt.want) + } + }) + } +} + +func TestShortCommitHash(t *testing.T) { + t.Parallel() + + if got := shortCommitHash("1234567890abcdef"); got != "1234567" { + t.Fatalf("shortCommitHash() = %q, want %q", got, "1234567") + } + if got := shortCommitHash("1234567"); got != "1234567" { + t.Fatalf("shortCommitHash() preserved %q unexpectedly as %q", "1234567", got) + } +} + +func TestOwnerAndRepoFromURL(t *testing.T) { + t.Parallel() + + owner, repo, err := ownerAndRepoFromURL("https://gitea.example.test/org/repo.git") + if err != nil { + t.Fatalf("ownerAndRepoFromURL returned error: %v", err) + } + if owner != "org" || repo != "repo" { + t.Fatalf("ownerAndRepoFromURL returned %q/%q", owner, repo) + } +} + +func TestDeriveAPIBaseURL(t *testing.T) { + t.Parallel() + + got := deriveAPIBaseURL("https://gitea.example.test/org/repo") + want := "https://gitea.example.test/api/v1" + if got != want { + t.Fatalf("deriveAPIBaseURL() = %q, want %q", got, want) + } +} + +func TestNormalizeAPIBaseURL(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want string + }{ + {input: "https://gitea.example.test", want: "https://gitea.example.test/api/v1"}, + {input: "https://gitea.example.test/api/v1", want: "https://gitea.example.test/api/v1"}, + } + + for _, tt := range tests { + if got := normalizeAPIBaseURL(tt.input); got != tt.want { + t.Fatalf("normalizeAPIBaseURL(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestGenerateIncludesPullRequestsAndDirectPushes(t *testing.T) { + t.Parallel() + + repoRoot := t.TempDir() + runGit(t, repoRoot, "init") + runGit(t, repoRoot, "config", "user.name", "Test User") + runGit(t, repoRoot, "config", "user.email", "test@example.com") + + writeFile(t, filepath.Join(repoRoot, "content", "entry.txt"), "initial\n") + runGit(t, repoRoot, "add", ".") + runGit(t, repoRoot, "commit", "-m", "Initial import") + runGit(t, repoRoot, "tag", "v1.0.0") + + writeFile(t, filepath.Join(repoRoot, "content", "entry.txt"), "direct\n") + runGit(t, repoRoot, "add", ".") + runGit(t, repoRoot, "commit", "-m", "Fix direct push handling") + directHash := strings.TrimSpace(runGit(t, repoRoot, "rev-parse", "HEAD")) + + writeFile(t, filepath.Join(repoRoot, "content", "entry.txt"), "pull\n") + runGit(t, repoRoot, "add", ".") + runGit(t, repoRoot, "commit", "-m", "Add release summary (#12)") + runGit(t, repoRoot, "tag", "v1.1.0") + + configPath := filepath.Join(repoRoot, "scripts", "changelog.json") + writeFile(t, configPath, `{ + "repo_url": "REPO_URL", + "categories": [ + { + "title": "Content", + "sections": [ + { + "title": "Entries", + "paths": ["content/"] + } + ] + } + ] +}`) + + var pullRequests int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/repos/org/repo/pulls/12" { + t.Fatalf("unexpected API path %q", r.URL.Path) + } + pullRequests++ + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"user":{"full_name":"Patch Author","login":"patch-author"}}`) + })) + defer server.Close() + + repoURL := server.URL + "/org/repo" + configData, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("read changelog config: %v", err) + } + configData = bytes.ReplaceAll(configData, []byte("REPO_URL"), []byte(repoURL)) + if err := os.WriteFile(configPath, configData, 0o644); err != nil { + t.Fatalf("write changelog config: %v", err) + } + + var stdout bytes.Buffer + err = Generate(Options{ + RepoRoot: repoRoot, + ConfigPath: configPath, + CurrentTag: "v1.1.0", + PreviousTag: "v1.0.0", + APIBaseURL: server.URL + "/api/v1", + Stdout: &stdout, + }) + if err != nil { + t.Fatalf("Generate() returned error: %v", err) + } + + rendered := stdout.String() + if !strings.Contains(rendered, "- Add release summary ([#12]("+repoURL+"/pulls/12)) - From Patch Author") { + t.Fatalf("rendered changelog missing pull entry:\n%s", rendered) + } + if !strings.Contains(rendered, "- Fix direct push handling (["+shortCommitHash(directHash)+"]("+repoURL+"/commit/"+directHash+")) - From Test User") { + t.Fatalf("rendered changelog missing direct-push entry:\n%s", rendered) + } + if pullRequests != 1 { + t.Fatalf("expected 1 pull lookup, got %d", pullRequests) + } +} + +func runGit(t *testing.T, repoRoot string, args ...string) string { + t.Helper() + + cmd := exec.Command("git", args...) + cmd.Dir = repoRoot + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, output) + } + return string(output) +} + +func writeFile(t *testing.T, path string, content string) { + t.Helper() + + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("MkdirAll(%q): %v", path, err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile(%q): %v", path, err) + } +} diff --git a/internal/dispatch/dispatch.go b/internal/dispatch/dispatch.go index 62ff338..ece2879 100644 --- a/internal/dispatch/dispatch.go +++ b/internal/dispatch/dispatch.go @@ -2,12 +2,11 @@ // builders shared by the `crucible` dispatcher and the standalone // `crucible-` shims. // -// Scaffold contract (Phase 5): every builder is registered but UNWIRED. Running -// a builder fails closed (exit 70) with an operator-cutover message; it never -// fakes an artifact. The internal pipeline/topdata/erf/wiki/music packages from -// gitea/sow-tools are migrated by the operator at cutover (the workspace hard -// rules forbid transplanting source here). Once a builder's handler is wired, -// flip its Wired flag and register the handler. +// Cutover state (Phase 5 → 6): the internal pipeline/topdata/erf/wiki/music +// packages migrated from gitea/sow-tools now live in this tree, so wired +// builders delegate to internal/app's legacy command surface (mapped per +// docs/command-surface.md). A builder with no migrated logic yet (depot) keeps +// the Wired=false fail-closed path: exit 70, never a faked artifact. package dispatch import ( @@ -16,6 +15,7 @@ import ( "os" "text/tabwriter" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/app" "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo" ) @@ -33,6 +33,21 @@ type Builder struct { Bin string // standalone binary name, e.g. "crucible-module" Summary string // one-line description Legacy []string // nwn-tool commands this subsumes (migration aid; see docs/command-surface.md) + Extra []string // additional callable subcommands beyond Legacy (e.g. cross-listed "music") + Wired bool // true once the builder delegates to migrated internal/app logic +} + +// subcommands returns every legacy command a wired builder accepts: the +// canonical Legacy set plus any cross-listed Extra commands. +func (b Builder) subcommands() []string { return append(append([]string{}, b.Legacy...), b.Extra...) } + +func (b Builder) accepts(sub string) bool { + for _, s := range b.subcommands() { + if s == sub { + return true + } + } + return false } // Registry is the single source of truth for the Crucible command surface. @@ -43,30 +58,39 @@ var Registry = []Builder{ Bin: "crucible-depot", Summary: "verify/move content-addressed depot blobs (SeaweedFS)", Legacy: nil, + Wired: false, // no migrated logic yet; fails closed (exit 70) }, { Name: "hak", Bin: "crucible-hak", Summary: "pack/unpack ERF/HAK archives + hak manifests", Legacy: []string{"build-haks", "apply-hak-manifest"}, + Extra: []string{"music"}, // music BMUs are packed into HAKs (command-surface.md) + Wired: true, }, { Name: "module", Bin: "crucible-module", Summary: "build/extract/validate/compare the .mod", Legacy: []string{"build", "build-module", "extract", "validate", "compare"}, + // apply-hak-manifest is canonically a hak command but reachable here too; + // music is folded into the module build pipeline (command-surface.md). + Extra: []string{"apply-hak-manifest", "music"}, + Wired: true, }, { Name: "topdata", Bin: "crucible-topdata", Summary: "compile 2da/tlk topdata + packages", Legacy: []string{"validate-topdata", "build-topdata", "build-top-package", "compare-topdata", "convert-topdata"}, + Wired: true, }, { Name: "wiki", Bin: "crucible-wiki", Summary: "render + deploy mechanical wiki pages", Legacy: []string{"build-wiki", "deploy-wiki"}, + Wired: true, }, } @@ -102,6 +126,13 @@ func run(args []string, out, errw io.Writer) int { case "list": list(out) return exitOK + case "config": + // Global concern shared by every builder (command-surface.md): the legacy + // `config` command group, surfaced verbatim. + return delegateLegacy(args, errw) + case "changelog": + // Release tooling: global alias for the legacy `build-changelog` command. + return delegateLegacy(append([]string{"build-changelog"}, args[1:]...), errw) } return runBuilder(args[0], args[1:], out, errw) } @@ -120,19 +151,43 @@ func runBuilder(name string, args []string, out, errw io.Writer) int { return exitOK } } - // Every builder is unwired in the Phase 5 scaffold: fail closed, never fake. - fmt.Fprintf(errw, unwiredMsg, b.Name, b.Bin) - return exitUnwired + if !b.Wired { + // No migrated logic yet (depot): fail closed, never fake an artifact. + fmt.Fprintf(errw, unwiredMsg, b.Name, b.Bin) + return exitUnwired + } + if len(args) == 0 { + fmt.Fprintf(errw, "crucible %s: a subcommand is required\n\n", b.Name) + builderHelp(errw, b) + return exitUsage + } + sub := args[0] + if !b.accepts(sub) { + fmt.Fprintf(errw, "crucible %s: unknown subcommand %q\n\n", b.Name, sub) + builderHelp(errw, b) + return exitUsage + } + // Delegate to the migrated legacy command surface. args[0] is already the + // legacy command name, so it is forwarded unchanged. + return delegateLegacy(args, errw) +} + +// delegateLegacy runs a migrated nwn-tool command via internal/app and maps its +// (code, err) result onto the dispatcher's exit code. app writes its own output +// to os.Stdout/os.Stderr (the writers the entrypoints pass through), so only the +// returned error is surfaced here. +func delegateLegacy(args []string, errw io.Writer) int { + code, err := app.Run(args) + if err != nil { + fmt.Fprintln(errw, err) + } + return code } const unwiredMsg = `crucible %[1]s: not wired yet. -The Crucible suite is scaffolded (Phase 5). The %[2]s implementation is migrated -from gitea/sow-tools internal packages at operator cutover; the workspace hard -rules forbid transplanting that source into this tree automatically. - -This command fails closed (exit 70) rather than producing a fake artifact. -See docs/migration-from-nwn-tool.md. +The %[2]s builder has no migrated logic in this tree yet, so it fails closed +(exit 70) rather than producing a fake artifact. See docs/command-surface.md. ` func usage(w io.Writer) { @@ -147,6 +202,8 @@ func usage(w io.Writer) { fmt.Fprintf(w, " help | -h show this help\n") fmt.Fprintf(w, " version | -V show the build version\n") fmt.Fprintf(w, " list list builders (one per line: namebinsummary)\n") + fmt.Fprintf(w, " config [args] inspect/validate effective configuration\n") + fmt.Fprintf(w, " changelog [args] generate the release changelog\n") fmt.Fprintf(w, "\nNWN_ROOT must be passed explicitly (env or flag); crucible never guesses\n") fmt.Fprintf(w, "from $HOME. See docs/consumer-contract.md.\n") } @@ -161,15 +218,15 @@ func list(w io.Writer) { func builderHelp(w io.Writer, b Builder) { fmt.Fprintf(w, "crucible %s (%s) — %s\n\n", b.Name, b.Bin, b.Summary) - if len(b.Legacy) > 0 { - fmt.Fprintf(w, "subsumes nwn-tool commands: ") - for i, c := range b.Legacy { - if i > 0 { - fmt.Fprintf(w, ", ") - } - fmt.Fprintf(w, "%s", c) - } - fmt.Fprintf(w, "\n\n") + if !b.Wired { + fmt.Fprintf(w, "status: not wired — no migrated logic yet; fails closed (exit 70).\n") + fmt.Fprintf(w, "See docs/command-surface.md.\n") + return } - fmt.Fprintf(w, "status: scaffolded, not wired (Phase 5). See docs/command-surface.md.\n") + fmt.Fprintf(w, "usage: crucible %s [args] (or: %s [args])\n\n", b.Name, b.Bin) + fmt.Fprintf(w, "subcommands:\n") + for _, c := range b.subcommands() { + fmt.Fprintf(w, " %s\n", c) + } + fmt.Fprintf(w, "\nstatus: wired to migrated nwn-tool logic. See docs/command-surface.md.\n") } diff --git a/internal/dispatch/dispatch_test.go b/internal/dispatch/dispatch_test.go index bb46dc4..f5ce543 100644 --- a/internal/dispatch/dispatch_test.go +++ b/internal/dispatch/dispatch_test.go @@ -55,8 +55,11 @@ func TestUnknownBuilderFailsUsage(t *testing.T) { } } -func TestEveryBuilderFailsClosed(t *testing.T) { +func TestUnwiredBuilderFailsClosed(t *testing.T) { for _, b := range Registry { + if b.Wired { + continue + } var out, errw bytes.Buffer // Via dispatcher. if code := run([]string{b.Name}, &out, &errw); code != exitUnwired { @@ -74,6 +77,28 @@ func TestEveryBuilderFailsClosed(t *testing.T) { } } +func TestWiredBuilderRejectsBadInvocation(t *testing.T) { + for _, b := range Registry { + if !b.Wired { + continue + } + // No subcommand is a usage error (it must never silently delegate). + var out, errw bytes.Buffer + if code := runBuilder(b.Name, nil, &out, &errw); code != exitUsage { + t.Errorf("crucible %s (no subcommand): exit=%d want %d", b.Name, code, exitUsage) + } + // Unknown subcommand is a usage error, not a delegate. + out.Reset() + errw.Reset() + if code := runBuilder(b.Name, []string{"frobnicate"}, &out, &errw); code != exitUsage { + t.Errorf("crucible %s frobnicate: exit=%d want %d", b.Name, code, exitUsage) + } + if !strings.Contains(errw.String(), "unknown subcommand") { + t.Errorf("crucible %s frobnicate: stderr=%q", b.Name, errw.String()) + } + } +} + func TestBuilderHelpIsOK(t *testing.T) { for _, b := range Registry { var out, errw bytes.Buffer @@ -85,3 +110,28 @@ func TestBuilderHelpIsOK(t *testing.T) { } } } + +// Every legacy command named in command-surface.md must have exactly one +// canonical home (Builder.Legacy), so the migrated surface has no gaps or +// ambiguous homes. +func TestLegacyCommandsHaveExactlyOneHome(t *testing.T) { + legacy := []string{ + "build", "build-module", "extract", "validate", "compare", + "build-haks", "apply-hak-manifest", + "validate-topdata", "build-topdata", "build-top-package", "compare-topdata", "convert-topdata", + "build-wiki", "deploy-wiki", + } + for _, cmd := range legacy { + homes := 0 + for _, b := range Registry { + for _, c := range b.Legacy { + if c == cmd { + homes++ + } + } + } + if homes != 1 { + t.Errorf("legacy command %q has %d canonical homes, want 1", cmd, homes) + } + } +} diff --git a/internal/erf/erf.go b/internal/erf/erf.go new file mode 100644 index 0000000..dd71df0 --- /dev/null +++ b/internal/erf/erf.go @@ -0,0 +1,424 @@ +package erf + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" + "sort" + "strings" +) + +const ( + headerSize = 160 + versionV10 = "V1.0" + strRefNone = 0xFFFFFFFF + typeMOD = "MOD " + typeHAK = "HAK " +) + +type Archive struct { + FileType string + Version string + Resources []Resource +} + +type Resource struct { + Name string + Type uint16 + Data []byte + SourcePath string + Size int64 +} + +type header struct { + FileType [4]byte + Version [4]byte + LanguageCount uint32 + LocalizedStringSize uint32 + EntryCount uint32 + LocalizedStringOffset uint32 + KeyListOffset uint32 + ResourceListOffset uint32 + BuildYear uint32 + BuildDay uint32 + DescriptionStrRef uint32 + Reserved [116]byte +} + +type keyEntry struct { + ResRef [16]byte + ResourceID uint32 + ResourceType uint16 + Unused uint16 +} + +type resourceEntry struct { + Offset uint32 + Size uint32 +} + +var extensionTypes = map[string]uint16{ + "res": 0x0000, + "bmp": 0x0001, + "mve": 0x0002, + "tga": 0x0003, + "wav": 0x0004, + "plt": 0x0006, + "ini": 0x0007, + "bmu": 0x0008, + "txt": 0x000A, + "mdl": 0x07D2, + "nss": 0x07D9, + "ncs": 0x07DA, + "are": 0x07DC, + "set": 0x07DD, + "ifo": 0x07DE, + "bic": 0x07DF, + "wok": 0x07E0, + "2da": 0x07E1, + "tlk": 0x07E2, + "txi": 0x07E6, + "git": 0x07E7, + "uti": 0x07E9, + "utc": 0x07EB, + "dlg": 0x07ED, + "itp": 0x07EE, + "utt": 0x07F0, + "dds": 0x07F1, + "uts": 0x07F3, + "fac": 0x07F6, + "gff": 0x07F7, + "ute": 0x07F8, + "utd": 0x07FA, + "utp": 0x07FC, + "dfa": 0x07FD, + "gic": 0x07FE, + "gui": 0x07FF, + "utm": 0x0803, + "dwk": 0x0804, + "pwk": 0x0805, + "utg": 0x0807, + "jrl": 0x0808, + "utw": 0x080A, + "ssf": 0x080C, + "hak": 0x080D, + "nwm": 0x080E, + "bik": 0x080F, + "ndb": 0x0810, + "ptm": 0x0811, + "ptt": 0x0812, + "ltr": 0x0813, + "shd": 0x0815, + "mdb": 0x0816, + "mtr": 0x0818, + "jpg": 0x081C, + "lod": 0x081E, + "png": 0x0820, + "lyt": 0x0BB8, + "vis": 0x0BB9, + "mdx": 0x0BC0, + "wlk": 0x0BCC, + "xml": 0x0BCD, + "gr2": 0x0FA3, +} + +var typeExtensions = map[uint16]string{ + 0x0000: "res", + 0x0001: "bmp", + 0x0002: "mve", + 0x0003: "tga", + 0x0004: "wav", + 0x0006: "plt", + 0x0007: "ini", + 0x0008: "bmu", + 0x000A: "txt", + 0x07D2: "mdl", + 0x07D9: "nss", + 0x07DA: "ncs", + 0x07DC: "are", + 0x07DD: "set", + 0x07DE: "ifo", + 0x07DF: "bic", + 0x07E0: "wok", + 0x07E1: "2da", + 0x07E2: "tlk", + 0x07E6: "txi", + 0x07E7: "git", + 0x07E9: "uti", + 0x07EB: "utc", + 0x07ED: "dlg", + 0x07EE: "itp", + 0x07F0: "utt", + 0x07F1: "dds", + 0x07F3: "uts", + 0x07F6: "fac", + 0x07F7: "gff", + 0x07F8: "ute", + 0x07FA: "utd", + 0x07FC: "utp", + 0x07FD: "dfa", + 0x07FE: "gic", + 0x07FF: "gui", + 0x0803: "utm", + 0x0804: "dwk", + 0x0805: "pwk", + 0x0807: "utg", + 0x0808: "jrl", + 0x080A: "utw", + 0x080C: "ssf", + 0x080D: "hak", + 0x080E: "nwm", + 0x080F: "bik", + 0x0810: "ndb", + 0x0811: "ptm", + 0x0812: "ptt", + 0x0813: "ltr", + 0x0815: "shd", + 0x0816: "mdb", + 0x0818: "mtr", + 0x081C: "jpg", + 0x081E: "lod", + 0x0820: "png", + 0x0BB8: "lyt", + 0x0BB9: "vis", + 0x0BC0: "mdx", + 0x0BCC: "wlk", + 0x0BCD: "xml", + 0x0FA3: "gr2", +} + +func init() { + for ext, resourceType := range extensionTypes { + canonicalExt, ok := typeExtensions[resourceType] + if !ok { + panic(fmt.Sprintf("missing canonical extension for resource type 0x%04X", resourceType)) + } + if canonicalExt == ext { + continue + } + } +} + +func New(fileType string, resources []Resource) Archive { + normalized := make([]Resource, len(resources)) + copy(normalized, resources) + sort.Slice(normalized, func(i, j int) bool { + if normalized[i].Name == normalized[j].Name { + return normalized[i].Type < normalized[j].Type + } + return normalized[i].Name < normalized[j].Name + }) + + return Archive{ + FileType: padFour(fileType), + Version: versionV10, + Resources: normalized, + } +} + +func ArchiveSize(resources []Resource) int64 { + return int64(headerSize+len(resources)*24+len(resources)*8) + totalResourceBytes(resources) +} + +func Write(w io.Writer, archive Archive) error { + if archive.Version == "" { + archive.Version = versionV10 + } + if archive.FileType == "" { + archive.FileType = typeMOD + } + + keys := make([]keyEntry, 0, len(archive.Resources)) + entries := make([]resourceEntry, 0, len(archive.Resources)) + + dataOffset := uint32(headerSize + len(archive.Resources)*24 + len(archive.Resources)*8) + for index, resource := range archive.Resources { + if len(resource.Name) > 16 { + return fmt.Errorf("resource %q exceeds 16-byte resref limit", resource.Name) + } + size, err := payloadSize(resource) + if err != nil { + return err + } + if size > int64(^uint32(0)) { + return fmt.Errorf("resource %q exceeds 4 GiB resource size limit", resource.Name) + } + + entry := resourceEntry{ + Offset: dataOffset, + Size: uint32(size), + } + entries = append(entries, entry) + dataOffset += uint32(size) + + var key keyEntry + copy(key.ResRef[:], []byte(resource.Name)) + key.ResourceID = uint32(index) + key.ResourceType = resource.Type + keys = append(keys, key) + } + + hdr := header{ + LanguageCount: 0, + LocalizedStringSize: 0, + EntryCount: uint32(len(archive.Resources)), + LocalizedStringOffset: headerSize, + KeyListOffset: headerSize, + ResourceListOffset: uint32(headerSize + len(keys)*24), + BuildYear: 0, + BuildDay: 0, + DescriptionStrRef: strRefNone, + } + copy(hdr.FileType[:], []byte(padFour(archive.FileType))) + copy(hdr.Version[:], []byte(padFour(archive.Version))) + + if err := binary.Write(w, binary.LittleEndian, hdr); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, keys); err != nil { + return err + } + if err := binary.Write(w, binary.LittleEndian, entries); err != nil { + return err + } + for _, resource := range archive.Resources { + if err := writeResourceData(w, resource); err != nil { + return err + } + } + return nil +} + +func Read(r io.Reader) (Archive, error) { + data, err := io.ReadAll(r) + if err != nil { + return Archive{}, fmt.Errorf("read erf: %w", err) + } + if len(data) < headerSize { + return Archive{}, fmt.Errorf("erf file too small: %d bytes", len(data)) + } + + var hdr header + if err := binary.Read(bytes.NewReader(data[:headerSize]), binary.LittleEndian, &hdr); err != nil { + return Archive{}, fmt.Errorf("decode erf header: %w", err) + } + + keyStart := int(hdr.KeyListOffset) + keyEnd := keyStart + int(hdr.EntryCount)*24 + if keyEnd > len(data) { + return Archive{}, fmt.Errorf("erf key list exceeds file bounds") + } + keys := make([]keyEntry, hdr.EntryCount) + if err := binary.Read(bytes.NewReader(data[keyStart:keyEnd]), binary.LittleEndian, &keys); err != nil { + return Archive{}, fmt.Errorf("decode key list: %w", err) + } + + resourceStart := int(hdr.ResourceListOffset) + resourceEnd := resourceStart + int(hdr.EntryCount)*8 + if resourceEnd > len(data) { + return Archive{}, fmt.Errorf("erf resource list exceeds file bounds") + } + entries := make([]resourceEntry, hdr.EntryCount) + if err := binary.Read(bytes.NewReader(data[resourceStart:resourceEnd]), binary.LittleEndian, &entries); err != nil { + return Archive{}, fmt.Errorf("decode resource list: %w", err) + } + + resources := make([]Resource, 0, hdr.EntryCount) + for index, key := range keys { + entry := entries[index] + start := int(entry.Offset) + end := start + int(entry.Size) + if end > len(data) { + return Archive{}, fmt.Errorf("resource %d exceeds file bounds", index) + } + + resref := string(bytes.TrimRight(key.ResRef[:], "\x00")) + payload := make([]byte, entry.Size) + copy(payload, data[start:end]) + resources = append(resources, Resource{ + Name: resref, + Type: key.ResourceType, + Data: payload, + Size: int64(entry.Size), + }) + } + + return Archive{ + FileType: string(hdr.FileType[:]), + Version: string(hdr.Version[:]), + Resources: resources, + }, nil +} + +func ResourceTypeForExtension(extension string) (uint16, bool) { + resourceType, ok := extensionTypes[strings.TrimPrefix(strings.ToLower(extension), ".")] + return resourceType, ok +} + +func HAKResourceTypeForExtension(extension string) (uint16, bool) { + return ResourceTypeForExtension(extension) +} + +func ExtensionForResourceType(resourceType uint16) (string, bool) { + extension, ok := typeExtensions[resourceType] + return extension, ok +} + +func padFour(value string) string { + value = strings.ToUpper(value) + if len(value) >= 4 { + return value[:4] + } + return value + strings.Repeat(" ", 4-len(value)) +} + +func totalResourceBytes(resources []Resource) int64 { + var total int64 + for _, resource := range resources { + switch { + case resource.Size > 0: + total += resource.Size + default: + total += int64(len(resource.Data)) + } + } + return total +} + +func payloadSize(resource Resource) (int64, error) { + if resource.Size > 0 { + return resource.Size, nil + } + if resource.SourcePath != "" && len(resource.Data) == 0 { + info, err := os.Stat(resource.SourcePath) + if err != nil { + return 0, fmt.Errorf("stat resource %q: %w", resource.SourcePath, err) + } + return info.Size(), nil + } + return int64(len(resource.Data)), nil +} + +func writeResourceData(w io.Writer, resource Resource) error { + if len(resource.Data) > 0 || resource.SourcePath == "" { + _, err := w.Write(resource.Data) + return err + } + + file, err := os.Open(resource.SourcePath) + if err != nil { + return fmt.Errorf("open resource %q: %w", resource.SourcePath, err) + } + defer file.Close() + + written, err := io.Copy(w, file) + if err != nil { + return fmt.Errorf("copy resource %q: %w", resource.SourcePath, err) + } + if resource.Size > 0 && written != resource.Size { + return fmt.Errorf("copy resource %q: expected %d bytes, wrote %d", resource.SourcePath, resource.Size, written) + } + return nil +} diff --git a/internal/erf/erf_test.go b/internal/erf/erf_test.go new file mode 100644 index 0000000..2dbe828 --- /dev/null +++ b/internal/erf/erf_test.go @@ -0,0 +1,130 @@ +package erf + +import ( + "bytes" + "testing" +) + +func TestArchiveRoundTrip(t *testing.T) { + archive := New("MOD ", []Resource{ + {Name: "module", Type: 0x07DE, Data: []byte("ifo")}, + {Name: "start", Type: 0x07DC, Data: []byte("are")}, + }) + + var buf bytes.Buffer + if err := Write(&buf, archive); err != nil { + t.Fatalf("write: %v", err) + } + + decoded, err := Read(bytes.NewReader(buf.Bytes())) + if err != nil { + t.Fatalf("read: %v", err) + } + + if len(decoded.Resources) != 2 { + t.Fatalf("expected 2 resources, got %d", len(decoded.Resources)) + } + if decoded.Resources[0].Name != "module" || string(decoded.Resources[0].Data) != "ifo" { + t.Fatalf("unexpected first resource: %#v", decoded.Resources[0]) + } +} + +func TestExtensionMappingsKeep2DAAndTLKDistinct(t *testing.T) { + resourceType, ok := ResourceTypeForExtension("2da") + if !ok { + t.Fatal("expected 2da resource type") + } + if resourceType != 0x07E1 { + t.Fatalf("expected 2da type 0x07E1, got 0x%04X", resourceType) + } + extension, ok := ExtensionForResourceType(resourceType) + if !ok { + t.Fatal("expected canonical extension for 2da type") + } + if extension != "2da" { + t.Fatalf("expected 2da extension, got %q", extension) + } + + tlkType, ok := ResourceTypeForExtension("tlk") + if !ok { + t.Fatal("expected tlk resource type") + } + if tlkType != 0x07E2 { + t.Fatalf("expected tlk type 0x07E2, got 0x%04X", tlkType) + } + if tlkType == resourceType { + t.Fatal("expected 2da and tlk resource types to stay distinct") + } +} + +func TestHAKResourceTypeForExtensionSupportsPNG(t *testing.T) { + if resourceType, ok := HAKResourceTypeForExtension("png"); !ok || resourceType != 0x0820 { + t.Fatalf("expected png restype 0x0820 (2080) for HAK generation, got ok=%v type=0x%04X", ok, resourceType) + } + if resourceType, ok := HAKResourceTypeForExtension("2da"); !ok || resourceType != 0x07E1 { + t.Fatalf("expected 2da to stay valid for HAK generation, got ok=%v type=0x%04X", ok, resourceType) + } +} + +func TestExtensionMappingsSupportModernAssetTypes(t *testing.T) { + cases := map[string]uint16{ + "mtr": 0x0818, + "shd": 0x0815, + "txi": 0x07E6, + "jpg": 0x081C, + "mdb": 0x0816, + "lyt": 0x0BB8, + "vis": 0x0BB9, + "mdx": 0x0BC0, + "xml": 0x0BCD, + "wlk": 0x0BCC, + "gr2": 0x0FA3, + } + for ext, wantType := range cases { + gotType, ok := ResourceTypeForExtension(ext) + if !ok { + t.Fatalf("expected %s to resolve to a resource type", ext) + } + if gotType != wantType { + t.Fatalf("expected %s type 0x%04X, got 0x%04X", ext, wantType, gotType) + } + gotExt, ok := ExtensionForResourceType(wantType) + if !ok { + t.Fatalf("expected type 0x%04X to resolve back to an extension", wantType) + } + if gotExt != ext { + t.Fatalf("expected type 0x%04X to resolve to %s, got %s", wantType, ext, gotExt) + } + } +} + +func TestExtensionMappingsSupportAllUTBlueprintTypes(t *testing.T) { + cases := map[string]uint16{ + "uti": 0x07E9, + "utc": 0x07EB, + "utt": 0x07F0, + "uts": 0x07F3, + "ute": 0x07F8, + "utd": 0x07FA, + "utp": 0x07FC, + "utm": 0x0803, + "utg": 0x0807, + "utw": 0x080A, + } + for ext, wantType := range cases { + gotType, ok := ResourceTypeForExtension(ext) + if !ok { + t.Fatalf("expected %s to resolve to a resource type", ext) + } + if gotType != wantType { + t.Fatalf("expected %s type 0x%04X, got 0x%04X", ext, wantType, gotType) + } + gotExt, ok := ExtensionForResourceType(wantType) + if !ok { + t.Fatalf("expected type 0x%04X to resolve back to an extension", wantType) + } + if gotExt != ext { + t.Fatalf("expected type 0x%04X to resolve to %s, got %s", wantType, ext, gotExt) + } + } +} diff --git a/internal/gff/binary.go b/internal/gff/binary.go new file mode 100644 index 0000000..729dc50 --- /dev/null +++ b/internal/gff/binary.go @@ -0,0 +1,675 @@ +package gff + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math" +) + +const ( + headerSize = 56 +) + +type header struct { + FileType [4]byte + FileVersion [4]byte + StructOffset uint32 + StructCount uint32 + FieldOffset uint32 + FieldCount uint32 + LabelOffset uint32 + LabelCount uint32 + FieldDataOffset uint32 + FieldDataCount uint32 + FieldIndicesOffset uint32 + FieldIndicesCount uint32 + ListIndicesOffset uint32 + ListIndicesCount uint32 +} + +type rawStruct struct { + Type uint32 + DataOrOffset uint32 + FieldCount uint32 +} + +type rawField struct { + Type uint32 + LabelIndex uint32 + DataOrOffset uint32 +} + +func Read(r io.Reader) (Document, error) { + data, err := io.ReadAll(r) + if err != nil { + return Document{}, fmt.Errorf("read gff: %w", err) + } + if len(data) < headerSize { + return Document{}, fmt.Errorf("gff file too small: %d bytes", len(data)) + } + + var hdr header + if err := binary.Read(bytes.NewReader(data[:headerSize]), binary.LittleEndian, &hdr); err != nil { + return Document{}, fmt.Errorf("decode header: %w", err) + } + + reader := binaryReader{data: data, hdr: hdr} + rawStructs, err := reader.readStructs() + if err != nil { + return Document{}, err + } + rawFields, err := reader.readFields() + if err != nil { + return Document{}, err + } + labels, err := reader.readLabels() + if err != nil { + return Document{}, err + } + + root, err := reader.decodeStruct(0, rawStructs, rawFields, labels) + if err != nil { + return Document{}, err + } + + return Document{ + FileType: string(hdr.FileType[:]), + FileVersion: string(hdr.FileVersion[:]), + Root: root, + }, nil +} + +func Write(w io.Writer, doc Document) error { + encoder := newBinaryEncoder(doc) + data, err := encoder.encode() + if err != nil { + return err + } + _, err = w.Write(data) + return err +} + +type binaryReader struct { + data []byte + hdr header +} + +func (r binaryReader) readStructs() ([]rawStruct, error) { + return readTable[rawStruct](r.data, r.hdr.StructOffset, r.hdr.StructCount) +} + +func (r binaryReader) readFields() ([]rawField, error) { + return readTable[rawField](r.data, r.hdr.FieldOffset, r.hdr.FieldCount) +} + +func (r binaryReader) readLabels() ([]string, error) { + start := int(r.hdr.LabelOffset) + end := start + int(r.hdr.LabelCount)*16 + if end > len(r.data) { + return nil, fmt.Errorf("label table exceeds file bounds") + } + + labels := make([]string, 0, r.hdr.LabelCount) + for offset := start; offset < end; offset += 16 { + chunk := r.data[offset : offset+16] + n := bytes.IndexByte(chunk, 0) + if n == -1 { + n = len(chunk) + } + labels = append(labels, string(chunk[:n])) + } + return labels, nil +} + +func (r binaryReader) decodeStruct(index uint32, structs []rawStruct, fields []rawField, labels []string) (Struct, error) { + if index >= uint32(len(structs)) { + return Struct{}, fmt.Errorf("struct index %d out of range", index) + } + + raw := structs[index] + fieldIndices, err := r.fieldIndices(raw) + if err != nil { + return Struct{}, err + } + + out := Struct{ + Type: raw.Type, + Fields: make([]Field, 0, len(fieldIndices)), + } + for _, fieldIndex := range fieldIndices { + if fieldIndex >= uint32(len(fields)) { + return Struct{}, fmt.Errorf("field index %d out of range", fieldIndex) + } + field, err := r.decodeField(fields[fieldIndex], structs, fields, labels) + if err != nil { + return Struct{}, err + } + out.Fields = append(out.Fields, field) + } + + return out, nil +} + +func (r binaryReader) fieldIndices(s rawStruct) ([]uint32, error) { + switch s.FieldCount { + case 0: + return nil, nil + case 1: + return []uint32{s.DataOrOffset}, nil + default: + start := int(r.hdr.FieldIndicesOffset + s.DataOrOffset) + end := start + int(s.FieldCount)*4 + if end > len(r.data) { + return nil, fmt.Errorf("field indices exceed file bounds") + } + + out := make([]uint32, s.FieldCount) + reader := bytes.NewReader(r.data[start:end]) + if err := binary.Read(reader, binary.LittleEndian, &out); err != nil { + return nil, fmt.Errorf("decode field indices: %w", err) + } + return out, nil + } +} + +func (r binaryReader) decodeField(raw rawField, structs []rawStruct, fields []rawField, labels []string) (Field, error) { + if raw.LabelIndex >= uint32(len(labels)) { + return Field{}, fmt.Errorf("label index %d out of range", raw.LabelIndex) + } + fieldType := FieldType(raw.Type) + value, err := r.decodeValue(fieldType, raw.DataOrOffset, structs, fields, labels) + if err != nil { + return Field{}, fmt.Errorf("decode field %q: %w", labels[raw.LabelIndex], err) + } + + return Field{ + Label: labels[raw.LabelIndex], + Type: fieldType, + Value: value, + }, nil +} + +func (r binaryReader) decodeValue(fieldType FieldType, data uint32, structs []rawStruct, fields []rawField, labels []string) (Value, error) { + switch fieldType { + case TypeByte: + return ByteValue(uint8(data)), nil + case TypeChar: + return CharValue(int8(data)), nil + case TypeWord: + return WordValue(uint16(data)), nil + case TypeShort: + return ShortValue(int16(data)), nil + case TypeDWord: + return DWordValue(data), nil + case TypeInt: + return IntValue(int32(data)), nil + case TypeFloat: + return FloatValue(math.Float32frombits(data)), nil + case TypeDWord64: + raw, err := r.sliceFieldData(data, 8) + if err != nil { + return nil, err + } + return DWord64Value(binary.LittleEndian.Uint64(raw)), nil + case TypeInt64: + raw, err := r.sliceFieldData(data, 8) + if err != nil { + return nil, err + } + return Int64Value(int64(binary.LittleEndian.Uint64(raw))), nil + case TypeDouble: + raw, err := r.sliceFieldData(data, 8) + if err != nil { + return nil, err + } + return DoubleValue(math.Float64frombits(binary.LittleEndian.Uint64(raw))), nil + case TypeCExoString: + raw, err := r.prefixedFieldData(data) + if err != nil { + return nil, err + } + return StringValue(string(raw)), nil + case TypeResRef: + raw, err := r.prefixedByteFieldData(data, 1) + if err != nil { + return nil, err + } + return ResRefValue(string(raw)), nil + case TypeCExoLocString: + raw, err := r.locStringFieldData(data) + if err != nil { + return nil, err + } + return decodeLocString(raw) + case TypeVoid: + raw, err := r.prefixedFieldData(data) + if err != nil { + return nil, err + } + return VoidValue(raw), nil + case TypeStruct: + return r.decodeStruct(data, structs, fields, labels) + case TypeList: + return r.decodeList(data, structs, fields, labels) + case TypeOrientation: + raw, err := r.sliceFieldData(data, 16) + if err != nil { + return nil, err + } + return Orientation{ + X: math.Float32frombits(binary.LittleEndian.Uint32(raw[0:4])), + Y: math.Float32frombits(binary.LittleEndian.Uint32(raw[4:8])), + Z: math.Float32frombits(binary.LittleEndian.Uint32(raw[8:12])), + W: math.Float32frombits(binary.LittleEndian.Uint32(raw[12:16])), + }, nil + case TypeVector: + raw, err := r.sliceFieldData(data, 12) + if err != nil { + return nil, err + } + return Vector{ + X: math.Float32frombits(binary.LittleEndian.Uint32(raw[0:4])), + Y: math.Float32frombits(binary.LittleEndian.Uint32(raw[4:8])), + Z: math.Float32frombits(binary.LittleEndian.Uint32(raw[8:12])), + }, nil + default: + return nil, fmt.Errorf("unsupported field type %d", fieldType) + } +} + +func (r binaryReader) decodeList(offset uint32, structs []rawStruct, fields []rawField, labels []string) (ListValue, error) { + start := int(r.hdr.ListIndicesOffset + offset) + if start+4 > len(r.data) { + return nil, fmt.Errorf("list data exceeds file bounds") + } + count := binary.LittleEndian.Uint32(r.data[start : start+4]) + start += 4 + end := start + int(count)*4 + if end > len(r.data) { + return nil, fmt.Errorf("list indices exceed file bounds") + } + + list := make(ListValue, 0, count) + for pos := start; pos < end; pos += 4 { + index := binary.LittleEndian.Uint32(r.data[pos : pos+4]) + child, err := r.decodeStruct(index, structs, fields, labels) + if err != nil { + return nil, err + } + list = append(list, child) + } + return list, nil +} + +func (r binaryReader) sliceFieldData(offset uint32, size int) ([]byte, error) { + start := int(r.hdr.FieldDataOffset + offset) + end := start + size + if end > len(r.data) { + return nil, fmt.Errorf("field data exceeds file bounds") + } + return r.data[start:end], nil +} + +func (r binaryReader) prefixedFieldData(offset uint32) ([]byte, error) { + sizeRaw, err := r.sliceFieldData(offset, 4) + if err != nil { + return nil, err + } + size := int(binary.LittleEndian.Uint32(sizeRaw)) + return r.sliceFieldData(offset+4, size) +} + +func (r binaryReader) prefixedByteFieldData(offset uint32, prefixBytes uint32) ([]byte, error) { + header, err := r.sliceFieldData(offset, int(prefixBytes)) + if err != nil { + return nil, err + } + size := int(header[0]) + return r.sliceFieldData(offset+prefixBytes, size) +} + +func (r binaryReader) locStringFieldData(offset uint32) ([]byte, error) { + sizeRaw, err := r.sliceFieldData(offset, 4) + if err != nil { + return nil, err + } + size := int(binary.LittleEndian.Uint32(sizeRaw)) + return r.sliceFieldData(offset, size+4) +} + +func readTable[T any](data []byte, offset uint32, count uint32) ([]T, error) { + if count == 0 { + return nil, nil + } + var row T + size := binary.Size(row) + start := int(offset) + end := start + int(count)*size + if end > len(data) { + return nil, fmt.Errorf("table exceeds file bounds") + } + + out := make([]T, count) + reader := bytes.NewReader(data[start:end]) + if err := binary.Read(reader, binary.LittleEndian, &out); err != nil { + return nil, fmt.Errorf("decode table: %w", err) + } + return out, nil +} + +func decodeLocString(data []byte) (LocString, error) { + if len(data) < 12 { + return LocString{}, fmt.Errorf("locstring too small") + } + totalSize := binary.LittleEndian.Uint32(data[0:4]) + if totalSize+4 != uint32(len(data)) { + return LocString{}, fmt.Errorf("locstring size mismatch") + } + stringRef := binary.LittleEndian.Uint32(data[4:8]) + count := binary.LittleEndian.Uint32(data[8:12]) + pos := 12 + entries := make([]LocStringEntry, 0, count) + for i := uint32(0); i < count; i++ { + if pos+8 > len(data) { + return LocString{}, fmt.Errorf("locstring entry header truncated") + } + id := binary.LittleEndian.Uint32(data[pos : pos+4]) + length := binary.LittleEndian.Uint32(data[pos+4 : pos+8]) + pos += 8 + if pos+int(length) > len(data) { + return LocString{}, fmt.Errorf("locstring entry %d truncated", i) + } + entries = append(entries, LocStringEntry{ + ID: id, + Value: string(data[pos : pos+int(length)]), + }) + pos += int(length) + } + + return LocString{ + StringRef: stringRef, + Entries: entries, + }, nil +} + +type binaryEncoder struct { + document Document + structs []rawStruct + fields []rawField + labels []string + labelIndex map[string]uint32 + fieldData bytes.Buffer + fieldIndices []uint32 + listIndices []uint32 +} + +func newBinaryEncoder(doc Document) *binaryEncoder { + return &binaryEncoder{ + document: doc, + labelIndex: map[string]uint32{}, + } +} + +func (e *binaryEncoder) encode() ([]byte, error) { + if err := e.addStruct(e.document.Root); err != nil { + return nil, err + } + + var buf bytes.Buffer + hdr := header{} + copy(hdr.FileType[:], []byte(padFour(e.document.FileType))) + copy(hdr.FileVersion[:], []byte(padFour(defaultString(e.document.FileVersion, "V3.2")))) + + hdr.StructOffset = headerSize + hdr.StructCount = uint32(len(e.structs)) + hdr.FieldOffset = hdr.StructOffset + uint32(len(e.structs))*12 + hdr.FieldCount = uint32(len(e.fields)) + hdr.LabelOffset = hdr.FieldOffset + uint32(len(e.fields))*12 + hdr.LabelCount = uint32(len(e.labels)) + hdr.FieldDataOffset = hdr.LabelOffset + uint32(len(e.labels))*16 + hdr.FieldDataCount = uint32(e.fieldData.Len()) + hdr.FieldIndicesOffset = hdr.FieldDataOffset + uint32(e.fieldData.Len()) + hdr.FieldIndicesCount = uint32(len(e.fieldIndices) * 4) + hdr.ListIndicesOffset = hdr.FieldIndicesOffset + uint32(len(e.fieldIndices))*4 + hdr.ListIndicesCount = uint32(len(e.listIndices) * 4) + + if err := binary.Write(&buf, binary.LittleEndian, hdr); err != nil { + return nil, err + } + if err := binary.Write(&buf, binary.LittleEndian, e.structs); err != nil { + return nil, err + } + if err := binary.Write(&buf, binary.LittleEndian, e.fields); err != nil { + return nil, err + } + for _, label := range e.labels { + var raw [16]byte + copy(raw[:], []byte(label)) + if _, err := buf.Write(raw[:]); err != nil { + return nil, err + } + } + if _, err := buf.Write(e.fieldData.Bytes()); err != nil { + return nil, err + } + if err := binary.Write(&buf, binary.LittleEndian, e.fieldIndices); err != nil { + return nil, err + } + if err := binary.Write(&buf, binary.LittleEndian, e.listIndices); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (e *binaryEncoder) addStruct(s Struct) error { + index := uint32(len(e.structs)) + e.structs = append(e.structs, rawStruct{Type: s.Type}) + + fieldIndices := make([]uint32, 0, len(s.Fields)) + for _, field := range s.Fields { + fieldIndex, err := e.addField(field) + if err != nil { + return err + } + fieldIndices = append(fieldIndices, fieldIndex) + } + + entry := &e.structs[index] + entry.FieldCount = uint32(len(fieldIndices)) + switch len(fieldIndices) { + case 0: + entry.DataOrOffset = 0 + case 1: + entry.DataOrOffset = fieldIndices[0] + default: + entry.DataOrOffset = uint32(len(e.fieldIndices) * 4) + e.fieldIndices = append(e.fieldIndices, fieldIndices...) + } + return nil +} + +func (e *binaryEncoder) addField(field Field) (uint32, error) { + labelIndex, err := e.indexLabel(field.Label) + if err != nil { + return 0, err + } + data, err := e.encodeValue(field.Value) + if err != nil { + return 0, fmt.Errorf("encode field %q: %w", field.Label, err) + } + + entry := rawField{ + Type: uint32(field.Type), + LabelIndex: labelIndex, + DataOrOffset: data, + } + index := uint32(len(e.fields)) + e.fields = append(e.fields, entry) + return index, nil +} + +func (e *binaryEncoder) encodeValue(value Value) (uint32, error) { + switch typed := value.(type) { + case ByteValue: + return uint32(uint8(typed)), nil + case CharValue: + return uint32(uint8(typed)), nil + case WordValue: + return uint32(uint16(typed)), nil + case ShortValue: + return uint32(uint16(typed)), nil + case DWordValue: + return uint32(typed), nil + case IntValue: + return uint32(int32(typed)), nil + case FloatValue: + return math.Float32bits(float32(typed)), nil + case DWord64Value: + return e.writeFieldData(func(buf *bytes.Buffer) error { + return binary.Write(buf, binary.LittleEndian, uint64(typed)) + }), nil + case Int64Value: + return e.writeFieldData(func(buf *bytes.Buffer) error { + return binary.Write(buf, binary.LittleEndian, int64(typed)) + }), nil + case DoubleValue: + return e.writeFieldData(func(buf *bytes.Buffer) error { + return binary.Write(buf, binary.LittleEndian, math.Float64bits(float64(typed))) + }), nil + case StringValue: + return e.writeLengthPrefixed([]byte(typed)), nil + case ResRefValue: + if len(typed) > 255 { + return 0, fmt.Errorf("resref exceeds 255 bytes") + } + return e.writeFieldData(func(buf *bytes.Buffer) error { + if err := buf.WriteByte(byte(len(typed))); err != nil { + return err + } + _, err := buf.Write([]byte(typed)) + return err + }), nil + case LocString: + return e.writeFieldData(func(buf *bytes.Buffer) error { + payload := bytes.Buffer{} + if err := binary.Write(&payload, binary.LittleEndian, typed.StringRef); err != nil { + return err + } + if err := binary.Write(&payload, binary.LittleEndian, uint32(len(typed.Entries))); err != nil { + return err + } + for _, entry := range typed.Entries { + if err := binary.Write(&payload, binary.LittleEndian, entry.ID); err != nil { + return err + } + if err := binary.Write(&payload, binary.LittleEndian, uint32(len(entry.Value))); err != nil { + return err + } + if _, err := payload.Write([]byte(entry.Value)); err != nil { + return err + } + } + if err := binary.Write(buf, binary.LittleEndian, uint32(payload.Len())); err != nil { + return err + } + if _, err := buf.Write(payload.Bytes()); err != nil { + return err + } + return nil + }), nil + case VoidValue: + return e.writeLengthPrefixed([]byte(typed)), nil + case Struct: + index := uint32(len(e.structs)) + if err := e.addStruct(typed); err != nil { + return 0, err + } + return index, nil + case ListValue: + return e.writeList(typed) + case Orientation: + return e.writeFieldData(func(buf *bytes.Buffer) error { + for _, component := range []float32{typed.X, typed.Y, typed.Z, typed.W} { + if err := binary.Write(buf, binary.LittleEndian, math.Float32bits(component)); err != nil { + return err + } + } + return nil + }), nil + case Vector: + return e.writeFieldData(func(buf *bytes.Buffer) error { + for _, component := range []float32{typed.X, typed.Y, typed.Z} { + if err := binary.Write(buf, binary.LittleEndian, math.Float32bits(component)); err != nil { + return err + } + } + return nil + }), nil + default: + return 0, fmt.Errorf("unsupported value type %T", value) + } +} + +func (e *binaryEncoder) writeList(list ListValue) (uint32, error) { + indices := make([]uint32, 0, len(list)) + for _, item := range list { + index := uint32(len(e.structs)) + if err := e.addStruct(item); err != nil { + return 0, err + } + indices = append(indices, index) + } + + offset := uint32(len(e.listIndices) * 4) + e.listIndices = append(e.listIndices, uint32(len(list))) + e.listIndices = append(e.listIndices, indices...) + return offset, nil +} + +func (e *binaryEncoder) writeFieldData(write func(*bytes.Buffer) error) uint32 { + offset := uint32(e.fieldData.Len()) + if err := write(&e.fieldData); err != nil { + panic(err) + } + return offset +} + +func (e *binaryEncoder) writeLengthPrefixed(data []byte) uint32 { + return e.writeFieldData(func(buf *bytes.Buffer) error { + if err := binary.Write(buf, binary.LittleEndian, uint32(len(data))); err != nil { + return err + } + _, err := buf.Write(data) + return err + }) +} + +func (e *binaryEncoder) indexLabel(label string) (uint32, error) { + if len(label) > 16 { + return 0, fmt.Errorf("label %q exceeds 16 bytes", label) + } + if index, ok := e.labelIndex[label]; ok { + return index, nil + } + index := uint32(len(e.labels)) + e.labels = append(e.labels, label) + e.labelIndex[label] = index + return index, nil +} + +func padFour(value string) string { + if len(value) >= 4 { + return value[:4] + } + return value + string(bytes.Repeat([]byte(" "), 4-len(value))) +} + +func defaultString(value, fallback string) string { + if value == "" { + return fallback + } + return value +} diff --git a/internal/gff/binary_test.go b/internal/gff/binary_test.go new file mode 100644 index 0000000..141e77c --- /dev/null +++ b/internal/gff/binary_test.go @@ -0,0 +1,55 @@ +package gff + +import ( + "bytes" + "encoding/json" + "testing" +) + +func TestRoundTripBinaryAndJSON(t *testing.T) { + original := Document{ + FileType: "UTC ", + FileVersion: "V3.2", + Root: Struct{ + Type: 0, + Fields: []Field{ + NewField("Tag", StringValue("test_creature")), + NewField("TemplateResRef", ResRefValue("nw_test")), + NewField("HP", IntValue(12)), + NewField("Position", Vector{X: 1.25, Y: 2.5, Z: 3.75}), + NewField("Inventory", ListValue{ + { + Type: 1, + Fields: []Field{ + NewField("Slot", DWordValue(0)), + NewField("ResRef", ResRefValue("itm_sword")), + }, + }, + }), + }, + }, + } + + var binaryBuf bytes.Buffer + if err := Write(&binaryBuf, original); err != nil { + t.Fatalf("write binary: %v", err) + } + + decoded, err := Read(bytes.NewReader(binaryBuf.Bytes())) + if err != nil { + t.Fatalf("read binary: %v", err) + } + + left, err := json.Marshal(original) + if err != nil { + t.Fatalf("marshal original json: %v", err) + } + right, err := json.Marshal(decoded) + if err != nil { + t.Fatalf("marshal decoded json: %v", err) + } + + if string(left) != string(right) { + t.Fatalf("roundtrip mismatch\noriginal: %s\ndecoded: %s", left, right) + } +} diff --git a/internal/gff/json.go b/internal/gff/json.go new file mode 100644 index 0000000..d7e4c3f --- /dev/null +++ b/internal/gff/json.go @@ -0,0 +1,317 @@ +package gff + +import ( + "encoding/base64" + "encoding/json" + "fmt" +) + +type jsonDocument struct { + FileType string `json:"file_type"` + FileVersion string `json:"file_version"` + Root jsonStruct `json:"root"` +} + +type jsonStruct struct { + StructType uint32 `json:"struct_type"` + Fields []jsonField `json:"fields"` +} + +type jsonField struct { + Label string `json:"label"` + Type string `json:"type"` + Value json.RawMessage `json:"value"` +} + +type jsonLocString struct { + StringRef uint32 `json:"string_ref"` + Entries []LocStringEntry `json:"entries"` +} + +func (s Struct) MarshalJSON() ([]byte, error) { + payload, err := marshalStruct(s) + if err != nil { + return nil, err + } + return json.Marshal(payload) +} + +func (s *Struct) UnmarshalJSON(data []byte) error { + var payload jsonStruct + if err := json.Unmarshal(data, &payload); err != nil { + return err + } + decoded, err := unmarshalStruct(payload) + if err != nil { + return err + } + *s = decoded + return nil +} + +func (d Document) MarshalJSON() ([]byte, error) { + root, err := marshalStruct(d.Root) + if err != nil { + return nil, err + } + + return json.Marshal(jsonDocument{ + FileType: d.FileType, + FileVersion: d.FileVersion, + Root: root, + }) +} + +func (d *Document) UnmarshalJSON(data []byte) error { + var payload jsonDocument + if err := json.Unmarshal(data, &payload); err != nil { + return err + } + + root, err := unmarshalStruct(payload.Root) + if err != nil { + return err + } + + d.FileType = payload.FileType + d.FileVersion = payload.FileVersion + d.Root = root + return nil +} + +func marshalStruct(in Struct) (jsonStruct, error) { + fields := make([]jsonField, 0, len(in.Fields)) + for _, field := range in.Fields { + payload, err := marshalValue(field.Value) + if err != nil { + return jsonStruct{}, fmt.Errorf("marshal field %q: %w", field.Label, err) + } + fields = append(fields, jsonField{ + Label: field.Label, + Type: field.Type.String(), + Value: payload, + }) + } + + return jsonStruct{ + StructType: in.Type, + Fields: fields, + }, nil +} + +func unmarshalStruct(in jsonStruct) (Struct, error) { + fields := make([]Field, 0, len(in.Fields)) + for _, field := range in.Fields { + ft, err := parseFieldType(field.Type) + if err != nil { + return Struct{}, fmt.Errorf("field %q: %w", field.Label, err) + } + + value, err := unmarshalValue(ft, field.Value) + if err != nil { + return Struct{}, fmt.Errorf("field %q: %w", field.Label, err) + } + + fields = append(fields, Field{ + Label: field.Label, + Type: ft, + Value: value, + }) + } + + return Struct{ + Type: in.StructType, + Fields: fields, + }, nil +} + +func marshalValue(value Value) (json.RawMessage, error) { + switch typed := value.(type) { + case ByteValue: + return json.Marshal(uint8(typed)) + case CharValue: + return json.Marshal(int8(typed)) + case WordValue: + return json.Marshal(uint16(typed)) + case ShortValue: + return json.Marshal(int16(typed)) + case DWordValue: + return json.Marshal(uint32(typed)) + case IntValue: + return json.Marshal(int32(typed)) + case DWord64Value: + return json.Marshal(uint64(typed)) + case Int64Value: + return json.Marshal(int64(typed)) + case FloatValue: + return json.Marshal(float32(typed)) + case DoubleValue: + return json.Marshal(float64(typed)) + case StringValue: + return json.Marshal(string(typed)) + case ResRefValue: + return json.Marshal(string(typed)) + case LocString: + return json.Marshal(jsonLocString(typed)) + case VoidValue: + return json.Marshal(base64.StdEncoding.EncodeToString([]byte(typed))) + case Struct: + payload, err := marshalStruct(typed) + if err != nil { + return nil, err + } + return json.Marshal(payload) + case ListValue: + payload := make([]jsonStruct, 0, len(typed)) + for _, item := range typed { + entry, err := marshalStruct(item) + if err != nil { + return nil, err + } + payload = append(payload, entry) + } + return json.Marshal(payload) + case Orientation: + return json.Marshal(typed) + case Vector: + return json.Marshal(typed) + default: + return nil, fmt.Errorf("unsupported value type %T", value) + } +} + +func unmarshalValue(ft FieldType, data []byte) (Value, error) { + switch ft { + case TypeByte: + var out uint8 + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return ByteValue(out), nil + case TypeChar: + var out int8 + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return CharValue(out), nil + case TypeWord: + var out uint16 + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return WordValue(out), nil + case TypeShort: + var out int16 + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return ShortValue(out), nil + case TypeDWord: + var out uint32 + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return DWordValue(out), nil + case TypeInt: + var out int32 + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return IntValue(out), nil + case TypeDWord64: + var out uint64 + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return DWord64Value(out), nil + case TypeInt64: + var out int64 + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return Int64Value(out), nil + case TypeFloat: + var out float32 + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return FloatValue(out), nil + case TypeDouble: + var out float64 + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return DoubleValue(out), nil + case TypeCExoString: + var out string + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return StringValue(out), nil + case TypeResRef: + var out string + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return ResRefValue(out), nil + case TypeCExoLocString: + var out jsonLocString + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return LocString(out), nil + case TypeVoid: + var out string + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + decoded, err := base64.StdEncoding.DecodeString(out) + if err != nil { + return nil, fmt.Errorf("decode base64 void: %w", err) + } + return VoidValue(decoded), nil + case TypeStruct: + var payload jsonStruct + if err := json.Unmarshal(data, &payload); err != nil { + return nil, err + } + return unmarshalStruct(payload) + case TypeList: + var payload []jsonStruct + if err := json.Unmarshal(data, &payload); err != nil { + return nil, err + } + out := make(ListValue, 0, len(payload)) + for _, item := range payload { + decoded, err := unmarshalStruct(item) + if err != nil { + return nil, err + } + out = append(out, decoded) + } + return out, nil + case TypeOrientation: + var out Orientation + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return out, nil + case TypeVector: + var out Vector + if err := json.Unmarshal(data, &out); err != nil { + return nil, err + } + return out, nil + default: + return nil, fmt.Errorf("unsupported field type %d", ft) + } +} + +func parseFieldType(name string) (FieldType, error) { + for kind, candidate := range fieldTypeNames { + if candidate == name { + return kind, nil + } + } + return 0, fmt.Errorf("unknown field type %q", name) +} diff --git a/internal/gff/types.go b/internal/gff/types.go new file mode 100644 index 0000000..d3cbb6b --- /dev/null +++ b/internal/gff/types.go @@ -0,0 +1,140 @@ +package gff + +import "fmt" + +type FieldType uint32 + +const ( + TypeByte FieldType = iota + TypeChar + TypeWord + TypeShort + TypeDWord + TypeInt + TypeDWord64 + TypeInt64 + TypeFloat + TypeDouble + TypeCExoString + TypeResRef + TypeCExoLocString + TypeVoid + TypeStruct + TypeList + TypeOrientation + TypeVector +) + +var fieldTypeNames = map[FieldType]string{ + TypeByte: "Byte", + TypeChar: "Char", + TypeWord: "Word", + TypeShort: "Short", + TypeDWord: "DWord", + TypeInt: "Int", + TypeDWord64: "DWord64", + TypeInt64: "Int64", + TypeFloat: "Float", + TypeDouble: "Double", + TypeCExoString: "CExoString", + TypeResRef: "ResRef", + TypeCExoLocString: "CExoLocString", + TypeVoid: "Void", + TypeStruct: "Struct", + TypeList: "List", + TypeOrientation: "Orientation", + TypeVector: "Vector", +} + +type Document struct { + FileType string `json:"file_type"` + FileVersion string `json:"file_version"` + Root Struct `json:"root"` +} + +type Struct struct { + Type uint32 `json:"struct_type"` + Fields []Field `json:"fields"` +} + +type Field struct { + Label string `json:"label"` + Type FieldType `json:"-"` + Value Value `json:"-"` +} + +type Value interface { + fieldType() FieldType +} + +type LocString struct { + StringRef uint32 `json:"string_ref"` + Entries []LocStringEntry `json:"entries"` +} + +type LocStringEntry struct { + ID uint32 `json:"id"` + Value string `json:"value"` +} + +type Vector struct { + X float32 `json:"x"` + Y float32 `json:"y"` + Z float32 `json:"z"` +} + +type Orientation struct { + X float32 `json:"x"` + Y float32 `json:"y"` + Z float32 `json:"z"` + W float32 `json:"w"` +} + +type ByteValue uint8 +type CharValue int8 +type WordValue uint16 +type ShortValue int16 +type DWordValue uint32 +type IntValue int32 +type DWord64Value uint64 +type Int64Value int64 +type FloatValue float32 +type DoubleValue float64 +type StringValue string +type ResRefValue string +type VoidValue []byte +type ListValue []Struct + +func (ByteValue) fieldType() FieldType { return TypeByte } +func (CharValue) fieldType() FieldType { return TypeChar } +func (WordValue) fieldType() FieldType { return TypeWord } +func (ShortValue) fieldType() FieldType { return TypeShort } +func (DWordValue) fieldType() FieldType { return TypeDWord } +func (IntValue) fieldType() FieldType { return TypeInt } +func (DWord64Value) fieldType() FieldType { return TypeDWord64 } +func (Int64Value) fieldType() FieldType { return TypeInt64 } +func (FloatValue) fieldType() FieldType { return TypeFloat } +func (DoubleValue) fieldType() FieldType { return TypeDouble } +func (StringValue) fieldType() FieldType { return TypeCExoString } +func (ResRefValue) fieldType() FieldType { return TypeResRef } +func (LocString) fieldType() FieldType { return TypeCExoLocString } +func (VoidValue) fieldType() FieldType { return TypeVoid } +func (Struct) fieldType() FieldType { return TypeStruct } +func (ListValue) fieldType() FieldType { return TypeList } +func (Orientation) fieldType() FieldType { return TypeOrientation } +func (Vector) fieldType() FieldType { return TypeVector } + +func (t FieldType) String() string { + if name, ok := fieldTypeNames[t]; ok { + return name + } + return fmt.Sprintf("FieldType(%d)", t) +} + +func NewField(label string, value Value) Field { + return Field{ + Label: label, + Type: value.fieldType(), + Value: value, + } +} diff --git a/internal/music/config.go b/internal/music/config.go new file mode 100644 index 0000000..0423f0f --- /dev/null +++ b/internal/music/config.go @@ -0,0 +1,47 @@ +package music + +import ( + "fmt" + "path/filepath" + "strings" +) + +type DatasetConfig struct { + Source string `json:"source" yaml:"source"` + Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"` + StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` + CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` +} + +type DefaultsConfig struct { + StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` + CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` + CreditsOverlay string `json:"credits_overlay,omitempty" yaml:"credits_overlay,omitempty"` + MaxStemLength *int `json:"max_stem_length,omitempty" yaml:"max_stem_length,omitempty"` +} + +func ValidateDatasetSource(field, source string) error { + trimmed := strings.TrimSpace(source) + if trimmed == "" { + return fmt.Errorf("%s.source is required", field) + } + if strings.Contains(trimmed, "\x00") { + return fmt.Errorf("%s.source must not contain NUL bytes", field) + } + clean := filepath.Clean(filepath.FromSlash(trimmed)) + if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return fmt.Errorf("%s.source must not escape project root", field) + } + return nil +} + +func ValidateDatasetID(id string) error { + trimmed := strings.TrimSpace(id) + if trimmed == "" { + return fmt.Errorf("dataset id must not be empty") + } + if strings.Contains(trimmed, "/") || strings.Contains(trimmed, "\\") { + return fmt.Errorf("dataset id %q must not contain path separators", trimmed) + } + return nil +} diff --git a/internal/music/credits.go b/internal/music/credits.go new file mode 100644 index 0000000..b76f6ba --- /dev/null +++ b/internal/music/credits.go @@ -0,0 +1,292 @@ +package music + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func ParseCreditsOverlay(path string) (CreditsOverlay, error) { + entries, err := ParseCreditsMarkdown(path) + if err != nil { + return CreditsOverlay{}, err + } + overlay := CreditsOverlay{ + ByOriginal: make(map[string]CreditsEntry), + ByOutput: make(map[string]CreditsEntry), + Entries: entries, + } + for _, entry := range entries { + if entry.OriginalFile != "" { + key := strings.ToLower(strings.TrimSpace(entry.OriginalFile)) + if _, exists := overlay.ByOriginal[key]; exists { + return CreditsOverlay{}, fmt.Errorf("duplicate manual credits row for original file %s", entry.OriginalFile) + } + overlay.ByOriginal[key] = entry + } + if entry.OutputFile != "" { + key := strings.ToLower(strings.TrimSpace(entry.OutputFile)) + if _, exists := overlay.ByOutput[key]; exists { + return CreditsOverlay{}, fmt.Errorf("duplicate manual credits row for output file %s", entry.OutputFile) + } + overlay.ByOutput[key] = entry + } + } + return overlay, nil +} + +func (o CreditsOverlay) Match(originalFile, outputFile string) *CreditsEntry { + if entry, ok := o.ByOutput[strings.ToLower(strings.TrimSpace(outputFile))]; ok { + entryCopy := entry + return &entryCopy + } + if entry, ok := o.ByOriginal[strings.ToLower(strings.TrimSpace(originalFile))]; ok { + entryCopy := entry + return &entryCopy + } + return nil +} + +func ValidateOverlayEntries(dir string, overlay CreditsOverlay, generated []CreditsEntry) error { + if len(overlay.Entries) == 0 { + return nil + } + validOriginal := make(map[string]struct{}, len(generated)) + validOutput := make(map[string]struct{}, len(generated)) + for _, entry := range generated { + validOriginal[strings.ToLower(entry.OriginalFile)] = struct{}{} + validOutput[strings.ToLower(entry.OutputFile)] = struct{}{} + } + for _, entry := range overlay.Entries { + if entry.OriginalFile != "" { + if _, ok := validOriginal[strings.ToLower(entry.OriginalFile)]; !ok { + return fmt.Errorf("manual credits entry in %s references unknown original file %s", dir, entry.OriginalFile) + } + } + if entry.OutputFile != "" { + if _, ok := validOutput[strings.ToLower(entry.OutputFile)]; !ok { + return fmt.Errorf("manual credits entry in %s references unknown output file %s", dir, entry.OutputFile) + } + } + } + return nil +} + +func ParseCreditsMarkdown(path string) ([]CreditsEntry, error) { + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + lines := strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") + header := []string(nil) + entries := make([]CreditsEntry, 0) + for _, line := range lines { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "|") { + continue + } + cells := ParseMarkdownTableRow(line) + if len(cells) == 0 { + continue + } + if header == nil { + header = make([]string, len(cells)) + for i, cell := range cells { + header[i] = NormalizeCreditsHeader(cell) + } + continue + } + if IsMarkdownSeparatorRow(cells) { + continue + } + entry := CreditsEntry{} + for i, key := range header { + if i >= len(cells) { + continue + } + value := CleanCreditsCell(cells[i]) + switch key { + case "artist": + entry.Artist = value + case "title": + entry.Title = value + case "output_file": + entry.OutputFile = value + case "original_file": + entry.OriginalFile = value + case "album": + entry.Album = value + case "date": + entry.Date = value + case "rights": + entry.Rights = value + case "notes": + entry.Notes = value + } + } + if entry.Artist == "" && entry.Title == "" && entry.OutputFile == "" && entry.OriginalFile == "" { + continue + } + entries = append(entries, entry) + } + return entries, nil +} + +func ParseMarkdownTableRow(line string) []string { + trimmed := strings.TrimSpace(line) + trimmed = strings.TrimPrefix(trimmed, "|") + trimmed = strings.TrimSuffix(trimmed, "|") + parts := strings.Split(trimmed, "|") + cells := make([]string, 0, len(parts)) + for _, part := range parts { + cells = append(cells, strings.TrimSpace(strings.ReplaceAll(part, `\|`, "|"))) + } + return cells +} + +func IsMarkdownSeparatorRow(cells []string) bool { + if len(cells) == 0 { + return false + } + for _, cell := range cells { + cell = strings.TrimSpace(cell) + cell = strings.ReplaceAll(cell, "-", "") + cell = strings.ReplaceAll(cell, ":", "") + if cell != "" { + return false + } + } + return true +} + +func NormalizeCreditsHeader(value string) string { + switch strings.ToLower(CleanCreditsCell(value)) { + case "artist": + return "artist" + case "title": + return "title" + case "output file": + return "output_file" + case "original file": + return "original_file" + case "album": + return "album" + case "date": + return "date" + case "license / copyright": + return "rights" + case "notes": + return "notes" + default: + return "" + } +} + +func CleanCreditsCell(value string) string { + value = strings.TrimSpace(value) + value = strings.Trim(value, "`") + value = strings.ReplaceAll(value, "
", "\n") + return strings.TrimSpace(value) +} + +func RenderCreditsMarkdown(entries []CreditsEntry) string { + var builder strings.Builder + builder.WriteString(CreditsHeader) + for _, entry := range entries { + builder.WriteString("| ") + builder.WriteString(EscapeMarkdownCell(entry.Artist)) + builder.WriteString(" | ") + builder.WriteString(EscapeMarkdownCell(entry.Title)) + builder.WriteString(" | ") + builder.WriteString(EscapeMarkdownCell(WrapCodeCell(entry.OutputFile))) + builder.WriteString(" | ") + builder.WriteString(EscapeMarkdownCell(WrapCodeCell(entry.OriginalFile))) + builder.WriteString(" | ") + builder.WriteString(EscapeMarkdownCell(entry.Album)) + builder.WriteString(" | ") + builder.WriteString(EscapeMarkdownCell(entry.Date)) + builder.WriteString(" | ") + builder.WriteString(EscapeMarkdownCell(entry.Rights)) + builder.WriteString(" | ") + builder.WriteString(EscapeMarkdownCell(entry.Notes)) + builder.WriteString(" |\n") + } + return builder.String() +} + +func WrapCodeCell(value string) string { + if strings.TrimSpace(value) == "" { + return "" + } + return "`" + value + "`" +} + +func EscapeMarkdownCell(value string) string { + value = strings.ReplaceAll(value, `\`, `\\`) + value = strings.ReplaceAll(value, "|", `\|`) + value = strings.ReplaceAll(value, "\n", "
") + return value +} + +func ApplyCreditsOverlay(entry *CreditsEntry, overlay *CreditsEntry) { + if overlay == nil { + return + } + entry.Artist = FirstNonEmpty(overlay.Artist, entry.Artist) + entry.Title = FirstNonEmpty(overlay.Title, entry.Title) + entry.Album = FirstNonEmpty(overlay.Album, entry.Album) + entry.Date = FirstNonEmpty(overlay.Date, entry.Date) + entry.Rights = FirstNonEmpty(overlay.Rights, entry.Rights) + entry.Notes = FirstNonEmpty(overlay.Notes, entry.Notes) +} + +func SyncGeneratedCreditsArtifacts(root string, desiredFiles map[string][]byte) (int, error) { + changedFiles := 0 + existingFiles := make(map[string]struct{}) + if err := filepath.WalkDir(root, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + if os.IsNotExist(walkErr) { + return nil + } + return walkErr + } + if d.IsDir() { + return nil + } + existingFiles[path] = struct{}{} + return nil + }); err != nil { + return 0, fmt.Errorf("scan credits dir: %w", err) + } + + for path, content := range desiredFiles { + current, err := os.ReadFile(path) + if err != nil && !os.IsNotExist(err) { + return 0, fmt.Errorf("read credits artifact %s: %w", path, err) + } + if err == nil && string(current) == string(content) { + delete(existingFiles, path) + continue + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return 0, fmt.Errorf("create generated credits dir: %w", err) + } + if err := os.WriteFile(path, content, 0o644); err != nil { + return 0, fmt.Errorf("write credits artifact %s: %w", path, err) + } + changedFiles++ + delete(existingFiles, path) + } + + for path := range existingFiles { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return 0, fmt.Errorf("remove stale credits artifact %s: %w", path, err) + } + changedFiles++ + } + return changedFiles, nil +} diff --git a/internal/music/metadata.go b/internal/music/metadata.go new file mode 100644 index 0000000..5114c36 --- /dev/null +++ b/internal/music/metadata.go @@ -0,0 +1,62 @@ +package music + +import ( + "encoding/json" + "fmt" + "os/exec" + "strings" +) + +func ProbeMetadata(ffprobePath, sourcePath string) (Metadata, error) { + cmd := exec.Command(ffprobePath, "-v", "quiet", "-print_format", "json", "-show_format", sourcePath) + output, err := cmd.Output() + if err != nil { + return Metadata{}, err + } + var payload struct { + Format struct { + Tags map[string]string `json:"tags"` + } `json:"format"` + } + if err := json.Unmarshal(output, &payload); err != nil { + return Metadata{}, err + } + tags := make(map[string]string, len(payload.Format.Tags)) + for key, value := range payload.Format.Tags { + tags[strings.ToLower(strings.TrimSpace(key))] = strings.TrimSpace(value) + } + return Metadata{ + Title: FirstNonEmpty(tags["title"]), + Artist: FirstNonEmpty(tags["artist"], tags["album_artist"], tags["composer"]), + Album: FirstNonEmpty(tags["album"]), + Date: FirstNonEmpty(tags["date"], tags["year"]), + Comment: FirstNonEmpty(tags["comment"], tags["description"]), + Copyright: FirstNonEmpty(tags["copyright"]), + License: FirstNonEmpty(tags["license"], tags["license_url"]), + }, nil +} + +func ConvertSource(ffmpegPath, sourcePath, outputPath string) error { + cmd := exec.Command( + ffmpegPath, "-y", + "-i", sourcePath, + "-vn", + "-map_metadata", "-1", + "-ar", "44100", + "-ac", "2", + "-b:a", "192k", + "-codec:a", "libmp3lame", + "-write_xing", "0", + "-f", "mp3", + outputPath, + ) + output, err := cmd.CombinedOutput() + if err != nil { + message := strings.TrimSpace(string(output)) + if message == "" { + message = err.Error() + } + return fmt.Errorf("%s", message) + } + return nil +} diff --git a/internal/music/music.go b/internal/music/music.go new file mode 100644 index 0000000..64584e5 --- /dev/null +++ b/internal/music/music.go @@ -0,0 +1,226 @@ +package music + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" +) + +const ( + MaxStemLen = 16 + CreditsHeader = "# NWN Music Pack Credits\n\nOriginal rights remain with the respective composers and licensors.\n\n## Processed Files\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n" +) + +var ( + defaultConvertExtensions = []string{ + ".flac", + ".m4a", + ".mp3", + ".ogg", + ".wav", + } + ConvertExtensions = map[string]struct{}{ + ".flac": {}, + ".m4a": {}, + ".mp3": {}, + ".ogg": {}, + ".wav": {}, + } + PassthroughExtensions = map[string]struct{}{ + ".bmu": {}, + ".wav": {}, + } + DropWords = map[string]struct{}{ + "mp3": {}, "wav": {}, "flac": {}, "ogg": {}, "m4a": {}, + "music": {}, "track": {}, "audio": {}, "song": {}, "soundtrack": {}, + "final": {}, "version": {}, "ver": {}, "v": {}, "mix": {}, "master": {}, "remaster": {}, + "free": {}, "royalty": {}, "copyright": {}, "background": {}, "bgm": {}, + "loop": {}, "loops": {}, "loopable": {}, "seamless": {}, + "official": {}, "download": {}, "preview": {}, + "the": {}, "a": {}, "an": {}, "and": {}, "of": {}, "to": {}, "in": {}, "for": {}, "with": {}, "by": {}, + } + BracketRE = regexp.MustCompile(`\[[^\]]*\]|\([^)]*\)|\{[^}]*\}`) + SplitRE = regexp.MustCompile(`[^A-Za-z0-9]+`) + YearRE = regexp.MustCompile(`^\d{4,}$`) + VowelRE = regexp.MustCompile(`[aeiou]`) +) + +func DefaultConvertExtensions() []string { + return append([]string(nil), defaultConvertExtensions...) +} + +type Metadata struct { + Title string + Artist string + Album string + Date string + Rights string + Notes string + Comment string + Copyright string + License string +} + +type CreditsEntry struct { + Artist string `json:"artist"` + Title string `json:"title"` + OutputFile string `json:"output_file"` + OriginalFile string `json:"original_file"` + Album string `json:"album"` + Date string `json:"date"` + Rights string `json:"rights"` + Notes string `json:"notes"` +} + +type CreditsSource struct { + Path string `json:"path"` + Kind string `json:"kind"` + Entries []CreditsEntry `json:"entries"` +} + +type CreditsInventory struct { + Sources []CreditsSource `json:"sources"` +} + +type CreditsOverlay struct { + ByOriginal map[string]CreditsEntry + ByOutput map[string]CreditsEntry + Entries []CreditsEntry +} + +func ResolveFFmpeg() (string, error) { + return ResolveFFmpegWithConfig("") +} + +func ResolveFFmpegWithConfig(configuredPath string) (string, error) { + if configured := strings.TrimSpace(configuredPath); configured != "" { + return configured, nil + } + if configured := strings.TrimSpace(os.Getenv("SOW_FFMPEG")); configured != "" { + return configured, nil + } + path, err := exec.LookPath("ffmpeg") + if err != nil { + return "", err + } + return path, nil +} + +func ResolveFFprobe() (string, error) { + return ResolveFFprobeWithConfig("") +} + +func ResolveFFprobeWithConfig(configuredPath string) (string, error) { + if configured := strings.TrimSpace(configuredPath); configured != "" { + return configured, nil + } + if configured := strings.TrimSpace(os.Getenv("SOW_FFPROBE")); configured != "" { + return configured, nil + } + path, err := exec.LookPath("ffprobe") + if err != nil { + return "", err + } + return path, nil +} + +func IsMusicAssetPath(rel string) bool { + rel = filepath.ToSlash(strings.TrimSpace(rel)) + return rel == "envi/music" || strings.HasPrefix(rel, "envi/music/") +} + +func PathIsUnder(root, rel string) bool { + root = TrimSlashes(filepath.ToSlash(root)) + rel = TrimSlashes(filepath.ToSlash(rel)) + if root == "" || rel == "" { + return false + } + return rel == root || strings.HasPrefix(rel, root+"/") +} + +func PrefixForDir(prefixes map[string]string, dir string) string { + dir = TrimSlashes(filepath.ToSlash(dir)) + bestPrefix := "" + bestLen := -1 + keys := make([]string, 0, len(prefixes)) + for key := range prefixes { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + value := prefixes[key] + normalizedKey := TrimSlashes(filepath.ToSlash(key)) + if normalizedKey == "" { + continue + } + if dir != normalizedKey && !strings.HasPrefix(dir, normalizedKey+"/") { + continue + } + if len(normalizedKey) > bestLen { + bestLen = len(normalizedKey) + bestPrefix = SanitizePrefix(value) + } + } + return bestPrefix +} + +func SanitizePrefix(prefix string) string { + prefix = strings.ToLower(strings.TrimSpace(prefix)) + var builder strings.Builder + for _, r := range prefix { + switch { + case r >= 'a' && r <= 'z': + builder.WriteRune(r) + case r >= '0' && r <= '9': + builder.WriteRune(r) + case r == '_': + builder.WriteRune(r) + } + } + result := builder.String() + if strings.TrimSpace(prefix) != "" && strings.HasSuffix(strings.TrimSpace(prefix), "_") && !strings.HasSuffix(result, "_") { + result += "_" + } + return result +} + +func TrimSlashes(value string) string { + return strings.Trim(strings.TrimSpace(value), "/") +} + +func Min(a, b int) int { + if a < b { + return a + } + return b +} + +func Max(a, b int) int { + if a > b { + return a + } + return b +} + +func FirstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func JoinNonEmpty(sep string, values ...string) string { + parts := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + parts = append(parts, value) + } + } + return strings.Join(parts, sep) +} diff --git a/internal/music/music_test.go b/internal/music/music_test.go new file mode 100644 index 0000000..ba33de3 --- /dev/null +++ b/internal/music/music_test.go @@ -0,0 +1,404 @@ +package music + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestUniqueNameHandlesShortStemCollisions(t *testing.T) { + used := map[string]struct{}{ + "mus_wg_mystc": {}, + } + + got := UniqueName("mus_wg_mystc", used) + if got != "mus_wg_mystc_1" { + t.Fatalf("unexpected collision result: got %q want %q", got, "mus_wg_mystc_1") + } +} + +func TestUniqueNameNoCollision(t *testing.T) { + used := map[string]struct{}{} + got := UniqueName("new_stem", used) + if got != "new_stem" { + t.Fatalf("expected 'new_stem', got %q", got) + } +} + +func TestUniqueNameMultipleCollisions(t *testing.T) { + used := map[string]struct{}{ + "stem": {}, + "stem_1": {}, + "stem_2": {}, + } + got := UniqueName("stem", used) + if got != "stem_3" { + t.Fatalf("expected 'stem_3', got %q", got) + } +} + +func TestGenerateStem(t *testing.T) { + used := map[string]struct{}{ + "existing": {}, + } + stem, err := GenerateStem("My Cool Track.mp3", "mus_", used) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if stem == "" { + t.Fatal("expected non-empty stem") + } + if len(stem) > MaxStemLen { + t.Fatalf("stem %q exceeds max length %d", stem, MaxStemLen) + } + if _, exists := used[stem]; !exists { + t.Fatal("expected stem to be registered in used set") + } +} + +func TestGenerateStemPrefixTooLong(t *testing.T) { + _, err := GenerateStem("test.mp3", "this_prefix_is_way_too_long_for_stem_", nil) + if err == nil { + t.Fatal("expected error for too-long prefix") + } +} + +func TestSlugWords(t *testing.T) { + words := SlugWords("My Cool Music Track (Official) [HD]") + if len(words) == 0 { + t.Fatal("expected non-empty words") + } +} + +func TestSlugWordsStripsBrackets(t *testing.T) { + words := SlugWords("Song [Explicit] (Remix)") + for _, w := range words { + if w == "explicit" || w == "remix" { + t.Fatalf("bracket content should be removed, got word %q", w) + } + } +} + +func TestSanitizePrefix(t *testing.T) { + if got := SanitizePrefix("Mus_WG_"); got != "mus_wg_" { + t.Fatalf("unexpected prefix: %q", got) + } + if got := SanitizePrefix(" Test "); got != "test" { + t.Fatalf("unexpected prefix: %q", got) + } + if got := SanitizePrefix(""); got != "" { + t.Fatalf("expected empty prefix, got %q", got) + } +} + +func TestReserveName(t *testing.T) { + used := map[string]struct{}{} + if err := ReserveName("testname", used); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, exists := used["testname"]; !exists { + t.Fatal("expected name to be reserved") + } + if err := ReserveName("testname", used); err == nil { + t.Fatal("expected error for duplicate name") + } +} + +func TestValidateManualOutputFile(t *testing.T) { + if err := ValidateManualOutputFile("test.bmu"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err := ValidateManualOutputFile("test.txt"); err == nil { + t.Fatal("expected error for non-bmu extension") + } + if err := ValidateManualOutputFile(".bmu"); err == nil { + t.Fatal("expected error for empty stem") + } +} + +func TestParseCreditsMarkdownMissingFile(t *testing.T) { + entries, err := ParseCreditsMarkdown("nonexistent.md") + if err != nil { + t.Fatalf("unexpected error for missing file: %v", err) + } + if entries != nil { + t.Fatalf("expected nil entries for missing file, got %v", entries) + } +} + +func TestParseCreditsMarkdownRealContent(t *testing.T) { + content := "# Header\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Test Artist | Test Title | `output.bmu` | `original.mp3` | Test Album | 2024 | MIT | test |\n" + path := filepath.Join(t.TempDir(), "CREDITS.md") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write test credits: %v", err) + } + + entries, err := ParseCreditsMarkdown(path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(entries) != 1 { + t.Fatalf("expected 1 entry, got %d", len(entries)) + } + if entries[0].Artist != "Test Artist" { + t.Fatalf("expected 'Test Artist', got %q", entries[0].Artist) + } + if entries[0].Title != "Test Title" { + t.Fatalf("expected 'Test Title', got %q", entries[0].Title) + } + if entries[0].OutputFile != "output.bmu" { + t.Fatalf("expected 'output.bmu', got %q", entries[0].OutputFile) + } +} + +func TestRenderCreditsMarkdown(t *testing.T) { + entries := []CreditsEntry{ + { + Artist: "Test Artist", + Title: "Test Title", + OutputFile: "output.bmu", + OriginalFile: "original.mp3", + Album: "Test Album", + Date: "2024", + Rights: "MIT", + Notes: "test note", + }, + } + + result := RenderCreditsMarkdown(entries) + if !strings.Contains(result, "Test Artist") { + t.Fatal("expected rendered output to contain artist") + } + if !strings.Contains(result, "output.bmu") { + t.Fatal("expected rendered output to contain output file") + } +} + +func TestParseCreditsOverlay(t *testing.T) { + content := "# Header\n\n| Artist | Title | Output File | Original File |\n|---|---|---|---|\n| Overlay Artist | Ovr Title | `override.bmu` | `original.mp3` |\n" + path := filepath.Join(t.TempDir(), "CREDITS.md") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write test overlay: %v", err) + } + + overlay, err := ParseCreditsOverlay(path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(overlay.Entries) != 1 { + t.Fatalf("expected 1 overlay entry, got %d", len(overlay.Entries)) + } + + matched := overlay.Match("original.mp3", "") + if matched == nil { + t.Fatal("expected overlay match by original file") + } + if matched.Artist != "Overlay Artist" { + t.Fatalf("expected 'Overlay Artist', got %q", matched.Artist) + } +} + +func TestApplyCreditsOverlay(t *testing.T) { + entry := &CreditsEntry{ + Artist: "Original Artist", + Title: "Original Title", + } + overlay := &CreditsEntry{ + Artist: "Overlay Artist", + } + + ApplyCreditsOverlay(entry, overlay) + if entry.Artist != "Overlay Artist" { + t.Fatalf("expected 'Overlay Artist', got %q", entry.Artist) + } + if entry.Title != "Original Title" { + t.Fatalf("expected 'Original Title' preserved, got %q", entry.Title) + } + + ApplyCreditsOverlay(entry, nil) + if entry.Artist != "Overlay Artist" { + t.Fatalf("expected nil overlay to be no-op, got %q", entry.Artist) + } +} + +func TestValidateOverlayEntries(t *testing.T) { + generated := []CreditsEntry{ + {OriginalFile: "track1.mp3", OutputFile: "track1.bmu"}, + } + overlay := CreditsOverlay{ + Entries: []CreditsEntry{ + {OriginalFile: "track1.mp3"}, + }, + } + if err := ValidateOverlayEntries("test", overlay, generated); err != nil { + t.Fatalf("unexpected validation error: %v", err) + } + + badOverlay := CreditsOverlay{ + Entries: []CreditsEntry{ + {OriginalFile: "nonexistent.mp3"}, + }, + } + if err := ValidateOverlayEntries("test", badOverlay, generated); err == nil { + t.Fatal("expected validation error for unknown original file") + } +} + +func TestIsMusicAssetPath(t *testing.T) { + if !IsMusicAssetPath("envi/music") { + t.Fatal("expected 'envi/music' to be music path") + } + if !IsMusicAssetPath("envi/music/westgate") { + t.Fatal("expected 'envi/music/westgate' to be music path") + } + if IsMusicAssetPath("envi/textures") { + t.Fatal("expected 'envi/textures' not to be music path") + } + if IsMusicAssetPath("") { + t.Fatal("expected empty string not to be music path") + } +} + +func TestFirstNonEmpty(t *testing.T) { + if got := FirstNonEmpty("", "hello", "world"); got != "hello" { + t.Fatalf("expected 'hello', got %q", got) + } + if got := FirstNonEmpty("", "", ""); got != "" { + t.Fatalf("expected empty, got %q", got) + } + if got := FirstNonEmpty("first"); got != "first" { + t.Fatalf("expected 'first', got %q", got) + } +} + +func TestJoinNonEmpty(t *testing.T) { + if got := JoinNonEmpty(", ", "a", "", "b"); got != "a, b" { + t.Fatalf("expected 'a, b', got %q", got) + } + if got := JoinNonEmpty(", "); got != "" { + t.Fatalf("expected empty, got %q", got) + } +} + +func TestTrimSlashes(t *testing.T) { + if got := TrimSlashes("/foo/bar/"); got != "foo/bar" { + t.Fatalf("expected 'foo/bar', got %q", got) + } + if got := TrimSlashes("///"); got != "" { + t.Fatalf("expected empty, got %q", got) + } +} + +func TestEscapeMarkdownCell(t *testing.T) { + result := EscapeMarkdownCell("a|b\nc") + if !strings.Contains(result, `\|`) { + t.Fatal("expected pipe to be escaped") + } + if !strings.Contains(result, "
") { + t.Fatal("expected newline to be replaced with
") + } +} + +func TestWrapCodeCell(t *testing.T) { + if got := WrapCodeCell("test.bmu"); got != "`test.bmu`" { + t.Fatalf("expected '`test.bmu`', got %q", got) + } + if got := WrapCodeCell(""); got != "" { + t.Fatalf("expected empty string, got %q", got) + } +} + +func TestSyncGeneratedCreditsArtifactsWritesNewFiles(t *testing.T) { + root := t.TempDir() + desired := map[string][]byte{ + filepath.Join(root, "test.md"): []byte("content"), + } + changed, err := SyncGeneratedCreditsArtifacts(root, desired) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if changed != 1 { + t.Fatalf("expected 1 changed file, got %d", changed) + } +} + +func TestSyncGeneratedCreditsArtifactsNoChanges(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "test.md") + if err := os.WriteFile(path, []byte("content"), 0o644); err != nil { + t.Fatalf("write file: %v", err) + } + desired := map[string][]byte{ + path: []byte("content"), + } + changed, err := SyncGeneratedCreditsArtifacts(root, desired) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if changed != 0 { + t.Fatalf("expected 0 changed files, got %d", changed) + } +} + +func TestSyncGeneratedCreditsArtifactsRemovesStale(t *testing.T) { + root := t.TempDir() + stalePath := filepath.Join(root, "stale.md") + if err := os.WriteFile(stalePath, []byte("stale"), 0o644); err != nil { + t.Fatalf("write stale file: %v", err) + } + desired := map[string][]byte{} + changed, err := SyncGeneratedCreditsArtifacts(root, desired) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if changed != 1 { + t.Fatalf("expected 1 removed file, got %d", changed) + } +} + +func TestPrefixForDir(t *testing.T) { + prefixes := map[string]string{ + "envi/music/westgate": "mus_wg_", + } + if got := PrefixForDir(prefixes, "envi/music/westgate"); got != "mus_wg_" { + t.Fatalf("expected 'mus_wg_', got %q", got) + } + if got := PrefixForDir(prefixes, "envi/music/other"); got != "" { + t.Fatalf("expected empty prefix, got %q", got) + } + if got := PrefixForDir(nil, "envi/music"); got != "" { + t.Fatalf("expected empty prefix for nil map, got %q", got) + } +} + +func TestMaxMin(t *testing.T) { + if got := Min(3, 5); got != 3 { + t.Fatalf("Min(3,5) = %d, want 3", got) + } + if got := Max(3, 5); got != 5 { + t.Fatalf("Max(3,5) = %d, want 5", got) + } +} + +func TestResolveFFmpegEnv(t *testing.T) { + t.Setenv("SOW_FFMPEG", "/custom/ffmpeg") + path, err := ResolveFFmpeg() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if path != "/custom/ffmpeg" { + t.Fatalf("expected '/custom/ffmpeg', got %q", path) + } +} + +func TestResolveFFprobeEnv(t *testing.T) { + t.Setenv("SOW_FFPROBE", "/custom/ffprobe") + path, err := ResolveFFprobe() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if path != "/custom/ffprobe" { + t.Fatalf("expected '/custom/ffprobe', got %q", path) + } +} diff --git a/internal/music/naming.go b/internal/music/naming.go new file mode 100644 index 0000000..798f190 --- /dev/null +++ b/internal/music/naming.go @@ -0,0 +1,197 @@ +package music + +import ( + "crypto/sha1" + "fmt" + "path/filepath" + "strings" +) + +func GenerateStem(fileName, prefix string, used map[string]struct{}) (string, error) { + return GenerateStemWithMax(fileName, prefix, MaxStemLen, used) +} + +func GenerateStemWithMax(fileName, prefix string, maxStemLen int, used map[string]struct{}) (string, error) { + if maxStemLen <= 0 { + maxStemLen = MaxStemLen + } + maxCore := maxStemLen - len(prefix) + if maxCore < 3 { + return "", fmt.Errorf("prefix too long: %q", prefix) + } + base := strings.TrimSuffix(fileName, filepath.Ext(fileName)) + words := SlugWords(base) + core := MakeMusicCore(words, maxCore) + if core == "" { + sum := sha1.Sum([]byte(fileName)) + core = fmt.Sprintf("%x", sum[:])[:maxCore] + } + stem := strings.TrimRight((prefix + core)[:Min(maxStemLen, len(prefix)+len(core))], "_") + if stem == "" { + return "", fmt.Errorf("could not derive music stem for %s", fileName) + } + return UniqueNameWithMax(stem, maxStemLen, used), nil +} + +func SlugWords(value string) []string { + value = BracketRE.ReplaceAllString(value, " ") + value = strings.ReplaceAll(value, "&", " and ") + value = strings.ReplaceAll(value, "'", "") + value = strings.ReplaceAll(value, "\u2019", "") + value = strings.ToLower(value) + var ascii strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z': + ascii.WriteRune(r) + case r >= '0' && r <= '9': + ascii.WriteRune(r) + default: + ascii.WriteRune(' ') + } + } + rawWords := SplitRE.Split(ascii.String(), -1) + words := make([]string, 0, len(rawWords)) + for _, word := range rawWords { + if word == "" { + continue + } + if _, drop := DropWords[word]; drop { + continue + } + if YearRE.MatchString(word) { + continue + } + words = append(words, word) + } + return words +} + +func MakeMusicCore(words []string, maxCore int) string { + core := "" + for _, word := range words { + candidate := word + if core != "" { + candidate = core + "_" + word + } + if len(candidate) <= maxCore { + core = candidate + continue + } + break + } + if core != "" { + return core + } + abbrev := make([]string, 0, Min(4, len(words))) + for _, word := range words { + if len(abbrev) == 4 { + break + } + abbrev = append(abbrev, AbbreviateWord(word, 5)) + } + for _, word := range abbrev { + candidate := word + if core != "" { + candidate = core + "_" + word + } + if len(candidate) <= maxCore { + core = candidate + continue + } + break + } + if core != "" { + return core + } + if len(words) > 0 { + return strings.Trim(words[0][:Min(maxCore, len(words[0]))], "_") + } + return "" +} + +func AbbreviateWord(word string, maxLen int) string { + if len(word) <= maxLen { + return word + } + consonants := VowelRE.ReplaceAllString(word, "") + candidate := word[:1] + if len(consonants) > 1 { + candidate += consonants[1:] + } + if len(candidate) >= 3 { + return candidate[:Min(maxLen, len(candidate))] + } + return word[:Min(maxLen, len(word))] +} + +func UniqueName(stem string, used map[string]struct{}) string { + return UniqueNameWithMax(stem, MaxStemLen, used) +} + +func UniqueNameWithMax(stem string, maxStemLen int, used map[string]struct{}) string { + if maxStemLen <= 0 { + maxStemLen = MaxStemLen + } + if _, exists := used[stem]; !exists { + used[stem] = struct{}{} + return stem + } + for index := 1; ; index++ { + suffix := fmt.Sprintf("_%d", index) + prefixLen := Min(len(stem), Max(0, maxStemLen-len(suffix))) + candidate := strings.TrimRight(stem[:prefixLen], "_") + suffix + if _, exists := used[candidate]; exists { + continue + } + used[candidate] = struct{}{} + return candidate + } +} + +func ReserveName(stem string, used map[string]struct{}) error { + stem = strings.ToLower(strings.TrimSpace(stem)) + if stem == "" { + return fmt.Errorf("output filename is empty") + } + if _, exists := used[stem]; exists { + return fmt.Errorf("output filename %s collides with an existing asset", stem) + } + used[stem] = struct{}{} + return nil +} + +func ValidateManualOutputFile(name string) error { + return ValidateManualOutputFileWithRules(name, ".bmu", MaxStemLen) +} + +func ValidateManualOutputFileWithRules(name, extension string, maxStemLen int) error { + name = strings.ToLower(strings.TrimSpace(name)) + extension = strings.ToLower(strings.TrimSpace(extension)) + if extension == "" { + extension = ".bmu" + } + if maxStemLen <= 0 { + maxStemLen = MaxStemLen + } + if !strings.HasSuffix(name, extension) { + return fmt.Errorf("output file must end with %s", extension) + } + stem := strings.TrimSuffix(name, extension) + if stem == "" { + return fmt.Errorf("output file stem is empty") + } + if len(stem) > maxStemLen { + return fmt.Errorf("output file stem %q exceeds %d characters", stem, maxStemLen) + } + for _, r := range stem { + switch { + case r >= 'a' && r <= 'z': + case r >= '0' && r <= '9': + case r == '_': + default: + return fmt.Errorf("output file stem %q contains unsupported character %q", stem, string(r)) + } + } + return nil +} diff --git a/internal/pipeline/apply_manifest.go b/internal/pipeline/apply_manifest.go new file mode 100644 index 0000000..50d7411 --- /dev/null +++ b/internal/pipeline/apply_manifest.go @@ -0,0 +1,70 @@ +package pipeline + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +type ApplyManifestResult struct { + ManifestPath string + ModuleSource string + HAKCount int +} + +func ApplyHAKManifest(p *project.Project, manifestPath string) (ApplyManifestResult, error) { + if manifestPath == "" { + manifestPath = p.HAKManifestPath() + } + if strings.TrimSpace(p.EffectiveConfig().Paths.Source) == "" { + return ApplyManifestResult{}, fmt.Errorf("cannot apply hak manifest: paths.source is not configured") + } + sourceRoot := filepath.Clean(p.SourceDir()) + if sourceRoot == "." || sourceRoot == string(filepath.Separator) || sourceRoot == filepath.Clean(p.Root) { + return ApplyManifestResult{}, fmt.Errorf("cannot apply hak manifest: paths.source resolves to unsafe source root %s", sourceRoot) + } + + raw, err := os.ReadFile(manifestPath) + if err != nil { + return ApplyManifestResult{}, fmt.Errorf("read hak manifest: %w", err) + } + + var manifest BuildManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return ApplyManifestResult{}, fmt.Errorf("parse hak manifest: %w", err) + } + + moduleSource := filepath.Join(sourceRoot, "module", "module.ifo.json") + sourceRaw, err := os.ReadFile(moduleSource) + if err != nil { + return ApplyManifestResult{}, fmt.Errorf("read module ifo source: %w", err) + } + + var document gff.Document + if err := json.Unmarshal(sourceRaw, &document); err != nil { + return ApplyManifestResult{}, fmt.Errorf("parse module ifo source: %w", err) + } + + setModuleHAKList(&document, manifest.ModuleHAKs) + + formatted, err := json.MarshalIndent(document, "", " ") + if err != nil { + return ApplyManifestResult{}, fmt.Errorf("marshal updated module ifo: %w", err) + } + formatted = append(formatted, '\n') + + if err := os.WriteFile(moduleSource, formatted, 0o644); err != nil { + return ApplyManifestResult{}, fmt.Errorf("write module ifo source: %w", err) + } + + return ApplyManifestResult{ + ManifestPath: manifestPath, + ModuleSource: moduleSource, + HAKCount: len(manifest.ModuleHAKs), + }, nil +} diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go new file mode 100644 index 0000000..99abd54 --- /dev/null +++ b/internal/pipeline/build.go @@ -0,0 +1,1627 @@ +package pipeline + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/topdata" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/validator" +) + +type BuildResult struct { + ModulePath string + HAKPaths []string + Manifest string + AutogenManifestPaths []string + CreditsArtifactPaths []string + CreditsSummary CreditsRefreshSummary + HAKSummary HAKArchiveSummary + Resources int + HAKAssets int + TopPackageHAK string + TopPackageTLK string + TopPackageFiles int +} + +type CreditsRefreshSummary struct { + ScannedDirs []string + TrackCount int + ArtifactPaths []string + GeneratedPaths []string + InventoryPath string + ChangedFiles int + Mappings []MusicFileMapping +} + +type MusicFileMapping struct { + Source string + Output string +} + +type HAKArchiveSummary struct { + Total int + Reused int + Written int + Actions []HAKArchiveAction +} + +type HAKArchiveAction struct { + Name string + AssetCount int + Reused bool + OutputPath string +} + +type BuildManifest struct { + ModuleHAKs []string `json:"module_haks"` + HAKs []BuildManifestHAK `json:"haks"` +} + +type BuildManifestHAK struct { + Name string `json:"name"` + Group string `json:"group"` + Priority int `json:"priority"` + MaxBytes int64 `json:"max_bytes"` + SizeBytes int64 `json:"size_bytes"` + Optional bool `json:"optional,omitempty"` + Assets []string `json:"assets"` + ContentHash string `json:"content_hash,omitempty"` +} + +type assetResource struct { + Rel string + Resource erf.Resource + Size int64 + CreatedAt time.Time + ContentID string +} + +type lfsAssetInfo struct { + OID string + Size int64 + ObjectPath string +} + +type hakChunk struct { + Config project.HAKConfig + Index int + Name string + Assets []assetResource + Size int64 +} + +type ProgressFunc func(string) + +type BuildHAKOptions struct { + Progress ProgressFunc + ArchiveNames []string + SourceManifestPath string + SkipMusic bool + MusicDatasetIDs []string +} + +func Build(p *project.Project) (BuildResult, error) { + var topPackage topdata.PackageResult + var err error + if p.HasTopData() { + topPackage, err = topdata.BuildAndPackage(p, nil) + if err != nil { + return BuildResult{}, err + } + } + + moduleResult, err := BuildModule(p) + if err != nil { + return BuildResult{}, err + } + hakResult, err := BuildHAKs(p) + if err != nil { + return BuildResult{}, err + } + moduleResult.HAKPaths = hakResult.HAKPaths + moduleResult.Manifest = hakResult.Manifest + moduleResult.HAKAssets = hakResult.HAKAssets + moduleResult.TopPackageHAK = topPackage.OutputHAKPath + moduleResult.TopPackageTLK = topPackage.OutputTLKPath + moduleResult.TopPackageFiles = topPackage.HAKResources + return moduleResult, nil +} + +func BuildModule(p *project.Project) (BuildResult, error) { + return buildModule(p, nil) +} + +func BuildModuleWithProgress(p *project.Project, progress ProgressFunc) (BuildResult, error) { + return buildModule(p, progress) +} + +func buildModule(p *project.Project, progress ProgressFunc) (BuildResult, error) { + progressf(progress, "Validating project...") + if err := validateForBuild(p); err != nil { + return BuildResult{}, err + } + + progressf(progress, "Resolving module HAK order...") + moduleHakOrder, err := plannedModuleHAKOrder(p) + if err != nil { + return BuildResult{}, err + } + + progressf(progress, "Collecting module resources...") + moduleResources, err := collectModuleResources(p, moduleHakOrder) + if err != nil { + return BuildResult{}, err + } + + progressf(progress, "Writing module archive...") + outputPath := p.ModuleArchivePath() + if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create module output dir: %w", err) + } + output, err := os.Create(outputPath) + if err != nil { + return BuildResult{}, fmt.Errorf("create module archive: %w", err) + } + defer output.Close() + + if err := erf.Write(output, erf.New("MOD ", moduleResources)); err != nil { + return BuildResult{}, fmt.Errorf("write module archive: %w", err) + } + + return BuildResult{ + ModulePath: outputPath, + Resources: len(moduleResources), + }, nil +} + +func BuildHAKs(p *project.Project) (BuildResult, error) { + return buildHAKs(p, nil) +} + +func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResult, error) { + return buildHAKs(p, progress) +} + +func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) { + return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs) +} + +func PlanHAKs(p *project.Project) (BuildResult, error) { + return planHAKs(p, nil) +} + +func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResult, error) { + return planHAKs(p, progress) +} + +func PlanHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) { + return planOrBuildHAKs(p, opts.Progress, false, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs) +} + +func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { + return planOrBuildHAKs(p, progress, true, nil, "", false, nil) +} + +func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { + return planOrBuildHAKs(p, progress, false, nil, "", false, nil) +} + +func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string, skipMusic bool, musicDatasetIDs []string) (result BuildResult, err error) { + preserveExistingHAKs := p.EffectiveConfig().Build.KeepExistingHAKs || envBool("SOW_BUILD_HAKS_KEEP_EXISTING") + + progressf(progress, "Validating project...") + if err := validateForBuild(p); err != nil { + return BuildResult{}, err + } + + progressf(progress, "Collecting asset resources...") + allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(sourceManifestPath) + if err != nil { + return BuildResult{}, err + } + musicAssets := &preparedMusicAssets{SkipSourceRel: map[string]struct{}{}} + if !skipMusic { + musicAssets, err = prepareMusicAssetsWithOptions(p, musicPrepareOptions{ + datasetIDs: musicDatasetIDs, + write: writeArchives, + }) + if err != nil { + return BuildResult{}, err + } + } + defer func() { + if cleanupErr := cleanupPreparedMusicAssets(musicAssets); cleanupErr != nil && err == nil { + err = cleanupErr + } + }() + generated2DAAssets, err := topdata.BuildGenerated2DAAssets(p, progress) + if err != nil { + return BuildResult{}, err + } + assetResources, err := collectAssetResources(p, false, allowedAssets, musicAssets, generated2DAAssets) + if err != nil { + return BuildResult{}, err + } + autogenManifests, err := topdata.ProduceAutogenManifests(p) + if err != nil { + return BuildResult{}, err + } + var previousManifest *BuildManifest + if sourceManifest == nil || writeArchives { + previousManifest, err = loadPreviousBuildManifest(p.HAKManifestPath()) + if err != nil { + return BuildResult{}, err + } + } + + result = BuildResult{HAKAssets: len(assetResources)} + result.CreditsArtifactPaths = append(result.CreditsArtifactPaths, musicAssets.Artifacts...) + result.CreditsSummary = musicAssets.Summary + if err := writeAutogenManifestOutputs(progress, autogenManifests); err != nil { + return BuildResult{}, err + } + for _, manifest := range autogenManifests { + result.AutogenManifestPaths = append(result.AutogenManifestPaths, manifest.OutputPath) + } + if len(assetResources) == 0 { + progressf(progress, "Cleaning previous generated HAKs...") + if err := cleanupGeneratedHAKs(p); err != nil { + return BuildResult{}, err + } + return result, nil + } + + progressf(progress, "Planning HAK chunks...") + var chunks []hakChunk + if sourceManifest != nil { + chunks, err = chunksFromManifest(assetResources, sourceManifest.HAKs) + } else { + chunks, err = planHAKChunksWithPrevious(p, assetResources, previousManifest) + } + if err != nil { + return BuildResult{}, err + } + chunks, err = filterHAKChunksByName(chunks, archiveNames) + if err != nil { + return BuildResult{}, err + } + if writeArchives { + chunks, err = hydrateHAKChunks(p, chunks) + if err != nil { + return BuildResult{}, err + } + } + progressf(progress, "Resolving module HAK order...") + moduleHakOrder := []string{} + if sourceManifest != nil { + moduleHakOrder = append(moduleHakOrder, sourceManifest.ModuleHAKs...) + } else { + moduleHakOrder, err = resolveModuleHAKOrder(p, chunks) + if err != nil { + return BuildResult{}, err + } + } + + manifest := BuildManifest{ + ModuleHAKs: moduleHakOrder, + HAKs: make([]BuildManifestHAK, 0, len(chunks)), + } + currentNames := make(map[string]struct{}, len(chunks)) + for _, chunk := range chunks { + manifestEntry := buildManifestEntry(chunk) + currentNames[chunk.Name] = struct{}{} + manifest.HAKs = append(manifest.HAKs, manifestEntry) + } + + manifestPath := p.HAKManifestPath() + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return BuildResult{}, fmt.Errorf("marshal hak manifest: %w", err) + } + manifestBytes = append(manifestBytes, '\n') + + if !writeArchives { + progressf(progress, "Cleaning previous generated HAKs...") + if err := cleanupGeneratedHAKs(p); err != nil { + return BuildResult{}, err + } + progressf(progress, "Writing HAK manifest...") + if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create hak manifest dir: %w", err) + } + if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil { + return BuildResult{}, fmt.Errorf("write hak manifest: %w", err) + } + result.Manifest = manifestPath + return result, nil + } + + progressf(progress, "Reusing unchanged generated HAKs where possible...") + + result.HAKSummary.Total = len(chunks) + result.HAKPaths = make([]string, 0, len(chunks)) + for index, chunk := range chunks { + hakPath := p.HAKArchivePath(chunk.Name) + manifestEntry := manifest.HAKs[index] + reused, err := reuseExistingHAK(previousManifest, manifestEntry, hakPath) + if err != nil { + return BuildResult{}, err + } + if reused { + result.HAKSummary.Reused++ + progressf(progress, fmt.Sprintf("Reusing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets))) + } else { + result.HAKSummary.Written++ + progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets))) + if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err) + } + hakOutput, err := os.Create(hakPath) + if err != nil { + return BuildResult{}, fmt.Errorf("create hak archive: %w", err) + } + if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil { + hakOutput.Close() + return BuildResult{}, fmt.Errorf("write hak archive: %w", err) + } + if err := hakOutput.Close(); err != nil { + return BuildResult{}, fmt.Errorf("close hak archive: %w", err) + } + } + + result.HAKSummary.Actions = append(result.HAKSummary.Actions, HAKArchiveAction{ + Name: chunk.Name, + AssetCount: len(chunk.Assets), + Reused: reused, + OutputPath: hakPath, + }) + result.HAKPaths = append(result.HAKPaths, hakPath) + } + progressf(progress, "Cleaning stale generated HAKs...") + if !preserveExistingHAKs { + if err := cleanupStaleGeneratedHAKs(p, currentNames); err != nil { + return BuildResult{}, err + } + } + + progressf(progress, "Writing HAK manifest...") + if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create hak manifest dir: %w", err) + } + if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil { + return BuildResult{}, fmt.Errorf("write hak manifest: %w", err) + } + result.Manifest = manifestPath + return result, nil +} + +func writeAutogenManifestOutputs(progress ProgressFunc, manifests []topdata.ProducedAutogenManifest) error { + if len(manifests) == 0 { + return nil + } + for _, manifest := range manifests { + progressf(progress, fmt.Sprintf("Writing autogen manifest %s...", filepath.Base(manifest.OutputPath))) + if err := os.MkdirAll(filepath.Dir(manifest.OutputPath), 0o755); err != nil { + return fmt.Errorf("create autogen manifest dir: %w", err) + } + if err := os.WriteFile(manifest.OutputPath, manifest.Payload, 0o644); err != nil { + return fmt.Errorf("write autogen manifest %s: %w", manifest.AssetName, err) + } + } + return nil +} + +func filterHAKChunksByName(chunks []hakChunk, archiveNames []string) ([]hakChunk, error) { + if len(archiveNames) == 0 { + return chunks, nil + } + + wanted := make(map[string]struct{}, len(archiveNames)) + for _, name := range archiveNames { + trimmed := strings.TrimSpace(strings.ToLower(name)) + if trimmed == "" { + continue + } + wanted[trimmed] = struct{}{} + } + if len(wanted) == 0 { + return chunks, nil + } + + filtered := make([]hakChunk, 0, len(wanted)) + found := make(map[string]struct{}, len(wanted)) + for _, chunk := range chunks { + key := strings.ToLower(strings.TrimSpace(chunk.Name)) + if _, ok := wanted[key]; !ok { + continue + } + filtered = append(filtered, chunk) + found[key] = struct{}{} + } + + var missing []string + for _, name := range archiveNames { + key := strings.TrimSpace(strings.ToLower(name)) + if key == "" { + continue + } + if _, ok := found[key]; !ok { + missing = append(missing, name) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf("unknown hak archive(s): %s", strings.Join(missing, ", ")) + } + return filtered, nil +} + +func hydrateHAKChunks(p *project.Project, chunks []hakChunk) ([]hakChunk, error) { + if len(chunks) == 0 { + return chunks, nil + } + + lfsAssets, err := collectLFSAssetInfo(p) + if err != nil { + return nil, err + } + + hydrated := make([]hakChunk, 0, len(chunks)) + for _, chunk := range chunks { + nextChunk := chunk + nextChunk.Assets = make([]assetResource, 0, len(chunk.Assets)) + for _, asset := range chunk.Assets { + if !strings.HasPrefix(asset.ContentID, "lfs:") { + nextChunk.Assets = append(nextChunk.Assets, asset) + continue + } + + info, ok := lfsAssets[filepath.ToSlash(asset.Rel)] + if !ok { + return nil, fmt.Errorf("missing LFS metadata for %s", asset.Rel) + } + resource, err := lfsPathResource(filepath.Join(p.AssetsDir(), filepath.FromSlash(asset.Rel)), info, true) + if err != nil { + return nil, err + } + asset.Resource = resource + nextChunk.Assets = append(nextChunk.Assets, asset) + } + hydrated = append(hydrated, nextChunk) + } + + return hydrated, nil +} + +func progressf(progress ProgressFunc, message string) { + if progress != nil { + progress(message) + } +} + +func envBool(name string) bool { + value := strings.TrimSpace(strings.ToLower(os.Getenv(name))) + switch value { + case "1", "true", "yes", "on": + return true + default: + return false + } +} + +func collectModuleResources(p *project.Project, moduleHakOrder []string) ([]erf.Resource, error) { + var moduleResources []erf.Resource + + for _, rel := range p.Inventory.SourceFiles { + abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) + resource, err := resourceFromJSON(abs, moduleHakOrder) + if err != nil { + return nil, err + } + moduleResources = append(moduleResources, resource) + } + + for _, rel := range p.Inventory.ScriptFiles { + abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) + resource, err := rawResource(abs) + if err != nil { + return nil, err + } + moduleResources = append(moduleResources, resource) + } + + sortResources(moduleResources) + return moduleResources, nil +} + +func collectAssetResources(p *project.Project, requireContent bool, allowed map[string]struct{}, musicAssets *preparedMusicAssets, generated2DAAssets []topdata.Generated2DAAsset) ([]assetResource, error) { + var hakResources []assetResource + assetCreatedAt, err := collectGitAssetCreationTimes(p) + if err != nil { + return nil, err + } + lfsAssets, err := collectLFSAssetInfo(p) + if err != nil { + return nil, err + } + assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir()) + if err != nil { + return nil, fmt.Errorf("resolve assets dir relative path: %w", err) + } + assetsRelPath = filepath.ToSlash(assetsRelPath) + + for _, rel := range p.Inventory.AssetFiles { + if musicAssets != nil { + if _, skip := musicAssets.SkipSourceRel[filepath.ToSlash(rel)]; skip { + continue + } + } + if len(allowed) > 0 { + if _, ok := allowed[filepath.ToSlash(rel)]; !ok { + continue + } + } + abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)) + resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent) + if err != nil { + return nil, err + } + info, err := os.Stat(abs) + if err != nil { + return nil, fmt.Errorf("stat %s: %w", abs, err) + } + repoRel := filepath.ToSlash(filepath.Join(assetsRelPath, filepath.FromSlash(rel))) + createdAt := assetCreatedAt[repoRel] + if createdAt.IsZero() { + createdAt = info.ModTime() + } + hakResources = append(hakResources, assetResource{ + Rel: rel, + Resource: resourceInfo.Resource, + Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}), + CreatedAt: createdAt, + ContentID: resourceInfo.ContentID, + }) + } + if musicAssets != nil { + for _, asset := range musicAssets.Generated { + if len(allowed) > 0 { + if _, ok := allowed[filepath.ToSlash(asset.Rel)]; !ok { + continue + } + } + hakResources = append(hakResources, asset) + } + } + for _, asset := range generated2DAAssets { + rel := filepath.ToSlash(asset.Rel) + if len(allowed) > 0 { + if _, ok := allowed[rel]; !ok { + continue + } + } + resourceInfo, err := assetResourceFromPath(asset.SourcePath, rel, lfsAssetInfo{}, requireContent) + if err != nil { + return nil, err + } + info, err := os.Stat(asset.SourcePath) + if err != nil { + return nil, fmt.Errorf("stat %s: %w", asset.SourcePath, err) + } + hakResources = append(hakResources, assetResource{ + Rel: rel, + Resource: resourceInfo.Resource, + Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}), + CreatedAt: info.ModTime(), + ContentID: resourceInfo.ContentID, + }) + } + + slices.SortFunc(hakResources, func(a, b assetResource) int { + return compareResourceKeys(a.Resource, b.Resource) + }) + return hakResources, nil +} + +func loadSourceManifestAssetSet(path string) (map[string]struct{}, *BuildManifest, error) { + if strings.TrimSpace(path) == "" { + return nil, nil, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("read source manifest %s: %w", path, err) + } + var manifest BuildManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, nil, fmt.Errorf("parse source manifest %s: %w", path, err) + } + allowed := make(map[string]struct{}) + for _, hak := range manifest.HAKs { + for _, rel := range hak.Assets { + allowed[filepath.ToSlash(rel)] = struct{}{} + } + } + return allowed, &manifest, nil +} + +func chunksFromManifest(assets []assetResource, entries []BuildManifestHAK) ([]hakChunk, error) { + assetByRel := make(map[string]assetResource, len(assets)) + for _, asset := range assets { + assetByRel[filepath.ToSlash(asset.Rel)] = asset + } + + chunks := make([]hakChunk, 0, len(entries)) + for index, entry := range entries { + chunkAssets := make([]assetResource, 0, len(entry.Assets)) + for _, rel := range entry.Assets { + asset, ok := assetByRel[filepath.ToSlash(rel)] + if !ok { + return nil, fmt.Errorf("source manifest references missing asset %s in archive %s", rel, entry.Name) + } + chunkAssets = append(chunkAssets, asset) + } + chunks = append(chunks, hakChunk{ + Config: project.HAKConfig{ + Name: entry.Group, + Priority: entry.Priority, + MaxBytes: entry.MaxBytes, + Optional: entry.Optional, + }, + Index: index, + Name: entry.Name, + Assets: chunkAssets, + Size: erf.ArchiveSize(resourceSlice(chunkAssets)), + }) + } + return chunks, nil +} + +func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) { + result := make(map[string]lfsAssetInfo) + lfsObjectRoot, err := resolveLFSObjectRoot(p.Root) + if err != nil { + return nil, err + } + for _, rel := range p.Inventory.AssetFiles { + rel = filepath.ToSlash(rel) + abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)) + info, ok, err := parseLFSPointerFile(abs, lfsObjectRoot) + if err != nil { + return nil, err + } + if ok { + result[rel] = info + } + } + + return result, nil +} + +func resolveLFSObjectRoot(repoRoot string) (string, error) { + output, err := gitOutput(repoRoot, "rev-parse", "--git-common-dir") + if err != nil { + return filepath.Join(repoRoot, ".git", "lfs", "objects"), nil + } + commonDir := strings.TrimSpace(output) + if commonDir == "" { + return filepath.Join(repoRoot, ".git", "lfs", "objects"), nil + } + if !filepath.IsAbs(commonDir) { + commonDir = filepath.Join(repoRoot, commonDir) + } + return filepath.Join(filepath.Clean(commonDir), "lfs", "objects"), nil +} + +func parseLFSPointerFile(path, lfsObjectRoot string) (lfsAssetInfo, bool, error) { + entry, err := os.Lstat(path) + if err != nil { + return lfsAssetInfo{}, false, fmt.Errorf("stat %s: %w", path, err) + } + if entry.Mode()&os.ModeSymlink != 0 || entry.Size() > 1024 { + return lfsAssetInfo{}, false, nil + } + + raw, err := os.ReadFile(path) + if err != nil { + return lfsAssetInfo{}, false, fmt.Errorf("read %s: %w", path, err) + } + text := strings.ReplaceAll(string(raw), "\r\n", "\n") + if !strings.HasPrefix(text, "version https://git-lfs.github.com/spec/v1\n") { + return lfsAssetInfo{}, false, nil + } + + info := lfsAssetInfo{} + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "oid sha256:"): + info.OID = strings.TrimSpace(strings.TrimPrefix(line, "oid sha256:")) + case strings.HasPrefix(line, "size "): + size, parseErr := strconv.ParseInt(strings.TrimSpace(strings.TrimPrefix(line, "size ")), 10, 64) + if parseErr != nil { + return lfsAssetInfo{}, false, fmt.Errorf("parse LFS pointer size in %s: %w", path, parseErr) + } + info.Size = size + } + } + if info.OID == "" || info.Size <= 0 { + return lfsAssetInfo{}, false, nil + } + if len(info.OID) >= 4 { + info.ObjectPath = filepath.Join(lfsObjectRoot, info.OID[:2], info.OID[2:4], info.OID) + } + return info, true, nil +} + +func assetResourceFromPath(path, rel string, lfsInfo lfsAssetInfo, requireContent bool) (assetResource, error) { + if lfsInfo.OID != "" { + resource, err := lfsPathResource(path, lfsInfo, requireContent) + if err != nil { + return assetResource{}, err + } + return assetResource{ + Rel: rel, + Resource: resource, + ContentID: "lfs:" + lfsInfo.OID, + }, nil + } + + resource, err := pathResource(path) + if err != nil { + return assetResource{}, err + } + fileHash, err := fileSHA256(path) + if err != nil { + return assetResource{}, err + } + return assetResource{ + Rel: rel, + Resource: resource, + ContentID: "file:" + fileHash, + }, nil +} + +func lfsPathResource(path string, info lfsAssetInfo, requireContent bool) (erf.Resource, error) { + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") + resourceType, ok := erf.HAKResourceTypeForExtension(extension) + if !ok { + return erf.Resource{}, fmt.Errorf("unsupported HAK resource extension %q", filepath.Ext(path)) + } + name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) + sourcePath := "" + if requireContent { + if info.ObjectPath == "" { + return erf.Resource{}, fmt.Errorf("missing downloaded LFS object for %s", path) + } + if _, err := os.Stat(info.ObjectPath); err != nil { + return erf.Resource{}, fmt.Errorf("missing downloaded LFS object for %s: %w", path, err) + } + sourcePath = info.ObjectPath + } + return erf.Resource{ + Name: name, + Type: resourceType, + SourcePath: sourcePath, + Size: info.Size, + }, nil +} + +func fileSHA256(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read %s: %w", path, err) + } + sum := sha256.Sum256(raw) + return fmt.Sprintf("%x", sum[:]), nil +} + +func collectGitAssetCreationTimes(p *project.Project) (map[string]time.Time, error) { + result := make(map[string]time.Time, len(p.Inventory.AssetFiles)) + if len(p.Inventory.AssetFiles) == 0 { + return result, nil + } + + assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir()) + if err != nil { + return result, fmt.Errorf("resolve assets dir relative path: %w", err) + } + assetsRelPath = filepath.ToSlash(assetsRelPath) + + cacheFile := filepath.Join(p.Root, ".git", "asset-creation-times.json") + if cache, err := loadCreationTimeCache(cacheFile, p.Root, assetsRelPath, p.Inventory.AssetFiles); err == nil { + return cache, nil + } + + output, err := gitOutput(p.Root, "log", "--diff-filter=A", "--format=%ct", "--name-only", "--", assetsRelPath) + if err != nil { + return result, nil + } + + wanted := make(map[string]struct{}, len(p.Inventory.AssetFiles)) + for _, rel := range p.Inventory.AssetFiles { + wanted[filepath.ToSlash(filepath.Join(assetsRelPath, filepath.FromSlash(rel)))] = struct{}{} + } + + var currentCommitTime time.Time + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if sec, err := strconv.ParseInt(line, 10, 64); err == nil { + currentCommitTime = time.Unix(sec, 0).UTC() + continue + } + if currentCommitTime.IsZero() { + continue + } + if _, ok := wanted[line]; !ok { + continue + } + if _, exists := result[line]; exists { + continue + } + result[line] = currentCommitTime + if len(result) == len(wanted) { + break + } + } + + _ = saveCreationTimeCache(cacheFile, p.Root, result) + return result, nil +} + +type creationTimeCache struct { + Head string `json:"head"` + Times map[string]int64 `json:"times"` +} + +func loadCreationTimeCache(cacheFile, root, assetsRel string, assetFiles []string) (map[string]time.Time, error) { + data, err := os.ReadFile(cacheFile) + if err != nil { + return nil, err + } + var cache creationTimeCache + if err := json.Unmarshal(data, &cache); err != nil { + return nil, err + } + currentHead, err := gitOutput(root, "rev-parse", "HEAD") + if err != nil { + return nil, err + } + currentHead = strings.TrimSpace(currentHead) + if cache.Head != currentHead { + return nil, fmt.Errorf("cache stale") + } + result := make(map[string]time.Time, len(cache.Times)) + for k, v := range cache.Times { + result[k] = time.Unix(v, 0).UTC() + } + return result, nil +} + +func saveCreationTimeCache(cacheFile, root string, times map[string]time.Time) error { + head, err := gitOutput(root, "rev-parse", "HEAD") + if err != nil { + return err + } + intTimes := make(map[string]int64, len(times)) + for k, v := range times { + intTimes[k] = v.Unix() + } + cache := creationTimeCache{ + Head: strings.TrimSpace(head), + Times: intTimes, + } + data, err := json.Marshal(cache) + if err != nil { + return err + } + return os.WriteFile(cacheFile, data, 0o644) +} + +func gitOutput(dir string, args ...string) (string, error) { + cmd := exec.Command("git", args...) + if dir != "" { + cmd.Dir = dir + } + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output))) + } + return string(output), nil +} + +func sortResources(resources []erf.Resource) { + slices.SortFunc(resources, func(a, b erf.Resource) int { + return compareResourceKeys(a, b) + }) +} + +func compareResourceKeys(a, b erf.Resource) int { + if cmp := strings.Compare(a.Name, b.Name); cmp != 0 { + return cmp + } + if a.Type < b.Type { + return -1 + } + if a.Type > b.Type { + return 1 + } + return 0 +} + +func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error) { + name, extension, err := splitSourceName(path) + if err != nil { + return erf.Resource{}, err + } + + resourceType, ok := erf.ResourceTypeForExtension(extension) + if !ok { + return erf.Resource{}, fmt.Errorf("unsupported source extension %q", extension) + } + + raw, err := os.ReadFile(path) + if err != nil { + return erf.Resource{}, fmt.Errorf("read %s: %w", path, err) + } + + var document gff.Document + if err := json.Unmarshal(raw, &document); err != nil { + return erf.Resource{}, fmt.Errorf("parse gff json %s: %w", path, err) + } + if document.FileType == "" { + document.FileType = strings.ToUpper(strings.TrimPrefix(extension, ".")) + } + if document.FileVersion == "" { + document.FileVersion = "V3.2" + } + if extension == ".ifo" && name == "module" && len(moduleHakOrder) > 0 { + setModuleHAKList(&document, moduleHakOrder) + } + + var buf bytes.Buffer + if err := gff.Write(&buf, document); err != nil { + return erf.Resource{}, fmt.Errorf("encode gff %s: %w", path, err) + } + + return erf.Resource{ + Name: strings.ToLower(name), + Type: resourceType, + Data: buf.Bytes(), + }, nil +} + +func pathResource(path string) (erf.Resource, error) { + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") + resourceType, ok := erf.HAKResourceTypeForExtension(extension) + if !ok { + return erf.Resource{}, fmt.Errorf("unsupported HAK resource extension %q", filepath.Ext(path)) + } + name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) + info, err := os.Stat(path) + if err != nil { + return erf.Resource{}, fmt.Errorf("stat %s: %w", path, err) + } + return erf.Resource{ + Name: name, + Type: resourceType, + SourcePath: path, + Size: info.Size(), + }, nil +} + +func rawResource(path string) (erf.Resource, error) { + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") + resourceType, ok := erf.HAKResourceTypeForExtension(extension) + if !ok { + return erf.Resource{}, fmt.Errorf("unsupported HAK resource extension %q", filepath.Ext(path)) + } + name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) + data, err := os.ReadFile(path) + if err != nil { + return erf.Resource{}, fmt.Errorf("read %s: %w", path, err) + } + return erf.Resource{ + Name: strings.ToLower(name), + Type: resourceType, + Data: data, + }, nil +} + +func splitSourceName(path string) (string, string, error) { + base := filepath.Base(path) + if !strings.HasSuffix(base, ".json") { + return "", "", fmt.Errorf("expected .json source file: %s", path) + } + stem := strings.TrimSuffix(base, ".json") + extension := filepath.Ext(stem) + if extension == "" { + return "", "", fmt.Errorf("source file must include target resource extension before .json: %s", path) + } + return strings.ToLower(strings.TrimSuffix(stem, extension)), strings.ToLower(extension), nil +} + +func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, error) { + return planHAKChunksWithPrevious(p, assets, nil) +} + +func planHAKChunksWithPrevious(p *project.Project, assets []assetResource, previousManifest *BuildManifest) ([]hakChunk, error) { + if len(assets) == 0 { + return nil, nil + } + if len(p.Config.HAKs) == 0 { + chunk := hakChunk{ + Config: project.HAKConfig{ + Name: p.Config.Module.ResRef, + Priority: 1, + }, + Index: 1, + Name: p.Config.Module.ResRef, + Size: erf.ArchiveSize(resourceSlice(assets)), + Assets: assets, + } + return []hakChunk{chunk}, nil + } + + configs := make([]project.HAKConfig, len(p.Config.HAKs)) + copy(configs, p.Config.HAKs) + slices.SortFunc(configs, func(a, b project.HAKConfig) int { + if a.Priority != b.Priority { + if a.Priority < b.Priority { + return -1 + } + return 1 + } + return strings.Compare(a.Name, b.Name) + }) + + grouped := make([][]assetResource, len(configs)) + for _, asset := range assets { + matched := false + for index, cfg := range configs { + if matchesAnyPattern(asset.Rel, cfg.Include) { + grouped[index] = append(grouped[index], asset) + matched = true + break + } + } + if !matched { + return nil, fmt.Errorf("asset %s does not match any hak include pattern", asset.Rel) + } + } + + var chunks []hakChunk + for index, cfg := range configs { + groupAssets := grouped[index] + if len(groupAssets) == 0 { + continue + } + slices.SortFunc(groupAssets, compareAssetBuildOrder) + + groupChunks, err := splitHAKGroupWithPrevious(cfg, groupAssets, previousManifest) + if err != nil { + return nil, err + } + chunks = append(chunks, groupChunks...) + } + return chunks, nil +} + +func splitHAKGroupWithPrevious(cfg project.HAKConfig, assets []assetResource, previousManifest *BuildManifest) ([]hakChunk, error) { + if previousManifest == nil || cfg.MaxBytes == 0 || !cfg.Split { + return splitHAKGroup(cfg, assets) + } + + previousEntries := make([]BuildManifestHAK, 0) + for _, entry := range previousManifest.HAKs { + if entry.Group != cfg.Name { + continue + } + if entry.Priority != cfg.Priority || entry.MaxBytes != cfg.MaxBytes || entry.Optional != cfg.Optional { + return splitHAKGroup(cfg, assets) + } + previousEntries = append(previousEntries, entry) + } + if len(previousEntries) == 0 { + return splitHAKGroup(cfg, assets) + } + + assetByRel := make(map[string]assetResource, len(assets)) + for _, asset := range assets { + assetByRel[filepath.ToSlash(asset.Rel)] = asset + } + + assigned := make(map[string]struct{}, len(assets)) + chunks := make([]hakChunk, 0, len(previousEntries)) + for _, entry := range previousEntries { + chunkAssets := make([]assetResource, 0, len(entry.Assets)) + for _, rel := range entry.Assets { + key := filepath.ToSlash(rel) + asset, ok := assetByRel[key] + if !ok { + continue + } + if _, exists := assigned[key]; exists { + return splitHAKGroup(cfg, assets) + } + chunkAssets = append(chunkAssets, asset) + assigned[key] = struct{}{} + } + if len(chunkAssets) == 0 { + continue + } + + chunk := hakChunk{ + Config: cfg, + Index: len(chunks) + 1, + Name: entry.Name, + Assets: chunkAssets, + Size: erf.ArchiveSize(resourceSlice(chunkAssets)), + } + if chunk.Size > cfg.MaxBytes { + return splitHAKGroup(cfg, assets) + } + if err := ensureUniqueChunkResources(chunk); err != nil { + return splitHAKGroup(cfg, assets) + } + chunks = append(chunks, chunk) + } + + newAssets := make([]assetResource, 0) + for _, asset := range assets { + key := filepath.ToSlash(asset.Rel) + if _, exists := assigned[key]; exists { + continue + } + newAssets = append(newAssets, asset) + } + slices.SortFunc(newAssets, compareAssetBuildOrder) + + for _, asset := range newAssets { + inserted := false + for index := len(chunks) - 1; index >= 0; index-- { + candidate, ok := chunkWithAppendedAsset(chunks[index], asset, cfg.MaxBytes) + if !ok { + continue + } + chunks[index] = candidate + inserted = true + break + } + if inserted { + continue + } + + chunk := hakChunk{ + Config: cfg, + Index: len(chunks) + 1, + Name: nextChunkName(cfg.Name, chunks), + Assets: []assetResource{asset}, + Size: erf.ArchiveSize([]erf.Resource{asset.Resource}), + } + if chunk.Size > cfg.MaxBytes { + return nil, fmt.Errorf("asset %s alone exceeds max_bytes for hak %s", asset.Rel, cfg.Name) + } + if err := ensureUniqueChunkResources(chunk); err != nil { + return nil, err + } + chunks = append(chunks, chunk) + } + + return chunks, nil +} + +func nextChunkName(base string, chunks []hakChunk) string { + used := make(map[string]struct{}, len(chunks)) + for _, chunk := range chunks { + used[chunk.Name] = struct{}{} + } + for index := len(chunks) + 1; ; index++ { + name := chunkName(base, index, true) + if _, exists := used[name]; !exists { + return name + } + } +} + +func chunkWithAppendedAsset(chunk hakChunk, asset assetResource, maxBytes int64) (hakChunk, bool) { + candidateAssets := append(append([]assetResource{}, chunk.Assets...), asset) + candidate := chunk + candidate.Assets = candidateAssets + candidate.Size = erf.ArchiveSize(resourceSlice(candidateAssets)) + if candidate.Size > maxBytes { + return hakChunk{}, false + } + if err := ensureUniqueChunkResources(candidate); err != nil { + return hakChunk{}, false + } + return candidate, true +} + +func splitHAKGroup(cfg project.HAKConfig, assets []assetResource) ([]hakChunk, error) { + maxBytes := cfg.MaxBytes + if maxBytes == 0 { + chunk := hakChunk{ + Config: cfg, + Index: 1, + Name: cfg.Name, + Assets: assets, + Size: erf.ArchiveSize(resourceSlice(assets)), + } + if err := ensureUniqueChunkResources(chunk); err != nil { + return nil, err + } + return []hakChunk{chunk}, nil + } + + var chunks []hakChunk + current := hakChunk{Config: cfg, Index: 1} + for _, asset := range assets { + candidateAssets := append(append([]assetResource{}, current.Assets...), asset) + candidateResources := resourceSlice(candidateAssets) + candidateSize := erf.ArchiveSize(candidateResources) + + if len(current.Assets) == 0 { + if candidateSize > maxBytes { + return nil, fmt.Errorf("asset %s alone exceeds max_bytes for hak %s", asset.Rel, cfg.Name) + } + current.Assets = candidateAssets + current.Size = candidateSize + continue + } + + if candidateSize <= maxBytes { + current.Assets = candidateAssets + current.Size = candidateSize + continue + } + + if !cfg.Split { + return nil, fmt.Errorf("hak %s exceeds max_bytes and split is disabled", cfg.Name) + } + + current.Name = chunkName(cfg.Name, current.Index, true) + if err := ensureUniqueChunkResources(current); err != nil { + return nil, err + } + chunks = append(chunks, current) + + current = hakChunk{ + Config: cfg, + Index: current.Index + 1, + Assets: []assetResource{asset}, + Size: erf.ArchiveSize([]erf.Resource{asset.Resource}), + } + if current.Size > maxBytes { + return nil, fmt.Errorf("asset %s alone exceeds max_bytes for hak %s", asset.Rel, cfg.Name) + } + } + + if len(current.Assets) > 0 { + current.Name = chunkName(cfg.Name, current.Index, cfg.Split || len(chunks) > 0) + if err := ensureUniqueChunkResources(current); err != nil { + return nil, err + } + chunks = append(chunks, current) + } + return chunks, nil +} + +func compareAssetBuildOrder(a, b assetResource) int { + if cmp := a.CreatedAt.Compare(b.CreatedAt); cmp != 0 { + return cmp + } + if cmp := compareResourceKeys(a.Resource, b.Resource); cmp != 0 { + return cmp + } + return strings.Compare(a.Rel, b.Rel) +} + +func chunkName(base string, index int, split bool) string { + if !split { + return base + } + return fmt.Sprintf("%s_%02d", base, index) +} + +func resourceSlice(assets []assetResource) []erf.Resource { + out := make([]erf.Resource, 0, len(assets)) + for _, asset := range assets { + out = append(out, asset.Resource) + } + return out +} + +func buildManifestEntry(chunk hakChunk) BuildManifestHAK { + assets := make([]string, 0, len(chunk.Assets)) + for _, asset := range chunk.Assets { + assets = append(assets, asset.Rel) + } + return BuildManifestHAK{ + Name: chunk.Name, + Group: chunk.Config.Name, + Priority: chunk.Config.Priority, + MaxBytes: chunk.Config.MaxBytes, + SizeBytes: chunk.Size, + Optional: chunk.Config.Optional, + Assets: assets, + ContentHash: chunkContentHash(chunk), + } +} + +func chunkContentHash(chunk hakChunk) string { + sum := sha256.New() + fmt.Fprintf(sum, "name:%s\n", chunk.Name) + fmt.Fprintf(sum, "group:%s\n", chunk.Config.Name) + fmt.Fprintf(sum, "priority:%d\n", chunk.Config.Priority) + fmt.Fprintf(sum, "max_bytes:%d\n", chunk.Config.MaxBytes) + fmt.Fprintf(sum, "optional:%t\n", chunk.Config.Optional) + fmt.Fprintf(sum, "size_bytes:%d\n", chunk.Size) + for _, asset := range chunk.Assets { + fmt.Fprintf(sum, "asset:%s\n", asset.Rel) + fmt.Fprintf(sum, "resref:%s\n", asset.Resource.Name) + fmt.Fprintf(sum, "restype:%d\n", asset.Resource.Type) + fmt.Fprintf(sum, "datasize:%d\n", asset.Resource.Size) + fmt.Fprintf(sum, "content:%s\n", asset.ContentID) + } + return fmt.Sprintf("%x", sum.Sum(nil)) +} + +func loadPreviousBuildManifest(manifestPath string) (*BuildManifest, error) { + raw, err := os.ReadFile(manifestPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("read previous hak manifest: %w", err) + } + var manifest BuildManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("parse previous hak manifest: %w", err) + } + return &manifest, nil +} + +func reuseExistingHAK(previousManifest *BuildManifest, current BuildManifestHAK, hakPath string) (bool, error) { + if previousManifest == nil { + return false, nil + } + var previous *BuildManifestHAK + for index := range previousManifest.HAKs { + entry := &previousManifest.HAKs[index] + if entry.Name == current.Name { + previous = entry + break + } + } + if previous == nil { + return false, nil + } + if previous.Group != current.Group || + previous.Priority != current.Priority || + previous.MaxBytes != current.MaxBytes || + previous.SizeBytes != current.SizeBytes || + previous.ContentHash != current.ContentHash || + !slices.Equal(previous.Assets, current.Assets) { + return false, nil + } + if _, err := os.Stat(hakPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, fmt.Errorf("stat existing hak archive %s: %w", hakPath, err) + } + return true, nil +} + +func cleanupStaleGeneratedHAKs(p *project.Project, keepNames map[string]struct{}) error { + paths, err := manifestHAKPaths(p) + if err != nil { + return err + } + for _, path := range paths { + name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + if _, keep := keepNames[name]; keep { + continue + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove stale generated hak %s: %w", path, err) + } + } + + legacy := p.HAKArchivePath(p.Config.Module.ResRef) + if _, keep := keepNames[p.Config.Module.ResRef]; !keep { + if err := os.Remove(legacy); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove legacy generated hak %s: %w", legacy, err) + } + } + return nil +} + +func matchesAnyPattern(path string, patterns []string) bool { + for _, pattern := range patterns { + if matchPathPattern(filepath.ToSlash(path), filepath.ToSlash(pattern)) { + return true + } + } + return false +} + +func matchPathPattern(path, pattern string) bool { + pathSegs := strings.Split(strings.Trim(path, "/"), "/") + patternSegs := strings.Split(strings.Trim(pattern, "/"), "/") + return matchSegments(pathSegs, patternSegs) +} + +func cleanupGeneratedHAKs(p *project.Project) error { + paths, err := manifestHAKPaths(p) + if err != nil { + return err + } + for _, path := range paths { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove previous generated hak %s: %w", path, err) + } + } + + legacy := p.HAKArchivePath(p.Config.Module.ResRef) + if err := os.Remove(legacy); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove legacy generated hak %s: %w", legacy, err) + } + + manifest := p.HAKManifestPath() + if err := os.Remove(manifest); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove previous hak manifest: %w", err) + } + return nil +} + +func plannedModuleHAKOrder(p *project.Project) (order []string, err error) { + manifest, err := loadPreviousBuildManifest(p.HAKManifestPath()) + if err != nil { + return nil, err + } + if manifest != nil && len(manifest.ModuleHAKs) > 0 { + return append([]string(nil), manifest.ModuleHAKs...), nil + } + + musicAssets, err := prepareMusicAssets(p) + if err != nil { + return nil, err + } + defer func() { + if cleanupErr := cleanupPreparedMusicAssets(musicAssets); cleanupErr != nil && err == nil { + err = cleanupErr + } + }() + generated2DAAssets, err := topdata.BuildGenerated2DAAssets(p, nil) + if err != nil { + return nil, err + } + assetResources, err := collectAssetResources(p, false, nil, musicAssets, generated2DAAssets) + if err != nil { + return nil, err + } + chunks, err := planHAKChunks(p, assetResources) + if err != nil { + return nil, err + } + return resolveModuleHAKOrder(p, chunks) +} + +func resolveModuleHAKOrder(p *project.Project, chunks []hakChunk) ([]string, error) { + if len(p.Config.Module.HAKOrder) == 0 { + return nil, nil + } + + byGroup := map[string][]string{} + optionalNames := map[string]struct{}{} + for _, chunk := range chunks { + if chunk.Config.Optional { + optionalNames[chunk.Name] = struct{}{} + continue + } + byGroup[chunk.Config.Name] = append(byGroup[chunk.Config.Name], chunk.Name) + } + + seen := map[string]struct{}{} + var ordered []string + for _, entry := range p.Config.Module.HAKOrder { + if strings.HasPrefix(entry, "group:") { + groupName := strings.TrimPrefix(entry, "group:") + names, ok := byGroup[groupName] + if !ok { + continue + } + for _, name := range names { + if _, exists := seen[name]; exists { + continue + } + ordered = append(ordered, name) + seen[name] = struct{}{} + } + continue + } + + if _, optional := optionalNames[entry]; optional { + continue + } + if _, exists := seen[entry]; exists { + continue + } + ordered = append(ordered, entry) + seen[entry] = struct{}{} + } + + return ordered, nil +} + +func setModuleHAKList(document *gff.Document, hakNames []string) { + list := make(gff.ListValue, 0, len(hakNames)) + for _, name := range hakNames { + list = append(list, gff.Struct{ + Type: 8, + Fields: []gff.Field{ + gff.NewField("Mod_Hak", gff.StringValue(name)), + }, + }) + } + + for index, field := range document.Root.Fields { + if field.Label == "Mod_HakList" { + document.Root.Fields[index] = gff.NewField("Mod_HakList", list) + return + } + } + + document.Root.Fields = append(document.Root.Fields, gff.NewField("Mod_HakList", list)) +} + +func validateForBuild(p *project.Project) error { + report := validator.ValidateProject(p) + if report.HasErrors() { + summary := report.SummaryLines(3) + if len(summary) > 0 { + return fmt.Errorf("validation failed with %d error(s): %s", report.ErrorCount(), summary[0]) + } + return fmt.Errorf("validation failed with %d error(s)", report.ErrorCount()) + } + return nil +} + +func matchSegments(pathSegs, patternSegs []string) bool { + if len(patternSegs) == 0 { + return len(pathSegs) == 0 + } + if patternSegs[0] == "**" { + if matchSegments(pathSegs, patternSegs[1:]) { + return true + } + if len(pathSegs) > 0 { + return matchSegments(pathSegs[1:], patternSegs) + } + return false + } + if len(pathSegs) == 0 { + return false + } + ok, err := filepath.Match(patternSegs[0], pathSegs[0]) + if err != nil || !ok { + return false + } + return matchSegments(pathSegs[1:], patternSegs[1:]) +} diff --git a/internal/pipeline/chunk_validation.go b/internal/pipeline/chunk_validation.go new file mode 100644 index 0000000..41a22d9 --- /dev/null +++ b/internal/pipeline/chunk_validation.go @@ -0,0 +1,27 @@ +package pipeline + +import ( + "fmt" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" +) + +func ensureUniqueChunkResources(chunk hakChunk) error { + seen := map[string]string{} + for _, asset := range chunk.Assets { + key := fmt.Sprintf("%s:%04x", asset.Resource.Name, asset.Resource.Type) + if previous, exists := seen[key]; exists { + return fmt.Errorf("resource %s from %s conflicts with %s inside generated hak %s", chunkResourceLabel(asset), asset.Rel, previous, chunk.Name) + } + seen[key] = asset.Rel + } + return nil +} + +func chunkResourceLabel(asset assetResource) string { + extension, ok := erf.ExtensionForResourceType(asset.Resource.Type) + if !ok { + return asset.Resource.Name + } + return asset.Resource.Name + "." + extension +} diff --git a/internal/pipeline/compare.go b/internal/pipeline/compare.go new file mode 100644 index 0000000..86174ef --- /dev/null +++ b/internal/pipeline/compare.go @@ -0,0 +1,359 @@ +package pipeline + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +type CompareResult struct { + ModulePath string + HAKPaths []string + Checked int +} + +type resourceExpectation struct { + Key string + Kind string + Bytes []byte +} + +func Compare(p *project.Project) (CompareResult, error) { + modulePath := p.ModuleArchivePath() + + var hakPaths []string + if len(p.Inventory.AssetFiles) > 0 { + var err error + hakPaths, err = manifestHAKPaths(p) + if err != nil { + return CompareResult{}, err + } + } + if err := ensureBuildIsCurrent(p, modulePath, hakPaths); err != nil { + return CompareResult{ModulePath: modulePath, HAKPaths: hakPaths}, err + } + + moduleArchive, err := readArchive(modulePath) + if err != nil { + return CompareResult{}, err + } + + expected, err := expectedResources(p) + if err != nil { + return CompareResult{}, err + } + + actual := archiveIndex(moduleArchive) + for _, hakPath := range hakPaths { + hakArchive, err := readArchive(hakPath) + if err != nil { + return CompareResult{}, err + } + for key, value := range archiveIndex(hakArchive) { + actual[key] = value + } + } + + var diagnostics []error + checked := 0 + + expectedKeys := make([]string, 0, len(expected)) + for key := range expected { + expectedKeys = append(expectedKeys, key) + } + slices.Sort(expectedKeys) + + for _, key := range expectedKeys { + want := expected[key] + got, ok := actual[key] + if !ok { + diagnostics = append(diagnostics, fmt.Errorf("missing resource in built archives: %s", key)) + continue + } + if !bytes.Equal(want.Bytes, got.Data) { + diagnostics = append(diagnostics, fmt.Errorf("resource content mismatch: %s", key)) + continue + } + checked++ + delete(actual, key) + } + + if len(actual) > 0 { + extraKeys := make([]string, 0, len(actual)) + for key := range actual { + extraKeys = append(extraKeys, key) + } + slices.Sort(extraKeys) + for _, key := range extraKeys { + diagnostics = append(diagnostics, fmt.Errorf("unexpected extra resource in built archives: %s", key)) + } + } + + result := CompareResult{ + ModulePath: modulePath, + HAKPaths: hakPaths, + Checked: checked, + } + if len(diagnostics) > 0 { + return result, errors.Join(diagnostics...) + } + return result, nil +} + +func expectedResources(p *project.Project) (map[string]resourceExpectation, error) { + out := map[string]resourceExpectation{} + moduleHakOrder, err := plannedModuleHAKOrder(p) + if err != nil { + return nil, err + } + + for _, rel := range p.Inventory.SourceFiles { + abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) + name, extension, err := splitSourceName(abs) + if err != nil { + return nil, err + } + + raw, err := os.ReadFile(abs) + if err != nil { + return nil, fmt.Errorf("read %s: %w", abs, err) + } + var document gff.Document + if err := json.Unmarshal(raw, &document); err != nil { + return nil, fmt.Errorf("parse gff json %s: %w", abs, err) + } + if extension == ".ifo" && strings.EqualFold(name, "module") && len(moduleHakOrder) > 0 { + setModuleHAKList(&document, moduleHakOrder) + } + + canonical, err := json.Marshal(document) + if err != nil { + return nil, fmt.Errorf("marshal canonical gff json %s: %w", abs, err) + } + + out[resourceKey(name, extension)] = resourceExpectation{ + Key: resourceKey(name, extension), + Kind: "gff-json", + Bytes: canonical, + } + } + + for _, rel := range p.Inventory.ScriptFiles { + abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) + data, err := os.ReadFile(abs) + if err != nil { + return nil, fmt.Errorf("read %s: %w", abs, err) + } + name := strings.ToLower(strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs))) + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(abs)), ".") + out[resourceKey(name, extension)] = resourceExpectation{ + Key: resourceKey(name, extension), + Kind: "raw", + Bytes: data, + } + } + + for _, rel := range p.Inventory.AssetFiles { + abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)) + data, err := os.ReadFile(abs) + if err != nil { + return nil, fmt.Errorf("read %s: %w", abs, err) + } + name := strings.ToLower(strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs))) + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(abs)), ".") + out[resourceKey(name, extension)] = resourceExpectation{ + Key: resourceKey(name, extension), + Kind: "raw", + Bytes: data, + } + } + + return out, nil +} + +func ensureBuildIsCurrent(p *project.Project, modulePath string, hakPaths []string) error { + archivePaths := make([]string, 0, 1+len(hakPaths)) + archivePaths = append(archivePaths, modulePath) + archivePaths = append(archivePaths, hakPaths...) + + oldestArchive := time.Time{} + for _, path := range archivePaths { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("stat archive %s: %w", path, err) + } + if oldestArchive.IsZero() || info.ModTime().Before(oldestArchive) { + oldestArchive = info.ModTime() + } + } + + sourcePaths := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles)) + for _, rel := range p.Inventory.SourceFiles { + sourcePaths = append(sourcePaths, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) + } + for _, rel := range p.Inventory.ScriptFiles { + sourcePaths = append(sourcePaths, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) + } + for _, rel := range p.Inventory.AssetFiles { + sourcePaths = append(sourcePaths, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) + } + + newestSource := time.Time{} + newestPath := "" + for _, path := range sourcePaths { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("stat source %s: %w", path, err) + } + if newestSource.IsZero() || info.ModTime().After(newestSource) { + newestSource = info.ModTime() + newestPath = path + } + } + + if !newestSource.IsZero() && newestSource.After(oldestArchive) { + return fmt.Errorf("built archives are older than source file %s; run build-module or build-haks before compare", newestPath) + } + return nil +} + +func readArchive(path string) (erf.Archive, error) { + input, err := os.Open(path) + if err != nil { + return erf.Archive{}, fmt.Errorf("open archive %s: %w", path, err) + } + defer input.Close() + + archive, err := erf.Read(input) + if err != nil { + return erf.Archive{}, fmt.Errorf("read archive %s: %w", path, err) + } + return archive, nil +} + +func archiveIndex(archive erf.Archive) map[string]erf.Resource { + out := map[string]erf.Resource{} + for _, resource := range archive.Resources { + extension, ok := erf.ExtensionForResourceType(resource.Type) + if !ok { + continue + } + + key := resourceKey(strings.ToLower(resource.Name), extension) + switch extension { + case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw", + "are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl": + document, err := gff.Read(bytes.NewReader(resource.Data)) + if err != nil { + out[key] = resource + continue + } + canonical, err := json.Marshal(document) + if err != nil { + out[key] = resource + continue + } + resource.Data = canonical + } + + out[key] = resource + } + return out +} + +func resourceKey(name, extension string) string { + return strings.ToLower(name) + "." + strings.TrimPrefix(strings.ToLower(extension), ".") +} + +func manifestHAKPaths(p *project.Project) ([]string, error) { + manifestPath := p.HAKManifestPath() + raw, err := os.ReadFile(manifestPath) + if err == nil { + var manifest BuildManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("parse hak manifest: %w", err) + } + paths := make([]string, 0, len(manifest.HAKs)) + for _, hak := range manifest.HAKs { + paths = append(paths, p.HAKArchivePath(hak.Name)) + } + return paths, nil + } + if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("read hak manifest: %w", err) + } + + legacy := filepath.Join(p.BuildDir(), "*.hak") + paths, err := filepath.Glob(legacy) + if err != nil { + return nil, fmt.Errorf("scan hak archives: %w", err) + } + filtered := make([]string, 0, len(paths)) + for _, path := range paths { + if filepath.Base(path) == p.TopDataPackageHAKName() { + continue + } + filtered = append(filtered, path) + } + slices.Sort(filtered) + return filtered, nil +} + +func topPackageSourcePaths(p *project.Project) ([]string, error) { + sourceDir := p.TopDataSourceDir() + generated2DADir := filepath.Join(sourceDir, "assets", "2da") + paths := make([]string, 0) + for _, candidate := range []string{ + filepath.Join(sourceDir, ".tlk_state.json"), + filepath.Join(sourceDir, "base_dialog.json"), + } { + if _, err := os.Stat(candidate); err == nil { + paths = append(paths, candidate) + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("stat topdata source %s: %w", candidate, err) + } + } + for _, dir := range []string{ + filepath.Join(sourceDir, "data"), + filepath.Join(sourceDir, "assets"), + } { + info, err := os.Stat(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + return nil, fmt.Errorf("stat topdata path %s: %w", dir, err) + } + if !info.IsDir() { + continue + } + err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() && path == generated2DADir { + return filepath.SkipDir + } + if d.IsDir() { + return nil + } + paths = append(paths, path) + return nil + }) + if err != nil { + return nil, fmt.Errorf("scan topdata path %s: %w", dir, err) + } + } + return paths, nil +} diff --git a/internal/pipeline/extract.go b/internal/pipeline/extract.go new file mode 100644 index 0000000..40d78c1 --- /dev/null +++ b/internal/pipeline/extract.go @@ -0,0 +1,621 @@ +package pipeline + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +type ExtractResult struct { + ModulePath string + HAKPaths []string + DeletedArchivePaths []string + Written int + Overwritten int + Removed int + Skipped int +} + +type extractionArchive struct { + Path string + Rel string + Extension string +} + +type extractionScope struct { + Source bool + Assets bool +} + +func Extract(p *project.Project, files ...string) (ExtractResult, error) { + var result ExtractResult + var failures []error + desired := map[string]struct{}{} + scope := extractionScope{} + + archives, err := resolveExtractionArchives(p, files) + if err != nil { + return result, err + } + for _, archivePath := range archives { + input, err := os.Open(archivePath.Path) + if err != nil { + failures = append(failures, fmt.Errorf("open archive %s: %w", archivePath.Path, err)) + continue + } + archive, err := erf.Read(input) + input.Close() + if err != nil { + failures = append(failures, fmt.Errorf("read archive %s: %w", archivePath.Path, err)) + continue + } + + switch archivePath.Extension { + case ".mod": + if result.ModulePath == "" { + result.ModulePath = archivePath.Path + } + case ".hak": + result.HAKPaths = append(result.HAKPaths, archivePath.Path) + } + + written, overwritten, skipped, extractedScope, errs := extractArchiveResources(p, archive, desired) + result.Written += written + result.Overwritten += overwritten + result.Skipped += skipped + scope.Source = scope.Source || extractedScope.Source + scope.Assets = scope.Assets || extractedScope.Assets + failures = append(failures, errs...) + } + + if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale { + removed, errs := cleanupStaleFiles(p, desired, scope) + result.Removed = removed + failures = append(failures, errs...) + } + + if len(failures) > 0 { + return result, errors.Join(failures...) + } + consumeAll, consumeModules := extractionConsumePolicy(p) + if consumeAll || consumeModules { + for _, archivePath := range archives { + if !consumeAll && archivePath.Extension != ".mod" { + continue + } + if err := os.Remove(archivePath.Path); err != nil { + return result, fmt.Errorf("delete consumed archive %s: %w", archivePath.Path, err) + } + result.DeletedArchivePaths = append(result.DeletedArchivePaths, archivePath.Path) + } + } + return result, nil +} + +func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, extractionScope, []error) { + var failures []error + writtenCount := 0 + overwrittenCount := 0 + skippedCount := 0 + scope := extractionScope{} + + ignored := make(map[string]bool) + for _, ext := range p.Config.Extract.IgnoreExtensions { + ext := strings.TrimPrefix(ext, ".") + ignored[ext] = true + } + + for _, resource := range archive.Resources { + ext, ok := erf.ExtensionForResourceType(resource.Type) + if !ok { + failures = append(failures, fmt.Errorf("unsupported resource type 0x%04X for %s", resource.Type, resource.Name)) + continue + } + if ignored[ext] { + skippedCount++ + continue + } + + target, data, err := extractedFile(p, resource, ext) + if err != nil { + failures = append(failures, err) + continue + } + desired[target] = struct{}{} + scope.Source = scope.Source || pathWithinRoot(target, p.SourceDir()) + scope.Assets = scope.Assets || pathWithinRoot(target, p.AssetsDir()) + + state, err := writeManagedFile(target, data) + if err != nil { + failures = append(failures, err) + continue + } + switch state { + case writeNew: + writtenCount++ + case writeOverwritten: + overwrittenCount++ + case writeSkipped: + skippedCount++ + } + } + + return writtenCount, overwrittenCount, skippedCount, scope, failures +} + +func resolveExtractionArchives(p *project.Project, overrides []string) ([]extractionArchive, error) { + patterns := p.EffectiveConfig().Extract.Archives + if len(overrides) > 0 { + patterns = overrides + } + if len(patterns) == 0 { + return nil, fmt.Errorf("extract.archives must contain at least one build-relative archive pattern") + } + + buildRoot := filepath.Clean(p.BuildDir()) + byPath := map[string]extractionArchive{} + for _, rawPattern := range patterns { + pattern, err := normalizeExtractionArchivePattern(p, rawPattern) + if err != nil { + return nil, err + } + matched, err := matchExtractionArchives(buildRoot, pattern) + if err != nil { + return nil, err + } + if len(matched) == 0 { + return nil, fmt.Errorf("extract archive pattern %q matched no files under %s", rawPattern, buildRoot) + } + for _, archive := range matched { + byPath[archive.Path] = archive + } + } + + archives := make([]extractionArchive, 0, len(byPath)) + for _, archive := range byPath { + archives = append(archives, archive) + } + slices.SortFunc(archives, func(a, b extractionArchive) int { + return strings.Compare(a.Rel, b.Rel) + }) + return archives, nil +} + +func normalizeExtractionArchivePattern(p *project.Project, raw string) (string, error) { + pattern := strings.TrimSpace(raw) + if pattern == "" { + return "", fmt.Errorf("extract archive pattern must not be empty") + } + pattern = strings.NewReplacer( + "{module.resref}", strings.TrimSpace(p.Config.Module.ResRef), + ).Replace(pattern) + if strings.Contains(pattern, "\x00") { + return "", fmt.Errorf("extract archive pattern %q must not contain NUL bytes", raw) + } + if filepath.IsAbs(pattern) { + return "", fmt.Errorf("extract archive pattern %q must be relative to paths.build", raw) + } + clean := filepath.Clean(filepath.FromSlash(pattern)) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("extract archive pattern %q must not escape paths.build", raw) + } + return filepath.ToSlash(clean), nil +} + +func matchExtractionArchives(buildRoot, pattern string) ([]extractionArchive, error) { + var archives []extractionArchive + err := filepath.WalkDir(buildRoot, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + rel, err := filepath.Rel(buildRoot, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if !matchPathPattern(rel, pattern) { + return nil + } + archive, err := validateExtractionArchive(buildRoot, rel) + if err != nil { + return err + } + archives = append(archives, archive) + return nil + }) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("scan build directory %s: %w", buildRoot, err) + } + return nil, err + } + return archives, nil +} + +func validateExtractionArchive(buildRoot, rel string) (extractionArchive, error) { + cleanRel := filepath.Clean(filepath.FromSlash(rel)) + if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(filepath.Separator)) { + return extractionArchive{}, fmt.Errorf("extract archive %q must not escape paths.build", rel) + } + path := filepath.Join(buildRoot, cleanRel) + if !pathWithinRoot(path, buildRoot) { + return extractionArchive{}, fmt.Errorf("extract archive %q resolves outside paths.build", rel) + } + extension := strings.ToLower(filepath.Ext(cleanRel)) + switch extension { + case ".mod", ".hak": + return extractionArchive{Path: path, Rel: filepath.ToSlash(cleanRel), Extension: extension}, nil + case ".erf": + return extractionArchive{}, fmt.Errorf("extract archive %q is an .erf; ERF extraction is unsafe because ERFs lack module.ifo context", rel) + default: + return extractionArchive{}, fmt.Errorf("extract archive %q uses unsupported extension %q", rel, extension) + } +} + +func extractionConsumePolicy(p *project.Project) (consumeAll bool, consumeModules bool) { + if configured := p.Config.Extract.ConsumeArchives; configured != nil { + return *configured, false + } + return false, p.Config.Extract.DeleteModuleArchiveAfterSuccess +} + +func pathWithinRoot(path, root string) bool { + root = filepath.Clean(root) + if root == "" || root == "." { + return false + } + rel, err := filepath.Rel(root, path) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel) +} + +func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) { + resref := strings.ToLower(resource.Name) + effective := p.EffectiveConfig() + + switch extension { + case "nss": + target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), "scripts", resref+".nss") + if err != nil { + return "", nil, err + } + return target, resource.Data, nil + case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw", + "are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl": + document, err := gff.Read(bytes.NewReader(resource.Data)) + if err != nil { + return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err) + } + target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json") + if err != nil { + return "", nil, err + } + if err := mergeExtractedGFFJSON(p, target, &document); err != nil { + return "", nil, fmt.Errorf("merge extracted gff json %s.%s: %w", resource.Name, extension, err) + } + formatted, err := json.MarshalIndent(document, "", " ") + if err != nil { + return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err) + } + formatted = append(formatted, '\n') + return target, formatted, nil + default: + target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension) + if err != nil { + return "", nil, err + } + return target, resource.Data, nil + } +} + +func extractionTarget(p *project.Project, field, configured, root string, parts ...string) (string, error) { + if strings.TrimSpace(configured) == "" { + return "", fmt.Errorf("cannot extract resource: %s is not configured", field) + } + + cleanRoot := filepath.Clean(root) + if cleanRoot == "." || cleanRoot == string(filepath.Separator) || cleanRoot == filepath.Clean(p.Root) { + return "", fmt.Errorf("cannot extract resource: %s resolves to unsafe extraction root %s", field, cleanRoot) + } + + target := filepath.Join(append([]string{cleanRoot}, parts...)...) + rel, err := filepath.Rel(cleanRoot, target) + if err != nil { + return "", fmt.Errorf("resolve extraction target %s: %w", target, err) + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return "", fmt.Errorf("refusing to extract resource outside %s: %s", cleanRoot, target) + } + return target, nil +} + +func mergeExtractedGFFJSON(p *project.Project, target string, extracted *gff.Document) error { + rule, ok, err := extractGFFJSONMergeRule(p, target) + if err != nil || !ok { + return err + } + + raw, err := os.ReadFile(target) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("read existing source %s: %w", target, err) + } + var existing gff.Document + if err := json.Unmarshal(raw, &existing); err != nil { + return fmt.Errorf("parse existing source %s: %w", target, err) + } + + for _, label := range rule.PreserveFields { + if field, ok := gffField(existing.Root, label); ok { + setGFFField(&extracted.Root, field) + } + } + for _, listRule := range rule.MergeLists { + if err := mergeGFFListByKey(&extracted.Root, existing.Root, listRule); err != nil { + return err + } + } + return nil +} + +func extractGFFJSONMergeRule(p *project.Project, target string) (project.ExtractGFFJSONMergeRule, bool, error) { + rel, err := filepath.Rel(filepath.Clean(p.SourceDir()), filepath.Clean(target)) + if err != nil { + return project.ExtractGFFJSONMergeRule{}, false, fmt.Errorf("resolve source-relative target %s: %w", target, err) + } + rel = filepath.ToSlash(rel) + if rel == ".." || strings.HasPrefix(rel, "../") { + return project.ExtractGFFJSONMergeRule{}, false, nil + } + for _, rule := range p.EffectiveConfig().Extract.Merge.GFFJSON { + if rule.Target == rel { + return rule, true, nil + } + } + return project.ExtractGFFJSONMergeRule{}, false, nil +} + +func gffField(s gff.Struct, label string) (gff.Field, bool) { + for _, field := range s.Fields { + if field.Label == label { + return field, true + } + } + return gff.Field{}, false +} + +func setGFFField(s *gff.Struct, replacement gff.Field) { + for index, field := range s.Fields { + if field.Label == replacement.Label { + s.Fields[index] = replacement + return + } + } + s.Fields = append(s.Fields, replacement) +} + +func mergeGFFListByKey(extracted *gff.Struct, existing gff.Struct, rule project.ExtractListMergeRule) error { + extractedField, ok := gffField(*extracted, rule.Field) + if !ok { + return fmt.Errorf("extracted field %q not found", rule.Field) + } + extractedList, ok := extractedField.Value.(gff.ListValue) + if !ok { + return fmt.Errorf("extracted field %q is %s, not List", rule.Field, extractedField.Type) + } + existingField, ok := gffField(existing, rule.Field) + if !ok { + return nil + } + existingList, ok := existingField.Value.(gff.ListValue) + if !ok { + return fmt.Errorf("existing field %q is %s, not List", rule.Field, existingField.Type) + } + + extractedByKey := map[string]gff.Struct{} + extractedOrder := make([]string, 0, len(extractedList)) + for _, item := range extractedList { + key, err := gffStructKey(item, rule.KeyField) + if err != nil { + return fmt.Errorf("extracted field %q: %w", rule.Field, err) + } + if _, exists := extractedByKey[key]; exists { + return fmt.Errorf("extracted field %q has duplicate %s key %q", rule.Field, rule.KeyField, key) + } + extractedByKey[key] = item + extractedOrder = append(extractedOrder, key) + } + + merged := make(gff.ListValue, 0, len(extractedList)) + seen := map[string]struct{}{} + for _, item := range existingList { + key, err := gffStructKey(item, rule.KeyField) + if err != nil { + return fmt.Errorf("existing field %q: %w", rule.Field, err) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("existing field %q has duplicate %s key %q", rule.Field, rule.KeyField, key) + } + if extractedItem, exists := extractedByKey[key]; exists { + merged = append(merged, extractedItem) + seen[key] = struct{}{} + } + } + for _, key := range extractedOrder { + if _, exists := seen[key]; exists { + continue + } + merged = append(merged, extractedByKey[key]) + } + + setGFFField(extracted, gff.NewField(rule.Field, merged)) + return nil +} + +func gffStructKey(s gff.Struct, keyField string) (string, error) { + field, ok := gffField(s, keyField) + if !ok { + return "", fmt.Errorf("key field %q not found", keyField) + } + switch value := field.Value.(type) { + case gff.ResRefValue: + return string(value), nil + case gff.StringValue: + return string(value), nil + default: + return "", fmt.Errorf("key field %q is %s, not ResRef or CExoString", keyField, field.Type) + } +} + +type writeState int + +const ( + writeSkipped writeState = iota + writeNew + writeOverwritten +) + +func writeManagedFile(path string, data []byte) (writeState, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return writeSkipped, fmt.Errorf("create parent directory for %s: %w", path, err) + } + + existing, err := os.ReadFile(path) + if err == nil { + if bytes.Equal(existing, data) { + return writeSkipped, nil + } + if err := os.WriteFile(path, data, 0o644); err != nil { + return writeSkipped, fmt.Errorf("overwrite %s: %w", path, err) + } + return writeOverwritten, nil + } + if !errors.Is(err, os.ErrNotExist) { + return writeSkipped, fmt.Errorf("check existing file %s: %w", path, err) + } + + if err := os.WriteFile(path, data, 0o644); err != nil { + return writeSkipped, fmt.Errorf("write %s: %w", path, err) + } + return writeNew, nil +} + +func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) { + candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles)) + if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) { + for _, rel := range p.Inventory.SourceFiles { + candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) + } + for _, rel := range p.Inventory.ScriptFiles { + candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) + } + } + if scope.Assets && safeCleanupRoot(p.AssetsDir(), p.Root) { + for _, rel := range p.Inventory.AssetFiles { + candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) + } + } + + removed := 0 + var failures []error + for _, path := range candidates { + if _, keep := desired[path]; keep { + continue + } + if err := os.Remove(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + failures = append(failures, fmt.Errorf("remove stale file %s: %w", path, err)) + continue + } + removed++ + cleanupEmptyParents(filepath.Dir(path), p.SourceDir(), p.AssetsDir()) + } + + return removed, failures +} + +func safeCleanupRoot(root, projectRoot string) bool { + cleanRoot := filepath.Clean(root) + return cleanRoot != "." && cleanRoot != string(filepath.Separator) && cleanRoot != filepath.Clean(projectRoot) +} + +func cleanupEmptyParents(dir string, roots ...string) { + for { + if dir == "." || dir == string(filepath.Separator) { + return + } + stop := false + for _, root := range roots { + if dir == root { + stop = true + break + } + } + if stop { + return + } + if err := os.Remove(dir); err != nil { + return + } + dir = filepath.Dir(dir) + } +} + +func sourceSubdir(extension string) string { + switch strings.ToLower(extension) { + case "are": + return "areas" + case "dlg": + return "dialogs" + case "fac": + return "factions" + case "gic": + return "instance" + case "git": + return "instance" + case "ifo": + return "module" + case "itp": + return "palettes" + case "jrl": + return "journal" + case "utc": + return filepath.Join("blueprints", "creatures") + case "utd": + return filepath.Join("blueprints", "doors") + case "ute": + return filepath.Join("blueprints", "encounters") + case "uti": + return filepath.Join("blueprints", "items") + case "utm": + return filepath.Join("blueprints", "merchants") + case "utp": + return filepath.Join("blueprints", "placeables") + case "uts": + return filepath.Join("blueprints", "sounds") + case "utt": + return filepath.Join("blueprints", "triggers") + case "utw": + return filepath.Join("blueprints", "waypoints") + default: + return extension + } +} diff --git a/internal/pipeline/music.go b/internal/pipeline/music.go new file mode 100644 index 0000000..d16b1f4 --- /dev/null +++ b/internal/pipeline/music.go @@ -0,0 +1,524 @@ +package pipeline + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +type preparedMusicAssets struct { + Generated []assetResource + SkipSourceRel map[string]struct{} + Artifacts []string + TempRoots []string + Summary CreditsRefreshSummary +} + +type MusicBuildOptions struct { + DatasetIDs []string + Write bool +} + +type musicPrepareOptions struct { + datasetIDs []string + write bool +} + +type musicSourceGroup struct { + DatasetID string + Dataset project.EffectiveMusicDataset + SourceDir string + OutputDir string + Sources []string +} + +type musicCreditGroup struct { + CreditsRoot string + SourceDir string + Entries []music.CreditsEntry +} + +func musicRootPath(p *project.Project, path string) string { + if strings.TrimSpace(path) == "" { + return p.Root + } + if filepath.IsAbs(path) { + return path + } + return filepath.Join(p.Root, filepath.FromSlash(path)) +} + +func effectiveMusicDatasets(p *project.Project, onlyIDs []string) (map[string]project.EffectiveMusicDataset, error) { + effective := p.EffectiveConfig() + wanted := map[string]struct{}{} + for _, id := range onlyIDs { + id = strings.TrimSpace(id) + if id != "" { + wanted[id] = struct{}{} + } + } + datasets := map[string]project.EffectiveMusicDataset{} + for id, dataset := range effective.Music.Datasets { + if len(wanted) > 0 { + if _, ok := wanted[id]; !ok { + continue + } + delete(wanted, id) + } + datasets[id] = dataset + } + if len(wanted) > 0 { + unknown := make([]string, 0, len(wanted)) + for id := range wanted { + unknown = append(unknown, id) + } + sort.Strings(unknown) + return nil, fmt.Errorf("unknown music dataset(s): %s", strings.Join(unknown, ", ")) + } + if len(datasets) == 0 && len(wanted) == 0 { + for _, rel := range p.Inventory.AssetFiles { + ext := strings.ToLower(filepath.Ext(rel)) + if !music.PathIsUnder("envi/music", rel) { + continue + } + if _, ok := extensionSet(effective.Music.ConvertExtensions)[ext]; ok { + datasets["legacy_envi_music"] = project.EffectiveMusicDataset{ + Source: "envi/music", + Output: "envi/music", + StageRoot: effective.Music.StageRoot, + CreditsRoot: effective.Music.CreditsRoot, + NamingScheme: effective.Music.NamingScheme, + OutputExtension: effective.Music.OutputExtension, + MaxStemLength: effective.Music.MaxStemLength, + PackageMode: "hak_asset", + ConvertExtensions: effective.Music.ConvertExtensions, + } + break + } + } + } + return datasets, nil +} + +func extensionSet(values []string) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.ToLower(strings.TrimSpace(value)) + if value != "" { + out[value] = struct{}{} + } + } + return out +} + +func datasetForMusicSource(datasets map[string]project.EffectiveMusicDataset, rel, ext string) (string, project.EffectiveMusicDataset, bool) { + bestID := "" + var best project.EffectiveMusicDataset + bestLen := -1 + for id, dataset := range datasets { + if _, ok := extensionSet(dataset.ConvertExtensions)[ext]; !ok { + continue + } + if !music.PathIsUnder(dataset.Source, rel) { + continue + } + if len(dataset.Source) > bestLen { + bestID = id + best = dataset + bestLen = len(dataset.Source) + } + } + return bestID, best, bestID != "" +} + +func plannedMusicAsset(outputRel, sourceRel string) (assetResource, error) { + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(outputRel)), ".") + resourceType, ok := erf.HAKResourceTypeForExtension(extension) + if !ok { + return assetResource{}, fmt.Errorf("unsupported generated music resource extension %q", filepath.Ext(outputRel)) + } + name := strings.ToLower(strings.TrimSuffix(filepath.Base(outputRel), filepath.Ext(outputRel))) + return assetResource{ + Rel: outputRel, + Resource: erf.Resource{ + Name: name, + Type: resourceType, + }, + ContentID: "music-plan:" + filepath.ToSlash(sourceRel), + }, nil +} + +func BuildMusic(p *project.Project, opts MusicBuildOptions) (BuildResult, error) { + prepared, err := prepareMusicAssetsWithOptions(p, musicPrepareOptions{datasetIDs: opts.DatasetIDs, write: opts.Write}) + if err != nil { + return BuildResult{}, err + } + if err := cleanupPreparedMusicAssets(prepared); err != nil { + return BuildResult{}, err + } + return BuildResult{ + CreditsArtifactPaths: prepared.Artifacts, + CreditsSummary: prepared.Summary, + HAKAssets: len(prepared.Generated), + }, nil +} + +func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) { + return prepareMusicAssetsWithOptions(p, musicPrepareOptions{write: true}) +} + +func prepareMusicAssetsWithOptions(p *project.Project, opts musicPrepareOptions) (*preparedMusicAssets, error) { + usedNames := make(map[string]struct{}) + groupsByKey := make(map[string]*musicSourceGroup) + datasets, err := effectiveMusicDatasets(p, opts.datasetIDs) + if err != nil { + return nil, err + } + for _, rel := range p.Inventory.AssetFiles { + rel = filepath.ToSlash(rel) + ext := strings.ToLower(filepath.Ext(rel)) + name := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))) + if datasetID, dataset, ok := datasetForMusicSource(datasets, rel, ext); ok { + sourceDir := filepath.ToSlash(filepath.Dir(rel)) + relDir, err := filepath.Rel(filepath.FromSlash(dataset.Source), filepath.FromSlash(sourceDir)) + if err != nil { + return nil, fmt.Errorf("resolve music source directory %s: %w", sourceDir, err) + } + relDir = filepath.ToSlash(relDir) + outputDir := dataset.Output + if relDir != "." && relDir != "" { + outputDir = filepath.ToSlash(filepath.Join(outputDir, filepath.FromSlash(relDir))) + } + key := datasetID + "\x00" + sourceDir + group := groupsByKey[key] + if group == nil { + group = &musicSourceGroup{ + DatasetID: datasetID, + Dataset: dataset, + SourceDir: sourceDir, + OutputDir: outputDir, + } + groupsByKey[key] = group + } + group.Sources = append(group.Sources, rel) + continue + } + if name != "" { + usedNames[name] = struct{}{} + } + } + + result := &preparedMusicAssets{ + SkipSourceRel: make(map[string]struct{}), + } + if len(groupsByKey) == 0 { + if !opts.write { + return result, nil + } + writeResult, err := writeCreditsArtifacts(p, nil) + if err != nil { + return nil, err + } + result.Artifacts = writeResult.Artifacts + result.Summary = writeResult.Summary + return result, nil + } + + groups := make([]*musicSourceGroup, 0, len(groupsByKey)) + for _, group := range groupsByKey { + sort.Strings(group.Sources) + groups = append(groups, group) + } + sort.Slice(groups, func(i, j int) bool { + if groups[i].SourceDir != groups[j].SourceDir { + return groups[i].SourceDir < groups[j].SourceDir + } + return groups[i].DatasetID < groups[j].DatasetID + }) + + creditGroups := make([]musicCreditGroup, 0, len(groups)) + for _, group := range groups { + result.Summary.ScannedDirs = append(result.Summary.ScannedDirs, group.SourceDir) + result.Summary.TrackCount += len(group.Sources) + } + ffmpegPath := "" + ffprobePath := "" + if opts.write { + var err error + ffmpegPath, err = music.ResolveFFmpegWithConfig(p.MusicFFmpegPath()) + if err != nil { + return nil, err + } + ffprobePath, err = music.ResolveFFprobeWithConfig(p.MusicFFprobePath()) + if err != nil { + return nil, err + } + } + + for _, group := range groups { + stageRoot := musicRootPath(p, group.Dataset.StageRoot) + result.TempRoots = append(result.TempRoots, stageRoot) + prefix := music.SanitizePrefix(group.Dataset.Prefix) + overlayPath := filepath.Join(p.AssetsDir(), filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay) + overlay, err := music.ParseCreditsOverlay(overlayPath) + if err != nil { + return nil, fmt.Errorf("parse credits overlay %s: %w", overlayPath, err) + } + + entries := make([]music.CreditsEntry, 0, len(group.Sources)) + for _, rel := range group.Sources { + base := filepath.Base(rel) + manual := overlay.Match(base, "") + outputFile := "" + switch { + case manual != nil && strings.TrimSpace(manual.OutputFile) != "": + outputFile = strings.ToLower(strings.TrimSpace(manual.OutputFile)) + if err := music.ValidateManualOutputFileWithRules(outputFile, group.Dataset.OutputExtension, group.Dataset.MaxStemLength); err != nil { + return nil, fmt.Errorf("manual credits output for %s: %w", rel, err) + } + if err := music.ReserveName(strings.TrimSuffix(outputFile, filepath.Ext(outputFile)), usedNames); err != nil { + return nil, fmt.Errorf("manual credits output for %s: %w", rel, err) + } + default: + stem, err := music.GenerateStemWithMax(base, prefix, group.Dataset.MaxStemLength, usedNames) + if err != nil { + return nil, fmt.Errorf("generate music name for %s: %w", rel, err) + } + outputFile = stem + group.Dataset.OutputExtension + } + outputRel := filepath.ToSlash(filepath.Join(group.OutputDir, outputFile)) + result.Summary.Mappings = append(result.Summary.Mappings, MusicFileMapping{ + Source: rel, + Output: outputRel, + }) + result.SkipSourceRel[rel] = struct{}{} + + if !opts.write { + if group.Dataset.PackageMode == "hak_asset" { + asset, err := plannedMusicAsset(outputRel, rel) + if err != nil { + return nil, err + } + result.Generated = append(result.Generated, asset) + } + continue + } + + metadata, err := music.ProbeMetadata(ffprobePath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) + if err != nil { + return nil, fmt.Errorf("probe music metadata for %s: %w", rel, err) + } + entry := music.CreditsEntry{ + Artist: metadata.Artist, + Title: music.FirstNonEmpty(metadata.Title, strings.TrimSuffix(base, filepath.Ext(base))), + OutputFile: outputFile, + OriginalFile: base, + Album: metadata.Album, + Date: metadata.Date, + Rights: music.FirstNonEmpty(metadata.Rights, music.JoinNonEmpty("; ", metadata.License, metadata.Copyright)), + Notes: music.FirstNonEmpty(metadata.Notes, metadata.Comment), + } + music.ApplyCreditsOverlay(&entry, manual) + + stagePath := filepath.Join(stageRoot, filepath.FromSlash(outputRel)) + if err := os.MkdirAll(filepath.Dir(stagePath), 0o755); err != nil { + return nil, fmt.Errorf("create music stage dir: %w", err) + } + if err := music.ConvertSource(ffmpegPath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)), stagePath); err != nil { + return nil, fmt.Errorf("convert music %s: %w", rel, err) + } + if group.Dataset.PackageMode == "hak_asset" { + resourceInfo, err := assetResourceFromPath(stagePath, outputRel, lfsAssetInfo{}, false) + if err != nil { + return nil, fmt.Errorf("load converted music %s: %w", outputRel, err) + } + info, err := os.Stat(stagePath) + if err != nil { + return nil, fmt.Errorf("stat converted music %s: %w", stagePath, err) + } + result.Generated = append(result.Generated, assetResource{ + Rel: outputRel, + Resource: resourceInfo.Resource, + Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}), + CreatedAt: info.ModTime(), + ContentID: resourceInfo.ContentID, + }) + } + entries = append(entries, entry) + } + if opts.write { + if err := music.ValidateOverlayEntries(group.SourceDir, overlay, entries); err != nil { + return nil, err + } + creditGroups = append(creditGroups, musicCreditGroup{ + CreditsRoot: group.Dataset.CreditsRoot, + SourceDir: group.SourceDir, + Entries: entries, + }) + } + } + + if !opts.write { + sort.Slice(result.Generated, func(i, j int) bool { + return compareResourceKeys(result.Generated[i].Resource, result.Generated[j].Resource) < 0 + }) + return result, nil + } + + writeResult, err := writeCreditsArtifacts(p, creditGroups) + if err != nil { + return nil, err + } + writeResult.Summary.ScannedDirs = append(writeResult.Summary.ScannedDirs, result.Summary.ScannedDirs...) + writeResult.Summary.TrackCount = result.Summary.TrackCount + result.Artifacts = writeResult.Artifacts + result.Summary = writeResult.Summary + sort.Slice(result.Generated, func(i, j int) bool { + return compareResourceKeys(result.Generated[i].Resource, result.Generated[j].Resource) < 0 + }) + return result, nil +} + +type creditsArtifactWriteResult struct { + Artifacts []string + Summary CreditsRefreshSummary +} + +func writeCreditsArtifacts(p *project.Project, generated []musicCreditGroup) (creditsArtifactWriteResult, error) { + defaultCreditsRoot := p.CreditsDir() + sources := make([]music.CreditsSource, 0) + desiredByRoot := map[string]map[string][]byte{} + summary := CreditsRefreshSummary{} + if generated != nil { + sort.Slice(generated, func(i, j int) bool { + if generated[i].SourceDir != generated[j].SourceDir { + return generated[i].SourceDir < generated[j].SourceDir + } + return generated[i].CreditsRoot < generated[j].CreditsRoot + }) + for _, group := range generated { + creditsRoot := musicRootPath(p, group.CreditsRoot) + if desiredByRoot[creditsRoot] == nil { + desiredByRoot[creditsRoot] = map[string][]byte{} + } + entries := append([]music.CreditsEntry(nil), group.Entries...) + sort.Slice(entries, func(i, j int) bool { + if strings.ToLower(entries[i].Artist) != strings.ToLower(entries[j].Artist) { + return strings.ToLower(entries[i].Artist) < strings.ToLower(entries[j].Artist) + } + if strings.ToLower(entries[i].Title) != strings.ToLower(entries[j].Title) { + return strings.ToLower(entries[i].Title) < strings.ToLower(entries[j].Title) + } + return strings.ToLower(entries[i].OutputFile) < strings.ToLower(entries[j].OutputFile) + }) + outputPath := filepath.Join(creditsRoot, filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay) + desiredByRoot[creditsRoot][outputPath] = []byte(music.RenderCreditsMarkdown(entries)) + summary.GeneratedPaths = append(summary.GeneratedPaths, outputPath) + for _, entry := range entries { + summary.Mappings = append(summary.Mappings, MusicFileMapping{ + Source: entry.OriginalFile, + Output: entry.OutputFile, + }) + } + sources = append(sources, music.CreditsSource{ + Path: filepath.ToSlash(outputPath), + Kind: "generated_music", + Entries: entries, + }) + } + } + + err := filepath.WalkDir(p.AssetsDir(), func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + if strings.ToLower(d.Name()) != "credits.md" { + return nil + } + entries, err := music.ParseCreditsMarkdown(path) + if err != nil { + return fmt.Errorf("parse credits %s: %w", path, err) + } + rel, err := filepath.Rel(p.Root, path) + if err != nil { + return err + } + sources = append(sources, music.CreditsSource{ + Path: filepath.ToSlash(rel), + Kind: "authored", + Entries: entries, + }) + return nil + }) + if err != nil { + return creditsArtifactWriteResult{}, err + } + + sort.Slice(sources, func(i, j int) bool { return sources[i].Path < sources[j].Path }) + payload, err := json.MarshalIndent(music.CreditsInventory{Sources: sources}, "", " ") + if err != nil { + return creditsArtifactWriteResult{}, fmt.Errorf("marshal credits inventory: %w", err) + } + payload = append(payload, '\n') + if len(desiredByRoot) == 0 { + desiredByRoot[defaultCreditsRoot] = map[string][]byte{} + } + artifacts := make([]string, 0) + changedFiles := 0 + roots := make([]string, 0, len(desiredByRoot)) + for root := range desiredByRoot { + roots = append(roots, root) + } + sort.Strings(roots) + for _, creditsRoot := range roots { + desiredFiles := desiredByRoot[creditsRoot] + inventoryPath := filepath.Join(creditsRoot, "credits.json") + desiredFiles[inventoryPath] = payload + if err := os.MkdirAll(creditsRoot, 0o755); err != nil { + return creditsArtifactWriteResult{}, fmt.Errorf("create credits dir: %w", err) + } + changed, err := music.SyncGeneratedCreditsArtifacts(creditsRoot, desiredFiles) + if err != nil { + return creditsArtifactWriteResult{}, err + } + changedFiles += changed + for path := range desiredFiles { + artifacts = append(artifacts, path) + } + if summary.InventoryPath == "" { + summary.InventoryPath = inventoryPath + } + } + sort.Strings(artifacts) + summary.ArtifactPaths = append(summary.ArtifactPaths, artifacts...) + summary.ChangedFiles = changedFiles + return creditsArtifactWriteResult{ + Artifacts: artifacts, + Summary: summary, + }, nil +} + +func cleanupPreparedMusicAssets(prepared *preparedMusicAssets) error { + if prepared == nil { + return nil + } + for _, root := range prepared.TempRoots { + if strings.TrimSpace(root) == "" { + continue + } + if err := os.RemoveAll(root); err != nil { + return fmt.Errorf("remove temporary music artifact root %s: %w", root, err) + } + } + return nil +} diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go new file mode 100644 index 0000000..23a3694 --- /dev/null +++ b/internal/pipeline/pipeline_test.go @@ -0,0 +1,4057 @@ +package pipeline + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func TestBuildThenExtract(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + + mustMkdir(t, filepath.Join(root, "src", "module")) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + } + ] + } +} +`) + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + buildResult, err := Build(p) + if err != nil { + t.Fatalf("build: %v", err) + } + if buildResult.Resources != 1 { + t.Fatalf("expected 1 resource, got %d", buildResult.Resources) + } + + if err := os.Remove(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil { + t.Fatalf("remove source file: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("rescan: %v", err) + } + + extractResult, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if extractResult.Written != 1 { + t.Fatalf("expected 1 extracted file, got %d", extractResult.Written) + } + + if _, err := os.Stat(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil { + t.Fatalf("expected extracted module file: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan after extract: %v", err) + } + if _, err := BuildModule(p); err != nil { + t.Fatalf("rebuild after extract: %v", err) + } + + compareResult, err := Compare(p) + if err != nil { + t.Fatalf("compare: %v", err) + } + if compareResult.Checked != 1 { + t.Fatalf("expected 1 compared resource, got %d", compareResult.Checked) + } +} + +func TestExtractReadsHAKAssets(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "extract": { + "archives": ["*.hak"] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + modFile, err := os.Create(filepath.Join(root, "build", "testmod.mod")) + if err != nil { + t.Fatalf("create mod: %v", err) + } + if err := erf.Write(modFile, erf.New("MOD ", nil)); err != nil { + t.Fatalf("write mod: %v", err) + } + if err := modFile.Close(); err != nil { + t.Fatalf("close mod: %v", err) + } + + hakFile, err := os.Create(filepath.Join(root, "build", "vfx.hak")) + if err != nil { + t.Fatalf("create hak: %v", err) + } + if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{ + {Name: "test_vfx", Type: 0x07D2, Data: []byte("mdl-data")}, + {Name: "test_icon", Type: 0x0006, Data: []byte("plt-data")}, + })); err != nil { + t.Fatalf("write hak: %v", err) + } + if err := hakFile.Close(); err != nil { + t.Fatalf("close hak: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(result.HAKPaths) != 1 { + t.Fatalf("expected 1 hak path, got %d", len(result.HAKPaths)) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "test_vfx.mdl")); err != nil { + t.Fatalf("expected extracted mdl asset: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "assets", "plt", "test_icon.plt")); err != nil { + t.Fatalf("expected extracted plt asset: %v", err) + } +} + +func TestExtractReadsSoundBlueprintResources(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + archiveFile, err := os.Create(filepath.Join(root, "build", "testmod.mod")) + if err != nil { + t.Fatalf("create mod: %v", err) + } + if err := erf.Write(archiveFile, erf.New("MOD ", []erf.Resource{ + {Name: "S_AL_CANDLE01", Type: 0x07F3, Data: minimalGFF(t, "UTS")}, + })); err != nil { + t.Fatalf("write mod: %v", err) + } + if err := archiveFile.Close(); err != nil { + t.Fatalf("close mod: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if result.Written != 1 { + t.Fatalf("expected 1 extracted sound blueprint, got %d", result.Written) + } + if _, err := os.Stat(filepath.Join(root, "src", "blueprints", "sounds", "s_al_candle01.uts.json")); err != nil { + t.Fatalf("expected extracted sound blueprint: %v", err) + } +} + +func TestExtractDefaultOnlyReadsConfiguredModuleArchive(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "mdl")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Original Module" + } + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "mdl", "stale_asset.mdl"), "keep") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + if err := os.Remove(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil { + t.Fatalf("remove source file: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("rescan before extract: %v", err) + } + + hakFile, err := os.Create(filepath.Join(root, "build", "vfx.hak")) + if err != nil { + t.Fatalf("create hak: %v", err) + } + if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{ + {Name: "test_vfx", Type: 0x07D2, Data: []byte("mdl-data")}, + })); err != nil { + t.Fatalf("write hak: %v", err) + } + if err := hakFile.Close(); err != nil { + t.Fatalf("close hak: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(result.HAKPaths) != 0 { + t.Fatalf("expected default extract to ignore haks, got %#v", result.HAKPaths) + } + if _, err := os.Stat(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil { + t.Fatalf("expected module source to be restored: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "stale_asset.mdl")); err != nil { + t.Fatalf("expected asset cleanup to be skipped for module-only extract: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "test_vfx.mdl")); !os.IsNotExist(err) { + t.Fatalf("expected hak asset not to be extracted by default, err=%v", err) + } +} + +func TestExtractRejectsERFArchiveTargets(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +extract: + archives: + - imported.erf +`) + + erfFile, err := os.Create(filepath.Join(root, "build", "imported.erf")) + if err != nil { + t.Fatalf("create erf: %v", err) + } + if err := erf.Write(erfFile, erf.New("ERF ", nil)); err != nil { + t.Fatalf("write erf: %v", err) + } + if err := erfFile.Close(); err != nil { + t.Fatalf("close erf: %v", err) + } + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + _, err = Extract(p) + if err == nil { + t.Fatal("expected erf extraction to fail") + } + if !strings.Contains(err.Error(), "ERF extraction is unsafe") { + t.Fatalf("expected unsafe erf error, got %v", err) + } +} + +func TestExtractRejectsEscapingArchiveOverride(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + _, err = Extract(p, "../outside.mod") + if err == nil { + t.Fatal("expected escaping archive override to fail") + } + if !strings.Contains(err.Error(), "must not escape paths.build") { + t.Fatalf("expected escaping archive error, got %v", err) + } +} + +func TestExtractConsumesConfiguredHAKArchiveAfterSuccess(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +extract: + archives: + - vfx.hak + consume_archives: true +`) + + hakPath := filepath.Join(root, "build", "vfx.hak") + hakFile, err := os.Create(hakPath) + if err != nil { + t.Fatalf("create hak: %v", err) + } + if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{ + {Name: "test_vfx", Type: 0x07D2, Data: []byte("mdl-data")}, + })); err != nil { + t.Fatalf("write hak: %v", err) + } + if err := hakFile.Close(); err != nil { + t.Fatalf("close hak: %v", err) + } + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(result.DeletedArchivePaths) != 1 || result.DeletedArchivePaths[0] != hakPath { + t.Fatalf("expected consumed hak path, got %#v", result.DeletedArchivePaths) + } + if _, err := os.Stat(hakPath); !os.IsNotExist(err) { + t.Fatalf("expected hak archive to be deleted, stat err=%v", err) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "test_vfx.mdl")); err != nil { + t.Fatalf("expected extracted hak asset: %v", err) + } +} + +func TestExtractConsumeArchivesFalseOverridesLegacyDeleteFlag(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "src", "areas")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +extract: + consume_archives: false + delete_module_archive_after_success: true +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Original Module" + } + ] + } +} +`) + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + buildResult, err := BuildModule(p) + if err != nil { + t.Fatalf("build module: %v", err) + } + if err := os.Remove(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil { + t.Fatalf("remove source file: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("rescan before extract: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(result.DeletedArchivePaths) != 0 { + t.Fatalf("expected no deleted archives, got %#v", result.DeletedArchivePaths) + } + if _, err := os.Stat(buildResult.ModulePath); err != nil { + t.Fatalf("expected module archive to remain, stat err=%v", err) + } +} + +func TestExtractRefusesAssetsWhenAssetsPathIsUnset(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "build")) + mustWriteFile(t, filepath.Join(root, "root_level_asset.mdl"), "must stay") + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + if len(p.Inventory.AssetFiles) != 0 { + t.Fatalf("expected unset assets path not to inventory repository-root assets, got %#v", p.Inventory.AssetFiles) + } + + modFile, err := os.Create(p.ModuleArchivePath()) + if err != nil { + t.Fatalf("create mod: %v", err) + } + if err := erf.Write(modFile, erf.New("MOD ", nil)); err != nil { + t.Fatalf("write mod: %v", err) + } + if err := modFile.Close(); err != nil { + t.Fatalf("close mod: %v", err) + } + + hakFile, err := os.Create(filepath.Join(root, "build", "vfx.hak")) + if err != nil { + t.Fatalf("create hak: %v", err) + } + if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{ + {Name: "test_vfx", Type: 0x07D2, Data: []byte("mdl-data")}, + })); err != nil { + t.Fatalf("write hak: %v", err) + } + if err := hakFile.Close(); err != nil { + t.Fatalf("close hak: %v", err) + } + + _, err = Extract(p, "vfx.hak") + if err == nil { + t.Fatal("expected extract to fail") + } + if !strings.Contains(err.Error(), "paths.assets is not configured") { + t.Fatalf("expected missing assets path error, got %v", err) + } + if _, statErr := os.Stat(filepath.Join(root, "mdl", "test_vfx.mdl")); !os.IsNotExist(statErr) { + t.Fatalf("expected no root-level extracted asset, stat err=%v", statErr) + } + if _, statErr := os.Stat(filepath.Join(root, "root_level_asset.mdl")); statErr != nil { + t.Fatalf("expected pre-existing root-level asset to remain, stat err=%v", statErr) + } +} + +func TestExtractRefusesAssetsWhenAssetsPathIsRepositoryRoot(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: . + build: build +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + + modFile, err := os.Create(p.ModuleArchivePath()) + if err != nil { + t.Fatalf("create mod: %v", err) + } + if err := erf.Write(modFile, erf.New("MOD ", nil)); err != nil { + t.Fatalf("write mod: %v", err) + } + if err := modFile.Close(); err != nil { + t.Fatalf("close mod: %v", err) + } + + hakFile, err := os.Create(filepath.Join(root, "build", "vfx.hak")) + if err != nil { + t.Fatalf("create hak: %v", err) + } + if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{ + {Name: "test_vfx", Type: 0x07D2, Data: []byte("mdl-data")}, + })); err != nil { + t.Fatalf("write hak: %v", err) + } + if err := hakFile.Close(); err != nil { + t.Fatalf("close hak: %v", err) + } + + _, err = Extract(p, "vfx.hak") + if err == nil { + t.Fatal("expected extract to fail") + } + if !strings.Contains(err.Error(), "paths.assets resolves to unsafe extraction root") { + t.Fatalf("expected unsafe assets root error, got %v", err) + } + if _, statErr := os.Stat(filepath.Join(root, "mdl", "test_vfx.mdl")); !os.IsNotExist(statErr) { + t.Fatalf("expected no root-level extracted asset, stat err=%v", statErr) + } +} + +func TestExtractRefusesSourceWhenSourcePathIsUnset(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + assets: assets + build: build +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + + modFile, err := os.Create(p.ModuleArchivePath()) + if err != nil { + t.Fatalf("create mod: %v", err) + } + if err := erf.Write(modFile, erf.New("MOD ", []erf.Resource{ + {Name: "test_script", Type: 0x07D9, Data: []byte("void main() {}\n")}, + })); err != nil { + t.Fatalf("write mod: %v", err) + } + if err := modFile.Close(); err != nil { + t.Fatalf("close mod: %v", err) + } + + _, err = Extract(p) + if err == nil { + t.Fatal("expected extract to fail") + } + if !strings.Contains(err.Error(), "paths.source is not configured") { + t.Fatalf("expected missing source path error, got %v", err) + } + if _, statErr := os.Stat(filepath.Join(root, "scripts", "test_script.nss")); !os.IsNotExist(statErr) { + t.Fatalf("expected no root-level extracted script, stat err=%v", statErr) + } +} + +func TestExtractConfiguredHAKDiscoveryIgnoresUnconfiguredArchives(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +extract: + archives: + - wanted.hak +haks: + - name: wanted + priority: 1 + include: + - wanted/** +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + + modFile, err := os.Create(p.ModuleArchivePath()) + if err != nil { + t.Fatalf("create mod: %v", err) + } + if err := erf.Write(modFile, erf.New("MOD ", nil)); err != nil { + t.Fatalf("write mod: %v", err) + } + if err := modFile.Close(); err != nil { + t.Fatalf("close mod: %v", err) + } + for _, name := range []string{"wanted", "ignored"} { + hakFile, err := os.Create(filepath.Join(root, "build", name+".hak")) + if err != nil { + t.Fatalf("create hak: %v", err) + } + if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{ + {Name: name + "_asset", Type: 0x07D2, Data: []byte("mdl-data")}, + })); err != nil { + t.Fatalf("write hak: %v", err) + } + if err := hakFile.Close(); err != nil { + t.Fatalf("close hak: %v", err) + } + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(result.HAKPaths) != 1 || filepath.Base(result.HAKPaths[0]) != "wanted.hak" { + t.Fatalf("expected only configured hak, got %#v", result.HAKPaths) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "wanted_asset.mdl")); err != nil { + t.Fatalf("expected wanted asset: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "ignored_asset.mdl")); !os.IsNotExist(err) { + t.Fatalf("expected ignored asset to remain absent, err=%v", err) + } +} + +func TestExtractDeletesConsumedModuleArchiveAfterSuccessWhenConfigured(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "extract": { + "delete_module_archive_after_success": true + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Original Module" + } + ] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + buildResult, err := BuildModule(p) + if err != nil { + t.Fatalf("build module: %v", err) + } + if err := os.Remove(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil { + t.Fatalf("remove source file: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("rescan: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(result.DeletedArchivePaths) != 1 { + t.Fatalf("expected 1 deleted archive path, got %#v", result.DeletedArchivePaths) + } + if result.DeletedArchivePaths[0] != buildResult.ModulePath { + t.Fatalf("expected deleted archive %s, got %#v", buildResult.ModulePath, result.DeletedArchivePaths) + } + if _, err := os.Stat(buildResult.ModulePath); !os.IsNotExist(err) { + t.Fatalf("expected consumed module archive to be removed, stat err=%v", err) + } + if _, err := os.Stat(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil { + t.Fatalf("expected extracted module file: %v", err) + } +} + +func TestExtractKeepsConsumedModuleArchiveWhenExtractionFails(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "extract": { + "delete_module_archive_after_success": true + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + modFile, err := os.Create(p.ModuleArchivePath()) + if err != nil { + t.Fatalf("create mod: %v", err) + } + if err := erf.Write(modFile, erf.New("MOD ", []erf.Resource{ + {Name: "broken", Type: 0xFFFF, Data: []byte("bad")}, + })); err != nil { + t.Fatalf("write mod: %v", err) + } + if err := modFile.Close(); err != nil { + t.Fatalf("close mod: %v", err) + } + + result, err := Extract(p) + if err == nil { + t.Fatal("expected extract to fail") + } + if len(result.DeletedArchivePaths) != 0 { + t.Fatalf("expected no deleted archive paths, got %#v", result.DeletedArchivePaths) + } + if _, statErr := os.Stat(p.ModuleArchivePath()); statErr != nil { + t.Fatalf("expected consumed module archive to remain after failed extract, stat err=%v", statErr) + } +} + +func TestApplyHAKManifestRefusesUnsetSourcePath(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + mustMkdir(t, filepath.Join(root, "module")) + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + assets: assets + build: build +`) + mustWriteFile(t, filepath.Join(root, "build", "haks.json"), `{"module_haks":["core"],"haks":[]}`) + mustWriteFile(t, filepath.Join(root, "module", "module.ifo.json"), "{}\n") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + + _, err = ApplyHAKManifest(p, filepath.Join(root, "build", "haks.json")) + if err == nil { + t.Fatal("expected apply manifest to fail") + } + if !strings.Contains(err.Error(), "paths.source is not configured") { + t.Fatalf("expected missing source path error, got %v", err) + } + raw, readErr := os.ReadFile(filepath.Join(root, "module", "module.ifo.json")) + if readErr != nil { + t.Fatalf("read root module file: %v", readErr) + } + if string(raw) != "{}\n" { + t.Fatalf("expected root module file to remain unchanged, got %q", string(raw)) + } +} + +func TestBuildSplitsConfiguredHAKs(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 300, + "split": true, + "include": ["core/**"] + } + ] +} +`) + mustMkdir(t, filepath.Join(root, "src", "module")) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), strings.Repeat("a", 80)) + mustWriteFile(t, filepath.Join(root, "assets", "core", "b.tga"), strings.Repeat("b", 80)) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if len(result.HAKPaths) != 2 { + t.Fatalf("expected 2 hak outputs, got %d", len(result.HAKPaths)) + } + if _, err := os.Stat(filepath.Join(root, "build", "core_01.hak")); err != nil { + t.Fatalf("expected first split hak: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "build", "core_02.hak")); err != nil { + t.Fatalf("expected second split hak: %v", err) + } + + rawManifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var manifest BuildManifest + if err := json.Unmarshal(rawManifest, &manifest); err != nil { + t.Fatalf("parse manifest: %v", err) + } + if len(manifest.HAKs) != 2 { + t.Fatalf("expected 2 manifest entries, got %d", len(manifest.HAKs)) + } + if len(manifest.ModuleHAKs) != 2 || manifest.ModuleHAKs[0] != "core_01" || manifest.ModuleHAKs[1] != "core_02" { + t.Fatalf("unexpected module hak order: %#v", manifest.ModuleHAKs) + } + + applyResult, err := ApplyHAKManifest(p, filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("apply hak manifest: %v", err) + } + if applyResult.HAKCount != 2 { + t.Fatalf("expected 2 applied hak entries, got %d", applyResult.HAKCount) + } + + updated, err := os.ReadFile(filepath.Join(root, "src", "module", "module.ifo.json")) + if err != nil { + t.Fatalf("read updated ifo: %v", err) + } + if !strings.Contains(string(updated), "\"value\": \"core_01\"") || !strings.Contains(string(updated), "\"value\": \"core_02\"") { + t.Fatalf("updated ifo did not contain expected hak entries:\n%s", string(updated)) + } +} + +func TestBuildBuildsTopPackageAndInjectsHAKList(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "repadjust")) + mustMkdir(t, filepath.Join(root, "topdata", "assets", "gui")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["sow_top"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "topdata": { + "source": "topdata", + "build": "build/topdata" + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + {"label": "Mod_Name", "type": "CExoString", "value": "Test Module"}, + {"label": "Mod_CustomTlk", "type": "CExoString", "value": "sow_tlk"} + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mustWriteFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.png"), "icon-data") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := Build(p) + if err != nil { + t.Fatalf("build: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "build", "sow_top.hak")); err != nil { + t.Fatalf("expected sow_top.hak: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "build", "sow_tlk.tlk")); err != nil { + t.Fatalf("expected sow_tlk.tlk: %v", err) + } + if result.TopPackageHAK == "" || result.TopPackageTLK == "" { + t.Fatalf("expected top package paths in build result: %#v", result) + } + + input, err := os.Open(result.ModulePath) + if err != nil { + t.Fatalf("open module: %v", err) + } + defer input.Close() + archive, err := erf.Read(input) + if err != nil { + t.Fatalf("read module archive: %v", err) + } + var ifoData []byte + for _, resource := range archive.Resources { + ext, _ := erf.ExtensionForResourceType(resource.Type) + if strings.ToLower(resource.Name)+"."+ext == "module.ifo" { + ifoData = resource.Data + break + } + } + if ifoData == nil { + t.Fatalf("expected module.ifo in built archive") + } + document, err := gff.Read(bytes.NewReader(ifoData)) + if err != nil { + t.Fatalf("decode module.ifo: %v", err) + } + foundHak := false + for _, field := range document.Root.Fields { + if field.Label != "Mod_HakList" { + continue + } + list, ok := field.Value.(gff.ListValue) + if !ok { + t.Fatalf("expected Mod_HakList list, got %T", field.Value) + } + for _, item := range list { + for _, nested := range item.Fields { + if nested.Label == "Mod_Hak" && nested.Value == gff.StringValue("sow_top") { + foundHak = true + } + } + } + } + if !foundHak { + t.Fatalf("expected built module to reference sow_top") + } +} + +func TestBuildModuleDoesNotCompileTopData(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "repadjust")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["sow_top"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "topdata": { + "source": "topdata", + "build": "build/topdata" + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + {"label": "Mod_Name", "type": "CExoString", "value": "Test Module"}, + {"label": "Mod_CustomTlk", "type": "CExoString", "value": "sow_tlk"} + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mustWriteFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + var progress []string + if _, err := BuildModuleWithProgress(p, func(message string) { + progress = append(progress, message) + }); err != nil { + t.Fatalf("build module: %v", err) + } + + joined := strings.Join(progress, "\n") + if got := strings.Count(joined, "Building native topdata datasets..."); got != 0 { + t.Fatalf("expected build-module to skip topdata compilation, got %d compile pass(es)\nprogress:\n%s", got, joined) + } + if _, err := os.Stat(filepath.Join(root, "build", "sow_top.hak")); !os.IsNotExist(err) { + t.Fatalf("expected build-module not to generate sow_top.hak, got err=%v", err) + } + if _, err := os.Stat(filepath.Join(root, "build", "sow_tlk.tlk")); !os.IsNotExist(err) { + t.Fatalf("expected build-module not to generate sow_tlk.tlk, got err=%v", err) + } +} + +func TestCompareIgnoresTopPackageStaleness(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "repadjust")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["sow_top"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "topdata": { + "source": "topdata", + "build": "build/topdata" + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + {"label": "Mod_Name", "type": "CExoString", "value": "Test Module"}, + {"label": "Mod_CustomTlk", "type": "CExoString", "value": "sow_tlk"} + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + topdataBase := filepath.Join(root, "topdata", "data", "repadjust", "base.json") + mustWriteFile(t, topdataBase, `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + newTime := time.Now().Add(2 * time.Second) + if err := os.Chtimes(topdataBase, newTime, newTime); err != nil { + t.Fatalf("touch topdata source: %v", err) + } + result, err := Compare(p) + if err != nil { + t.Fatalf("expected compare to ignore topdata staleness: %v", err) + } + if result.Checked != 1 { + t.Fatalf("expected 1 compared resource, got %d", result.Checked) + } +} + +func TestCompareNormalizesModuleHakListFromConfig(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["manual_top"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +}`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [ + { + "struct_type": 8, + "fields": [ + { + "label": "Mod_Hak", + "type": "CExoString", + "value": "stale_old_hak" + } + ] + } + ] + } + ] + } +}`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + if _, err := Compare(p); err != nil { + t.Fatalf("compare project: %v", err) + } +} + +func TestBuildOrdersMultipleHAKGroups(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "assets", "vfx")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core", "manual_top", "group:vfx"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 0, + "split": false, + "include": ["core/**"] + }, + { + "name": "vfx", + "priority": 2, + "max_bytes": 0, + "split": false, + "include": ["vfx/**"] + } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), "core-data") + mustWriteFile(t, filepath.Join(root, "assets", "vfx", "b.tga"), "vfx-data") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if len(result.HAKPaths) != 2 { + t.Fatalf("expected 2 hak outputs, got %d", len(result.HAKPaths)) + } + + rawManifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var manifest BuildManifest + if err := json.Unmarshal(rawManifest, &manifest); err != nil { + t.Fatalf("parse manifest: %v", err) + } + if len(manifest.ModuleHAKs) != 3 { + t.Fatalf("expected 3 module hak entries, got %#v", manifest.ModuleHAKs) + } + if manifest.ModuleHAKs[0] != "core" || manifest.ModuleHAKs[1] != "manual_top" || manifest.ModuleHAKs[2] != "vfx" { + t.Fatalf("unexpected module hak order: %#v", manifest.ModuleHAKs) + } +} + +func TestBuildSplitsMultipleHAKGroupsDeterministically(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "assets", "vfx")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core", "manual_top", "group:vfx"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 310, + "split": true, + "include": ["core/**"] + }, + { + "name": "vfx", + "priority": 2, + "max_bytes": 310, + "split": true, + "include": ["vfx/**"] + } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +} +`) + + mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), strings.Repeat("a", 80)) + mustWriteFile(t, filepath.Join(root, "assets", "core", "b.tga"), strings.Repeat("b", 80)) + mustWriteFile(t, filepath.Join(root, "assets", "vfx", "c.tga"), strings.Repeat("c", 80)) + mustWriteFile(t, filepath.Join(root, "assets", "vfx", "d.tga"), strings.Repeat("d", 80)) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if len(result.HAKPaths) != 4 { + t.Fatalf("expected 4 hak outputs, got %d", len(result.HAKPaths)) + } + + expectedFiles := []string{ + "core_01.hak", + "core_02.hak", + "vfx_01.hak", + "vfx_02.hak", + } + for _, name := range expectedFiles { + if _, err := os.Stat(filepath.Join(root, "build", name)); err != nil { + t.Fatalf("expected split hak %s: %v", name, err) + } + } + + rawManifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var manifest BuildManifest + if err := json.Unmarshal(rawManifest, &manifest); err != nil { + t.Fatalf("parse manifest: %v", err) + } + if len(manifest.HAKs) != 4 { + t.Fatalf("expected 4 manifest hak entries, got %d", len(manifest.HAKs)) + } + + actualManifestNames := make([]string, 0, len(manifest.HAKs)) + for _, hak := range manifest.HAKs { + actualManifestNames = append(actualManifestNames, hak.Name) + } + expectedManifestNames := []string{"core_01", "core_02", "vfx_01", "vfx_02"} + if strings.Join(actualManifestNames, ",") != strings.Join(expectedManifestNames, ",") { + t.Fatalf("unexpected manifest hak order: %#v", actualManifestNames) + } + + expectedModuleHAKs := []string{"core_01", "core_02", "manual_top", "vfx_01", "vfx_02"} + if strings.Join(manifest.ModuleHAKs, ",") != strings.Join(expectedModuleHAKs, ",") { + t.Fatalf("unexpected module hak order: %#v", manifest.ModuleHAKs) + } + + applyResult, err := ApplyHAKManifest(p, filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("apply hak manifest: %v", err) + } + if applyResult.HAKCount != 5 { + t.Fatalf("expected 5 applied hak entries, got %d", applyResult.HAKCount) + } + + updated, err := os.ReadFile(filepath.Join(root, "src", "module", "module.ifo.json")) + if err != nil { + t.Fatalf("read updated ifo: %v", err) + } + for _, name := range expectedModuleHAKs { + if !strings.Contains(string(updated), "\"value\": \""+name+"\"") { + t.Fatalf("updated ifo missing hak %s:\n%s", name, string(updated)) + } + } +} + +func TestBuildHAKsWithOptionsBuildsSingleArchiveByName(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 310, + "split": true, + "include": ["core/**"] + } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), strings.Repeat("a", 80)) + mustWriteFile(t, filepath.Join(root, "assets", "core", "b.tga"), strings.Repeat("b", 80)) + mustWriteFile(t, filepath.Join(root, "assets", "core", "c.tga"), strings.Repeat("c", 80)) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKsWithOptions(p, BuildHAKOptions{ArchiveNames: []string{"core_02"}}) + if err != nil { + t.Fatalf("build filtered haks: %v", err) + } + if len(result.HAKPaths) != 1 { + t.Fatalf("expected 1 hak output, got %d", len(result.HAKPaths)) + } + if got := filepath.Base(result.HAKPaths[0]); got != "core_02.hak" { + t.Fatalf("unexpected hak output %q", got) + } + if _, err := os.Stat(filepath.Join(root, "build", "core_01.hak")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expected core_01.hak to be absent, got err=%v", err) + } + if _, err := os.Stat(filepath.Join(root, "build", "core.hak")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expected unsplit core.hak to be absent, got err=%v", err) + } +} + +func TestBuildExcludesOptionalHAKsFromModuleOrder(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "assets", "optional")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core", "group:optional"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 0, + "split": false, + "include": ["core/**"] + }, + { + "name": "optional", + "priority": 2, + "max_bytes": 0, + "split": false, + "optional": true, + "include": ["optional/**"] + } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), "core-data") + mustWriteFile(t, filepath.Join(root, "assets", "optional", "b.tga"), "optional-data") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if len(result.HAKPaths) != 2 { + t.Fatalf("expected 2 hak outputs, got %d", len(result.HAKPaths)) + } + + rawManifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var manifest BuildManifest + if err := json.Unmarshal(rawManifest, &manifest); err != nil { + t.Fatalf("parse manifest: %v", err) + } + if got, want := strings.Join(manifest.ModuleHAKs, ","), "core"; got != want { + t.Fatalf("unexpected module hak order: got %q want %q", got, want) + } + if len(manifest.HAKs) != 2 { + t.Fatalf("expected 2 manifest hak entries, got %d", len(manifest.HAKs)) + } + if manifest.HAKs[0].Name != "core" || manifest.HAKs[0].Optional { + t.Fatalf("unexpected primary hak manifest entry: %#v", manifest.HAKs[0]) + } + if manifest.HAKs[1].Name != "optional" || !manifest.HAKs[1].Optional { + t.Fatalf("unexpected optional hak manifest entry: %#v", manifest.HAKs[1]) + } + + applyResult, err := ApplyHAKManifest(p, filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("apply hak manifest: %v", err) + } + if applyResult.HAKCount != 1 { + t.Fatalf("expected 1 applied hak entry, got %d", applyResult.HAKCount) + } + + updated, err := os.ReadFile(filepath.Join(root, "src", "module", "module.ifo.json")) + if err != nil { + t.Fatalf("read updated ifo: %v", err) + } + if !strings.Contains(string(updated), "\"value\": \"core\"") { + t.Fatalf("updated ifo missing primary hak:\n%s", string(updated)) + } + if strings.Contains(string(updated), "\"value\": \"optional\"") { + t.Fatalf("updated ifo should not contain optional hak:\n%s", string(updated)) + } +} + +func TestPlanHAKsWritesManifestWithoutArchives(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 0, + "split": false, + "include": ["core/**"] + } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), "core-data") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := PlanHAKs(p) + if err != nil { + t.Fatalf("plan haks: %v", err) + } + if len(result.HAKPaths) != 0 { + t.Fatalf("expected no hak outputs for plan-only, got %d", len(result.HAKPaths)) + } + + manifestPath := filepath.Join(root, "build", "haks.json") + rawManifest, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var manifest BuildManifest + if err := json.Unmarshal(rawManifest, &manifest); err != nil { + t.Fatalf("parse manifest: %v", err) + } + if len(manifest.HAKs) != 1 { + t.Fatalf("expected 1 manifest hak entry, got %d", len(manifest.HAKs)) + } + if manifest.HAKs[0].ContentHash == "" { + t.Fatal("expected plan-only manifest to include content hash") + } + if _, err := os.Stat(filepath.Join(root, "build", "core.hak")); !os.IsNotExist(err) { + t.Fatalf("expected no built hak archive, got err=%v", err) + } +} + +func TestBuildHAKsWritesConfiguredHeadVisualeffectsAutogenManifest(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "vfxs", "chest_accessories")) + mustMkdir(t, filepath.Join(root, "assets", "vfxs", "head_accessories")) + mustMkdir(t, filepath.Join(root, "assets", "vfxs", "head_decorations")) + mustMkdir(t, filepath.Join(root, "assets", "vfxs", "head_features", "hair")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "autogen": { + "producers": [ + { + "id": "accessory_visualeffects", + "root": "vfxs", + "include": ["chest_accessories/**/*.mdl", "head_accessories/**/*.mdl", "head_decorations/**/*.mdl", "head_features/**/*.mdl"], + "derive": { + "kind": "model_stem", + "group_from": "first_path_segment" + }, + "manifest": { + "release_tag": "head-vfx-manifest-current", + "asset_name": "sow-accessory-vfx-manifest.json", + "cache_name": "sow-accessory-vfx-manifest.json" + }, + "accessory_visualeffects": { + "groups": { + "head_accessories": { + "prefix": "head_accessories", + "model_columns": ["Imp_HeadCon_Node", "Imp_Root_H_Node"] + } + } + } + } + ] + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 0, + "split": false, + "include": ["vfxs/**"] + } + ] +} +`) + + mustWriteFile(t, filepath.Join(root, "assets", "vfxs", "chest_accessories", "tfx_sash.mdl"), "sash") + mustWriteFile(t, filepath.Join(root, "assets", "vfxs", "head_accessories", "hfx_bandana.mdl"), "bandana") + mustWriteFile(t, filepath.Join(root, "assets", "vfxs", "head_decorations", "hfx_laurel.mdl"), "laurel") + mustWriteFile(t, filepath.Join(root, "assets", "vfxs", "head_features", "hair", "hfx_hair_bangs.mdl"), "hair") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if len(result.AutogenManifestPaths) != 1 { + t.Fatalf("expected 1 autogen manifest, got %d", len(result.AutogenManifestPaths)) + } + if _, err := os.Stat(filepath.Join(root, "build", "sow-parts-manifest.json")); !os.IsNotExist(err) { + t.Fatalf("expected no parts manifest, got stat error %v", err) + } + + headRaw, err := os.ReadFile(filepath.Join(root, "build", "sow-accessory-vfx-manifest.json")) + if err != nil { + t.Fatalf("read head visualeffects manifest: %v", err) + } + text := string(headRaw) + for _, want := range []string{ + `"id": "accessory_visualeffects"`, + `"group": "chest_accessories"`, + `"model_stem": "tfx_sash"`, + `"group": "head_accessories"`, + `"model_stem": "hfx_bandana"`, + `"accessory_visualeffects": {`, + `"model_columns": [`, + `"Imp_Root_H_Node"`, + `"group": "head_decorations"`, + `"model_stem": "hfx_laurel"`, + `"group": "head_features"`, + `"model_stem": "hfx_hair_bangs"`, + } { + if !strings.Contains(text, want) { + t.Fatalf("expected head visualeffects manifest to contain %q, got:\n%s", want, text) + } + } +} + +func TestBuildHAKsWritesFolderCategoryInHeadVisualeffectsAutogenManifest(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "vfxs", "head_features", "sinfar", "ears_plt")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "autogen": { + "producers": [ + { + "id": "accessory_visualeffects", + "root": "vfxs", + "include": ["head_features/**/*.mdl"], + "derive": { + "kind": "model_stem", + "group_from": "first_path_segment" + }, + "manifest": { + "release_tag": "head-vfx-manifest-current", + "asset_name": "sow-accessory-vfx-manifest.json", + "cache_name": "sow-accessory-vfx-manifest.json" + }, + "accessory_visualeffects": { + "groups": { + "head_features": { + "model_columns": ["Imp_HeadCon_Node", "Imp_Root_H_Node"] + } + }, + "group_token_source": "folder_name", + "category_from": "immediate_parent", + "delimiter": "/", + "case": "preserve", + "strip_model_prefixes": ["hfx_"], + "key_format": "{dataset}:{group}{delimiter}{category_segment}{stem}", + "label_format": "{group}{delimiter}{category_segment}{stem}" + } + } + ] + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 0, + "split": false, + "include": ["vfxs/**"] + } + ] +} +`) + + mustWriteFile(t, filepath.Join(root, "assets", "vfxs", "head_features", "sinfar", "ears_plt", "hfx_Ear_EL_L.mdl"), "ear") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildHAKs(p); err != nil { + t.Fatalf("build haks: %v", err) + } + headRaw, err := os.ReadFile(filepath.Join(root, "build", "sow-accessory-vfx-manifest.json")) + if err != nil { + t.Fatalf("read head visualeffects manifest: %v", err) + } + text := string(headRaw) + for _, want := range []string{ + `"group": "head_features"`, + `"subgroup": "sinfar/ears_plt"`, + `"category": "ears_plt"`, + `"model_stem": "hfx_Ear_EL_L"`, + `"source": "head_features/sinfar/ears_plt/hfx_Ear_EL_L.mdl"`, + `"group_token_source": "folder_name"`, + `"category_from": "immediate_parent"`, + } { + if !strings.Contains(text, want) { + t.Fatalf("expected head visualeffects manifest to contain %q, got:\n%s", want, text) + } + } +} + +func TestBuildHAKsGeneratesParts2DAAssetsFromLocalModels(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "part", "belt")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "parts")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "parts", "modules")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:sow_part"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "generated_assets": { + "topdata_2da": [ + { + "id": "parts", + "source": "topdata", + "output": "{paths.cache}/generated-assets/parts-2da", + "include_datasets": ["parts/**"], + "package_root": "part", + "autogen": { + "id": "parts", + "mode": "parts_rows", + "root": "part", + "include": ["**/*.mdl"], + "derive": { + "kind": "trailing_numeric_suffix", + "group_from": "first_path_segment" + }, + "parts_rows": { + "row_defaults": {"COSTMODIFIER": "0"}, + "acbonus": { + "default": { + "strategy": "ascending_row_id_sort_key", + "max_row_id": 999, + "divisor": 100, + "format": "%.2f" + }, + "datasets": { + "chest": {"strategy": "fixed", "value": "0.00"} + } + } + } + } + } + ] + }, + "haks": [ + { + "name": "sow_part", + "priority": 1, + "max_bytes": 0, + "split": false, + "include": ["part/**"] + } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt018.mdl"), "belt") + mustWriteFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt117.mdl"), "belt") + mustWriteFile(t, filepath.Join(root, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER", "ACBONUS"], + "rows": [ + {"id": 0, "COSTMODIFIER": 1, "ACBONUS": "0.25"}, + {"id": 17, "COSTMODIFIER": 0, "ACBONUS": "0.90"}, + {"id": 19, "COSTMODIFIER": 0, "ACBONUS": "****"}, + {"id": 117, "COSTMODIFIER": 0, "ACBONUS": "****"} + ] +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "data", "parts", "modules", "ovr_belt_acbonus_zero.json"), `{ + "overrides": [ + {"id": 18, "ACBONUS": "0.75"} + ] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if len(result.HAKPaths) != 1 { + t.Fatalf("expected one HAK, got %#v", result.HAKPaths) + } + raw, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read haks manifest: %v", err) + } + manifestText := string(raw) + if !strings.Contains(manifestText, `"part/parts_belt.2da"`) { + t.Fatalf("expected generated parts 2da in manifest, got:\n%s", manifestText) + } + partsRaw, err := os.ReadFile(filepath.Join(root, ".cache", "generated-assets", "parts-2da", "parts_belt.2da")) + if err != nil { + t.Fatalf("read generated parts 2da: %v", err) + } + partsText := string(partsRaw) + for _, want := range []string{ + "0\t1\t0.00", + "17\t0\t0.17", + "18\t0\t0.75", + "19\t****\t****", + "117\t0\t1.17", + } { + if !strings.Contains(partsText, want) { + t.Fatalf("expected generated parts 2da to contain %q after normalization and overrides, got:\n%s", want, partsText) + } + } + if strings.Contains(partsText, "17\t0\t0.90") { + t.Fatalf("expected authored and generated rows in parts 2da, got:\n%s", partsText) + } +} + +func TestBuildHAKsPackagesStaticGeneratedTopDataByConfiguredRoot(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "tailmodel")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "skyboxes")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:sow_part", "group:sow_envi"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "generated_assets": { + "topdata_2da": [ + { + "id": "part_topdata", + "source": "topdata", + "output": "{paths.cache}/generated-assets/part-topdata-2da", + "include_datasets": ["tailmodel"], + "package_root": "part" + }, + { + "id": "envi_topdata", + "source": "topdata", + "output": "{paths.cache}/generated-assets/envi-topdata-2da", + "include_datasets": ["skyboxes"], + "package_root": "envi" + } + ] + }, + "haks": [ + { + "name": "sow_part", + "priority": 1, + "max_bytes": 0, + "split": false, + "include": ["part/**"] + }, + { + "name": "sow_envi", + "priority": 2, + "max_bytes": 0, + "split": false, + "include": ["envi/**"] + } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "data", "tailmodel", "base.json"), `{ + "output": "tailmodel.2da", + "columns": ["LABEL", "MODEL", "ENVMAP"], + "rows": [ + {"id": 0, "LABEL": "(None)", "MODEL": "****", "ENVMAP": "****"}, + {"id": 1, "LABEL": "Lizard", "MODEL": "c_tailliz", "ENVMAP": "default"} + ] +} +`) + mustWriteFile(t, filepath.Join(root, "topdata", "data", "skyboxes", "base.json"), `{ + "output": "skyboxes.2da", + "columns": ["LABEL", "STRING_REF", "CYCLICAL", "DAWN", "DAY", "DUSK", "NIGHT"], + "rows": [ + {"id": 0, "LABEL": "(None)", "STRING_REF": "****", "CYCLICAL": "****", "DAWN": "****", "DAY": "****", "DUSK": "****", "NIGHT": "****"}, + {"id": 1, "LABEL": "Grass_Clear", "STRING_REF": "****", "CYCLICAL": "1", "DAWN": "Skyda_001", "DAY": "Sky_001", "DUSK": "Skyd_001", "NIGHT": "Skyn_001"} + ] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if len(result.HAKPaths) != 2 { + t.Fatalf("expected two HAKs, got %#v", result.HAKPaths) + } + raw, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read haks manifest: %v", err) + } + manifestText := string(raw) + for _, want := range []string{ + `"part/tailmodel.2da"`, + `"envi/skyboxes.2da"`, + } { + if !strings.Contains(manifestText, want) { + t.Fatalf("expected generated static 2da asset %q in manifest, got:\n%s", want, manifestText) + } + } + if strings.Contains(manifestText, `"part/skyboxes.2da"`) || strings.Contains(manifestText, `"envi/tailmodel.2da"`) { + t.Fatalf("expected generated static 2da assets to follow configured package roots, got:\n%s", manifestText) + } + for _, generated := range []string{ + filepath.Join(root, ".cache", "generated-assets", "part-topdata-2da", "tailmodel.2da"), + filepath.Join(root, ".cache", "generated-assets", "envi-topdata-2da", "skyboxes.2da"), + } { + if _, err := os.Stat(generated); err != nil { + t.Fatalf("expected generated 2da %s: %v", generated, err) + } + } +} + +func TestBuildHAKsRejectsTLKBackedGeneratedParts2DAAssets(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "part", "belt")) + mustMkdir(t, filepath.Join(root, "topdata", "data", "parts")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": {"name": "Test Module", "resref": "testmod"}, + "paths": {"source": "src", "assets": "assets", "build": "build"}, + "generated_assets": { + "topdata_2da": [ + { + "id": "parts", + "source": "topdata", + "output": "{paths.cache}/generated-assets/parts-2da", + "include_datasets": ["parts/**"], + "package_root": "part", + "autogen": { + "id": "parts", + "mode": "parts_rows", + "root": "part", + "include": ["**/*.mdl"], + "derive": {"kind": "trailing_numeric_suffix", "group_from": "first_path_segment"} + } + } + ] + }, + "haks": [{"name": "sow_part", "priority": 1, "max_bytes": 0, "include": ["part/**"]}] +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt018.mdl"), "belt") + mustWriteFile(t, filepath.Join(root, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER", "ACBONUS", "NAME"], + "rows": [ + {"id": 0, "COSTMODIFIER": 1, "ACBONUS": "0.25", "NAME": {"tlk": {"text": "Belt"}}} + ] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + _, err = BuildHAKs(p) + if err == nil || !strings.Contains(err.Error(), "2DA-only generated asset") { + t.Fatalf("expected TLK-backed generated asset error, got %v", err) + } +} + +func TestBuildSkipsEmptyHAKGroupsInModuleOrder(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "assets", "vfx")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["manual_top", "group:core", "group:empty", "group:vfx"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 1024, + "split": false, + "include": ["core/**"] + }, + { + "name": "empty", + "priority": 2, + "max_bytes": 1024, + "split": false, + "include": ["empty/**"] + }, + { + "name": "vfx", + "priority": 3, + "max_bytes": 1024, + "split": false, + "include": ["vfx/**"] + } + ] +}`) + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a") + mustWriteFile(t, filepath.Join(root, "assets", "vfx", "vfx_a.tga"), "vfx-a") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + + rawManifest, err := os.ReadFile(result.Manifest) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var manifest BuildManifest + if err := json.Unmarshal(rawManifest, &manifest); err != nil { + t.Fatalf("parse manifest: %v", err) + } + if got, want := strings.Join(manifest.ModuleHAKs, ","), "manual_top,core,vfx"; got != want { + t.Fatalf("unexpected module hak order: got %q want %q", got, want) + } +} + +func TestBuildHAKsReusesUnchangedChunks(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "assets", "vfx")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core", "group:vfx"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 1024, + "split": false, + "include": ["core/**"] + }, + { + "name": "vfx", + "priority": 2, + "max_bytes": 1024, + "split": false, + "include": ["vfx/**"] + } + ] +}`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +}`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a") + mustWriteFile(t, filepath.Join(root, "assets", "vfx", "vfx_a.tga"), "vfx-a") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildHAKs(p); err != nil { + t.Fatalf("first build haks: %v", err) + } + corePath := filepath.Join(root, "build", "core.hak") + vfxPath := filepath.Join(root, "build", "vfx.hak") + coreInfoBefore, err := os.Stat(corePath) + if err != nil { + t.Fatalf("stat core before: %v", err) + } + vfxInfoBefore, err := os.Stat(vfxPath) + if err != nil { + t.Fatalf("stat vfx before: %v", err) + } + + time.Sleep(1100 * time.Millisecond) + + if _, err := BuildHAKs(p); err != nil { + t.Fatalf("second build haks: %v", err) + } + coreInfoAfter, err := os.Stat(corePath) + if err != nil { + t.Fatalf("stat core after: %v", err) + } + vfxInfoAfter, err := os.Stat(vfxPath) + if err != nil { + t.Fatalf("stat vfx after: %v", err) + } + + if !coreInfoAfter.ModTime().Equal(coreInfoBefore.ModTime()) { + t.Fatalf("expected unchanged core.hak to be reused") + } + if !vfxInfoAfter.ModTime().Equal(vfxInfoBefore.ModTime()) { + t.Fatalf("expected unchanged vfx.hak to be reused") + } + + rawManifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var manifest BuildManifest + if err := json.Unmarshal(rawManifest, &manifest); err != nil { + t.Fatalf("parse manifest: %v", err) + } + for _, hak := range manifest.HAKs { + if hak.ContentHash == "" { + t.Fatalf("expected content hash for %s", hak.Name) + } + } +} + +func TestBuildModulePrefersManifestModuleHAKOrder(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["manual_top", "group:core"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 1024, + "split": false, + "include": ["core/**"] + } + ] +}`) + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +} +`) + + mustWriteFile(t, filepath.Join(root, "build", "haks.json"), `{ + "module_haks": ["sow_top", "sow_core_01", "sow_appr_01"], + "haks": [] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildModule(p) + if err != nil { + t.Fatalf("build module: %v", err) + } + + archive, err := readArchive(result.ModulePath) + if err != nil { + t.Fatalf("read module archive: %v", err) + } + + var ifo erf.Resource + found := false + for _, resource := range archive.Resources { + if resource.Name == "module" { + ifo = resource + found = true + break + } + } + if !found { + t.Fatalf("module.ifo not found in built archive") + } + + document, err := gff.Read(bytes.NewReader(ifo.Data)) + if err != nil { + t.Fatalf("decode module ifo: %v", err) + } + + var got []string + for _, field := range document.Root.Fields { + if field.Label != "Mod_HakList" { + continue + } + list, ok := field.Value.(gff.ListValue) + if !ok { + t.Fatalf("expected Mod_HakList list, got %T", field.Value) + } + for _, item := range list { + for _, nested := range item.Fields { + if nested.Label == "Mod_Hak" { + got = append(got, string(nested.Value.(gff.StringValue))) + } + } + } + } + + if want := "sow_top,sow_core_01,sow_appr_01"; strings.Join(got, ",") != want { + t.Fatalf("unexpected module hak order: got %q want %q", strings.Join(got, ","), want) + } +} + +func TestBuildHAKsRebuildsOnlyChangedChunks(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "assets", "vfx")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core", "group:vfx"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 1024, + "split": false, + "include": ["core/**"] + }, + { + "name": "vfx", + "priority": 2, + "max_bytes": 1024, + "split": false, + "include": ["vfx/**"] + } + ] +}`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +}`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a") + mustWriteFile(t, filepath.Join(root, "assets", "vfx", "vfx_a.tga"), "vfx-a") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildHAKs(p); err != nil { + t.Fatalf("first build haks: %v", err) + } + corePath := filepath.Join(root, "build", "core.hak") + vfxPath := filepath.Join(root, "build", "vfx.hak") + coreInfoBefore, err := os.Stat(corePath) + if err != nil { + t.Fatalf("stat core before: %v", err) + } + vfxInfoBefore, err := os.Stat(vfxPath) + if err != nil { + t.Fatalf("stat vfx before: %v", err) + } + + time.Sleep(1100 * time.Millisecond) + mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a-updated") + if err := p.Scan(); err != nil { + t.Fatalf("rescan after change: %v", err) + } + + if _, err := BuildHAKs(p); err != nil { + t.Fatalf("second build haks: %v", err) + } + coreInfoAfter, err := os.Stat(corePath) + if err != nil { + t.Fatalf("stat core after: %v", err) + } + vfxInfoAfter, err := os.Stat(vfxPath) + if err != nil { + t.Fatalf("stat vfx after: %v", err) + } + + if !coreInfoAfter.ModTime().After(coreInfoBefore.ModTime()) { + t.Fatalf("expected changed core.hak to be rebuilt") + } + if !vfxInfoAfter.ModTime().Equal(vfxInfoBefore.ModTime()) { + t.Fatalf("expected unchanged vfx.hak to be reused") + } +} + +func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 400, + "split": true, + "include": ["core/**"] + } + ] +}`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + } + ] + } +} +`) + + runGitTest(t, root, "init") + runGitTest(t, root, "config", "user.name", "Test User") + runGitTest(t, root, "config", "user.email", "test@example.com") + + mustWriteFile(t, filepath.Join(root, "assets", "core", "old_a.tga"), strings.Repeat("a", 120)) + runGitCommitTest(t, root, "2001-01-01T00:00:00Z", "add old_a") + + mustWriteFile(t, filepath.Join(root, "assets", "core", "old_b.tga"), strings.Repeat("b", 120)) + runGitCommitTest(t, root, "2002-01-01T00:00:00Z", "add old_b") + + mustWriteFile(t, filepath.Join(root, "assets", "core", "new_c.tga"), strings.Repeat("c", 120)) + runGitCommitTest(t, root, "2003-01-01T00:00:00Z", "add new_c") + + now := time.Now() + if err := os.Chtimes(filepath.Join(root, "assets", "core", "old_a.tga"), now, now); err != nil { + t.Fatalf("chtimes old_a: %v", err) + } + if err := os.Chtimes(filepath.Join(root, "assets", "core", "old_b.tga"), now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil { + t.Fatalf("chtimes old_b: %v", err) + } + if err := os.Chtimes(filepath.Join(root, "assets", "core", "new_c.tga"), now.Add(-4*time.Hour), now.Add(-4*time.Hour)); err != nil { + t.Fatalf("chtimes new_c: %v", err) + } + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + assets, err := collectAssetResources(p, false, nil, nil, nil) + if err != nil { + t.Fatalf("collect asset resources: %v", err) + } + slices.SortFunc(assets, compareAssetBuildOrder) + twoAssetSize := erf.ArchiveSize(resourceSlice(assets[:2])) + threeAssetSize := erf.ArchiveSize(resourceSlice(assets)) + if threeAssetSize <= twoAssetSize { + t.Fatalf("expected three-asset archive to be larger than two-asset archive") + } + p.Config.HAKs[0].MaxBytes = twoAssetSize + + chunks, err := planHAKChunks(p, assets) + if err != nil { + t.Fatalf("plan hak chunks: %v", err) + } + if len(chunks) != 2 { + t.Fatalf("expected 2 chunks, got %d", len(chunks)) + } + + firstChunkAssets := make([]string, 0, len(chunks[0].Assets)) + for _, asset := range chunks[0].Assets { + firstChunkAssets = append(firstChunkAssets, asset.Rel) + } + lastChunkAssets := make([]string, 0, len(chunks[1].Assets)) + for _, asset := range chunks[1].Assets { + lastChunkAssets = append(lastChunkAssets, asset.Rel) + } + + if got, want := strings.Join(firstChunkAssets, ","), "core/old_a.tga,core/old_b.tga"; got != want { + t.Fatalf("unexpected first chunk assets: got %q want %q", got, want) + } + if got, want := strings.Join(lastChunkAssets, ","), "core/new_c.tga"; got != want { + t.Fatalf("unexpected last chunk assets: got %q want %q", got, want) + } +} + +func TestPlanHAKChunksKeepsPreviousSplitsAndPacksNewAssetsAtTail(t *testing.T) { + oldB := testAssetResource("core/old_b.tga", "old_b", "b", time.Date(2002, 1, 1, 0, 0, 0, 0, time.UTC)) + oldC := testAssetResource("core/old_c.tga", "old_c", "c", time.Date(2003, 1, 1, 0, 0, 0, 0, time.UTC)) + newA := testAssetResource("core/new_a.tga", "new_a", "a", time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC)) + + twoAssetSize := erf.ArchiveSize([]erf.Resource{oldB.Resource, oldC.Resource}) + p := &project.Project{ + Config: project.Config{ + HAKs: []project.HAKConfig{ + { + Name: "core", + Priority: 1, + MaxBytes: twoAssetSize, + Split: true, + Include: []string{"core/**"}, + }, + }, + }, + } + previous := &BuildManifest{ + HAKs: []BuildManifestHAK{ + {Name: "core_01", Group: "core", Priority: 1, MaxBytes: twoAssetSize, Assets: []string{"core/old_b.tga"}}, + {Name: "core_02", Group: "core", Priority: 1, MaxBytes: twoAssetSize, Assets: []string{"core/old_c.tga"}}, + }, + } + + chunks, err := planHAKChunksWithPrevious(p, []assetResource{oldB, oldC, newA}, previous) + if err != nil { + t.Fatalf("plan hak chunks: %v", err) + } + if len(chunks) != 2 { + t.Fatalf("expected 2 chunks, got %d", len(chunks)) + } + if got, want := strings.Join(chunkAssetRels(chunks[0]), ","), "core/old_b.tga"; got != want { + t.Fatalf("unexpected first chunk assets: got %q want %q", got, want) + } + if got, want := strings.Join(chunkAssetRels(chunks[1]), ","), "core/old_c.tga,core/new_a.tga"; got != want { + t.Fatalf("unexpected tail chunk assets: got %q want %q", got, want) + } +} + +func testAssetResource(rel, name, payload string, createdAt time.Time) assetResource { + resource := erf.Resource{ + Name: strings.ToLower(name), + Type: 0x0003, + Data: []byte(strings.Repeat(payload, 80)), + } + return assetResource{ + Rel: rel, + Resource: resource, + Size: erf.ArchiveSize([]erf.Resource{resource}), + CreatedAt: createdAt, + ContentID: "test:" + name, + } +} + +func chunkAssetRels(chunk hakChunk) []string { + rels := make([]string, 0, len(chunk.Assets)) + for _, asset := range chunk.Assets { + rels = append(rels, asset.Rel) + } + return rels +} + +func TestCollectLFSAssetInfoUsesGitCommonDirForWorktree(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + + parent := t.TempDir() + repoRoot := filepath.Join(parent, "repo") + worktreeRoot := filepath.Join(parent, "worktree") + mustMkdir(t, repoRoot) + runGitTest(t, repoRoot, "init") + mustWriteFile(t, filepath.Join(repoRoot, "README.md"), "test\n") + runGitTest(t, repoRoot, "add", ".") + runGitTest(t, repoRoot, "config", "user.name", "Test User") + runGitTest(t, repoRoot, "config", "user.email", "test@example.com") + runGitTest(t, repoRoot, "commit", "-m", "initial") + runGitTest(t, repoRoot, "worktree", "add", worktreeRoot, "HEAD") + + oid := strings.Repeat("a", 64) + assetRel := filepath.Join("core", "foo.mdl") + assetPath := filepath.Join(worktreeRoot, "assets", assetRel) + mustMkdir(t, filepath.Dir(assetPath)) + mustWriteFile(t, assetPath, "version https://git-lfs.github.com/spec/v1\n"+"oid sha256:"+oid+"\n"+"size 5\n") + + lfsObjectPath := filepath.Join(repoRoot, ".git", "lfs", "objects", oid[:2], oid[2:4], oid) + mustMkdir(t, filepath.Dir(lfsObjectPath)) + mustWriteFile(t, lfsObjectPath, "model") + + proj := &project.Project{ + Root: worktreeRoot, + Config: project.Config{ + Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"}, + }, + Inventory: project.Inventory{ + AssetFiles: []string{filepath.ToSlash(assetRel)}, + }, + } + + infos, err := collectLFSAssetInfo(proj) + if err != nil { + t.Fatalf("collect lfs asset info: %v", err) + } + info, ok := infos[filepath.ToSlash(assetRel)] + if !ok { + t.Fatal("expected LFS asset info") + } + if got, want := info.ObjectPath, lfsObjectPath; got != want { + t.Fatalf("unexpected LFS object path:\ngot %s\nwant %s", got, want) + } + if _, err := lfsPathResource(assetPath, info, true); err != nil { + t.Fatalf("expected worktree LFS object to resolve: %v", err) + } +} + +func runGitCommitTest(t *testing.T, dir, timestamp, message string) { + t.Helper() + runGitTest(t, dir, "add", ".") + cmd := exec.Command("git", "commit", "-m", message) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_DATE="+timestamp, + "GIT_COMMITTER_DATE="+timestamp, + ) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git commit failed: %v\n%s", err, string(output)) + } +} + +func runGitTest(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + if dir != "" { + cmd.Dir = dir + } + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v failed: %v\n%s", args, err, string(output)) + } +} + +func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "src", "areas")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Original Module" + } + ] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Changed In Toolset" + } + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "areas", "area001.are.json"), `{"stale": true}`) + + if err := p.Scan(); err != nil { + t.Fatalf("rescan before extract: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if result.Overwritten != 1 { + t.Fatalf("expected 1 overwritten file, got %d", result.Overwritten) + } + if result.Removed != 1 { + t.Fatalf("expected 1 removed stale file, got %d", result.Removed) + } + + raw, err := os.ReadFile(filepath.Join(root, "src", "module", "module.ifo.json")) + if err != nil { + t.Fatalf("read extracted module file: %v", err) + } + if !strings.Contains(string(raw), "Original Module") { + t.Fatalf("expected extracted module file to be overwritten with archive contents:\n%s", string(raw)) + } + if _, err := os.Stat(filepath.Join(root, "src", "areas", "area001.are.json")); !os.IsNotExist(err) { + t.Fatalf("expected stale area file to be removed, stat err=%v", err) + } +} + +func TestExtractMergesConfiguredGFFJSONFieldsAndLists(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "src", "areas")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +extract: + merge: + gff_json: + - target: module/module.ifo.json + preserve_fields: + - Mod_Entry_Area + - Mod_Entry_X + merge_lists: + - field: Mod_Area_list + key_field: Area_Name + strategy: preserve_existing_order_append_new +`) + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_Entry_Area", + "type": "ResRef", + "value": "area_a" + }, + { + "label": "Mod_Entry_X", + "type": "Float", + "value": 1 + }, + { + "label": "Mod_Area_list", + "type": "List", + "value": [ + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "area_a" + } + ] + }, + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "area_b" + } + ] + }, + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "area_c" + } + ] + } + ] + } + ] + } +} +`) + for _, area := range []string{"area_a", "area_b", "area_c"} { + mustWriteFile(t, filepath.Join(root, "src", "areas", area+".are.json"), `{ + "file_type": "ARE ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [] + } +} +`) + } + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Entry_Area", + "type": "ResRef", + "value": "area_b" + }, + { + "label": "Mod_Entry_X", + "type": "Float", + "value": 99 + }, + { + "label": "Mod_Area_list", + "type": "List", + "value": [ + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "area_b" + } + ] + }, + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "area_a" + } + ] + }, + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "removed_area" + } + ] + } + ] + } + ] + } +} +`) + + if err := p.Scan(); err != nil { + t.Fatalf("rescan before extract: %v", err) + } + if _, err := Extract(p); err != nil { + t.Fatalf("extract: %v", err) + } + + document := readGFFJSON(t, filepath.Join(root, "src", "module", "module.ifo.json")) + if got, want := fieldValue(t, document.Root, "Mod_Entry_Area"), gff.ResRefValue("area_b"); got != want { + t.Fatalf("expected preserved Mod_Entry_Area %#v, got %#v", want, got) + } + if got, want := fieldValue(t, document.Root, "Mod_Entry_X"), gff.FloatValue(99); got != want { + t.Fatalf("expected preserved Mod_Entry_X %#v, got %#v", want, got) + } + if got, want := keyedListValues(t, document.Root, "Mod_Area_list", "Area_Name"), []string{"area_b", "area_a", "area_c"}; !slices.Equal(got, want) { + t.Fatalf("expected merged area order %#v, got %#v", want, got) + } +} + +func TestExtractNormalizesResourceNamesToLowercase(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "blueprints", "items")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + + mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json"), `{ + "file_type": "UTI ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "LocalizedName", + "type": "CExoString", + "value": "Elven Chain" + } + ] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + if err := os.Remove(filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json")); err != nil { + t.Fatalf("remove uppercase source file: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("rescan before extract: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if result.Written != 1 { + t.Fatalf("expected 1 written file, got %d", result.Written) + } + if _, err := os.Stat(filepath.Join(root, "src", "blueprints", "items", "i_elvenchain.uti.json")); err != nil { + t.Fatalf("expected lowercase extracted resource: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json")); !os.IsNotExist(err) { + t.Fatalf("expected uppercase extracted resource path to be absent, stat err=%v", err) + } +} + +func TestCompareFailsWhenBuildIsStale(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Original Module" + } + ] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + modulePath := filepath.Join(root, "build", "testmod.mod") + moduleInfo, err := os.Stat(modulePath) + if err != nil { + t.Fatalf("stat built module: %v", err) + } + staleSourceTime := moduleInfo.ModTime().Add(2 * time.Second) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Updated Module" + } + ] + } +} +`) + if err := os.Chtimes(filepath.Join(root, "src", "module", "module.ifo.json"), staleSourceTime, staleSourceTime); err != nil { + t.Fatalf("chtimes updated source: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("rescan: %v", err) + } + + _, err = Compare(p) + if err == nil { + t.Fatal("expected stale compare error") + } + if !strings.Contains(err.Error(), "built archives are older than source file") { + t.Fatalf("unexpected compare error: %v", err) + } +} + +func mustMkdir(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } +} + +func mustWriteFile(t *testing.T, path, data string) { + t.Helper() + if err := os.WriteFile(path, []byte(data), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func readGFFJSON(t *testing.T, path string) gff.Document { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + var document gff.Document + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + return document +} + +func fieldValue(t *testing.T, s gff.Struct, label string) gff.Value { + t.Helper() + for _, field := range s.Fields { + if field.Label == label { + return field.Value + } + } + t.Fatalf("field %s not found", label) + return nil +} + +func keyedListValues(t *testing.T, s gff.Struct, listLabel, keyLabel string) []string { + t.Helper() + list, ok := fieldValue(t, s, listLabel).(gff.ListValue) + if !ok { + t.Fatalf("field %s is not a list", listLabel) + } + values := make([]string, 0, len(list)) + for _, item := range list { + value := fieldValue(t, item, keyLabel) + switch typed := value.(type) { + case gff.ResRefValue: + values = append(values, string(typed)) + case gff.StringValue: + values = append(values, string(typed)) + default: + t.Fatalf("field %s has unsupported key type %T", keyLabel, value) + } + } + return values +} + +func minimalGFF(t *testing.T, fileType string) []byte { + t.Helper() + document := gff.Document{ + FileType: fileType + " ", + FileVersion: "V3.2", + Root: gff.Struct{Type: 0}, + } + var buf bytes.Buffer + if err := gff.Write(&buf, document); err != nil { + t.Fatalf("write %s gff: %v", fileType, err) + } + return buf.Bytes() +} + +func TestBuildHAKsFailsForDuplicateResourcesInSameHAK(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "core", "a")) + mustMkdir(t, filepath.Join(root, "assets", "core", "b")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 0, + "split": false, + "include": ["core/**"] + } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "a", "shared.mdl"), "one") + mustWriteFile(t, filepath.Join(root, "assets", "core", "b", "shared.mdl"), "two") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + _, err = BuildHAKs(p) + if err == nil { + t.Fatal("expected duplicate same-hak build error") + } + if !strings.Contains(err.Error(), "validation failed with 1 error(s)") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestBuildHAKsConvertsMusicSourcesAndWritesCreditsArtifacts(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "assets", "envi", "music", "westgate")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Assets", + "resref": "testassets" + }, + "paths": { + "assets": "assets", + "build": "build" + }, + "music": { + "prefixes": { + "envi/music/westgate": "mus_wg_" + } + }, + "haks": [ + { + "name": "envi", + "priority": 1, + "max_bytes": 1048576, + "split": false, + "include": ["envi/**"] + } + ] +} +`) + + mustWriteFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "AleandAnecdotes.mp3"), "source-mp3") + mustWriteFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "CREDITS.md"), "# NWN Music Pack Credits\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Darren Curtis | Ale and Anecdotes | `mus_wg_andnc.bmu` | `AleandAnecdotes.mp3` | Darren's Commercially Free Music | | Darren Curtis Music | |\n") + + ffprobePath := filepath.Join(root, "ffprobe") + mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Unknown\",\"title\":\"Broken Metadata\"}}}'\n") + if err := os.Chmod(ffprobePath, 0o755); err != nil { + t.Fatalf("chmod ffprobe: %v", err) + } + + ffmpegPath := filepath.Join(root, "ffmpeg") + mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'bmu-data' > \"$last\"\n") + if err := os.Chmod(ffmpegPath, 0o755); err != nil { + t.Fatalf("chmod ffmpeg: %v", err) + } + + t.Setenv("SOW_FFMPEG", ffmpegPath) + t.Setenv("SOW_FFPROBE", ffprobePath) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if len(result.CreditsArtifactPaths) == 0 { + t.Fatal("expected credits artifacts to be written") + } + + manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + if !strings.Contains(string(manifestRaw), "envi/music/westgate/mus_wg_andnc.bmu") { + t.Fatalf("expected converted bmu in manifest, got %s", string(manifestRaw)) + } + if strings.Contains(string(manifestRaw), "AleandAnecdotes.mp3") { + t.Fatalf("did not expect source mp3 in manifest, got %s", string(manifestRaw)) + } + + creditsRaw, err := os.ReadFile(filepath.Join(root, ".cache", "credits", "envi", "music", "westgate", "CREDITS.md")) + if err != nil { + t.Fatalf("read generated credits: %v", err) + } + creditsText := string(creditsRaw) + if !strings.Contains(creditsText, "Darren Curtis") || !strings.Contains(creditsText, "mus_wg_andnc.bmu") { + t.Fatalf("unexpected generated credits contents: %s", creditsText) + } + if strings.Contains(creditsText, "Broken Metadata") { + t.Fatalf("manual credits overlay should override bad metadata: %s", creditsText) + } + + inventoryRaw, err := os.ReadFile(filepath.Join(root, ".cache", "credits", "credits.json")) + if err != nil { + t.Fatalf("read credits inventory: %v", err) + } + inventoryText := string(inventoryRaw) + if !strings.Contains(inventoryText, "assets/envi/music/westgate/CREDITS.md") { + t.Fatalf("expected authored credits source in inventory: %s", inventoryText) + } + if !strings.Contains(inventoryText, ".cache/credits/envi/music/westgate/CREDITS.md") { + t.Fatalf("expected generated credits source in inventory: %s", inventoryText) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "music")); !os.IsNotExist(err) { + t.Fatalf("expected temporary music staging to be cleaned, got err=%v", err) + } + + secondResult, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks second pass: %v", err) + } + if secondResult.CreditsSummary.ChangedFiles != 0 { + t.Fatalf("expected second credits refresh to be unchanged, got %d changed files", secondResult.CreditsSummary.ChangedFiles) + } + if secondResult.CreditsSummary.TrackCount != 1 { + t.Fatalf("expected one tracked music conversion, got %d", secondResult.CreditsSummary.TrackCount) + } +} + +func TestBuildHAKsUsesExplicitMusicDatasetOutsideLegacyRoot(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate")) + mustMkdir(t, filepath.Join(root, "build")) + + ffprobePath := filepath.Join(root, "tools", "ffprobe") + ffmpegPath := filepath.Join(root, "tools", "ffmpeg") + mustMkdir(t, filepath.Dir(ffprobePath)) + mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Dataset Artist\",\"title\":\"Dataset Title\"}}}'\n") + mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'dataset-bmu' > \"$last\"\n") + if err := os.Chmod(ffprobePath, 0o755); err != nil { + t.Fatalf("chmod ffprobe: %v", err) + } + if err := os.Chmod(ffmpegPath, 0o755); err != nil { + t.Fatalf("chmod ffmpeg: %v", err) + } + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Assets + resref: testassets +paths: + assets: assets + build: build +music: + tools: + ffmpeg: tools/ffmpeg + ffprobe: tools/ffprobe + defaults: + credits_root: custom-credits + datasets: + westgate_audio: + source: audio/westgate + output: generated/music + prefix: wg_ +haks: + - name: music + priority: 1 + max_bytes: 1048576 + split: false + include: + - generated/music/** +`) + mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if result.CreditsSummary.TrackCount != 1 { + t.Fatalf("expected one music track, got %#v", result.CreditsSummary) + } + manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + manifestText := string(manifestRaw) + if !strings.Contains(manifestText, "generated/music/wg_theme.bmu") { + t.Fatalf("expected dataset output resource in manifest, got:\n%s", manifestText) + } + if strings.Contains(manifestText, "audio/westgate/Theme Song.mp3") { + t.Fatalf("did not expect source mp3 in manifest, got:\n%s", manifestText) + } + creditsRaw, err := os.ReadFile(filepath.Join(root, "custom-credits", "audio", "westgate", "CREDITS.md")) + if err != nil { + t.Fatalf("read generated dataset credits: %v", err) + } + if !strings.Contains(string(creditsRaw), "Dataset Artist") || !strings.Contains(string(creditsRaw), "wg_theme.bmu") { + t.Fatalf("unexpected generated credits:\n%s", string(creditsRaw)) + } +} + +func TestBuildHAKsConvertsFLACMusicSourcesByDefault(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate")) + mustMkdir(t, filepath.Join(root, "build")) + + ffprobePath := filepath.Join(root, "tools", "ffprobe") + ffmpegPath := filepath.Join(root, "tools", "ffmpeg") + mustMkdir(t, filepath.Dir(ffprobePath)) + mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"FLAC Artist\",\"title\":\"FLAC Title\"}}}'\n") + mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'dataset-bmu' > \"$last\"\n") + if err := os.Chmod(ffprobePath, 0o755); err != nil { + t.Fatalf("chmod ffprobe: %v", err) + } + if err := os.Chmod(ffmpegPath, 0o755); err != nil { + t.Fatalf("chmod ffmpeg: %v", err) + } + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Assets + resref: testassets +paths: + assets: assets + build: build +music: + tools: + ffmpeg: tools/ffmpeg + ffprobe: tools/ffprobe + datasets: + westgate_audio: + source: audio/westgate + output: generated/music + prefix: wg_ +haks: + - name: music + priority: 1 + max_bytes: 1048576 + split: false + include: + - generated/music/** +`) + mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.flac"), "source-flac") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if result.CreditsSummary.TrackCount != 1 { + t.Fatalf("expected one music track, got %#v", result.CreditsSummary) + } + manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + manifestText := string(manifestRaw) + if !strings.Contains(manifestText, "generated/music/wg_theme.bmu") { + t.Fatalf("expected generated FLAC music in manifest, got:\n%s", manifestText) + } + if strings.Contains(manifestText, "audio/westgate/Theme Song.flac") { + t.Fatalf("did not expect source flac in manifest, got:\n%s", manifestText) + } +} + +func TestBuildHAKsUsesYAMLMusicConvertExtensionsForDiscovery(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate")) + mustMkdir(t, filepath.Join(root, "build")) + + ffprobePath := filepath.Join(root, "tools", "ffprobe") + ffmpegPath := filepath.Join(root, "tools", "ffmpeg") + mustMkdir(t, filepath.Dir(ffprobePath)) + mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"AAC Artist\",\"title\":\"AAC Title\"}}}'\n") + mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'dataset-bmu' > \"$last\"\n") + if err := os.Chmod(ffprobePath, 0o755); err != nil { + t.Fatalf("chmod ffprobe: %v", err) + } + if err := os.Chmod(ffmpegPath, 0o755); err != nil { + t.Fatalf("chmod ffmpeg: %v", err) + } + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Assets + resref: testassets +paths: + assets: assets + build: build +music: + defaults: + convert_extensions: + - .aac + tools: + ffmpeg: tools/ffmpeg + ffprobe: tools/ffprobe + datasets: + westgate_audio: + source: audio/westgate + output: generated/music + prefix: wg_ +haks: + - name: music + priority: 1 + max_bytes: 1048576 + split: false + include: + - generated/music/** +`) + mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.aac"), "source-aac") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if result.CreditsSummary.TrackCount != 1 { + t.Fatalf("expected one music track discovered through YAML convert_extensions, got %#v", result.CreditsSummary) + } +} + +func TestPlanHAKsDoesNotTranscodeMusicSources(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate")) + mustMkdir(t, filepath.Join(root, "build")) + + ffmpegPath := filepath.Join(root, "tools", "ffmpeg") + mustMkdir(t, filepath.Dir(ffmpegPath)) + mustWriteFile(t, ffmpegPath, "#!/bin/sh\necho should-not-run >&2\nexit 42\n") + if err := os.Chmod(ffmpegPath, 0o755); err != nil { + t.Fatalf("chmod ffmpeg: %v", err) + } + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Assets + resref: testassets +paths: + assets: assets + build: build +music: + tools: + ffmpeg: tools/ffmpeg + ffprobe: tools/ffprobe + datasets: + westgate_audio: + source: audio/westgate + output: generated/music + prefix: wg_ +haks: + - name: music + priority: 1 + max_bytes: 1048576 + split: false + include: + - generated/music/** +`) + mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := PlanHAKs(p) + if err != nil { + t.Fatalf("plan haks: %v", err) + } + if result.CreditsSummary.TrackCount != 1 { + t.Fatalf("expected one planned music track, got %#v", result.CreditsSummary) + } + manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + if !strings.Contains(string(manifestRaw), "generated/music/wg_theme.bmu") { + t.Fatalf("expected planned generated music in manifest, got:\n%s", string(manifestRaw)) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "music")); !os.IsNotExist(err) { + t.Fatalf("plan-only should not stage music, stat err=%v", err) + } +} diff --git a/internal/project/effective.go b/internal/project/effective.go new file mode 100644 index 0000000..f6b6d99 --- /dev/null +++ b/internal/project/effective.go @@ -0,0 +1,738 @@ +package project + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music" +) + +const ( + DefaultBuildPath = "build" + DefaultCachePath = ".cache" + DefaultToolsPath = "tools" + DefaultModuleArchiveTemplate = "{module.resref}.mod" + DefaultHAKManifest = "haks.json" + DefaultHAKArchiveTemplate = "{hak.name}.hak" + DefaultSourceJSONPattern = "{resref}.{extension}.json" + DefaultValidationProfile = "nwn_module" + DefaultMusicStageRoot = "{paths.cache}/music" + DefaultCreditsRoot = "{paths.cache}/credits" + DefaultCreditsOverlayFile = "CREDITS.md" + DefaultMusicNamingScheme = "nwn_bmu" + DefaultMusicOutputExtension = ".bmu" + DefaultTopDataBuild = "{paths.build}/topdata" + DefaultTopDataCompiled2DADir = "2da" + DefaultTopDataCompiledTLK = "sow_tlk.tlk" + DefaultTopDataWikiOutputRoot = "wiki" + DefaultTopDataWikiPagesDir = "pages" + DefaultTopDataWikiStateFile = "state.json" + DefaultTopDataWikiSource = "topdata/wiki" + DefaultTopDataWikiRenderer = "nodebb_tiptap_html" + DefaultTopDataWikiLinkStrategy = "preserve_westgate_wiki_links" + DefaultTopDataWikiNamespacesFile = "namespaces.yaml" + DefaultTopDataWikiTablesFile = "tables.yaml" + DefaultTopDataWikiVisibilityFile = "visibility.yaml" + DefaultTopDataWikiDataFile = "data.yaml" + DefaultTopDataWikiPagePathsFile = "wiki.yaml" + DefaultTopDataWikiTemplatesDir = "templates" + DefaultTopDataWikiPageTemplatesDir = "templates/pages" + DefaultTopDataWikiManualSectionsDir = "manual-sections" + DefaultTopDataWikiManualSectionsFile = "manual-sections/default.yaml" + DefaultTopDataWikiPageIndexFile = "page-index.json" + DefaultTopDataWikiStaleDefault = "report" + DefaultTopDataWikiStaleLiveCleanup = "archive" + DefaultTopDataWikiMarkerFormat = "html_comments" + DefaultTopDataWikiPageMarkerPrefix = "sow-topdata-wiki" + DefaultTopDataWikiTitlePrefixMinLength = 3 + DefaultTopDataWikiStatusPages = true + DefaultTopDataWikiStatusListingScope = "all" + DefaultTopDataWikiAlignmentLinkFormat = "wiki_markup" + DefaultWikiDeployManifest = ".wiki_deploy_manifest.json" + DefaultWikiDeployEditSummary = "Auto-generated from native builder" + DefaultTopDataPackageHAK = "sow_top.hak" + DefaultTopDataPackageTLK = "sow_tlk.tlk" + DefaultAutogenCacheRoot = "{paths.cache}" + DefaultAutogenCacheMaxAge = time.Hour + DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH" + DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH" + DefaultAutogenReleaseProvider = "gitea" + DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL" + DefaultAutogenRepoEnv = "SOW_ASSETS_REPO" + DefaultExtractLayout = "nwn_canonical_json" + DefaultExtractHAKDiscovery = "build_glob" +) + +type ConfigProvenance map[string]ConfigValueProvenance + +type ConfigValueProvenance struct { + Source string `json:"source" yaml:"source"` + Detail string `json:"detail,omitempty" yaml:"detail,omitempty"` + Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"` +} + +type EffectiveConfig struct { + ConfigSource ConfigSource `json:"config_source" yaml:"config_source"` + Module ModuleConfig `json:"module" yaml:"module"` + Paths EffectivePathConfig `json:"paths" yaml:"paths"` + Outputs EffectiveOutputConfig `json:"outputs" yaml:"outputs"` + Build BuildConfig `json:"build" yaml:"build"` + Generated GeneratedConfig `json:"generated_assets" yaml:"generated_assets"` + Inventory InventoryConfig `json:"inventory" yaml:"inventory"` + Validation ValidationConfig `json:"validation" yaml:"validation"` + Music EffectiveMusicConfig `json:"music" yaml:"music"` + TopData EffectiveTopDataConfig `json:"topdata" yaml:"topdata"` + Extract ExtractConfig `json:"extract" yaml:"extract"` + Autogen EffectiveAutogenConfig `json:"autogen" yaml:"autogen"` + HAKs []HAKConfig `json:"haks" yaml:"haks"` + Provenance ConfigProvenance `json:"provenance" yaml:"provenance"` + Overrides []ConfigOverride `json:"overrides,omitempty" yaml:"overrides,omitempty"` +} + +type EffectivePathConfig struct { + Source string `json:"source" yaml:"source"` + Assets string `json:"assets" yaml:"assets"` + Build string `json:"build" yaml:"build"` + Cache string `json:"cache" yaml:"cache"` + Tools string `json:"tools" yaml:"tools"` +} + +type EffectiveOutputConfig struct { + ModuleArchive string `json:"module_archive" yaml:"module_archive"` + HAKManifest string `json:"hak_manifest" yaml:"hak_manifest"` + HAKArchive string `json:"hak_archive" yaml:"hak_archive"` +} + +type EffectiveMusicConfig struct { + Prefixes map[string]string `json:"prefixes" yaml:"prefixes"` + Tools MusicToolsConfig `json:"tools" yaml:"tools"` + StageRoot string `json:"stage_root" yaml:"stage_root"` + CreditsRoot string `json:"credits_root" yaml:"credits_root"` + CreditsOverlay string `json:"credits_overlay" yaml:"credits_overlay"` + MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"` + NamingScheme string `json:"naming_scheme" yaml:"naming_scheme"` + OutputExtension string `json:"output_extension" yaml:"output_extension"` + ConvertExtensions []string `json:"convert_extensions" yaml:"convert_extensions"` + Datasets map[string]EffectiveMusicDataset `json:"datasets,omitempty" yaml:"datasets,omitempty"` +} + +type EffectiveMusicDataset struct { + Source string `json:"source" yaml:"source"` + Output string `json:"output" yaml:"output"` + Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"` + StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` + CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` + NamingScheme string `json:"naming_scheme" yaml:"naming_scheme"` + OutputExtension string `json:"output_extension" yaml:"output_extension"` + MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"` + PackageMode string `json:"package_mode" yaml:"package_mode"` + ConvertExtensions []string `json:"convert_extensions" yaml:"convert_extensions"` +} + +type EffectiveTopDataConfig struct { + Source string `json:"source" yaml:"source"` + Build string `json:"build" yaml:"build"` + ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"` + Assets string `json:"assets" yaml:"assets"` + Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"` + CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"` + PackageHAK string `json:"package_hak" yaml:"package_hak"` + PackageTLK string `json:"package_tlk" yaml:"package_tlk"` + ValueEncodings []TopDataValueEncodingConfig `json:"value_encodings" yaml:"value_encodings"` + ValueDefaults []TopDataValueDefaultConfig `json:"value_defaults" yaml:"value_defaults"` + RowGeneration []TopDataRowGenerationConfig `json:"row_generation" yaml:"row_generation"` + RowExtensions []TopDataRowExtensionConfig `json:"row_extensions" yaml:"row_extensions"` + ClassFeatInjections TopDataClassFeatInjectionConfig `json:"class_feat_injections" yaml:"class_feat_injections"` + Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"` +} + +type EffectiveAutogenConfig struct { + Producers []AutogenProducerConfig `json:"producers" yaml:"producers"` + Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"` + Cache EffectiveAutogenCacheConfig `json:"cache" yaml:"cache"` + ReleaseSource EffectiveAutogenReleaseSourceConfig `json:"release_source" yaml:"release_source"` +} + +type EffectiveAutogenCacheConfig struct { + Root string `json:"root" yaml:"root"` + MaxAge string `json:"max_age" yaml:"max_age"` + RefreshEnv string `json:"refresh_env" yaml:"refresh_env"` + PartsManifestRefreshEnv string `json:"parts_manifest_refresh_env" yaml:"parts_manifest_refresh_env"` +} + +type EffectiveAutogenReleaseSourceConfig struct { + Provider string `json:"provider" yaml:"provider"` + ServerURLEnv string `json:"server_url_env" yaml:"server_url_env"` + RepoEnv string `json:"repo_env" yaml:"repo_env"` +} + +type ConfigOverride struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + Source string `json:"source" yaml:"source"` +} + +func (p *Project) EffectiveConfig() EffectiveConfig { + provenance := cloneProvenance(p.Provenance) + markMissingDefaults(provenance) + + paths := EffectivePathConfig{ + Source: strings.TrimSpace(p.Config.Paths.Source), + Assets: strings.TrimSpace(p.Config.Paths.Assets), + Build: defaultString(p.Config.Paths.Build, DefaultBuildPath), + Cache: defaultString(p.Config.Paths.Cache, DefaultCachePath), + Tools: defaultString(p.Config.Paths.Tools, DefaultToolsPath), + } + outputs := EffectiveOutputConfig{ + ModuleArchive: defaultString(p.Config.Outputs.ModuleArchive, DefaultModuleArchiveTemplate), + HAKManifest: defaultString(p.Config.Outputs.HAKManifest, DefaultHAKManifest), + HAKArchive: defaultString(p.Config.Outputs.HAKArchive, DefaultHAKArchiveTemplate), + } + inventory := InventoryConfig{ + SourceExtensions: defaultStringSlice(p.Config.Inventory.SourceExtensions, SourceExtensions), + AssetExtensions: defaultStringSlice(p.Config.Inventory.AssetExtensions, AssetExtensions), + SourceJSONPattern: defaultString(p.Config.Inventory.SourceJSONPattern, DefaultSourceJSONPattern), + } + validation := ValidationConfig{ + Profile: defaultString(p.Config.Validation.Profile, DefaultValidationProfile), + BuiltinScriptPrefixes: normalizeLowerStringSlice(defaultStringSlice(p.Config.Validation.BuiltinScriptPrefixes, DefaultBuiltinScriptPrefixes())), + RequiredFields: defaultRequiredFields(p.Config.Validation.RequiredFields), + } + topBuild := expandPathTemplate(defaultString(p.Config.TopData.Build, DefaultTopDataBuild), paths) + top := EffectiveTopDataConfig{ + Source: strings.TrimSpace(p.Config.TopData.Source), + Build: topBuild, + ReferenceBuilder: strings.TrimSpace(p.Config.TopData.ReferenceBuilder), + Assets: strings.TrimSpace(p.Config.TopData.Assets), + Compiled2DADir: defaultString(p.Config.TopData.Compiled2DADir, DefaultTopDataCompiled2DADir), + CompiledTLK: defaultString(p.Config.TopData.CompiledTLK, DefaultTopDataCompiledTLK), + PackageHAK: defaultString(p.Config.TopData.PackageHAK, DefaultTopDataPackageHAK), + PackageTLK: defaultString(p.Config.TopData.PackageTLK, DefaultTopDataPackageTLK), + ValueEncodings: cloneTopDataValueEncodings(p.Config.TopData.ValueEncodings), + ValueDefaults: cloneTopDataValueDefaults(p.Config.TopData.ValueDefaults), + RowGeneration: cloneTopDataRowGeneration(p.Config.TopData.RowGeneration), + RowExtensions: cloneTopDataRowExtensions(p.Config.TopData.RowExtensions), + ClassFeatInjections: cloneTopDataClassFeatInjections(p.Config.TopData.ClassFeatInjections), + Wiki: TopDataWikiConfig{ + OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot), + PagesDir: defaultString(p.Config.TopData.Wiki.PagesDir, DefaultTopDataWikiPagesDir), + StateFile: defaultString(p.Config.TopData.Wiki.StateFile, DefaultTopDataWikiStateFile), + ManagedNamespaces: defaultStringSlice(p.Config.TopData.Wiki.ManagedNamespaces, []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}), + DeployManifest: defaultString(p.Config.TopData.Wiki.DeployManifest, DefaultWikiDeployManifest), + DeployEditSummary: defaultString(p.Config.TopData.Wiki.DeployEditSummary, DefaultWikiDeployEditSummary), + Source: defaultString(p.Config.TopData.Wiki.Source, DefaultTopDataWikiSource), + Renderer: defaultString(p.Config.TopData.Wiki.Renderer, DefaultTopDataWikiRenderer), + LinkStrategy: defaultString(p.Config.TopData.Wiki.LinkStrategy, DefaultTopDataWikiLinkStrategy), + NamespacesFile: defaultString(p.Config.TopData.Wiki.NamespacesFile, DefaultTopDataWikiNamespacesFile), + TablesFile: defaultString(p.Config.TopData.Wiki.TablesFile, DefaultTopDataWikiTablesFile), + VisibilityFile: defaultString(p.Config.TopData.Wiki.VisibilityFile, DefaultTopDataWikiVisibilityFile), + DataFile: defaultString(p.Config.TopData.Wiki.DataFile, DefaultTopDataWikiDataFile), + PagePathsFile: defaultString(p.Config.TopData.Wiki.PagePathsFile, DefaultTopDataWikiPagePathsFile), + TemplatesDir: defaultString(p.Config.TopData.Wiki.TemplatesDir, DefaultTopDataWikiTemplatesDir), + PageTemplatesDir: defaultWikiPageTemplatesDir(p.Config.TopData.Wiki), + ManualSectionsDir: defaultString(p.Config.TopData.Wiki.ManualSectionsDir, DefaultTopDataWikiManualSectionsDir), + ManualSectionsFile: defaultWikiManualSectionsFile(p.Config.TopData.Wiki), + PageIndexFile: defaultString(p.Config.TopData.Wiki.PageIndexFile, DefaultTopDataWikiPageIndexFile), + AlignmentLinks: TopDataWikiAlignmentLinks{ + TargetPattern: strings.TrimSpace(p.Config.TopData.Wiki.AlignmentLinks.TargetPattern), + LinkFormat: defaultString(p.Config.TopData.Wiki.AlignmentLinks.LinkFormat, DefaultTopDataWikiAlignmentLinkFormat), + }, + TitlePrefixMinLength: defaultInt(p.Config.TopData.Wiki.TitlePrefixMinLength, DefaultTopDataWikiTitlePrefixMinLength), + StatusPages: defaultBoolPointer(p.Config.TopData.Wiki.StatusPages, DefaultTopDataWikiStatusPages), + StatusListingScope: defaultString(p.Config.TopData.Wiki.StatusListingScope, DefaultTopDataWikiStatusListingScope), + StalePages: TopDataWikiStalePagesConfig{ + Default: defaultString(p.Config.TopData.Wiki.StalePages.Default, DefaultTopDataWikiStaleDefault), + LiveCleanup: defaultString(p.Config.TopData.Wiki.StalePages.LiveCleanup, DefaultTopDataWikiStaleLiveCleanup), + }, + ManagedRegion: TopDataWikiManagedRegionConfig{ + MarkerFormat: defaultString(p.Config.TopData.Wiki.ManagedRegion.MarkerFormat, DefaultTopDataWikiMarkerFormat), + PageMarkerPrefix: defaultString(p.Config.TopData.Wiki.ManagedRegion.PageMarkerPrefix, DefaultTopDataWikiPageMarkerPrefix), + }, + }, + } + generated := p.Config.Generated + for index := range generated.TopData2DA { + generated.TopData2DA[index].Output = expandPathTemplate(defaultString(generated.TopData2DA[index].Output, "{paths.cache}/generated-assets/"+generated.TopData2DA[index].ID), paths) + } + + extract := p.Config.Extract + extract.Layout = defaultString(extract.Layout, DefaultExtractLayout) + extract.HAKDiscovery = defaultString(extract.HAKDiscovery, DefaultExtractHAKDiscovery) + if len(extract.Archives) == 0 { + extract.Archives = []string{outputs.ModuleArchiveName(p.Config.Module)} + } + if extract.CleanupStale == nil { + defaultCleanup := true + extract.CleanupStale = &defaultCleanup + } + if extract.ConsumeArchives == nil { + defaultConsume := false + extract.ConsumeArchives = &defaultConsume + } + + musicStageRoot := expandPathTemplate(DefaultMusicStageRoot, paths) + musicCreditsRoot := expandPathTemplate(DefaultCreditsRoot, paths) + musicCreditsOverlay := DefaultCreditsOverlayFile + musicMaxStemLength := 16 + musicNamingScheme := DefaultMusicNamingScheme + musicOutputExtension := DefaultMusicOutputExtension + musicConvertExtensions := music.DefaultConvertExtensions() + if d := p.Config.Music.Defaults; d != nil { + if d.StageRoot != "" { + musicStageRoot = expandPathTemplate(d.StageRoot, paths) + } + if d.CreditsRoot != "" { + musicCreditsRoot = expandPathTemplate(d.CreditsRoot, paths) + } + if d.CreditsOverlay != "" { + musicCreditsOverlay = d.CreditsOverlay + } + if d.MaxStemLength != nil { + musicMaxStemLength = *d.MaxStemLength + } + if d.NamingScheme != "" { + musicNamingScheme = d.NamingScheme + } + if d.OutputExtension != "" { + musicOutputExtension = normalizeExtension(d.OutputExtension) + } + if len(d.ConvertExtensions) > 0 { + musicConvertExtensions = normalizeExtensionSlice(d.ConvertExtensions) + } + } + musicPrefixes := cloneStringMap(p.Config.Music.Prefixes) + if len(musicPrefixes) == 0 { + musicPrefixes = make(map[string]string, len(p.Config.Music.Datasets)) + for _, ds := range p.Config.Music.Datasets { + if ds.Source != "" && ds.Prefix != "" { + musicPrefixes[ds.Source] = ds.Prefix + } + } + } + effectiveDatasets := make(map[string]EffectiveMusicDataset, len(p.Config.Music.Datasets)) + for id, ds := range p.Config.Music.Datasets { + dsStageRoot := ds.StageRoot + if dsStageRoot == "" { + dsStageRoot = musicStageRoot + } + dsCreditsRoot := ds.CreditsRoot + if dsCreditsRoot == "" { + dsCreditsRoot = musicCreditsRoot + } + dsOutput := strings.TrimSpace(ds.Output) + if dsOutput == "" { + dsOutput = ds.Source + } + dsNamingScheme := defaultString(ds.NamingScheme, musicNamingScheme) + dsOutputExtension := musicOutputExtension + if ds.OutputExtension != "" { + dsOutputExtension = normalizeExtension(ds.OutputExtension) + } + dsConvertExtensions := musicConvertExtensions + if len(ds.ConvertExtensions) > 0 { + dsConvertExtensions = normalizeExtensionSlice(ds.ConvertExtensions) + } + effectiveDatasets[id] = EffectiveMusicDataset{ + Source: ds.Source, + Output: dsOutput, + Prefix: ds.Prefix, + StageRoot: dsStageRoot, + CreditsRoot: dsCreditsRoot, + NamingScheme: dsNamingScheme, + OutputExtension: dsOutputExtension, + MaxStemLength: musicMaxStemLength, + PackageMode: defaultString(ds.PackageMode, "hak_asset"), + ConvertExtensions: dsConvertExtensions, + } + } + effective := EffectiveConfig{ + ConfigSource: p.ConfigSource, + Module: p.Config.Module, + Paths: paths, + Outputs: outputs, + Build: p.Config.Build, + Generated: generated, + Inventory: inventory, + Validation: validation, + Music: EffectiveMusicConfig{ + Prefixes: musicPrefixes, + Tools: p.Config.Music.Tools, + StageRoot: musicStageRoot, + CreditsRoot: musicCreditsRoot, + CreditsOverlay: musicCreditsOverlay, + MaxStemLength: musicMaxStemLength, + NamingScheme: musicNamingScheme, + OutputExtension: musicOutputExtension, + ConvertExtensions: musicConvertExtensions, + Datasets: effectiveDatasets, + }, + TopData: top, + Extract: extract, + Autogen: EffectiveAutogenConfig{ + Producers: slices.Clone(p.Config.Autogen.Producers), + Consumers: slices.Clone(p.Config.Autogen.Consumers), + Cache: EffectiveAutogenCacheConfig{ + Root: expandPathTemplate(defaultString(p.Config.Autogen.Cache.Root, DefaultAutogenCacheRoot), paths), + MaxAge: defaultString(p.Config.Autogen.Cache.MaxAge, DefaultAutogenCacheMaxAge.String()), + RefreshEnv: defaultString(p.Config.Autogen.Cache.RefreshEnv, DefaultAutogenRefreshEnv), + PartsManifestRefreshEnv: DefaultPartsManifestRefreshEnv, + }, + ReleaseSource: EffectiveAutogenReleaseSourceConfig{ + Provider: defaultString(p.Config.Autogen.ReleaseSource.Provider, DefaultAutogenReleaseProvider), + ServerURLEnv: defaultString(p.Config.Autogen.ReleaseSource.ServerURLEnv, DefaultAutogenServerURLEnv), + RepoEnv: defaultString(p.Config.Autogen.ReleaseSource.RepoEnv, DefaultAutogenRepoEnv), + }, + }, + HAKs: slices.Clone(p.Config.HAKs), + Provenance: provenance, + } + effective.Overrides = activeOverrides(effective) + return effective +} + +func cloneTopDataValueEncodings(values []TopDataValueEncodingConfig) []TopDataValueEncodingConfig { + if len(values) == 0 { + return nil + } + out := make([]TopDataValueEncodingConfig, len(values)) + copy(out, values) + return out +} + +func cloneTopDataValueDefaults(values []TopDataValueDefaultConfig) []TopDataValueDefaultConfig { + if len(values) == 0 { + return nil + } + out := make([]TopDataValueDefaultConfig, len(values)) + copy(out, values) + return out +} + +func cloneTopDataRowGeneration(values []TopDataRowGenerationConfig) []TopDataRowGenerationConfig { + if len(values) == 0 { + return nil + } + out := make([]TopDataRowGenerationConfig, len(values)) + copy(out, values) + for index := range out { + if strings.TrimSpace(out[index].Dataset) == "" { + out[index].Dataset = strings.TrimSpace(out[index].Namespace) + } + if strings.TrimSpace(out[index].Namespace) == "" { + out[index].Namespace = strings.TrimSpace(out[index].Dataset) + } + if strings.TrimSpace(out[index].Mode) == "" { + out[index].Mode = "after_base" + } + out[index].Dataset = filepath.ToSlash(strings.TrimSpace(out[index].Dataset)) + out[index].Namespace = filepath.ToSlash(strings.TrimSpace(out[index].Namespace)) + out[index].Mode = strings.TrimSpace(out[index].Mode) + } + return out +} + +func cloneTopDataRowExtensions(values []TopDataRowExtensionConfig) []TopDataRowExtensionConfig { + if len(values) == 0 { + return nil + } + out := make([]TopDataRowExtensionConfig, len(values)) + copy(out, values) + for index := range out { + out[index].Dataset = filepath.ToSlash(strings.TrimSpace(out[index].Dataset)) + out[index].Mode = strings.TrimSpace(out[index].Mode) + out[index].LevelColumn = strings.TrimSpace(out[index].LevelColumn) + } + return out +} + +func cloneTopDataClassFeatInjections(value TopDataClassFeatInjectionConfig) TopDataClassFeatInjectionConfig { + out := TopDataClassFeatInjectionConfig{ + GlobalFeats: slices.Clone(value.GlobalFeats), + ClassSkillMasterfeats: slices.Clone(value.ClassSkillMasterfeats), + } + for index := range out.GlobalFeats { + out.GlobalFeats[index].RequirePresent = slices.Clone(value.GlobalFeats[index].RequirePresent) + out.GlobalFeats[index].UnlessPresent = slices.Clone(value.GlobalFeats[index].UnlessPresent) + } + return out +} + +func (p *Project) EffectiveConfigJSON() ([]byte, error) { + return json.MarshalIndent(p.EffectiveConfig(), "", " ") +} + +func (p *Project) ExplainConfig(key string) (any, ConfigValueProvenance, bool) { + effective := p.EffectiveConfig() + value, ok := lookupEffectiveValue(effective, key) + if !ok { + return nil, ConfigValueProvenance{}, false + } + prov := effective.Provenance[key] + if prov.Source == "" { + prov = ConfigValueProvenance{Source: "yaml", Detail: p.ConfigSource.Name} + } + return value, prov, true +} + +func lookupEffectiveValue(effective EffectiveConfig, key string) (any, bool) { + raw, err := json.Marshal(effective) + if err != nil { + return nil, false + } + var data any + if err := json.Unmarshal(raw, &data); err != nil { + return nil, false + } + current := data + for _, part := range strings.Split(key, ".") { + object, ok := current.(map[string]any) + if !ok { + return nil, false + } + current, ok = object[part] + if !ok { + return nil, false + } + } + return current, true +} + +func markMissingDefaults(provenance ConfigProvenance) { + defaults := map[string]string{ + "paths.build": "build", + "paths.cache": ".cache", + "paths.tools": "tools", + "outputs.module_archive": DefaultModuleArchiveTemplate, + "outputs.hak_manifest": DefaultHAKManifest, + "outputs.hak_archive": DefaultHAKArchiveTemplate, + "generated_assets.topdata_2da": "no generated topdata 2DA assets", + "inventory.source_extensions": "NWN source resource extensions", + "inventory.asset_extensions": "NWN asset resource extensions", + "inventory.source_json_pattern": DefaultSourceJSONPattern, + "validation.profile": DefaultValidationProfile, + "validation.builtin_script_prefixes": "NWN built-in script prefixes", + "validation.required_fields": "NWN required GFF fields", + "music.stage_root": DefaultMusicStageRoot, + "music.credits_root": DefaultCreditsRoot, + "music.credits_overlay": DefaultCreditsOverlayFile, + "music.max_stem_length": "16", + "music.naming_scheme": DefaultMusicNamingScheme, + "music.output_extension": DefaultMusicOutputExtension, + "music.convert_extensions": strings.Join(music.DefaultConvertExtensions(), ", "), + "topdata.build": DefaultTopDataBuild, + "topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir, + "topdata.compiled_tlk": DefaultTopDataCompiledTLK, + "topdata.package_hak": DefaultTopDataPackageHAK, + "topdata.package_tlk": DefaultTopDataPackageTLK, + "topdata.wiki.output_root": DefaultTopDataWikiOutputRoot, + "topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir, + "topdata.wiki.state_file": DefaultTopDataWikiStateFile, + "topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces", + "topdata.wiki.deploy_manifest": DefaultWikiDeployManifest, + "topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary, + "topdata.wiki.source": DefaultTopDataWikiSource, + "topdata.wiki.renderer": DefaultTopDataWikiRenderer, + "topdata.wiki.link_strategy": DefaultTopDataWikiLinkStrategy, + "topdata.wiki.namespaces_file": DefaultTopDataWikiNamespacesFile, + "topdata.wiki.tables_file": DefaultTopDataWikiTablesFile, + "topdata.wiki.visibility_file": DefaultTopDataWikiVisibilityFile, + "topdata.wiki.data_file": DefaultTopDataWikiDataFile, + "topdata.wiki.page_paths_file": DefaultTopDataWikiPagePathsFile, + "topdata.wiki.templates_dir": DefaultTopDataWikiTemplatesDir, + "topdata.wiki.page_templates_dir": DefaultTopDataWikiPageTemplatesDir, + "topdata.wiki.manual_sections_dir": DefaultTopDataWikiManualSectionsDir, + "topdata.wiki.manual_sections_file": DefaultTopDataWikiManualSectionsFile, + "topdata.wiki.page_index_file": DefaultTopDataWikiPageIndexFile, + "topdata.wiki.title_prefix_min_length": "3", + "topdata.wiki.status_pages": "true", + "topdata.wiki.status_listing_scope": DefaultTopDataWikiStatusListingScope, + "topdata.wiki.alignment_links.link_format": DefaultTopDataWikiAlignmentLinkFormat, + "topdata.wiki.stale_pages.default": DefaultTopDataWikiStaleDefault, + "topdata.wiki.stale_pages.live_cleanup": DefaultTopDataWikiStaleLiveCleanup, + "topdata.wiki.managed_region.marker_format": DefaultTopDataWikiMarkerFormat, + "topdata.wiki.managed_region.page_marker_prefix": DefaultTopDataWikiPageMarkerPrefix, + "extract.layout": DefaultExtractLayout, + "extract.hak_discovery": DefaultExtractHAKDiscovery, + "extract.archives": DefaultModuleArchiveTemplate, + "extract.cleanup_stale": "true", + "extract.consume_archives": "false", + "autogen.cache.root": DefaultAutogenCacheRoot, + "autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(), + "autogen.cache.refresh_env": DefaultAutogenRefreshEnv, + "autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv, + "autogen.release_source.provider": DefaultAutogenReleaseProvider, + "autogen.release_source.server_url_env": DefaultAutogenServerURLEnv, + "autogen.release_source.repo_env": DefaultAutogenRepoEnv, + } + for key, detail := range defaults { + if _, ok := provenance[key]; !ok { + provenance[key] = ConfigValueProvenance{Source: "toolkit default", Detail: detail} + } + } +} + +func cloneProvenance(input ConfigProvenance) ConfigProvenance { + out := ConfigProvenance{} + for key, value := range input { + out[key] = value + } + return out +} + +func defaultString(value, fallback string) string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return fallback + } + return trimmed +} + +func defaultInt(value, fallback int) int { + if value == 0 { + return fallback + } + return value +} + +func defaultBoolPointer(value *bool, fallback bool) *bool { + out := fallback + if value != nil { + out = *value + } + return &out +} + +func defaultStringSlice(value, fallback []string) []string { + if len(value) == 0 { + return slices.Clone(fallback) + } + return slices.Clone(value) +} + +func defaultWikiPageTemplatesDir(cfg TopDataWikiConfig) string { + if strings.TrimSpace(cfg.PageTemplatesDir) != "" { + return strings.TrimSpace(cfg.PageTemplatesDir) + } + if strings.TrimSpace(cfg.TemplatesDir) != "" { + return filepath.ToSlash(filepath.Join(filepath.FromSlash(strings.TrimSpace(cfg.TemplatesDir)), "pages")) + } + return DefaultTopDataWikiPageTemplatesDir +} + +func defaultWikiManualSectionsFile(cfg TopDataWikiConfig) string { + if strings.TrimSpace(cfg.ManualSectionsFile) != "" { + return strings.TrimSpace(cfg.ManualSectionsFile) + } + if strings.TrimSpace(cfg.ManualSectionsDir) != "" { + return filepath.ToSlash(filepath.Join(filepath.FromSlash(strings.TrimSpace(cfg.ManualSectionsDir)), "default.yaml")) + } + return DefaultTopDataWikiManualSectionsFile +} + +func defaultRequiredFields(configured map[string][]string) map[string][]string { + if len(configured) > 0 { + return cloneStringSliceMap(normalizeRequiredFields(configured)) + } + return DefaultRequiredFields() +} + +func cloneStringSliceMap(input map[string][]string) map[string][]string { + out := make(map[string][]string, len(input)) + for key, values := range input { + out[key] = slices.Clone(values) + } + return out +} + +func cloneStringMap(input map[string]string) map[string]string { + if len(input) == 0 { + return nil + } + out := make(map[string]string, len(input)) + for key, value := range input { + out[key] = value + } + return out +} + +func expandPathTemplate(template string, paths EffectivePathConfig) string { + replacer := strings.NewReplacer( + "{paths.source}", paths.Source, + "{paths.assets}", paths.Assets, + "{paths.build}", paths.Build, + "{paths.cache}", paths.Cache, + "{paths.tools}", paths.Tools, + ) + return filepath.ToSlash(replacer.Replace(template)) +} + +func expandPathTemplates(templates []string, paths EffectivePathConfig) []string { + out := make([]string, 0, len(templates)) + for _, template := range templates { + out = append(out, expandPathTemplate(template, paths)) + } + return out +} + +func (p *Project) ActiveOverrides() []ConfigOverride { + return activeOverrides(p.EffectiveConfig()) +} + +func activeOverrides(effective EffectiveConfig) []ConfigOverride { + envs := map[string]string{ + "build.keep_existing_haks": effectiveBuildKeepExistingEnv, + "autogen.cache.refresh": effective.Autogen.Cache.RefreshEnv, + "autogen.cache.parts_manifest_refresh": effective.Autogen.Cache.PartsManifestRefreshEnv, + "autogen.release_source.server_url": effective.Autogen.ReleaseSource.ServerURLEnv, + "autogen.release_source.repo": effective.Autogen.ReleaseSource.RepoEnv, + "music.ffmpeg": "SOW_FFMPEG", + "music.ffprobe": "SOW_FFPROBE", + "wiki.endpoint": "NODEBB_API_ENDPOINT", + "wiki.token": "NODEBB_API_TOKEN", + "wiki.categories": "NODEBB_WIKI_CATEGORIES", + "release.version": "GITHUB_REF_NAME", + } + keys := make([]string, 0, len(envs)) + for key := range envs { + keys = append(keys, key) + } + slices.Sort(keys) + overrides := make([]ConfigOverride, 0) + for _, key := range keys { + env := envs[key] + if value := strings.TrimSpace(os.Getenv(env)); value != "" { + if isSensitiveOverrideKey(key) || isSensitiveOverrideKey(env) { + value = "" + } + overrides = append(overrides, ConfigOverride{Key: key, Value: value, Source: "env:" + env}) + } + } + return overrides +} + +const effectiveBuildKeepExistingEnv = "SOW_BUILD_HAKS_KEEP_EXISTING" + +func isSensitiveOverrideKey(key string) bool { + lower := strings.ToLower(key) + return strings.Contains(lower, "token") || strings.Contains(lower, "password") +} + +func (o EffectiveOutputConfig) ModuleArchiveName(module ModuleConfig) string { + return strings.NewReplacer("{module.resref}", strings.TrimSpace(module.ResRef)).Replace(o.ModuleArchive) +} + +func (o EffectiveOutputConfig) HAKArchiveName(name string) string { + return strings.NewReplacer("{hak.name}", strings.TrimSpace(name)).Replace(o.HAKArchive) +} + +func (p ConfigValueProvenance) String() string { + if p.Detail == "" { + return p.Source + } + return fmt.Sprintf("%s (%s)", p.Source, p.Detail) +} diff --git a/internal/project/project.go b/internal/project/project.go new file mode 100644 index 0000000..6f719f8 --- /dev/null +++ b/internal/project/project.go @@ -0,0 +1,2158 @@ +package project + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + ConfigFile = "nwn-tool.yaml" + ConfigFileYML = "nwn-tool.yml" + LegacyConfigFile = "nwn-tool.json" +) + +var ConfigFileCandidates = []string{ConfigFile, ConfigFileYML, LegacyConfigFile} + +var SourceExtensions = []string{ + ".are", ".dlg", ".fac", ".git", ".ifo", ".jrl", + ".utc", ".utd", ".ute", ".uti", ".utm", ".utp", ".uts", ".utt", ".utw", +} + +var AssetExtensions = []string{ + ".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".gr2", ".itp", ".jpg", ".lod", ".lyt", ".mdb", ".mdl", ".mdx", ".mp3", ".mtr", ".ogg", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wlk", ".wok", ".xml", +} + +var BuiltinScriptPrefixes = []string{ + "nw_", + "x0_", + "x1_", + "x2_", + "x3_", + "ga_", + "gc_", + "gen_", + "gui_", + "nwg_", + "ta_", +} + +var RequiredFields = map[string][]string{ + "ifo": {"Mod_Name"}, +} + +func DefaultBuiltinScriptPrefixes() []string { + return slices.Clone(BuiltinScriptPrefixes) +} + +func DefaultRequiredFields() map[string][]string { + out := make(map[string][]string, len(RequiredFields)) + for extension, fields := range RequiredFields { + out[extension] = slices.Clone(fields) + } + return out +} + +type Project struct { + Root string + Config Config + ConfigSource ConfigSource + Provenance ConfigProvenance + Inventory Inventory +} + +type ConfigSource struct { + Path string + Name string + Format string + Legacy bool +} + +type Config struct { + Module ModuleConfig `json:"module" yaml:"module"` + Paths PathConfig `json:"paths" yaml:"paths"` + Outputs OutputConfig `json:"outputs" yaml:"outputs"` + Build BuildConfig `json:"build" yaml:"build"` + Generated GeneratedConfig `json:"generated_assets" yaml:"generated_assets"` + HAKs []HAKConfig `json:"haks" yaml:"haks"` + Inventory InventoryConfig `json:"inventory" yaml:"inventory"` + Validation ValidationConfig `json:"validation" yaml:"validation"` + Music MusicConfig `json:"music" yaml:"music"` + TopData TopDataConfig `json:"topdata" yaml:"topdata"` + Extract ExtractConfig `json:"extract" yaml:"extract"` + Autogen AutogenConfig `json:"autogen" yaml:"autogen"` +} + +type ModuleConfig struct { + Name string `json:"name" yaml:"name"` + ResRef string `json:"resref" yaml:"resref"` + Description string `json:"description" yaml:"description"` + HAKOrder []string `json:"hak_order" yaml:"hak_order"` +} + +type PathConfig struct { + Source string `json:"source" yaml:"source"` + Assets string `json:"assets" yaml:"assets"` + Build string `json:"build" yaml:"build"` + Cache string `json:"cache" yaml:"cache"` + Tools string `json:"tools" yaml:"tools"` +} + +type OutputConfig struct { + ModuleArchive string `json:"module_archive" yaml:"module_archive"` + HAKManifest string `json:"hak_manifest" yaml:"hak_manifest"` + HAKArchive string `json:"hak_archive" yaml:"hak_archive"` +} + +type BuildConfig struct { + KeepExistingHAKs bool `json:"keep_existing_haks,omitempty" yaml:"keep_existing_haks,omitempty"` +} + +type GeneratedConfig struct { + TopData2DA []GeneratedTopData2DAConfig `json:"topdata_2da" yaml:"topdata_2da"` +} + +type GeneratedTopData2DAConfig struct { + ID string `json:"id" yaml:"id"` + Source string `json:"source" yaml:"source"` + Output string `json:"output" yaml:"output"` + IncludeDatasets []string `json:"include_datasets" yaml:"include_datasets"` + PackageRoot string `json:"package_root" yaml:"package_root"` + Autogen AutogenConsumerConfig `json:"autogen" yaml:"autogen"` +} + +type InventoryConfig struct { + SourceExtensions []string `json:"source_extensions" yaml:"source_extensions"` + AssetExtensions []string `json:"asset_extensions" yaml:"asset_extensions"` + SourceJSONPattern string `json:"source_json_pattern" yaml:"source_json_pattern"` +} + +type ValidationConfig struct { + Profile string `json:"profile" yaml:"profile"` + BuiltinScriptPrefixes []string `json:"builtin_script_prefixes" yaml:"builtin_script_prefixes"` + RequiredFields map[string][]string `json:"required_fields" yaml:"required_fields"` +} + +type HAKConfig struct { + Name string `json:"name" yaml:"name"` + Priority int `json:"priority" yaml:"priority"` + MaxBytes int64 `json:"max_bytes" yaml:"max_bytes"` + Split bool `json:"split" yaml:"split"` + Optional bool `json:"optional,omitempty" yaml:"optional,omitempty"` + Include []string `json:"include" yaml:"include"` +} + +type MusicConfig struct { + Prefixes map[string]string `json:"prefixes,omitempty" yaml:"prefixes,omitempty"` + Tools MusicToolsConfig `json:"tools,omitempty" yaml:"tools,omitempty"` + Defaults *MusicDefaultsConfig `json:"defaults,omitempty" yaml:"defaults,omitempty"` + Datasets map[string]MusicDatasetConfig `json:"datasets,omitempty" yaml:"datasets,omitempty"` +} + +type MusicToolsConfig struct { + FFmpeg string `json:"ffmpeg,omitempty" yaml:"ffmpeg,omitempty"` + FFprobe string `json:"ffprobe,omitempty" yaml:"ffprobe,omitempty"` +} + +type MusicDefaultsConfig struct { + StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` + CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` + CreditsOverlay string `json:"credits_overlay,omitempty" yaml:"credits_overlay,omitempty"` + MaxStemLength *int `json:"max_stem_length,omitempty" yaml:"max_stem_length,omitempty"` + NamingScheme string `json:"naming_scheme,omitempty" yaml:"naming_scheme,omitempty"` + OutputExtension string `json:"output_extension,omitempty" yaml:"output_extension,omitempty"` + ConvertExtensions []string `json:"convert_extensions,omitempty" yaml:"convert_extensions,omitempty"` +} + +type MusicDatasetConfig struct { + Source string `json:"source" yaml:"source"` + Output string `json:"output,omitempty" yaml:"output,omitempty"` + Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"` + StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` + CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` + NamingScheme string `json:"naming_scheme,omitempty" yaml:"naming_scheme,omitempty"` + OutputExtension string `json:"output_extension,omitempty" yaml:"output_extension,omitempty"` + PackageMode string `json:"package_mode,omitempty" yaml:"package_mode,omitempty"` + ConvertExtensions []string `json:"convert_extensions,omitempty" yaml:"convert_extensions,omitempty"` +} + +type TopDataConfig struct { + Source string `json:"source" yaml:"source"` + Build string `json:"build" yaml:"build"` + ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"` + Assets string `json:"assets" yaml:"assets"` + Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"` + CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"` + PackageHAK string `json:"package_hak" yaml:"package_hak"` + PackageTLK string `json:"package_tlk" yaml:"package_tlk"` + ValueEncodings []TopDataValueEncodingConfig `json:"value_encodings" yaml:"value_encodings"` + ValueDefaults []TopDataValueDefaultConfig `json:"value_defaults" yaml:"value_defaults"` + RowGeneration []TopDataRowGenerationConfig `json:"row_generation" yaml:"row_generation"` + RowExtensions []TopDataRowExtensionConfig `json:"row_extensions" yaml:"row_extensions"` + ClassFeatInjections TopDataClassFeatInjectionConfig `json:"class_feat_injections" yaml:"class_feat_injections"` + Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"` +} + +type TopDataRowGenerationConfig struct { + Dataset string `json:"dataset,omitempty" yaml:"dataset,omitempty"` + Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` + Mode string `json:"mode" yaml:"mode"` + MinimumRow int `json:"minimum_row,omitempty" yaml:"minimum_row,omitempty"` +} + +type TopDataRowExtensionConfig struct { + Dataset string `json:"dataset" yaml:"dataset"` + Mode string `json:"mode" yaml:"mode"` + LevelColumn string `json:"level_column" yaml:"level_column"` + TargetLevel int `json:"target_level" yaml:"target_level"` +} + +type TopDataClassFeatInjectionConfig struct { + GlobalFeats []TopDataClassFeatGlobalRule `json:"global_feats" yaml:"global_feats"` + ClassSkillMasterfeats []TopDataClassFeatMasterfeatRule `json:"class_skill_masterfeats" yaml:"class_skill_masterfeats"` +} + +type TopDataClassFeatGlobalRule struct { + Feat string `json:"feat" yaml:"feat"` + List string `json:"list" yaml:"list"` + GrantedOnLevel string `json:"granted_on_level" yaml:"granted_on_level"` + OnMenu string `json:"on_menu" yaml:"on_menu"` + RequirePresent []string `json:"require_present,omitempty" yaml:"require_present,omitempty"` + UnlessPresent []string `json:"unless_present,omitempty" yaml:"unless_present,omitempty"` +} + +type TopDataClassFeatMasterfeatRule struct { + Masterfeat string `json:"masterfeat" yaml:"masterfeat"` + List string `json:"list" yaml:"list"` + GrantedOnLevel string `json:"granted_on_level" yaml:"granted_on_level"` + OnMenu string `json:"on_menu" yaml:"on_menu"` +} + +type TopDataValueEncodingConfig struct { + Dataset string `json:"dataset" yaml:"dataset"` + Column string `json:"column" yaml:"column"` + Mode string `json:"mode" yaml:"mode"` + Min int `json:"min" yaml:"min"` + Max int `json:"max" yaml:"max"` + HexWidth int `json:"hex_width" yaml:"hex_width"` +} + +type TopDataValueDefaultConfig struct { + Dataset string `json:"dataset" yaml:"dataset"` + Column string `json:"column" yaml:"column"` + Value any `json:"value" yaml:"value"` +} + +type TopDataWikiConfig struct { + OutputRoot string `json:"output_root" yaml:"output_root"` + PagesDir string `json:"pages_dir" yaml:"pages_dir"` + StateFile string `json:"state_file" yaml:"state_file"` + ManagedNamespaces []string `json:"managed_namespaces" yaml:"managed_namespaces"` + DeployManifest string `json:"deploy_manifest" yaml:"deploy_manifest"` + DeployEditSummary string `json:"deploy_edit_summary" yaml:"deploy_edit_summary"` + Source string `json:"source" yaml:"source"` + Renderer string `json:"renderer" yaml:"renderer"` + LinkStrategy string `json:"link_strategy" yaml:"link_strategy"` + NamespacesFile string `json:"namespaces_file" yaml:"namespaces_file"` + TablesFile string `json:"tables_file" yaml:"tables_file"` + VisibilityFile string `json:"visibility_file" yaml:"visibility_file"` + DataFile string `json:"data_file" yaml:"data_file"` + PagePathsFile string `json:"page_paths_file" yaml:"page_paths_file"` + TemplatesDir string `json:"templates_dir" yaml:"templates_dir"` + PageTemplatesDir string `json:"page_templates_dir" yaml:"page_templates_dir"` + ManualSectionsDir string `json:"manual_sections_dir" yaml:"manual_sections_dir"` + ManualSectionsFile string `json:"manual_sections_file" yaml:"manual_sections_file"` + PageIndexFile string `json:"page_index_file" yaml:"page_index_file"` + AlignmentLinks TopDataWikiAlignmentLinks `json:"alignment_links" yaml:"alignment_links"` + TitlePrefixMinLength int `json:"title_prefix_min_length" yaml:"title_prefix_min_length"` + StatusPages *bool `json:"status_pages,omitempty" yaml:"status_pages,omitempty"` + StatusListingScope string `json:"status_listing_scope" yaml:"status_listing_scope"` + StalePages TopDataWikiStalePagesConfig `json:"stale_pages" yaml:"stale_pages"` + ManagedRegion TopDataWikiManagedRegionConfig `json:"managed_region" yaml:"managed_region"` +} + +type TopDataWikiAlignmentLinks struct { + TargetPattern string `json:"target_pattern" yaml:"target_pattern"` + LinkFormat string `json:"link_format" yaml:"link_format"` +} + +type TopDataWikiStalePagesConfig struct { + Default string `json:"default" yaml:"default"` + LiveCleanup string `json:"live_cleanup" yaml:"live_cleanup"` +} + +type TopDataWikiManagedRegionConfig struct { + MarkerFormat string `json:"marker_format" yaml:"marker_format"` + PageMarkerPrefix string `json:"page_marker_prefix" yaml:"page_marker_prefix"` +} + +type ExtractConfig struct { + IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"` + Archives []string `json:"archives,omitempty" yaml:"archives,omitempty"` + Layout string `json:"layout" yaml:"layout"` + HAKDiscovery string `json:"hak_discovery" yaml:"hak_discovery"` + CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"` + ConsumeArchives *bool `json:"consume_archives,omitempty" yaml:"consume_archives,omitempty"` + DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"` + Merge ExtractMergeConfig `json:"merge" yaml:"merge"` +} + +type ExtractMergeConfig struct { + GFFJSON []ExtractGFFJSONMergeRule `json:"gff_json" yaml:"gff_json"` +} + +type ExtractGFFJSONMergeRule struct { + Target string `json:"target" yaml:"target"` + PreserveFields []string `json:"preserve_fields" yaml:"preserve_fields"` + MergeLists []ExtractListMergeRule `json:"merge_lists" yaml:"merge_lists"` +} + +type ExtractListMergeRule struct { + Field string `json:"field" yaml:"field"` + KeyField string `json:"key_field" yaml:"key_field"` + Strategy string `json:"strategy" yaml:"strategy"` +} + +type AutogenConfig struct { + Producers []AutogenProducerConfig `json:"producers" yaml:"producers"` + Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"` + Cache AutogenCacheConfig `json:"cache" yaml:"cache"` + ReleaseSource AutogenReleaseSourceConfig `json:"release_source" yaml:"release_source"` +} + +type AutogenCacheConfig struct { + Root string `json:"root" yaml:"root"` + MaxAge string `json:"max_age" yaml:"max_age"` + RefreshEnv string `json:"refresh_env" yaml:"refresh_env"` +} + +type AutogenReleaseSourceConfig struct { + Provider string `json:"provider" yaml:"provider"` + ServerURLEnv string `json:"server_url_env" yaml:"server_url_env"` + RepoEnv string `json:"repo_env" yaml:"repo_env"` +} + +type AutogenProducerConfig struct { + ID string `json:"id" yaml:"id"` + Root string `json:"root" yaml:"root"` + Include []string `json:"include" yaml:"include"` + Derive AutogenDeriveConfig `json:"derive" yaml:"derive"` + Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"` + HeadVisualeffects HeadVisualeffectsConfig `json:"accessory_visualeffects" yaml:"accessory_visualeffects"` +} + +type AutogenConsumerConfig struct { + ID string `json:"id" yaml:"id"` + Producer string `json:"producer" yaml:"producer"` + Dataset string `json:"dataset" yaml:"dataset"` + Mode string `json:"mode" yaml:"mode"` + Optional bool `json:"optional,omitempty" yaml:"optional,omitempty"` + Root string `json:"root" yaml:"root"` + Include []string `json:"include" yaml:"include"` + Derive AutogenDeriveConfig `json:"derive" yaml:"derive"` + PartsRows PartsRowsConfig `json:"parts_rows" yaml:"parts_rows"` + HeadVisualeffects HeadVisualeffectsConfig `json:"accessory_visualeffects" yaml:"accessory_visualeffects"` + Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"` + LocalOverrideRoot string `json:"local_override_root,omitempty" yaml:"local_override_root,omitempty"` +} + +type HeadVisualeffectsConfig struct { + Groups map[string]HeadVisualeffectGroupConfig `json:"groups,omitempty" yaml:"groups"` + GroupTokenSource string `json:"group_token_source,omitempty" yaml:"group_token_source"` + CategoryFrom string `json:"category_from,omitempty" yaml:"category_from"` + Delimiter string `json:"delimiter,omitempty" yaml:"delimiter"` + Case string `json:"case,omitempty" yaml:"case"` + StripModelPrefixes []string `json:"strip_model_prefixes,omitempty" yaml:"strip_model_prefixes"` + KeyFormat string `json:"key_format,omitempty" yaml:"key_format"` + LabelFormat string `json:"label_format,omitempty" yaml:"label_format"` + ModelColumn string `json:"model_column,omitempty" yaml:"model_column"` + RowDefaults map[string]string `json:"row_defaults,omitempty" yaml:"row_defaults"` +} + +type HeadVisualeffectGroupConfig struct { + Prefix string `json:"prefix" yaml:"prefix"` + ModelColumn string `json:"model_column,omitempty" yaml:"model_column"` + ModelColumns []string `json:"model_columns,omitempty" yaml:"model_columns"` + RowDefaults map[string]string `json:"row_defaults,omitempty" yaml:"row_defaults"` +} + +type PartsRowsConfig struct { + RowDefaults map[string]string `json:"row_defaults" yaml:"row_defaults"` + ACBonus PartsRowsACBonusConfig `json:"acbonus" yaml:"acbonus"` + Datasets map[string]PartsRowsDatasetConfig `json:"datasets" yaml:"datasets"` +} + +type PartsRowsDatasetConfig struct { + ACBonus PartsRowsACBonusPolicy `json:"acbonus" yaml:"acbonus"` +} + +type PartsRowsACBonusConfig struct { + Default PartsRowsACBonusPolicy `json:"default" yaml:"default"` + Datasets map[string]PartsRowsACBonusPolicy `json:"datasets" yaml:"datasets"` +} + +type PartsRowsACBonusPolicy struct { + Strategy string `json:"strategy" yaml:"strategy"` + MaxRowID int `json:"max_row_id" yaml:"max_row_id"` + Divisor int `json:"divisor" yaml:"divisor"` + Format string `json:"format" yaml:"format"` + Value string `json:"value" yaml:"value"` +} + +type AutogenDeriveConfig struct { + Kind string `json:"kind" yaml:"kind"` + GroupFrom string `json:"group_from,omitempty" yaml:"group_from,omitempty"` +} + +type AutogenManifestConfig struct { + ReleaseTag string `json:"release_tag" yaml:"release_tag"` + AssetName string `json:"asset_name" yaml:"asset_name"` + CacheName string `json:"cache_name" yaml:"cache_name"` +} + +type Inventory struct { + SourceFiles []string + ScriptFiles []string + AssetFiles []string + Extensions []string +} + +type InventoryReport struct { + SourceFiles int + ScriptFiles int + AssetFiles int + Extensions []string +} + +func FindRoot(start string) (string, error) { + current := start + for { + for _, name := range ConfigFileCandidates { + candidate := filepath.Join(current, name) + if _, err := os.Stat(candidate); err == nil { + return current, nil + } + } + parent := filepath.Dir(current) + if parent == current { + return "", fmt.Errorf("could not find %s, %s, or legacy %s from %s", ConfigFile, ConfigFileYML, LegacyConfigFile, start) + } + current = parent + } +} + +func Load(root string) (*Project, error) { + source, err := findConfigSource(root) + if err != nil { + return nil, err + } + raw, err := os.ReadFile(source.Path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", source.Name, err) + } + + cfg := defaultConfig() + provenance, err := configProvenance(raw, source) + if err != nil { + return nil, err + } + if source.Format == "yaml" { + decoder := yaml.NewDecoder(bytes.NewReader(raw)) + decoder.KnownFields(true) + if err := decoder.Decode(&cfg); err != nil { + return nil, fmt.Errorf("parse %s: %w", source.Name, err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return nil, fmt.Errorf("parse %s: multiple YAML documents are not supported", source.Name) + } + } else { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&cfg); err != nil { + return nil, fmt.Errorf("parse legacy %s: %w", source.Name, err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return nil, fmt.Errorf("parse legacy %s: multiple JSON documents are not supported", source.Name) + } + } + normalizeConfig(&cfg) + + return &Project{ + Root: root, + Config: cfg, + ConfigSource: source, + Provenance: provenance, + }, nil +} + +func findConfigSource(root string) (ConfigSource, error) { + for _, name := range ConfigFileCandidates { + path := filepath.Join(root, name) + if _, err := os.Stat(path); err == nil { + format := "yaml" + legacy := false + if name == LegacyConfigFile { + format = "json" + legacy = true + } + return ConfigSource{ + Path: path, + Name: name, + Format: format, + Legacy: legacy, + }, nil + } + } + return ConfigSource{}, fmt.Errorf("could not find %s, %s, or legacy %s in %s", ConfigFile, ConfigFileYML, LegacyConfigFile, root) +} + +func configProvenance(raw []byte, source ConfigSource) (ConfigProvenance, error) { + var data any + if source.Format == "yaml" { + if err := yaml.Unmarshal(raw, &data); err != nil { + return nil, fmt.Errorf("parse %s for provenance: %w", source.Name, err) + } + } else { + if err := json.Unmarshal(raw, &data); err != nil { + return nil, fmt.Errorf("parse legacy %s for provenance: %w", source.Name, err) + } + } + provenance := ConfigProvenance{} + sourceName := "yaml" + if source.Legacy { + sourceName = "legacy json" + } + recordProvenance(provenance, "", data, ConfigValueProvenance{ + Source: sourceName, + Detail: source.Name, + Deprecated: source.Legacy, + }) + return provenance, nil +} + +func recordProvenance(provenance ConfigProvenance, prefix string, value any, source ConfigValueProvenance) { + switch typed := value.(type) { + case map[string]any: + for key, nested := range typed { + next := key + if prefix != "" { + next = prefix + "." + key + } + recordProvenance(provenance, next, nested, source) + } + case []any: + if prefix != "" { + provenance[prefix] = source + } + default: + if prefix != "" { + provenance[prefix] = source + } + } +} + +func (p *Project) ValidateLayout() error { + var failures []error + + if strings.TrimSpace(p.Config.Module.Name) == "" { + failures = append(failures, errors.New("module.name is required")) + } + if strings.TrimSpace(p.Config.Module.ResRef) == "" { + failures = append(failures, errors.New("module.resref is required")) + } + if len(p.Config.Module.ResRef) > 16 { + failures = append(failures, fmt.Errorf("module.resref %q exceeds 16 characters", p.Config.Module.ResRef)) + } + if strings.TrimSpace(p.Config.Paths.Source) == "" && strings.TrimSpace(p.Config.Paths.Assets) == "" && !p.HasTopData() { + failures = append(failures, errors.New("at least one of paths.source, paths.assets, or topdata.source is required")) + } + for field, value := range map[string]string{ + "paths.source": p.Config.Paths.Source, + "paths.assets": p.Config.Paths.Assets, + "paths.build": p.Config.Paths.Build, + "paths.cache": p.Config.Paths.Cache, + "paths.tools": p.Config.Paths.Tools, + "outputs.module_archive": p.Config.Outputs.ModuleArchive, + "outputs.hak_manifest": p.Config.Outputs.HAKManifest, + "outputs.hak_archive": p.Config.Outputs.HAKArchive, + "generated_assets.topdata_2da": "", + "validation.profile": p.Config.Validation.Profile, + "topdata.source": p.Config.TopData.Source, + "topdata.build": p.Config.TopData.Build, + "topdata.reference_builder": p.Config.TopData.ReferenceBuilder, + "topdata.assets": p.Config.TopData.Assets, + "topdata.compiled_2da_dir": p.Config.TopData.Compiled2DADir, + "topdata.compiled_tlk": p.Config.TopData.CompiledTLK, + "topdata.wiki.output_root": p.Config.TopData.Wiki.OutputRoot, + "topdata.wiki.pages_dir": p.Config.TopData.Wiki.PagesDir, + "topdata.wiki.state_file": p.Config.TopData.Wiki.StateFile, + "topdata.wiki.deploy_manifest": p.Config.TopData.Wiki.DeployManifest, + "topdata.wiki.deploy_edit_summary": p.Config.TopData.Wiki.DeployEditSummary, + "topdata.wiki.source": p.Config.TopData.Wiki.Source, + "topdata.wiki.namespaces_file": p.Config.TopData.Wiki.NamespacesFile, + "topdata.wiki.tables_file": p.Config.TopData.Wiki.TablesFile, + "topdata.wiki.visibility_file": p.Config.TopData.Wiki.VisibilityFile, + "topdata.wiki.data_file": p.Config.TopData.Wiki.DataFile, + "topdata.wiki.page_paths_file": p.Config.TopData.Wiki.PagePathsFile, + "topdata.wiki.templates_dir": p.Config.TopData.Wiki.TemplatesDir, + "topdata.wiki.page_templates_dir": p.Config.TopData.Wiki.PageTemplatesDir, + "topdata.wiki.manual_sections_dir": p.Config.TopData.Wiki.ManualSectionsDir, + "topdata.wiki.manual_sections_file": p.Config.TopData.Wiki.ManualSectionsFile, + "topdata.wiki.page_index_file": p.Config.TopData.Wiki.PageIndexFile, + "topdata.wiki.alignment_links.link_format": p.Config.TopData.Wiki.AlignmentLinks.LinkFormat, + "extract.layout": p.Config.Extract.Layout, + "extract.hak_discovery": p.Config.Extract.HAKDiscovery, + "autogen.cache.root": p.Config.Autogen.Cache.Root, + "autogen.cache.max_age": p.Config.Autogen.Cache.MaxAge, + "autogen.cache.refresh_env": p.Config.Autogen.Cache.RefreshEnv, + "autogen.release_source.provider": p.Config.Autogen.ReleaseSource.Provider, + "autogen.release_source.server_url_env": p.Config.Autogen.ReleaseSource.ServerURLEnv, + "autogen.release_source.repo_env": p.Config.Autogen.ReleaseSource.RepoEnv, + } { + if strings.Contains(value, "\x00") { + failures = append(failures, fmt.Errorf("%s must not contain NUL bytes", field)) + } + } + failures = append(failures, validateValidationConfig(p.Config.Validation)...) + effective := p.EffectiveConfig() + failures = append(failures, validateRelativePath("paths.cache", effective.Paths.Cache)...) + failures = append(failures, validateTreeRootPath("paths.source", effective.Paths.Source)...) + failures = append(failures, validateTreeRootPath("paths.assets", effective.Paths.Assets)...) + failures = append(failures, validateRelativePath("outputs.module_archive", effective.Outputs.ModuleArchive)...) + failures = append(failures, validateRelativePath("outputs.hak_manifest", effective.Outputs.HAKManifest)...) + failures = append(failures, validateRelativePath("outputs.hak_archive", effective.Outputs.HAKArchive)...) + failures = append(failures, validateGeneratedConfig(effective.Generated)...) + failures = append(failures, validateTopDataValueEncodings(effective.TopData.ValueEncodings)...) + failures = append(failures, validateTopDataValueDefaults(effective.TopData.ValueDefaults)...) + failures = append(failures, validateTopDataRowGeneration(effective.TopData.RowGeneration)...) + failures = append(failures, validateTopDataRowExtensions(effective.TopData.RowExtensions)...) + failures = append(failures, validateTopDataClassFeatInjections(effective.TopData.ClassFeatInjections)...) + failures = append(failures, validateRelativePath("topdata.compiled_2da_dir", effective.TopData.Compiled2DADir)...) + failures = append(failures, validateRelativePath("topdata.compiled_tlk", effective.TopData.CompiledTLK)...) + failures = append(failures, validateRelativePath("topdata.wiki.output_root", effective.TopData.Wiki.OutputRoot)...) + failures = append(failures, validateRelativePath("topdata.wiki.pages_dir", effective.TopData.Wiki.PagesDir)...) + failures = append(failures, validateRelativePath("topdata.wiki.state_file", effective.TopData.Wiki.StateFile)...) + failures = append(failures, validateRelativePath("topdata.wiki.deploy_manifest", effective.TopData.Wiki.DeployManifest)...) + failures = append(failures, validateTreeRootPath("topdata.wiki.source", effective.TopData.Wiki.Source)...) + failures = append(failures, validateRelativePath("topdata.wiki.namespaces_file", effective.TopData.Wiki.NamespacesFile)...) + failures = append(failures, validateRelativePath("topdata.wiki.tables_file", effective.TopData.Wiki.TablesFile)...) + failures = append(failures, validateRelativePath("topdata.wiki.visibility_file", effective.TopData.Wiki.VisibilityFile)...) + failures = append(failures, validateRelativePath("topdata.wiki.data_file", effective.TopData.Wiki.DataFile)...) + failures = append(failures, validateRelativePath("topdata.wiki.page_paths_file", effective.TopData.Wiki.PagePathsFile)...) + failures = append(failures, validateTreeRootPath("topdata.wiki.templates_dir", effective.TopData.Wiki.TemplatesDir)...) + failures = append(failures, validateRelativePath("topdata.wiki.page_templates_dir", effective.TopData.Wiki.PageTemplatesDir)...) + failures = append(failures, validateTreeRootPath("topdata.wiki.manual_sections_dir", effective.TopData.Wiki.ManualSectionsDir)...) + failures = append(failures, validateRelativePath("topdata.wiki.manual_sections_file", effective.TopData.Wiki.ManualSectionsFile)...) + failures = append(failures, validateRelativePath("topdata.wiki.page_index_file", effective.TopData.Wiki.PageIndexFile)...) + failures = append(failures, validateRelativePath("autogen.cache.root", effective.Autogen.Cache.Root)...) + if err := validateOutputFileName("outputs.module_archive", filepath.Base(effective.Outputs.ModuleArchive), ".mod"); err != nil { + failures = append(failures, err) + } + if err := validateOutputFileName("outputs.hak_manifest", filepath.Base(effective.Outputs.HAKManifest), ".json"); err != nil { + failures = append(failures, err) + } + if err := validateOutputFileName("outputs.hak_archive", filepath.Base(effective.Outputs.HAKArchive), ".hak"); err != nil { + failures = append(failures, err) + } + if err := validateOutputFileName("topdata.wiki.deploy_manifest", filepath.Base(effective.TopData.Wiki.DeployManifest), ".json"); err != nil { + failures = append(failures, err) + } + if err := validateOutputFileName("topdata.wiki.page_index_file", filepath.Base(effective.TopData.Wiki.PageIndexFile), ".json"); err != nil { + failures = append(failures, err) + } + switch effective.TopData.Wiki.Renderer { + case "nodebb_tiptap_html": + default: + failures = append(failures, fmt.Errorf("topdata.wiki.renderer %q is not supported", effective.TopData.Wiki.Renderer)) + } + switch effective.TopData.Wiki.LinkStrategy { + case "preserve_westgate_wiki_links": + default: + failures = append(failures, fmt.Errorf("topdata.wiki.link_strategy %q is not supported", effective.TopData.Wiki.LinkStrategy)) + } + switch effective.TopData.Wiki.AlignmentLinks.LinkFormat { + case "wiki_markup", "html_anchor": + default: + failures = append(failures, fmt.Errorf("topdata.wiki.alignment_links.link_format %q is not supported", effective.TopData.Wiki.AlignmentLinks.LinkFormat)) + } + if effective.TopData.Wiki.TitlePrefixMinLength < 1 { + failures = append(failures, fmt.Errorf("topdata.wiki.title_prefix_min_length must be at least 1")) + } + switch effective.TopData.Wiki.StatusListingScope { + case "all", "namespace": + default: + failures = append(failures, fmt.Errorf("topdata.wiki.status_listing_scope %q is not supported", effective.TopData.Wiki.StatusListingScope)) + } + for key, policy := range map[string]string{ + "topdata.wiki.stale_pages.default": effective.TopData.Wiki.StalePages.Default, + "topdata.wiki.stale_pages.live_cleanup": effective.TopData.Wiki.StalePages.LiveCleanup, + } { + switch policy { + case "report", "archive", "purge": + default: + failures = append(failures, fmt.Errorf("%s %q is not supported", key, policy)) + } + } + switch effective.Extract.Layout { + case "nwn_canonical_json": + default: + failures = append(failures, fmt.Errorf("extract.layout %q is not supported", effective.Extract.Layout)) + } + switch effective.Extract.HAKDiscovery { + case "build_glob", "configured_haks": + default: + failures = append(failures, fmt.Errorf("extract.hak_discovery %q is not supported", effective.Extract.HAKDiscovery)) + } + failures = append(failures, validateGlobList("extract.archives", effective.Extract.Archives)...) + failures = append(failures, validateExtractMergeConfig(effective.Extract.Merge)...) + if _, err := time.ParseDuration(effective.Autogen.Cache.MaxAge); err != nil { + failures = append(failures, fmt.Errorf("autogen.cache.max_age must be a duration: %w", err)) + } + if effective.Autogen.ReleaseSource.Provider != "gitea" { + failures = append(failures, fmt.Errorf("autogen.release_source.provider %q is not supported", effective.Autogen.ReleaseSource.Provider)) + } + if p.HasTopData() { + if err := validateOutputFileName("topdata.package_hak", p.TopDataPackageHAKName(), ".hak"); err != nil { + failures = append(failures, err) + } + if err := validateOutputFileName("topdata.package_tlk", p.TopDataPackageTLKName(), ".tlk"); err != nil { + failures = append(failures, err) + } + if err := validateOutputFileName("topdata.compiled_tlk", filepath.Base(p.TopDataCompiledTLKPath()), ".tlk"); err != nil { + failures = append(failures, err) + } + } + failures = append(failures, validateAutogenConfig(p.Config.Autogen)...) + failures = append(failures, validateMusicConfig(p.Config.Music)...) + + hakNames := map[string]struct{}{} + for index, hak := range p.Config.HAKs { + name := strings.TrimSpace(hak.Name) + if strings.TrimSpace(hak.Name) == "" { + failures = append(failures, fmt.Errorf("haks[%d].name is required", index)) + } else { + normalizedName := strings.ToLower(name) + if _, exists := hakNames[normalizedName]; exists { + failures = append(failures, fmt.Errorf("haks[%d].name %q is duplicated", index, name)) + } + hakNames[normalizedName] = struct{}{} + if err := validateOutputFileName(fmt.Sprintf("haks[%d].name", index), name+".hak", ".hak"); err != nil { + failures = append(failures, err) + } + } + if hak.Priority <= 0 { + failures = append(failures, fmt.Errorf("haks[%d].priority must be greater than zero", index)) + } + if hak.MaxBytes < 0 { + failures = append(failures, fmt.Errorf("haks[%d].max_bytes must be zero or greater", index)) + } + if len(hak.Include) == 0 { + failures = append(failures, fmt.Errorf("haks[%d].include must have at least one pattern", index)) + } + failures = append(failures, validateGlobList(fmt.Sprintf("haks[%d].include", index), hak.Include)...) + } + + for label, pathFn := range map[string]func() string{ + "paths.build": p.BuildDir, + } { + path := pathFn() + if path == "" || filepath.Clean(path) == p.Root { + continue + } + info, err := os.Stat(path) + if err != nil { + failures = append(failures, fmt.Errorf("%s: %w", label, err)) + continue + } + if !info.IsDir() { + failures = append(failures, fmt.Errorf("%s must be a directory: %s", label, path)) + } + } + + if len(failures) > 0 { + return errors.Join(failures...) + } + + return nil +} + +func validateTopDataValueEncodings(encodings []TopDataValueEncodingConfig) []error { + failures := []error{} + seen := map[string]struct{}{} + for index, encoding := range encodings { + prefix := fmt.Sprintf("topdata.value_encodings[%d]", index) + dataset := strings.TrimSpace(encoding.Dataset) + column := strings.TrimSpace(encoding.Column) + if dataset == "" { + failures = append(failures, fmt.Errorf("%s.dataset is required", prefix)) + } + if column == "" { + failures = append(failures, fmt.Errorf("%s.column is required", prefix)) + } + switch strings.TrimSpace(encoding.Mode) { + case "packed_hex_list", "external_hex_list", "alignment_hex_list", "alignment_set_mask", "id_hex_list": + default: + failures = append(failures, fmt.Errorf("%s.mode %q is not supported", prefix, encoding.Mode)) + } + if encoding.Min < 0 { + failures = append(failures, fmt.Errorf("%s.min must be zero or greater", prefix)) + } + if encoding.Max < encoding.Min { + failures = append(failures, fmt.Errorf("%s.max must be greater than or equal to min", prefix)) + } + if encoding.HexWidth <= 0 { + failures = append(failures, fmt.Errorf("%s.hex_width must be greater than zero", prefix)) + } else if encoding.Max >= intPow(16, encoding.HexWidth) { + failures = append(failures, fmt.Errorf("%s.max does not fit in hex_width %d", prefix, encoding.HexWidth)) + } + key := filepath.ToSlash(dataset) + "\x00" + strings.ToLower(column) + if _, ok := seen[key]; ok { + failures = append(failures, fmt.Errorf("%s duplicates an earlier dataset/column encoding", prefix)) + } + seen[key] = struct{}{} + } + return failures +} + +func validateTopDataValueDefaults(defaults []TopDataValueDefaultConfig) []error { + failures := []error{} + seen := map[string]struct{}{} + for index, valueDefault := range defaults { + prefix := fmt.Sprintf("topdata.value_defaults[%d]", index) + dataset := strings.TrimSpace(valueDefault.Dataset) + column := strings.TrimSpace(valueDefault.Column) + if dataset == "" { + failures = append(failures, fmt.Errorf("%s.dataset is required", prefix)) + } + if column == "" { + failures = append(failures, fmt.Errorf("%s.column is required", prefix)) + } + key := filepath.ToSlash(dataset) + "\x00" + strings.ToLower(column) + if _, ok := seen[key]; ok { + failures = append(failures, fmt.Errorf("%s duplicates an earlier dataset/column default", prefix)) + } + seen[key] = struct{}{} + } + return failures +} + +func validateTopDataRowGeneration(rules []TopDataRowGenerationConfig) []error { + failures := []error{} + seen := map[string]struct{}{} + for index, rule := range rules { + prefix := fmt.Sprintf("topdata.row_generation[%d]", index) + dataset := strings.TrimSpace(rule.Dataset) + namespace := strings.TrimSpace(rule.Namespace) + if dataset == "" { + dataset = namespace + } + if dataset == "" { + failures = append(failures, fmt.Errorf("%s.dataset is required", prefix)) + } + switch strings.TrimSpace(rule.Mode) { + case "", "after_base", "first_null_row": + default: + failures = append(failures, fmt.Errorf("%s.mode %q is not supported", prefix, rule.Mode)) + } + if rule.MinimumRow < 0 { + failures = append(failures, fmt.Errorf("%s.minimum_row must be zero or greater", prefix)) + } + key := filepath.ToSlash(dataset) + if _, ok := seen[key]; ok { + failures = append(failures, fmt.Errorf("%s duplicates an earlier dataset row generation rule", prefix)) + } + seen[key] = struct{}{} + } + return failures +} + +func validateTopDataRowExtensions(rules []TopDataRowExtensionConfig) []error { + failures := []error{} + seen := map[string]struct{}{} + for index, rule := range rules { + prefix := fmt.Sprintf("topdata.row_extensions[%d]", index) + dataset := strings.TrimSpace(rule.Dataset) + if dataset == "" { + failures = append(failures, fmt.Errorf("%s.dataset is required", prefix)) + } + switch strings.TrimSpace(rule.Mode) { + case "repeat_last": + default: + failures = append(failures, fmt.Errorf("%s.mode %q is not supported", prefix, rule.Mode)) + } + if strings.TrimSpace(rule.LevelColumn) == "" { + failures = append(failures, fmt.Errorf("%s.level_column is required", prefix)) + } + if rule.TargetLevel <= 0 { + failures = append(failures, fmt.Errorf("%s.target_level must be greater than zero", prefix)) + } + key := filepath.ToSlash(dataset) + if _, ok := seen[key]; ok { + failures = append(failures, fmt.Errorf("%s duplicates an earlier dataset row extension rule", prefix)) + } + seen[key] = struct{}{} + } + return failures +} + +func validateTopDataClassFeatInjections(config TopDataClassFeatInjectionConfig) []error { + failures := []error{} + for index, rule := range config.GlobalFeats { + prefix := fmt.Sprintf("topdata.class_feat_injections.global_feats[%d]", index) + if feat := strings.TrimSpace(rule.Feat); feat == "" { + failures = append(failures, fmt.Errorf("%s.feat is required", prefix)) + } else if !strings.HasPrefix(feat, "feat:") { + failures = append(failures, fmt.Errorf("%s.feat must use a feat: reference", prefix)) + } + failures = append(failures, validateClassFeatInjectionFields(prefix, rule.List, rule.GrantedOnLevel, rule.OnMenu)...) + failures = append(failures, validateFeatReferenceList(prefix+".require_present", rule.RequirePresent)...) + failures = append(failures, validateFeatReferenceList(prefix+".unless_present", rule.UnlessPresent)...) + } + for index, rule := range config.ClassSkillMasterfeats { + prefix := fmt.Sprintf("topdata.class_feat_injections.class_skill_masterfeats[%d]", index) + if masterfeat := strings.TrimSpace(rule.Masterfeat); masterfeat == "" { + failures = append(failures, fmt.Errorf("%s.masterfeat is required", prefix)) + } else if !strings.HasPrefix(masterfeat, "masterfeats:") { + failures = append(failures, fmt.Errorf("%s.masterfeat must use a masterfeats: reference", prefix)) + } + failures = append(failures, validateClassFeatInjectionFields(prefix, rule.List, rule.GrantedOnLevel, rule.OnMenu)...) + } + return failures +} + +func validateClassFeatInjectionFields(prefix, list, grantedOnLevel, onMenu string) []error { + failures := []error{} + for field, value := range map[string]string{ + "list": list, + "granted_on_level": grantedOnLevel, + "on_menu": onMenu, + } { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + failures = append(failures, fmt.Errorf("%s.%s is required", prefix, field)) + continue + } + if _, err := strconv.Atoi(trimmed); err != nil { + failures = append(failures, fmt.Errorf("%s.%s must be an integer", prefix, field)) + } + } + return failures +} + +func validateFeatReferenceList(prefix string, refs []string) []error { + failures := []error{} + for index, ref := range refs { + trimmed := strings.TrimSpace(ref) + if trimmed == "" { + failures = append(failures, fmt.Errorf("%s[%d] is required", prefix, index)) + } else if !strings.HasPrefix(trimmed, "feat:") { + failures = append(failures, fmt.Errorf("%s[%d] must use a feat: reference", prefix, index)) + } + } + return failures +} + +func intPow(base, exponent int) int { + result := 1 + for i := 0; i < exponent; i++ { + result *= base + } + return result +} + +func validateExtractMergeConfig(config ExtractMergeConfig) []error { + var failures []error + seenTargets := map[string]struct{}{} + for ruleIndex, rule := range config.GFFJSON { + prefix := fmt.Sprintf("extract.merge.gff_json[%d]", ruleIndex) + target := strings.TrimSpace(rule.Target) + switch { + case target == "": + failures = append(failures, fmt.Errorf("%s.target must not be empty", prefix)) + case filepath.IsAbs(filepath.FromSlash(target)): + failures = append(failures, fmt.Errorf("%s.target %q must be relative to paths.source", prefix, rule.Target)) + default: + clean := filepath.Clean(filepath.FromSlash(target)) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + failures = append(failures, fmt.Errorf("%s.target %q must not escape paths.source", prefix, rule.Target)) + } + normalized := filepath.ToSlash(clean) + if _, exists := seenTargets[normalized]; exists { + failures = append(failures, fmt.Errorf("extract.merge.gff_json target %q is configured more than once", normalized)) + } + seenTargets[normalized] = struct{}{} + } + + for listIndex, listRule := range rule.MergeLists { + listPrefix := fmt.Sprintf("%s.merge_lists[%d]", prefix, listIndex) + if strings.TrimSpace(listRule.Field) == "" { + failures = append(failures, fmt.Errorf("%s.field must not be empty", listPrefix)) + } + if strings.TrimSpace(listRule.KeyField) == "" { + failures = append(failures, fmt.Errorf("%s.key_field must not be empty", listPrefix)) + } + switch listRule.Strategy { + case "preserve_existing_order_append_new": + case "": + failures = append(failures, fmt.Errorf("%s.strategy must not be empty", listPrefix)) + default: + failures = append(failures, fmt.Errorf("%s.strategy %q is not supported", listPrefix, listRule.Strategy)) + } + } + } + return failures +} + +func (p *Project) Scan() error { + var sourceFiles []string + var sourceExts []string + effective := p.EffectiveConfig() + + sourceDir := p.SourceDir() + if sourceDir != "" && filepath.Clean(sourceDir) != filepath.Clean(p.Root) { + var err error + sourceFiles, sourceExts, err = scanDir(sourceDir, func(path string) bool { + ext := strings.ToLower(filepath.Ext(path)) + return ext == ".json" || ext == ".nss" + }) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("scan source tree: %w", err) + } + } + } + + assetDir := p.AssetsDir() + var assetFiles []string + if assetDir != "" && filepath.Clean(assetDir) != filepath.Clean(p.Root) { + musicSourceExts := make(map[string]struct{}) + for _, ext := range effective.Music.ConvertExtensions { + musicSourceExts[ext] = struct{}{} + } + musicSourceRoots := make(map[string]map[string]struct{}) + for _, ds := range effective.Music.Datasets { + if strings.TrimSpace(ds.Source) == "" { + continue + } + exts := musicSourceRoots[ds.Source] + if exts == nil { + exts = make(map[string]struct{}) + musicSourceRoots[ds.Source] = exts + } + for _, ext := range ds.ConvertExtensions { + exts[ext] = struct{}{} + } + for ext := range musicSourceExts { + exts[ext] = struct{}{} + } + } + var err error + assetFiles, _, err = scanDir(assetDir, func(path string) bool { + rel, err := filepath.Rel(assetDir, path) + if err != nil { + return false + } + rel = filepath.ToSlash(rel) + ext := strings.ToLower(filepath.Ext(rel)) + if slices.Contains(effective.Inventory.AssetExtensions, ext) { + return true + } + for root, exts := range musicSourceRoots { + if !pathIsUnder(root, rel) { + continue + } + _, ok := exts[ext] + return ok + } + return false + }) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + assetFiles = nil + } else { + return fmt.Errorf("scan assets tree: %w", err) + } + } + } + + var scripts []string + var sources []string + extensionSet := map[string]struct{}{} + for _, ext := range sourceExts { + extensionSet[ext] = struct{}{} + } + + for _, path := range sourceFiles { + switch strings.ToLower(filepath.Ext(path)) { + case ".nss": + scripts = append(scripts, path) + case ".json": + sources = append(sources, path) + } + } + + extensions := make([]string, 0, len(extensionSet)) + for ext := range extensionSet { + extensions = append(extensions, ext) + } + slices.Sort(extensions) + + p.Inventory = Inventory{ + SourceFiles: sources, + ScriptFiles: scripts, + AssetFiles: assetFiles, + Extensions: extensions, + } + return nil +} + +func (p *Project) SourceDir() string { + return p.rootPath(p.EffectiveConfig().Paths.Source) +} + +func (p *Project) AssetsDir() string { + assets := p.EffectiveConfig().Paths.Assets + if assets == "" { + return "" + } + return p.rootPath(assets) +} + +func (p *Project) BuildDir() string { + return p.rootPath(p.EffectiveConfig().Paths.Build) +} + +func (p *Project) CacheDir() string { + return p.rootPath(p.EffectiveConfig().Paths.Cache) +} + +func (p *Project) ModuleArchivePath() string { + effective := p.EffectiveConfig() + return filepath.Join(p.BuildDir(), effective.Outputs.ModuleArchiveName(effective.Module)) +} + +func (p *Project) HAKManifestPath() string { + return filepath.Join(p.BuildDir(), filepath.FromSlash(p.EffectiveConfig().Outputs.HAKManifest)) +} + +func (p *Project) HAKArchivePath(name string) string { + return filepath.Join(p.BuildDir(), filepath.FromSlash(p.EffectiveConfig().Outputs.HAKArchiveName(name))) +} + +func (p *Project) CloneWithHAKNames(names []string) (*Project, error) { + if len(names) == 0 { + return p, nil + } + + wanted := make(map[string]struct{}, len(names)) + for _, name := range names { + trimmed := strings.TrimSpace(strings.ToLower(name)) + if trimmed == "" { + continue + } + wanted[trimmed] = struct{}{} + } + if len(wanted) == 0 { + return p, nil + } + + filtered := make([]HAKConfig, 0, len(p.Config.HAKs)) + found := make(map[string]struct{}, len(wanted)) + for _, hak := range p.Config.HAKs { + key := strings.ToLower(strings.TrimSpace(hak.Name)) + if _, ok := wanted[key]; !ok { + continue + } + filtered = append(filtered, hak) + found[key] = struct{}{} + } + + var missing []string + for _, name := range names { + key := strings.TrimSpace(strings.ToLower(name)) + if key == "" { + continue + } + if _, ok := found[key]; !ok { + missing = append(missing, name) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf("unknown hak name(s): %s", strings.Join(missing, ", ")) + } + + clone := *p + clone.Config = p.Config + clone.Config.HAKs = filtered + clone.Inventory = p.Inventory + return &clone, nil +} + +func (p *Project) HasTopData() bool { + return strings.TrimSpace(p.Config.TopData.Source) != "" +} + +func (p *Project) TopDataSourceDir() string { + if !p.HasTopData() { + return "" + } + return filepath.Join(p.Root, p.Config.TopData.Source) +} + +func (p *Project) TopDataBuildDir() string { + return p.rootPath(p.EffectiveConfig().TopData.Build) +} + +func (p *Project) TopDataReferenceBuilderDir() string { + ref := strings.TrimSpace(p.Config.TopData.ReferenceBuilder) + if ref == "" { + return "" + } + if filepath.IsAbs(ref) { + return ref + } + return filepath.Join(p.Root, ref) +} + +func (p *Project) TopDataAssetsDir() string { + assets := strings.TrimSpace(p.Config.TopData.Assets) + if assets == "" { + return "" + } + if filepath.IsAbs(assets) { + return assets + } + absPath := filepath.Join(p.Root, assets) + if _, err := os.Stat(absPath); err == nil { + return absPath + } + return absPath +} + +func (p *Project) TopDataUsesRepoCache() bool { + return sameCleanPath(p.TopDataBuildDir(), p.CacheDir()) +} + +func (p *Project) TopDataCompiled2DADir() string { + return filepath.Join(p.TopDataBuildDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Compiled2DADir)) +} + +func (p *Project) TopDataCompiledTLKPath() string { + tlkName := p.EffectiveConfig().TopData.CompiledTLK + if p.TopDataUsesRepoCache() { + return filepath.Join(p.BuildDir(), filepath.FromSlash(tlkName)) + } + return filepath.Join(p.TopDataBuildDir(), filepath.FromSlash(tlkName)) +} + +func (p *Project) TopDataCompiledTLKDir() string { + return filepath.Dir(p.TopDataCompiledTLKPath()) +} + +func (p *Project) TopDataWikiRootDir() string { + wikiRoot := filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.OutputRoot) + if p.TopDataUsesRepoCache() { + return filepath.Join(p.CacheDir(), wikiRoot) + } + return filepath.Join(p.TopDataBuildDir(), wikiRoot) +} + +func (p *Project) TopDataWikiSourceDir() string { + source := filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.Source) + if filepath.IsAbs(source) { + return source + } + return filepath.Join(p.Root, source) +} + +func (p *Project) TopDataWikiPagesDir() string { + return filepath.Join(p.TopDataWikiRootDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.PagesDir)) +} + +func (p *Project) TopDataWikiStatePath() string { + return filepath.Join(p.TopDataWikiRootDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.StateFile)) +} + +func (p *Project) TopDataWikiPageIndexPath() string { + return filepath.Join(p.TopDataWikiRootDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.PageIndexFile)) +} + +func (p *Project) TopDataWikiDataPath() string { + return filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.DataFile)) +} + +func (p *Project) TopDataWikiPagePathsPath() string { + return filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.PagePathsFile)) +} + +func (p *Project) TopDataWikiPageTemplatesDir() string { + return filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.PageTemplatesDir)) +} + +func (p *Project) TopDataWikiManualSectionsPath() string { + return filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.ManualSectionsFile)) +} + +func (p *Project) TopDataPackageHAKName() string { + return p.EffectiveConfig().TopData.PackageHAK +} + +func (p *Project) TopDataPackageTLKName() string { + return p.EffectiveConfig().TopData.PackageTLK +} + +func (p *Project) TopDataPackageHAKPath() string { + return filepath.Join(p.BuildDir(), p.TopDataPackageHAKName()) +} + +func (p *Project) TopDataPackageTLKPath() string { + return filepath.Join(p.BuildDir(), p.TopDataPackageTLKName()) +} + +func (p *Project) MusicStageDir() string { + return p.rootPath(p.EffectiveConfig().Music.StageRoot) +} + +func (p *Project) CreditsDir() string { + return p.rootPath(p.EffectiveConfig().Music.CreditsRoot) +} + +func (p *Project) MusicToolPath(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return "" + } + if filepath.IsAbs(path) { + return path + } + return filepath.Join(p.Root, filepath.FromSlash(path)) +} + +func (p *Project) MusicFFmpegPath() string { + return p.MusicToolPath(p.EffectiveConfig().Music.Tools.FFmpeg) +} + +func (p *Project) MusicFFprobePath() string { + return p.MusicToolPath(p.EffectiveConfig().Music.Tools.FFprobe) +} + +func (p *Project) AutogenCacheDir() string { + return p.rootPath(p.EffectiveConfig().Autogen.Cache.Root) +} + +func (p *Project) AutogenCachePath(name string) string { + return filepath.Join(p.AutogenCacheDir(), filepath.FromSlash(strings.TrimSpace(name))) +} + +func (p *Project) AutogenCacheMaxAge() time.Duration { + duration, err := time.ParseDuration(p.EffectiveConfig().Autogen.Cache.MaxAge) + if err != nil { + return DefaultAutogenCacheMaxAge + } + return duration +} + +func (p *Project) AutogenRefreshEnv() string { + return p.EffectiveConfig().Autogen.Cache.RefreshEnv +} + +func (p *Project) PartsManifestRefreshEnv() string { + return p.EffectiveConfig().Autogen.Cache.PartsManifestRefreshEnv +} + +func (p *Project) AssetsServerURLEnv() string { + return p.EffectiveConfig().Autogen.ReleaseSource.ServerURLEnv +} + +func (p *Project) AssetsRepoEnv() string { + return p.EffectiveConfig().Autogen.ReleaseSource.RepoEnv +} + +func (p *Project) rootPath(path string) string { + if strings.TrimSpace(path) == "" { + return p.Root + } + if filepath.IsAbs(path) { + return path + } + return filepath.Join(p.Root, filepath.FromSlash(path)) +} + +func (i Inventory) Report() InventoryReport { + return InventoryReport{ + SourceFiles: len(i.SourceFiles), + ScriptFiles: len(i.ScriptFiles), + AssetFiles: len(i.AssetFiles), + Extensions: i.Extensions, + } +} + +func defaultConfig() Config { + return Config{ + Paths: PathConfig{ + Build: DefaultBuildPath, + }, + } +} + +func normalizeConfig(cfg *Config) { + cfg.Inventory.SourceExtensions = normalizeExtensionSlice(cfg.Inventory.SourceExtensions) + cfg.Inventory.AssetExtensions = normalizeExtensionSlice(cfg.Inventory.AssetExtensions) + cfg.Validation.BuiltinScriptPrefixes = normalizeLowerStringSlice(cfg.Validation.BuiltinScriptPrefixes) + cfg.Validation.Profile = strings.ToLower(strings.TrimSpace(cfg.Validation.Profile)) + cfg.Validation.RequiredFields = normalizeRequiredFields(cfg.Validation.RequiredFields) + for i := range cfg.HAKs { + cfg.HAKs[i].Include = normalizeStringSlice(cfg.HAKs[i].Include) + } + cfg.Extract.IgnoreExtensions = normalizeStringSlice(cfg.Extract.IgnoreExtensions) + cfg.Extract.Archives = normalizeStringSlice(cfg.Extract.Archives) + for i := range cfg.Extract.Merge.GFFJSON { + target := strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].Target) + if target != "" { + target = filepath.ToSlash(filepath.Clean(filepath.FromSlash(target))) + } + cfg.Extract.Merge.GFFJSON[i].Target = target + cfg.Extract.Merge.GFFJSON[i].PreserveFields = normalizeStringSlice(cfg.Extract.Merge.GFFJSON[i].PreserveFields) + for j := range cfg.Extract.Merge.GFFJSON[i].MergeLists { + cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Field = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Field) + cfg.Extract.Merge.GFFJSON[i].MergeLists[j].KeyField = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].KeyField) + cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Strategy = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Strategy) + } + } + for i := range cfg.Autogen.Producers { + cfg.Autogen.Producers[i].Include = normalizeStringSlice(cfg.Autogen.Producers[i].Include) + } + for i := range cfg.Autogen.Consumers { + cfg.Autogen.Consumers[i].Include = normalizeStringSlice(cfg.Autogen.Consumers[i].Include) + } + normalizeMusicConfig(&cfg.Music) +} + +func normalizeMusicConfig(music *MusicConfig) { + if music.Defaults != nil { + music.Defaults.ConvertExtensions = normalizeExtensionSlice(music.Defaults.ConvertExtensions) + music.Defaults.OutputExtension = normalizeExtension(music.Defaults.OutputExtension) + } + if len(music.Prefixes) > 0 && len(music.Datasets) == 0 { + datasets := make(map[string]MusicDatasetConfig, len(music.Prefixes)) + for dir, prefix := range music.Prefixes { + source := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(dir))) + if source == "" { + continue + } + key := strings.ReplaceAll(source, "/", "_") + datasets[key] = MusicDatasetConfig{ + Source: source, + Prefix: strings.TrimSpace(prefix), + } + } + music.Datasets = datasets + } + for id, ds := range music.Datasets { + ds.Source = trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Source))) + ds.Output = trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Output))) + ds.OutputExtension = normalizeExtension(ds.OutputExtension) + ds.ConvertExtensions = normalizeExtensionSlice(ds.ConvertExtensions) + music.Datasets[id] = ds + } +} + +func normalizeStringSlice(input []string) []string { + if len(input) == 0 { + return input + } + out := slices.Clone(input) + slices.SortFunc(out, func(a, b string) int { + return strings.Compare(strings.TrimSpace(a), strings.TrimSpace(b)) + }) + return out +} + +func normalizeExtensionSlice(input []string) []string { + if len(input) == 0 { + return input + } + out := make([]string, 0, len(input)) + for _, value := range input { + trimmed := strings.ToLower(strings.TrimSpace(value)) + if trimmed != "" && !strings.HasPrefix(trimmed, ".") { + trimmed = "." + trimmed + } + out = append(out, trimmed) + } + slices.Sort(out) + return out +} + +func normalizeExtension(input string) string { + trimmed := strings.ToLower(strings.TrimSpace(input)) + if trimmed != "" && !strings.HasPrefix(trimmed, ".") { + trimmed = "." + trimmed + } + return trimmed +} + +func normalizeLowerStringSlice(input []string) []string { + if len(input) == 0 { + return input + } + out := make([]string, 0, len(input)) + for _, value := range input { + trimmed := strings.ToLower(strings.TrimSpace(value)) + if trimmed != "" { + out = append(out, trimmed) + } + } + slices.Sort(out) + return out +} + +func normalizeRequiredFields(input map[string][]string) map[string][]string { + if len(input) == 0 { + return input + } + out := make(map[string][]string, len(input)) + for extension, labels := range input { + key := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(extension)), ".") + if key == "" { + continue + } + normalizedLabels := make([]string, 0, len(labels)) + for _, label := range labels { + label = strings.TrimSpace(label) + if label != "" { + normalizedLabels = append(normalizedLabels, label) + } + } + slices.Sort(normalizedLabels) + out[key] = normalizedLabels + } + return out +} + +func validateValidationConfig(cfg ValidationConfig) []error { + var failures []error + switch strings.TrimSpace(cfg.Profile) { + case "", DefaultValidationProfile: + default: + failures = append(failures, fmt.Errorf("validation.profile %q is not supported", cfg.Profile)) + } + seen := map[string]struct{}{} + for index, prefix := range cfg.BuiltinScriptPrefixes { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + failures = append(failures, fmt.Errorf("validation.builtin_script_prefixes[%d] must not be empty", index)) + continue + } + if strings.Contains(prefix, "\x00") { + failures = append(failures, fmt.Errorf("validation.builtin_script_prefixes[%d] must not contain NUL bytes", index)) + } + normalized := strings.ToLower(prefix) + if _, exists := seen[normalized]; exists { + failures = append(failures, fmt.Errorf("validation.builtin_script_prefixes[%d] %q is duplicated", index, prefix)) + } + seen[normalized] = struct{}{} + } + seenExtensions := map[string]struct{}{} + for extension, labels := range cfg.RequiredFields { + normalizedExtension := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(extension)), ".") + if normalizedExtension == "" { + failures = append(failures, errors.New("validation.required_fields contains an empty extension")) + continue + } + if strings.Contains(normalizedExtension, "\x00") { + failures = append(failures, fmt.Errorf("validation.required_fields[%q] must not contain NUL bytes", extension)) + } + if _, exists := seenExtensions[normalizedExtension]; exists { + failures = append(failures, fmt.Errorf("validation.required_fields[%q] is duplicated after normalization", extension)) + } + seenExtensions[normalizedExtension] = struct{}{} + seenLabels := map[string]struct{}{} + for index, label := range labels { + label = strings.TrimSpace(label) + if label == "" { + failures = append(failures, fmt.Errorf("validation.required_fields[%q][%d] must not be empty", extension, index)) + continue + } + if strings.Contains(label, "\x00") { + failures = append(failures, fmt.Errorf("validation.required_fields[%q][%d] must not contain NUL bytes", extension, index)) + } + if _, exists := seenLabels[label]; exists { + failures = append(failures, fmt.Errorf("validation.required_fields[%q][%d] %q is duplicated", extension, index, label)) + } + seenLabels[label] = struct{}{} + } + } + return failures +} + +func validateAutogenConfig(cfg AutogenConfig) []error { + var failures []error + producerIDs := map[string]struct{}{} + consumerIDs := map[string]struct{}{} + datasetIDs := map[string]struct{}{} + + for index, producer := range cfg.Producers { + fieldPrefix := fmt.Sprintf("autogen.producers[%d]", index) + id := strings.TrimSpace(producer.ID) + if id == "" { + failures = append(failures, fmt.Errorf("%s.id is required", fieldPrefix)) + } else { + if _, exists := producerIDs[id]; exists { + failures = append(failures, fmt.Errorf("%s.id %q is duplicated", fieldPrefix, id)) + } + producerIDs[id] = struct{}{} + } + if strings.TrimSpace(producer.Root) == "" { + failures = append(failures, fmt.Errorf("%s.root is required", fieldPrefix)) + } + if len(producer.Include) == 0 { + failures = append(failures, fmt.Errorf("%s.include must contain at least one glob", fieldPrefix)) + } + failures = append(failures, validateGlobList(fieldPrefix+".include", producer.Include)...) + failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", producer.Derive)...) + failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", producer.Manifest)...) + if headVisualeffectsConfigConfigured(producer.HeadVisualeffects) { + failures = append(failures, validateHeadVisualeffectsConfig(fieldPrefix+".accessory_visualeffects", producer.HeadVisualeffects)...) + } + } + + for index, consumer := range cfg.Consumers { + fieldPrefix := fmt.Sprintf("autogen.consumers[%d]", index) + id := strings.TrimSpace(consumer.ID) + if id == "" { + failures = append(failures, fmt.Errorf("%s.id is required", fieldPrefix)) + } else { + if _, exists := consumerIDs[id]; exists { + failures = append(failures, fmt.Errorf("%s.id %q is duplicated", fieldPrefix, id)) + } + consumerIDs[id] = struct{}{} + } + if strings.TrimSpace(consumer.Producer) == "" { + failures = append(failures, fmt.Errorf("%s.producer is required", fieldPrefix)) + } + if strings.TrimSpace(consumer.Dataset) == "" { + failures = append(failures, fmt.Errorf("%s.dataset is required", fieldPrefix)) + } else { + dataset := strings.TrimSpace(consumer.Dataset) + if _, exists := datasetIDs[dataset]; exists { + failures = append(failures, fmt.Errorf("%s.dataset %q is duplicated", fieldPrefix, dataset)) + } + datasetIDs[dataset] = struct{}{} + } + if strings.TrimSpace(consumer.Producer) != "" { + if _, exists := producerIDs[strings.TrimSpace(consumer.Producer)]; len(producerIDs) > 0 && !exists { + failures = append(failures, fmt.Errorf("%s.producer %q does not match a configured producer", fieldPrefix, consumer.Producer)) + } + } + switch strings.TrimSpace(consumer.Mode) { + case "parts_rows", "accessory_visualeffects": + case "": + failures = append(failures, fmt.Errorf("%s.mode is required", fieldPrefix)) + default: + failures = append(failures, fmt.Errorf("%s.mode %q is not supported", fieldPrefix, consumer.Mode)) + } + if strings.TrimSpace(consumer.Root) == "" { + failures = append(failures, fmt.Errorf("%s.root is required", fieldPrefix)) + } + if len(consumer.Include) == 0 { + failures = append(failures, fmt.Errorf("%s.include must contain at least one glob", fieldPrefix)) + } + failures = append(failures, validateGlobList(fieldPrefix+".include", consumer.Include)...) + failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", consumer.Derive)...) + if strings.TrimSpace(consumer.Mode) == "accessory_visualeffects" && headVisualeffectsConfigConfigured(consumer.HeadVisualeffects) { + failures = append(failures, validateHeadVisualeffectsConfig(fieldPrefix+".accessory_visualeffects", consumer.HeadVisualeffects)...) + } + failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", consumer.Manifest)...) + } + + return failures +} + +func validateGeneratedConfig(cfg GeneratedConfig) []error { + var failures []error + seen := map[string]struct{}{} + for index, top2da := range cfg.TopData2DA { + fieldPrefix := fmt.Sprintf("generated_assets.topdata_2da[%d]", index) + id := strings.TrimSpace(top2da.ID) + if id == "" { + failures = append(failures, fmt.Errorf("%s.id is required", fieldPrefix)) + } else { + if _, exists := seen[id]; exists { + failures = append(failures, fmt.Errorf("%s.id %q is duplicated", fieldPrefix, id)) + } + seen[id] = struct{}{} + } + if strings.TrimSpace(top2da.Source) == "" { + failures = append(failures, fmt.Errorf("%s.source is required", fieldPrefix)) + } + failures = append(failures, validateTreeRootPath(fieldPrefix+".source", top2da.Source)...) + if strings.TrimSpace(top2da.Output) == "" { + failures = append(failures, fmt.Errorf("%s.output is required", fieldPrefix)) + } + failures = append(failures, validateRelativePath(fieldPrefix+".output", top2da.Output)...) + if strings.TrimSpace(top2da.PackageRoot) == "" { + failures = append(failures, fmt.Errorf("%s.package_root is required", fieldPrefix)) + } + failures = append(failures, validateTreeRootPath(fieldPrefix+".package_root", top2da.PackageRoot)...) + if len(top2da.IncludeDatasets) == 0 { + failures = append(failures, fmt.Errorf("%s.include_datasets must contain at least one pattern", fieldPrefix)) + } + failures = append(failures, validateGlobList(fieldPrefix+".include_datasets", top2da.IncludeDatasets)...) + autogenMode := strings.TrimSpace(top2da.Autogen.Mode) + switch autogenMode { + case "parts_rows", "cachedmodels_rows": + case "": + continue + default: + failures = append(failures, fmt.Errorf("%s.autogen.mode %q is not supported", fieldPrefix, top2da.Autogen.Mode)) + } + if strings.TrimSpace(top2da.Autogen.Root) == "" { + failures = append(failures, fmt.Errorf("%s.autogen.root is required", fieldPrefix)) + } + failures = append(failures, validateTreeRootPath(fieldPrefix+".autogen.root", top2da.Autogen.Root)...) + if len(top2da.Autogen.Include) == 0 { + failures = append(failures, fmt.Errorf("%s.autogen.include must contain at least one glob", fieldPrefix)) + } + failures = append(failures, validateGlobList(fieldPrefix+".autogen.include", top2da.Autogen.Include)...) + failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".autogen.derive", top2da.Autogen.Derive)...) + if autogenMode == "parts_rows" && partsRowsConfigConfigured(top2da.Autogen.PartsRows) { + failures = append(failures, validatePartsRowsConfig(fieldPrefix+".autogen.parts_rows", top2da.Autogen.PartsRows)...) + } + } + return failures +} + +func headVisualeffectsConfigConfigured(cfg HeadVisualeffectsConfig) bool { + return len(cfg.Groups) > 0 || + strings.TrimSpace(cfg.GroupTokenSource) != "" || + strings.TrimSpace(cfg.CategoryFrom) != "" || + strings.TrimSpace(cfg.Delimiter) != "" || + strings.TrimSpace(cfg.Case) != "" || + len(cfg.StripModelPrefixes) > 0 || + strings.TrimSpace(cfg.KeyFormat) != "" || + strings.TrimSpace(cfg.LabelFormat) != "" || + strings.TrimSpace(cfg.ModelColumn) != "" || + len(cfg.RowDefaults) > 0 +} + +func validateHeadVisualeffectsConfig(fieldPrefix string, cfg HeadVisualeffectsConfig) []error { + var failures []error + groupTokenSource := strings.TrimSpace(cfg.GroupTokenSource) + switch groupTokenSource { + case "", "prefix", "folder_name": + default: + failures = append(failures, fmt.Errorf("%s.group_token_source %q is not supported", fieldPrefix, cfg.GroupTokenSource)) + } + categoryFrom := strings.TrimSpace(cfg.CategoryFrom) + switch categoryFrom { + case "", "none", "immediate_parent", "full_subgroup": + default: + failures = append(failures, fmt.Errorf("%s.category_from %q is not supported", fieldPrefix, cfg.CategoryFrom)) + } + if strings.TrimSpace(cfg.Delimiter) == "" && cfg.Delimiter != "" { + failures = append(failures, fmt.Errorf("%s.delimiter must not be only whitespace", fieldPrefix)) + } + if strings.ContainsAny(cfg.Delimiter, " \t\r\n\x00") { + failures = append(failures, fmt.Errorf("%s.delimiter must not contain whitespace or NUL bytes", fieldPrefix)) + } + caseMode := strings.TrimSpace(cfg.Case) + switch caseMode { + case "", "preserve", "lower", "upper": + default: + failures = append(failures, fmt.Errorf("%s.case %q is not supported", fieldPrefix, cfg.Case)) + } + for group, groupCfg := range cfg.Groups { + group = strings.TrimSpace(group) + if group == "" { + failures = append(failures, fmt.Errorf("%s.groups contains an empty group key", fieldPrefix)) + continue + } + failures = append(failures, validateTreeRootPath(fieldPrefix+".groups["+group+"]", group)...) + prefix := strings.TrimSpace(groupCfg.Prefix) + if prefix == "" && groupTokenSource != "folder_name" { + failures = append(failures, fmt.Errorf("%s.groups[%s].prefix is required", fieldPrefix, group)) + } + if strings.ContainsAny(prefix, " \t\r\n\x00") { + failures = append(failures, fmt.Errorf("%s.groups[%s].prefix must not contain whitespace or NUL bytes", fieldPrefix, group)) + } + if column := strings.TrimSpace(groupCfg.ModelColumn); column != "" && strings.ContainsAny(column, " \t\r\n\x00") { + failures = append(failures, fmt.Errorf("%s.groups[%s].model_column must not contain whitespace or NUL bytes", fieldPrefix, group)) + } + for index, column := range groupCfg.ModelColumns { + column = strings.TrimSpace(column) + if column == "" { + failures = append(failures, fmt.Errorf("%s.groups[%s].model_columns[%d] must not be empty", fieldPrefix, group, index)) + } + if strings.ContainsAny(column, " \t\r\n\x00") { + failures = append(failures, fmt.Errorf("%s.groups[%s].model_columns[%d] must not contain whitespace or NUL bytes", fieldPrefix, group, index)) + } + } + for column := range groupCfg.RowDefaults { + column = strings.TrimSpace(column) + if column == "" { + failures = append(failures, fmt.Errorf("%s.groups[%s].row_defaults contains an empty column name", fieldPrefix, group)) + } + if strings.ContainsAny(column, " \t\r\n\x00") { + failures = append(failures, fmt.Errorf("%s.groups[%s].row_defaults[%s] column name must not contain whitespace or NUL bytes", fieldPrefix, group, column)) + } + } + } + for index, prefix := range cfg.StripModelPrefixes { + if strings.TrimSpace(prefix) == "" { + failures = append(failures, fmt.Errorf("%s.strip_model_prefixes[%d] must not be empty", fieldPrefix, index)) + } + if strings.Contains(prefix, "\x00") { + failures = append(failures, fmt.Errorf("%s.strip_model_prefixes[%d] must not contain NUL bytes", fieldPrefix, index)) + } + } + if column := strings.TrimSpace(cfg.ModelColumn); column != "" && strings.ContainsAny(column, " \t\r\n\x00") { + failures = append(failures, fmt.Errorf("%s.model_column must not contain whitespace or NUL bytes", fieldPrefix)) + } + for column := range cfg.RowDefaults { + column = strings.TrimSpace(column) + if column == "" { + failures = append(failures, fmt.Errorf("%s.row_defaults contains an empty column name", fieldPrefix)) + } + if strings.ContainsAny(column, " \t\r\n\x00") { + failures = append(failures, fmt.Errorf("%s.row_defaults[%s] column name must not contain whitespace or NUL bytes", fieldPrefix, column)) + } + } + return failures +} + +func partsRowsConfigConfigured(cfg PartsRowsConfig) bool { + if len(cfg.RowDefaults) > 0 { + return true + } + if strings.TrimSpace(cfg.ACBonus.Default.Strategy) != "" { + return true + } + if len(cfg.ACBonus.Datasets) > 0 { + return true + } + if len(cfg.Datasets) > 0 { + return true + } + return false +} + +func validatePartsRowsConfig(fieldPrefix string, cfg PartsRowsConfig) []error { + var failures []error + if partsRowsProjectACBonusConfigured(cfg) { + failures = append(failures, validatePartsRowsACBonusPolicy(fieldPrefix+".acbonus.default", cfg.ACBonus.Default, false)...) + for dataset, policy := range cfg.ACBonus.Datasets { + dataset = strings.TrimSpace(dataset) + if dataset == "" { + failures = append(failures, fmt.Errorf("%s.acbonus.datasets contains an empty dataset key", fieldPrefix)) + continue + } + failures = append(failures, validatePartsRowsACBonusPolicy(fieldPrefix+".acbonus.datasets."+dataset, policy, true)...) + } + for dataset, datasetCfg := range cfg.Datasets { + dataset = strings.TrimSpace(dataset) + if dataset == "" { + failures = append(failures, fmt.Errorf("%s.datasets contains an empty dataset key", fieldPrefix)) + continue + } + failures = append(failures, validatePartsRowsACBonusPolicy(fieldPrefix+".datasets."+dataset+".acbonus", datasetCfg.ACBonus, true)...) + } + } + return failures +} + +func partsRowsProjectACBonusConfigured(cfg PartsRowsConfig) bool { + if strings.TrimSpace(cfg.ACBonus.Default.Strategy) != "" { + return true + } + if len(cfg.ACBonus.Datasets) > 0 { + return true + } + for _, dataset := range cfg.Datasets { + if strings.TrimSpace(dataset.ACBonus.Strategy) != "" { + return true + } + } + return false +} + +func validatePartsRowsACBonusPolicy(fieldPrefix string, policy PartsRowsACBonusPolicy, allowEmpty bool) []error { + strategy := strings.TrimSpace(policy.Strategy) + if strategy == "" { + if allowEmpty { + return nil + } + return []error{fmt.Errorf("%s.strategy is required", fieldPrefix)} + } + switch strategy { + case "ascending_row_id_sort_key", "descending_row_id_sort_key": + var failures []error + if strategy == "descending_row_id_sort_key" && policy.MaxRowID <= 0 { + failures = append(failures, fmt.Errorf("%s.max_row_id must be greater than 0", fieldPrefix)) + } + if policy.Divisor <= 0 { + failures = append(failures, fmt.Errorf("%s.divisor must be greater than 0", fieldPrefix)) + } + if err := validatePartsRowsFormat(policy.Format); err != nil { + failures = append(failures, fmt.Errorf("%s.format %q is invalid: %w", fieldPrefix, policy.Format, err)) + } + return failures + case "fixed": + if strings.TrimSpace(policy.Value) == "" { + return []error{fmt.Errorf("%s.value is required for fixed strategy", fieldPrefix)} + } + return nil + default: + return []error{fmt.Errorf("%s.strategy %q is not supported", fieldPrefix, policy.Strategy)} + } +} + +func validatePartsRowsFormat(format string) error { + if strings.TrimSpace(format) == "" { + return fmt.Errorf("must not be empty") + } + formatted := fmt.Sprintf(format, 1.25) + if strings.Contains(formatted, "%!") { + return fmt.Errorf("must format a numeric value") + } + return nil +} + +func validateMusicConfig(cfg MusicConfig) []error { + var failures []error + if cfg.Defaults != nil { + if cfg.Defaults.MaxStemLength != nil && *cfg.Defaults.MaxStemLength < 3 { + failures = append(failures, errors.New("music.defaults.max_stem_length must be at least 3")) + } + if cfg.Defaults.NamingScheme != "" && cfg.Defaults.NamingScheme != "nwn_bmu" && cfg.Defaults.NamingScheme != "passthrough" { + failures = append(failures, fmt.Errorf("music.defaults.naming_scheme %q is not supported", cfg.Defaults.NamingScheme)) + } + if cfg.Defaults.OutputExtension != "" && !strings.HasPrefix(cfg.Defaults.OutputExtension, ".") { + failures = append(failures, errors.New("music.defaults.output_extension must start with .")) + } + } + normalizedRoots := map[string]string{} + for root, prefix := range cfg.Prefixes { + normalizedRoot := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(root))) + if normalizedRoot == "" { + failures = append(failures, errors.New("music.prefixes contains an empty root")) + continue + } + if prior, exists := normalizedRoots[normalizedRoot]; exists { + failures = append(failures, fmt.Errorf("music.prefixes roots %q and %q normalize to the same path", prior, root)) + } + normalizedRoots[normalizedRoot] = root + if strings.TrimSpace(prefix) == "" { + failures = append(failures, fmt.Errorf("music.prefixes[%q] must not be empty", root)) + } + } + seenSources := map[string]string{} + for id, ds := range cfg.Datasets { + if strings.TrimSpace(id) == "" { + failures = append(failures, errors.New("music.datasets contains an empty key")) + } else if strings.Contains(id, "/") || strings.Contains(id, "\\") { + failures = append(failures, fmt.Errorf("music.datasets id %q must not contain path separators", id)) + } + source := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Source))) + if source == "" { + failures = append(failures, fmt.Errorf("music.datasets[%q].source is required", id)) + } else if pathEscapesRoot(source) { + failures = append(failures, fmt.Errorf("music.datasets[%q].source must not escape project root", id)) + } else if prior, exists := seenSources[source]; exists { + failures = append(failures, fmt.Errorf("music.datasets[%q].source %q conflicts with dataset %q", id, ds.Source, prior)) + } else { + seenSources[source] = id + } + output := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Output))) + if output != "" && pathEscapesRoot(output) { + failures = append(failures, fmt.Errorf("music.datasets[%q].output must not escape project root", id)) + } + if ds.NamingScheme != "" && ds.NamingScheme != "nwn_bmu" && ds.NamingScheme != "passthrough" { + failures = append(failures, fmt.Errorf("music.datasets[%q].naming_scheme %q is not supported", id, ds.NamingScheme)) + } + if ds.PackageMode != "" && ds.PackageMode != "hak_asset" && ds.PackageMode != "none" { + failures = append(failures, fmt.Errorf("music.datasets[%q].package_mode %q is not supported", id, ds.PackageMode)) + } + if ds.OutputExtension != "" && !strings.HasPrefix(ds.OutputExtension, ".") { + failures = append(failures, fmt.Errorf("music.datasets[%q].output_extension must start with .", id)) + } + } + return failures +} + +func pathEscapesRoot(path string) bool { + clean := filepath.Clean(filepath.FromSlash(path)) + return clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) +} + +func validateGlobList(field string, patterns []string) []error { + var failures []error + seen := map[string]struct{}{} + for index, pattern := range patterns { + trimmed := strings.TrimSpace(pattern) + if trimmed == "" { + failures = append(failures, fmt.Errorf("%s[%d] must not be empty", field, index)) + continue + } + if strings.Contains(trimmed, "\x00") { + failures = append(failures, fmt.Errorf("%s[%d] must not contain NUL bytes", field, index)) + } + if _, exists := seen[trimmed]; exists { + failures = append(failures, fmt.Errorf("%s[%d] duplicates %q", field, index, trimmed)) + } + seen[trimmed] = struct{}{} + } + return failures +} + +func trimConfigSlashes(path string) string { + return strings.Trim(path, "/") +} + +func validateAutogenDeriveConfig(fieldPrefix string, cfg AutogenDeriveConfig) []error { + var failures []error + switch strings.TrimSpace(cfg.Kind) { + case "trailing_numeric_suffix", "model_stem": + case "": + failures = append(failures, fmt.Errorf("%s.kind is required", fieldPrefix)) + default: + failures = append(failures, fmt.Errorf("%s.kind %q is not supported", fieldPrefix, cfg.Kind)) + } + switch strings.TrimSpace(cfg.GroupFrom) { + case "", "first_path_segment": + default: + failures = append(failures, fmt.Errorf("%s.group_from %q is not supported", fieldPrefix, cfg.GroupFrom)) + } + return failures +} + +func validateAutogenManifestConfig(fieldPrefix string, cfg AutogenManifestConfig) []error { + var failures []error + if strings.TrimSpace(cfg.ReleaseTag) == "" { + failures = append(failures, fmt.Errorf("%s.release_tag is required", fieldPrefix)) + } + if err := validateOutputFileName(fieldPrefix+".asset_name", cfg.AssetName, ".json"); err != nil { + failures = append(failures, err) + } + if err := validateOutputFileName(fieldPrefix+".cache_name", cfg.CacheName, ".json"); err != nil { + failures = append(failures, err) + } + return failures +} + +func scanDir(root string, include func(path string) bool) ([]string, []string, error) { + var files []string + extSet := map[string]struct{}{} + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if !include(path) { + return nil + } + + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + files = append(files, rel) + + if strings.HasSuffix(rel, ".json") { + base := strings.TrimSuffix(filepath.Base(rel), ".json") + ext := strings.ToLower(filepath.Ext(base)) + if ext != "" { + extSet[ext] = struct{}{} + } + } + + return nil + }) + if err != nil { + return nil, nil, err + } + + slices.Sort(files) + + exts := make([]string, 0, len(extSet)) + for ext := range extSet { + exts = append(exts, ext) + } + slices.Sort(exts) + return files, exts, nil +} + +func validateOutputFileName(field, name, expectedExt string) error { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + return fmt.Errorf("%s is required", field) + } + if filepath.Base(trimmed) != trimmed { + return fmt.Errorf("%s must be a file name, not a path: %q", field, name) + } + if !strings.EqualFold(filepath.Ext(trimmed), expectedExt) { + return fmt.Errorf("%s must use %s extension: %q", field, expectedExt, name) + } + return nil +} + +func pathIsUnder(root, rel string) bool { + root = strings.Trim(strings.TrimSpace(filepath.ToSlash(root)), "/") + rel = strings.Trim(strings.TrimSpace(filepath.ToSlash(rel)), "/") + if root == "" || rel == "" { + return false + } + return rel == root || strings.HasPrefix(rel, root+"/") +} + +func validateRelativePath(field, path string) []error { + var failures []error + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return failures + } + if filepath.IsAbs(trimmed) { + failures = append(failures, fmt.Errorf("%s must be relative to the repository root: %q", field, path)) + } + clean := filepath.Clean(filepath.FromSlash(trimmed)) + if clean == "." { + return failures + } + if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + failures = append(failures, fmt.Errorf("%s must not escape the repository root: %q", field, path)) + } + return failures +} + +func validateTreeRootPath(field, path string) []error { + failures := validateRelativePath(field, path) + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return failures + } + if filepath.Clean(filepath.FromSlash(trimmed)) == "." { + failures = append(failures, fmt.Errorf("%s must not resolve to the repository root: %q", field, path)) + } + return failures +} + +func sameCleanPath(a, b string) bool { + return filepath.Clean(a) == filepath.Clean(b) +} diff --git a/internal/project/project_test.go b/internal/project/project_test.go new file mode 100644 index 0000000..833c7cc --- /dev/null +++ b/internal/project/project_test.go @@ -0,0 +1,1830 @@ +package project + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" +) + +func TestLoadYAMLConfig(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: output +haks: + - name: core + priority: 1 + max_bytes: 1024 + split: false + include: + - core/** +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if proj.ConfigSource.Name != ConfigFile || proj.ConfigSource.Format != "yaml" || proj.ConfigSource.Legacy { + t.Fatalf("unexpected config source: %#v", proj.ConfigSource) + } + if got, want := proj.Config.Module.Name, "Test Module"; got != want { + t.Fatalf("expected module name %q, got %q", want, got) + } + if got, want := proj.BuildDir(), filepath.Join(root, "output"); got != want { + t.Fatalf("expected build dir %s, got %s", want, got) + } +} + +func TestEffectiveConfigAppliesVisibleToolkitDefaults(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + effective := proj.EffectiveConfig() + if got, want := effective.Paths.Build, "build"; got != want { + t.Fatalf("expected default build path %q, got %q", want, got) + } + if got, want := effective.Outputs.HAKManifest, "haks.json"; got != want { + t.Fatalf("expected default HAK manifest %q, got %q", want, got) + } + if got, want := strings.Join(effective.Extract.Archives, ","), "testmod.mod"; got != want { + t.Fatalf("expected default extract archives %q, got %q", want, got) + } + if effective.Extract.ConsumeArchives == nil || *effective.Extract.ConsumeArchives { + t.Fatalf("expected default extract consume_archives false, got %#v", effective.Extract.ConsumeArchives) + } + if got, want := effective.TopData.PackageHAK, "sow_top.hak"; got != want { + t.Fatalf("expected default topdata HAK %q, got %q", want, got) + } + if got, want := strings.Join(effective.Validation.BuiltinScriptPrefixes, ","), "ga_,gc_,gen_,gui_,nw_,nwg_,ta_,x0_,x1_,x2_,x3_"; got != want { + t.Fatalf("expected default built-in script prefixes %q, got %q", want, got) + } + if got := strings.Join(effective.Validation.RequiredFields["ifo"], ","); got != "Mod_Name" { + t.Fatalf("expected default IFO required fields, got %#v", effective.Validation.RequiredFields) + } + if got, want := strings.Join(effective.Music.ConvertExtensions, ","), ".flac,.m4a,.mp3,.ogg,.wav"; got != want { + t.Fatalf("expected default music convert extensions %q, got %q", want, got) + } + if prov := effective.Provenance["paths.build"]; prov.Source != "toolkit default" { + t.Fatalf("expected paths.build toolkit default provenance, got %#v", prov) + } + if prov := effective.Provenance["module.name"]; prov.Source != "yaml" { + t.Fatalf("expected module.name YAML provenance, got %#v", prov) + } +} + +func TestEffectiveConfigHonorsValidationConfig(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src +validation: + builtin_script_prefixes: + - custom_ + - NW_ + required_fields: + ifo: + - Mod_Name + - Mod_Hak +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + got := strings.Join(proj.EffectiveConfig().Validation.BuiltinScriptPrefixes, ",") + if want := "custom_,nw_"; got != want { + t.Fatalf("expected configured validation prefixes %q, got %q", want, got) + } + if got := strings.Join(proj.EffectiveConfig().Validation.RequiredFields["ifo"], ","); got != "Mod_Hak,Mod_Name" { + t.Fatalf("expected configured required fields, got %#v", proj.EffectiveConfig().Validation.RequiredFields) + } +} + +func TestEffectiveConfigIncludesTopDataValueEncodings(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +topdata: + source: topdata + value_encodings: + - dataset: racialtypes/core + column: AvailableSkinColors + mode: packed_hex_list + min: 0 + max: 175 + hex_width: 2 +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + encodings := proj.EffectiveConfig().TopData.ValueEncodings + if len(encodings) != 1 { + t.Fatalf("expected one topdata value encoding, got %#v", encodings) + } + got := encodings[0] + if got.Dataset != "racialtypes/core" || + got.Column != "AvailableSkinColors" || + got.Mode != "packed_hex_list" || + got.Min != 0 || + got.Max != 175 || + got.HexWidth != 2 { + t.Fatalf("unexpected topdata value encoding: %#v", got) + } +} + +func TestEffectiveConfigIncludesTopDataValueDefaults(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +topdata: + source: topdata + value_defaults: + - dataset: racialtypes/core + column: ECL + value: 0 +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + defaults := proj.EffectiveConfig().TopData.ValueDefaults + if len(defaults) != 1 { + t.Fatalf("expected one topdata value default, got %#v", defaults) + } + got := defaults[0] + if got.Dataset != "racialtypes/core" || got.Column != "ECL" || got.Value != 0 { + t.Fatalf("unexpected topdata value default: %#v", got) + } +} + +func TestEffectiveConfigIncludesTopDataRowGeneration(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +topdata: + source: topdata + row_generation: + - namespace: portraits + mode: first_null_row + minimum_row: 15000 +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + rules := proj.EffectiveConfig().TopData.RowGeneration + if len(rules) != 1 { + t.Fatalf("expected one topdata row generation rule, got %#v", rules) + } + got := rules[0] + if got.Dataset != "portraits" || got.Namespace != "portraits" || got.Mode != "first_null_row" || got.MinimumRow != 15000 { + t.Fatalf("unexpected topdata row generation rule: %#v", got) + } +} + +func TestEffectiveConfigIncludesTopDataRowExtensions(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +topdata: + source: topdata + row_extensions: + - dataset: classes/spellsgained/* + mode: repeat_last + level_column: Level + target_level: 60 +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + rules := proj.EffectiveConfig().TopData.RowExtensions + if len(rules) != 1 { + t.Fatalf("expected one topdata row extension rule, got %#v", rules) + } + got := rules[0] + if got.Dataset != "classes/spellsgained/*" || got.Mode != "repeat_last" || got.LevelColumn != "Level" || got.TargetLevel != 60 { + t.Fatalf("unexpected topdata row extension rule: %#v", got) + } +} + +func TestEffectiveConfigIncludesTopDataClassFeatInjections(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +topdata: + source: topdata + class_feat_injections: + global_feats: + - feat: feat:literate + list: 3 + granted_on_level: 1 + on_menu: 0 + unless_present: [feat:illiterate] + class_skill_masterfeats: + - masterfeat: masterfeats:skillfocus + list: 1 + granted_on_level: -1 + on_menu: 0 +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + injections := proj.EffectiveConfig().TopData.ClassFeatInjections + if len(injections.GlobalFeats) != 1 || len(injections.ClassSkillMasterfeats) != 1 { + t.Fatalf("expected configured class feat injections, got %#v", injections) + } + if got := injections.GlobalFeats[0]; got.Feat != "feat:literate" || got.List != "3" || got.GrantedOnLevel != "1" || got.OnMenu != "0" || len(got.UnlessPresent) != 1 || got.UnlessPresent[0] != "feat:illiterate" { + t.Fatalf("unexpected global class feat injection: %#v", got) + } + if got := injections.ClassSkillMasterfeats[0]; got.Masterfeat != "masterfeats:skillfocus" || got.List != "1" || got.GrantedOnLevel != "-1" || got.OnMenu != "0" { + t.Fatalf("unexpected class skill masterfeat injection: %#v", got) + } +} + +func TestValidateLayoutRejectsInvalidTopDataValueEncodings(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "topdata")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Build: "build"}, + TopData: TopDataConfig{ + Source: "topdata", + ValueEncodings: []TopDataValueEncodingConfig{ + {Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 256, HexWidth: 2}, + {Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 175, HexWidth: 2}, + {Dataset: "", Column: "AvailableHeadsMale", Mode: "unknown", Min: -1, Max: 999, HexWidth: 0}, + }, + ValueDefaults: []TopDataValueDefaultConfig{ + {Dataset: "racialtypes/core", Column: "ECL", Value: 0}, + {Dataset: "racialtypes/core", Column: "ECL", Value: 1}, + {Dataset: "", Column: "", Value: 0}, + }, + RowGeneration: []TopDataRowGenerationConfig{ + {Dataset: "portraits", Mode: "first_null_row"}, + {Dataset: "portraits", Mode: "after_base"}, + {Namespace: "placeabletypes", Mode: "unsupported"}, + {Mode: "first_null_row"}, + {Dataset: "classes", Mode: "first_null_row", MinimumRow: -1}, + }, + RowExtensions: []TopDataRowExtensionConfig{ + {Dataset: "classes/spellsgained/*", Mode: "repeat_last", LevelColumn: "Level", TargetLevel: 60}, + {Dataset: "classes/spellsgained/*", Mode: "repeat_last", LevelColumn: "Level", TargetLevel: 60}, + {Dataset: "", Mode: "unsupported", LevelColumn: "", TargetLevel: 0}, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected invalid topdata value encoding validation error") + } + text := err.Error() + for _, want := range []string{ + "topdata.value_encodings[0].max does not fit in hex_width 2", + "topdata.value_encodings[1] duplicates an earlier dataset/column encoding", + "topdata.value_encodings[2].dataset is required", + "topdata.value_encodings[2].mode \"unknown\" is not supported", + "topdata.value_encodings[2].min must be zero or greater", + "topdata.value_encodings[2].hex_width must be greater than zero", + "topdata.value_defaults[1] duplicates an earlier dataset/column default", + "topdata.value_defaults[2].dataset is required", + "topdata.value_defaults[2].column is required", + "topdata.row_generation[1] duplicates an earlier dataset row generation rule", + "topdata.row_generation[2].mode \"unsupported\" is not supported", + "topdata.row_generation[3].dataset is required", + "topdata.row_generation[4].minimum_row must be zero or greater", + "topdata.row_extensions[1] duplicates an earlier dataset row extension rule", + "topdata.row_extensions[2].dataset is required", + "topdata.row_extensions[2].mode \"unsupported\" is not supported", + "topdata.row_extensions[2].level_column is required", + "topdata.row_extensions[2].target_level must be greater than zero", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected validation error %q, got %v", want, err) + } + } +} + +func TestValidateLayoutRejectsInvalidTopDataClassFeatInjections(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "topdata")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Build: "build"}, + TopData: TopDataConfig{ + Source: "topdata", + ClassFeatInjections: TopDataClassFeatInjectionConfig{ + GlobalFeats: []TopDataClassFeatGlobalRule{ + {Feat: "literate", List: "3", GrantedOnLevel: "1", OnMenu: "0", UnlessPresent: []string{"illiterate"}}, + {Feat: "feat:valid", List: "", GrantedOnLevel: "x", OnMenu: "0"}, + }, + ClassSkillMasterfeats: []TopDataClassFeatMasterfeatRule{ + {Masterfeat: "skillfocus", List: "1", GrantedOnLevel: "-1", OnMenu: ""}, + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected invalid class feat injection validation error") + } + text := err.Error() + for _, want := range []string{ + "topdata.class_feat_injections.global_feats[0].feat", + "topdata.class_feat_injections.global_feats[0].unless_present[0]", + "topdata.class_feat_injections.global_feats[1].list", + "topdata.class_feat_injections.global_feats[1].granted_on_level", + "topdata.class_feat_injections.class_skill_masterfeats[0].masterfeat", + "topdata.class_feat_injections.class_skill_masterfeats[0].on_menu", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected validation error %q, got %v", want, err) + } + } +} + +func TestScanUsesYAMLMusicConvertExtensionsWithinConfiguredDatasetRoots(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "assets", "audio", "westgate"), 0o755); err != nil { + t.Fatalf("mkdir westgate audio: %v", err) + } + if err := os.MkdirAll(filepath.Join(root, "assets", "audio", "other"), 0o755); err != nil { + t.Fatalf("mkdir other audio: %v", err) + } + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets +music: + defaults: + convert_extensions: + - .flac + datasets: + westgate_audio: + source: audio/westgate + convert_extensions: + - .aac +`) + writeProjectFile(t, filepath.Join(root, "assets", "audio", "westgate", "theme.aac"), "aac") + writeProjectFile(t, filepath.Join(root, "assets", "audio", "westgate", "theme.flac"), "flac") + writeProjectFile(t, filepath.Join(root, "assets", "audio", "other", "ignored.aac"), "aac") + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if err := proj.Scan(); err != nil { + t.Fatalf("Scan returned error: %v", err) + } + if !slices.Contains(proj.Inventory.AssetFiles, "audio/westgate/theme.aac") { + t.Fatalf("expected configured dataset AAC file to be discovered, got %#v", proj.Inventory.AssetFiles) + } + if !slices.Contains(proj.Inventory.AssetFiles, "audio/westgate/theme.flac") { + t.Fatalf("expected configured dataset FLAC file to be discovered, got %#v", proj.Inventory.AssetFiles) + } + if slices.Contains(proj.Inventory.AssetFiles, "audio/other/ignored.aac") { + t.Fatalf("did not expect non-dataset AAC file to be discovered, got %#v", proj.Inventory.AssetFiles) + } +} + +func TestEffectiveConfigHonorsConfiguredOutputAndCacheFields(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + build: output + cache: cache +outputs: + module_archive: modules/{module.resref}.mod + hak_manifest: manifests/haks.json + hak_archive: haks/{hak.name}.hak +topdata: + source: topdata + build: "{paths.cache}" + compiled_2da_dir: tables + compiled_tlk: dialog.tlk + package_hak: custom_top.hak + package_tlk: custom_dialog.tlk + wiki: + output_root: docs + pages_dir: pages + state_file: wiki-state.json + page_index_file: manifest/page-index.json + source: docs-src/wiki + data_file: providers/wiki-data.yaml + page_paths_file: paths/wiki-paths.yaml + page_templates_dir: page-fragments + manual_sections_file: sections/defaults.yaml +autogen: + cache: + root: "{paths.cache}/autogen" +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if got, want := proj.ModuleArchivePath(), filepath.Join(root, "output", "modules", "testmod.mod"); got != want { + t.Fatalf("expected module archive path %s, got %s", want, got) + } + if got, want := proj.HAKManifestPath(), filepath.Join(root, "output", "manifests", "haks.json"); got != want { + t.Fatalf("expected HAK manifest path %s, got %s", want, got) + } + if got, want := proj.HAKArchivePath("core_01"), filepath.Join(root, "output", "haks", "core_01.hak"); got != want { + t.Fatalf("expected HAK archive path %s, got %s", want, got) + } + if got, want := proj.TopDataCompiled2DADir(), filepath.Join(root, "cache", "tables"); got != want { + t.Fatalf("expected topdata 2da path %s, got %s", want, got) + } + if got, want := proj.TopDataWikiStatePath(), filepath.Join(root, "cache", "docs", "wiki-state.json"); got != want { + t.Fatalf("expected wiki state path %s, got %s", want, got) + } + if got, want := proj.TopDataWikiPageIndexPath(), filepath.Join(root, "cache", "docs", "manifest", "page-index.json"); got != want { + t.Fatalf("expected wiki page-index path %s, got %s", want, got) + } + if got, want := proj.TopDataWikiDataPath(), filepath.Join(root, "docs-src", "wiki", "providers", "wiki-data.yaml"); got != want { + t.Fatalf("expected wiki data path %s, got %s", want, got) + } + if got, want := proj.TopDataWikiPagePathsPath(), filepath.Join(root, "docs-src", "wiki", "paths", "wiki-paths.yaml"); got != want { + t.Fatalf("expected wiki page paths path %s, got %s", want, got) + } + if got, want := proj.TopDataWikiPageTemplatesDir(), filepath.Join(root, "docs-src", "wiki", "page-fragments"); got != want { + t.Fatalf("expected wiki page template path %s, got %s", want, got) + } + if got, want := proj.TopDataWikiManualSectionsPath(), filepath.Join(root, "docs-src", "wiki", "sections", "defaults.yaml"); got != want { + t.Fatalf("expected wiki manual sections path %s, got %s", want, got) + } + if got, want := proj.AutogenCachePath("manifest.json"), filepath.Join(root, "cache", "autogen", "manifest.json"); got != want { + t.Fatalf("expected autogen cache path %s, got %s", want, got) + } +} + +func TestEffectiveConfigHonorsConfiguredExtractWikiAndAutogenPolicy(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + tools: custom-tools +extract: + layout: nwn_canonical_json + hak_discovery: configured_haks + cleanup_stale: false + merge: + gff_json: + - target: module/module.ifo.json + preserve_fields: + - Mod_Entry_Area + merge_lists: + - field: Mod_Area_list + key_field: Area_Name + strategy: preserve_existing_order_append_new +topdata: + wiki: + source: topdata/wiki + managed_namespaces: + - skills + renderer: nodebb_tiptap_html + link_strategy: preserve_westgate_wiki_links + namespaces_file: namespaces.yaml + tables_file: custom-tables.yaml + visibility_file: visibility.yaml + templates_dir: templates + manual_sections_dir: manual-sections + deploy_manifest: wiki-manifest.json + deploy_edit_summary: Custom summary + title_prefix_min_length: 4 + status_pages: false + status_listing_scope: namespace + stale_pages: + default: report + live_cleanup: archive +autogen: + cache: + root: "{paths.cache}/released" + max_age: 30m + refresh_env: TEST_AUTOGEN_REFRESH + release_source: + provider: gitea + server_url_env: TEST_ASSETS_SERVER + repo_env: TEST_ASSETS_REPO +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + effective := proj.EffectiveConfig() + if got, want := effective.Extract.HAKDiscovery, "configured_haks"; got != want { + t.Fatalf("expected extract hak discovery %q, got %q", want, got) + } + if effective.Extract.CleanupStale == nil || *effective.Extract.CleanupStale { + t.Fatalf("expected cleanup_stale false, got %#v", effective.Extract.CleanupStale) + } + if got, want := len(effective.Extract.Merge.GFFJSON), 1; got != want { + t.Fatalf("expected %d gff json merge rule, got %d", want, got) + } + rule := effective.Extract.Merge.GFFJSON[0] + if got, want := rule.Target, "module/module.ifo.json"; got != want { + t.Fatalf("expected merge target %q, got %q", want, got) + } + if got, want := rule.PreserveFields, []string{"Mod_Entry_Area"}; !reflect.DeepEqual(got, want) { + t.Fatalf("expected preserve fields %#v, got %#v", want, got) + } + if got, want := rule.MergeLists[0].Strategy, "preserve_existing_order_append_new"; got != want { + t.Fatalf("expected merge list strategy %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.DeployManifest, "wiki-manifest.json"; got != want { + t.Fatalf("expected wiki deploy manifest %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.Source, "topdata/wiki"; got != want { + t.Fatalf("expected wiki source %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.Renderer, "nodebb_tiptap_html"; got != want { + t.Fatalf("expected wiki renderer %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.LinkStrategy, "preserve_westgate_wiki_links"; got != want { + t.Fatalf("expected wiki link strategy %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.NamespacesFile, "namespaces.yaml"; got != want { + t.Fatalf("expected wiki namespaces file %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.TablesFile, "custom-tables.yaml"; got != want { + t.Fatalf("expected wiki tables file %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.VisibilityFile, "visibility.yaml"; got != want { + t.Fatalf("expected wiki visibility file %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.TemplatesDir, "templates"; got != want { + t.Fatalf("expected wiki templates dir %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.ManualSectionsDir, "manual-sections"; got != want { + t.Fatalf("expected wiki manual sections dir %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.StalePages.Default, "report"; got != want { + t.Fatalf("expected wiki stale default %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.StalePages.LiveCleanup, "archive"; got != want { + t.Fatalf("expected wiki stale live cleanup %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.TitlePrefixMinLength, 4; got != want { + t.Fatalf("expected wiki title prefix minimum length %d, got %d", want, got) + } + if effective.TopData.Wiki.StatusPages == nil || *effective.TopData.Wiki.StatusPages { + t.Fatalf("expected wiki status pages to be disabled, got %#v", effective.TopData.Wiki.StatusPages) + } + if got, want := effective.TopData.Wiki.StatusListingScope, "namespace"; got != want { + t.Fatalf("expected wiki status listing scope %q, got %q", want, got) + } + if got, want := effective.Autogen.Cache.MaxAge, "30m"; got != want { + t.Fatalf("expected autogen cache max age %q, got %q", want, got) + } + if got, want := proj.AutogenRefreshEnv(), "TEST_AUTOGEN_REFRESH"; got != want { + t.Fatalf("expected autogen refresh env %q, got %q", want, got) + } +} + +func TestProjectValidationAcceptsWikiStalePurgePolicy(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +topdata: + wiki: + stale_pages: + default: report + live_cleanup: purge +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := proj.ValidateLayout(); err != nil { + t.Fatalf("validate purge stale policy: %v", err) + } +} + +func TestProjectValidationRejectsUnsupportedWikiStalePolicies(t *testing.T) { + for _, policy := range []string{"unpublish", "remove-everything"} { + t.Run(policy, func(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + writeProjectFile(t, filepath.Join(root, ConfigFile), fmt.Sprintf(` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +topdata: + wiki: + stale_pages: + live_cleanup: %s +`, policy)) + + proj, err := Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + err = proj.ValidateLayout() + if err == nil || !strings.Contains(err.Error(), policy) { + t.Fatalf("expected %q stale policy validation failure, got %v", policy, err) + } + }) + } +} + +func TestValidateLayoutRejectsInvalidExtractMergeRules(t *testing.T) { + tests := []struct { + name string + config string + wantErr string + }{ + { + name: "empty target", + config: ` +extract: + merge: + gff_json: + - target: "" + preserve_fields: [Mod_Entry_Area] +`, + wantErr: "extract.merge.gff_json[0].target must not be empty", + }, + { + name: "duplicate target", + config: ` +extract: + merge: + gff_json: + - target: module/module.ifo.json + - target: module/module.ifo.json +`, + wantErr: "extract.merge.gff_json target \"module/module.ifo.json\" is configured more than once", + }, + { + name: "empty merge list field", + config: ` +extract: + merge: + gff_json: + - target: module/module.ifo.json + merge_lists: + - field: "" + key_field: Area_Name + strategy: preserve_existing_order_append_new +`, + wantErr: "extract.merge.gff_json[0].merge_lists[0].field must not be empty", + }, + { + name: "unsupported strategy", + config: ` +extract: + merge: + gff_json: + - target: module/module.ifo.json + merge_lists: + - field: Mod_Area_list + key_field: Area_Name + strategy: replace +`, + wantErr: "extract.merge.gff_json[0].merge_lists[0].strategy \"replace\" is not supported", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +`+tt.config) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + err = proj.ValidateLayout() + if err == nil { + t.Fatalf("expected validation error containing %q", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected validation error containing %q, got %v", tt.wantErr, err) + } + }) + } +} + +func TestActiveOverridesAreVisibleAndSensitiveValuesAreMasked(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src +`) + t.Setenv("SOW_BUILD_HAKS_KEEP_EXISTING", "1") + t.Setenv("NODEBB_API_TOKEN", "secret-token") + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + overrides := proj.ActiveOverrides() + var sawKeepExisting, sawMaskedToken bool + for _, override := range overrides { + if override.Key == "build.keep_existing_haks" && override.Value == "1" { + sawKeepExisting = true + } + if override.Key == "wiki.token" && override.Value == "" { + sawMaskedToken = true + } + } + if !sawKeepExisting { + t.Fatalf("expected build keep existing override in %#v", overrides) + } + if !sawMaskedToken { + t.Fatalf("expected masked wiki token override in %#v", overrides) + } +} + +func TestEffectiveConfigJSONIsDeterministic(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + first, err := proj.EffectiveConfigJSON() + if err != nil { + t.Fatalf("EffectiveConfigJSON returned error: %v", err) + } + second, err := proj.EffectiveConfigJSON() + if err != nil { + t.Fatalf("EffectiveConfigJSON returned error: %v", err) + } + if !bytes.Equal(first, second) { + t.Fatalf("effective config JSON is not deterministic") + } + if !bytes.Contains(first, []byte(`"hak_manifest": "haks.json"`)) { + t.Fatalf("effective config JSON missing HAK manifest default: %s", first) + } +} + +func TestLoadYMLConfig(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFileYML), ` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if proj.ConfigSource.Name != ConfigFileYML || proj.ConfigSource.Format != "yaml" || proj.ConfigSource.Legacy { + t.Fatalf("unexpected config source: %#v", proj.ConfigSource) + } +} + +func TestLoadLegacyJSONConfig(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, LegacyConfigFile), `{ + "module": { + "name": "Legacy Module", + "resref": "legacy" + }, + "paths": { + "source": "src", + "build": "build" + } +}`+"\n") + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if proj.ConfigSource.Name != LegacyConfigFile || proj.ConfigSource.Format != "json" || !proj.ConfigSource.Legacy { + t.Fatalf("unexpected config source: %#v", proj.ConfigSource) + } +} + +func TestLoadPrefersYAMLOverLegacyJSON(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, LegacyConfigFile), `{ + "module": { + "name": "Legacy Module", + "resref": "legacy" + } +}`+"\n") + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: YAML Module + resref: yamlmod +paths: + source: src + build: build +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if proj.ConfigSource.Name != ConfigFile { + t.Fatalf("expected YAML config to win, got %#v", proj.ConfigSource) + } + if got, want := proj.Config.Module.ResRef, "yamlmod"; got != want { + t.Fatalf("expected resref %q, got %q", want, got) + } +} + +func TestLoadRejectsMalformedYAML(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), "module:\n name: Test\n resref: bad\n") + + if _, err := Load(root); err == nil { + t.Fatal("expected malformed YAML error") + } +} + +func TestLoadRejectsUnknownYAMLFields(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +unexpected: true +`) + + err := loadAndValidate(root) + if err == nil { + t.Fatal("expected unknown field error") + } + if !strings.Contains(err.Error(), "field unexpected not found") { + t.Fatalf("expected unknown field error, got %v", err) + } +} + +func TestFindRootUsesYAMLConfig(t *testing.T) { + root := t.TempDir() + child := filepath.Join(root, "nested", "dir") + mkdirAll(t, child) + writeProjectFile(t, filepath.Join(root, ConfigFile), "module:\n name: Test\n resref: test\n") + + got, err := FindRoot(child) + if err != nil { + t.Fatalf("FindRoot returned error: %v", err) + } + if got != root { + t.Fatalf("expected root %s, got %s", root, got) + } +} + +func TestTopDataAssetsDirPrefersExplicitConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + mkdirAll(t, filepath.Join(root, "custom-assets")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: TopDataConfig{ + Assets: "custom-assets", + }, + }, + } + + result := proj.TopDataAssetsDir() + expected := filepath.Join(root, "custom-assets") + if result != expected { + t.Errorf("expected %s, got %s", expected, result) + } +} + +func TestTopDataAssetsDirDoesNotAssumeSiblingRepo(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + + parentDir := filepath.Dir(root) + siblingPath := filepath.Join(parentDir, "sow-assets") + mkdirAll(t, siblingPath) + mkdirAll(t, filepath.Join(siblingPath, "assets", "part", "belt")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: TopDataConfig{ + Assets: "sow-assets", + }, + }, + } + + result := proj.TopDataAssetsDir() + expected := filepath.Join(root, "sow-assets") + if result != expected { + t.Errorf("expected unresolved local path %s, got %s", expected, result) + } +} + +func TestTopDataAssetsDirReturnsEmptyForNoConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: TopDataConfig{}, + }, + } + + result := proj.TopDataAssetsDir() + if result != "" { + t.Errorf("expected empty string, got %s", result) + } +} + +func TestModuleArchivePathUsesConfiguredBuildDirAndResRef(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "output")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "testmod"}, + Paths: PathConfig{Build: "output"}, + }, + } + + got := proj.ModuleArchivePath() + want := filepath.Join(root, "output", "testmod.mod") + if got != want { + t.Fatalf("expected %s, got %s", want, got) + } +} + +func TestHAKPathsUseConfiguredBuildDir(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "output")) + + proj := &Project{ + Root: root, + Config: Config{ + Paths: PathConfig{Build: "output"}, + }, + } + + if got, want := proj.HAKManifestPath(), filepath.Join(root, "output", "haks.json"); got != want { + t.Fatalf("expected manifest path %s, got %s", want, got) + } + if got, want := proj.HAKArchivePath("core_01"), filepath.Join(root, "output", "core_01.hak"); got != want { + t.Fatalf("expected hak path %s, got %s", want, got) + } +} + +func TestTopDataPackagePathsUseConfiguredNamesAndBuildDir(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "output")) + + proj := &Project{ + Root: root, + Config: Config{ + Paths: PathConfig{Build: "output"}, + TopData: TopDataConfig{ + PackageHAK: "custom_top.hak", + PackageTLK: "custom_dialog.tlk", + }, + }, + } + + if got, want := proj.TopDataPackageHAKPath(), filepath.Join(root, "output", "custom_top.hak"); got != want { + t.Fatalf("expected top hak path %s, got %s", want, got) + } + if got, want := proj.TopDataPackageTLKPath(), filepath.Join(root, "output", "custom_dialog.tlk"); got != want { + t.Fatalf("expected top tlk path %s, got %s", want, got) + } +} + +func TestValidateLayoutAllowsMissingAssetsDir(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Assets: "assets", Build: "build"}, + }, + } + + if err := proj.ValidateLayout(); err != nil { + t.Fatalf("ValidateLayout returned error: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "assets")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expected missing assets dir to remain absent, got err=%v", err) + } +} + +func TestValidateLayoutRejectsInvalidTopDataPackageOutputNames(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + TopData: TopDataConfig{ + Source: "topdata", + PackageHAK: "nested/custom_top.hak", + PackageTLK: "custom_dialog.txt", + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected topdata output validation error") + } + if !strings.Contains(err.Error(), "topdata.package_hak") { + t.Fatalf("expected package_hak validation error, got %v", err) + } + if !strings.Contains(err.Error(), "topdata.package_tlk") { + t.Fatalf("expected package_tlk validation error, got %v", err) + } +} + +func TestValidateLayoutRejectsUnsafeGeneratedOutputPaths(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + Outputs: OutputConfig{ + HAKManifest: "../haks.json", + }, + TopData: TopDataConfig{ + Source: "topdata", + Compiled2DADir: "../2da", + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected unsafe output path validation error") + } + if !strings.Contains(err.Error(), "outputs.hak_manifest") { + t.Fatalf("expected HAK manifest path validation error, got %v", err) + } + if !strings.Contains(err.Error(), "topdata.compiled_2da_dir") { + t.Fatalf("expected topdata path validation error, got %v", err) + } +} + +func TestValidateLayoutRejectsInvalidTopDataWikiConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + TopData: TopDataConfig{ + Source: "topdata", + Wiki: TopDataWikiConfig{ + Source: "../wiki", + Renderer: "dokuwiki", + LinkStrategy: "rewrite_links", + StatusListingScope: "global", + NamespacesFile: "../namespaces.yaml", + TablesFile: "../tables.yaml", + VisibilityFile: "../visibility.yaml", + TemplatesDir: ".", + DataFile: "../data.yaml", + PagePathsFile: "../wiki.yaml", + PageTemplatesDir: "../templates/pages", + ManualSectionsFile: "../manual/default.yaml", + PageIndexFile: "../page-index.json", + ManualSectionsDir: "../manual", + AlignmentLinks: TopDataWikiAlignmentLinks{ + LinkFormat: "raw_url", + }, + StalePages: TopDataWikiStalePagesConfig{ + Default: "delete", + LiveCleanup: "delete", + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected invalid wiki config validation error") + } + for _, needle := range []string{ + "topdata.wiki.source", + "topdata.wiki.renderer", + "topdata.wiki.link_strategy", + "topdata.wiki.status_listing_scope", + "topdata.wiki.namespaces_file", + "topdata.wiki.tables_file", + "topdata.wiki.visibility_file", + "topdata.wiki.templates_dir", + "topdata.wiki.data_file", + "topdata.wiki.page_paths_file", + "topdata.wiki.page_templates_dir", + "topdata.wiki.manual_sections_file", + "topdata.wiki.page_index_file", + "topdata.wiki.manual_sections_dir", + "topdata.wiki.alignment_links.link_format", + "topdata.wiki.stale_pages.default", + "topdata.wiki.stale_pages.live_cleanup", + } { + if !strings.Contains(err.Error(), needle) { + t.Fatalf("expected validation error to mention %s, got %v", needle, err) + } + } +} + +func TestValidateLayoutRejectsRootSourceAndAssetPaths(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: ".", Assets: ".", Build: "build"}, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected root source and asset path validation error") + } + if !strings.Contains(err.Error(), "paths.source") { + t.Fatalf("expected source path validation error, got %v", err) + } + if !strings.Contains(err.Error(), "paths.assets") { + t.Fatalf("expected assets path validation error, got %v", err) + } +} + +func TestValidateLayoutRejectsInvalidAutogenConsumerConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + Autogen: AutogenConfig{ + Consumers: []AutogenConsumerConfig{ + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "unsupported", + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"}, + Manifest: AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "nested/sow-accessory-vfx-manifest.json", + CacheName: "sow-accessory-vfx-manifest.txt", + }, + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected autogen validation error") + } + if !strings.Contains(err.Error(), "autogen.consumers[0].mode") { + t.Fatalf("expected autogen mode validation error, got %v", err) + } + if !strings.Contains(err.Error(), "autogen.consumers[0].manifest.asset_name") { + t.Fatalf("expected autogen asset_name validation error, got %v", err) + } + if !strings.Contains(err.Error(), "autogen.consumers[0].manifest.cache_name") { + t.Fatalf("expected autogen cache_name validation error, got %v", err) + } +} + +func TestValidateLayoutRejectsInvalidHeadVisualeffectsConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + Autogen: AutogenConfig{ + Consumers: []AutogenConsumerConfig{ + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "accessory_visualeffects", + Root: "vfxs", + Include: []string{"head_jewels/**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"}, + Manifest: AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "sow-accessory-vfx-manifest.json", + CacheName: "sow-accessory-vfx-manifest.json", + }, + HeadVisualeffects: HeadVisualeffectsConfig{ + Groups: map[string]HeadVisualeffectGroupConfig{ + "head_jewels": { + ModelColumns: []string{"Imp Root M Node"}, + }, + }, + StripModelPrefixes: []string{""}, + ModelColumn: "Imp HeadCon Node", + }, + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected head visualeffects validation error") + } + for _, needle := range []string{ + "autogen.consumers[0].accessory_visualeffects.groups[head_jewels].prefix", + "autogen.consumers[0].accessory_visualeffects.groups[head_jewels].model_columns[0]", + "autogen.consumers[0].accessory_visualeffects.strip_model_prefixes[0]", + "autogen.consumers[0].accessory_visualeffects.model_column", + } { + if !strings.Contains(err.Error(), needle) { + t.Fatalf("expected validation error to mention %s, got %v", needle, err) + } + } +} + +func TestValidateLayoutRejectsInvalidHeadVisualeffectsNamingConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + Autogen: AutogenConfig{ + Consumers: []AutogenConsumerConfig{ + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "accessory_visualeffects", + Root: "vfxs", + Include: []string{"head_features/**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"}, + Manifest: AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "sow-accessory-vfx-manifest.json", + CacheName: "sow-accessory-vfx-manifest.json", + }, + HeadVisualeffects: HeadVisualeffectsConfig{ + Groups: map[string]HeadVisualeffectGroupConfig{ + "head_features": {}, + }, + GroupTokenSource: "unsupported", + CategoryFrom: "unsupported", + Delimiter: " ", + Case: "unsupported", + }, + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected head visualeffects naming validation error") + } + for _, needle := range []string{ + "autogen.consumers[0].accessory_visualeffects.group_token_source", + "autogen.consumers[0].accessory_visualeffects.category_from", + "autogen.consumers[0].accessory_visualeffects.delimiter", + "autogen.consumers[0].accessory_visualeffects.case", + } { + if !strings.Contains(err.Error(), needle) { + t.Fatalf("expected validation error to mention %s, got %v", needle, err) + } + } +} + +func TestLoadRejectsLegacyHeadVisualeffectsNamingFields(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src +autogen: + consumers: + - id: accessory_visualeffects + producer: accessory_visualeffects + dataset: visualeffects + mode: accessory_visualeffects + root: vfxs + include: + - head_features/**/*.mdl + derive: + kind: model_stem + group_from: first_path_segment + accessory_visualeffects: + groups: + head_features: + model_columns: + - Imp_HeadCon_Node + group_token_source: folder_name + category_from: immediate_parent + legacy_groups: + head_features: head_features + legacy_key_format: "{dataset}:{group}_{stem}" +`) + + _, err := Load(root) + if err == nil { + t.Fatal("expected legacy head visualeffects naming fields to be rejected") + } + if !strings.Contains(err.Error(), "legacy_groups") { + t.Fatalf("expected legacy_groups unknown field error, got %v", err) + } +} + +func TestValidateLayoutAcceptsGeneratedTopData2DAConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + proj := Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "testmod"}, + Paths: PathConfig{Assets: "assets"}, + Generated: GeneratedConfig{ + TopData2DA: []GeneratedTopData2DAConfig{ + { + ID: "parts", + Source: "topdata", + Output: "{paths.cache}/generated-assets/parts-2da", + IncludeDatasets: []string{"parts/**"}, + PackageRoot: "part", + Autogen: AutogenConsumerConfig{ + ID: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "trailing_numeric_suffix", GroupFrom: "first_path_segment"}, + PartsRows: PartsRowsConfig{ + RowDefaults: map[string]string{"COSTMODIFIER": "0"}, + ACBonus: PartsRowsACBonusConfig{ + Default: PartsRowsACBonusPolicy{Strategy: "ascending_row_id_sort_key", Divisor: 100, Format: "%.2f"}, + Datasets: map[string]PartsRowsACBonusPolicy{ + "chest": {Strategy: "fixed", Value: "0.00"}, + }, + }, + }, + }, + }, + { + ID: "cachedmodels", + Source: "topdata", + Output: "{paths.cache}/generated-assets/cachedmodels-2da", + IncludeDatasets: []string{"cachedmodels"}, + PackageRoot: "vfxs", + Autogen: AutogenConsumerConfig{ + ID: "cachedmodels", + Mode: "cachedmodels_rows", + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"}, + }, + }, + { + ID: "static_topdata", + Source: "topdata", + Output: "{paths.cache}/generated-assets/static-topdata-2da", + IncludeDatasets: []string{"tailmodel", "wingmodel"}, + PackageRoot: "part", + }, + }, + }, + }, + } + + if err := proj.ValidateLayout(); err != nil { + t.Fatalf("ValidateLayout returned error: %v", err) + } + effective := proj.EffectiveConfig() + if got, want := effective.Generated.TopData2DA[0].Output, ".cache/generated-assets/parts-2da"; got != want { + t.Fatalf("expected expanded output %q, got %q", want, got) + } +} + +func TestValidateLayoutRejectsUnsupportedGeneratedTopDataAutogenMode(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + proj := Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "testmod"}, + Paths: PathConfig{Assets: "assets"}, + Generated: GeneratedConfig{ + TopData2DA: []GeneratedTopData2DAConfig{ + { + ID: "bad", + Source: "topdata", + Output: "{paths.cache}/generated-assets/bad-2da", + IncludeDatasets: []string{"tailmodel"}, + PackageRoot: "part", + Autogen: AutogenConsumerConfig{ + Mode: "unknown_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"}, + }, + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil || !strings.Contains(err.Error(), `generated_assets.topdata_2da[0].autogen.mode "unknown_rows" is not supported`) { + t.Fatalf("expected unsupported generated autogen mode validation error, got %v", err) + } +} + +func TestValidateLayoutRejectsEscapingGeneratedTopData2DAConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + proj := Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "testmod"}, + Paths: PathConfig{Assets: "assets"}, + Generated: GeneratedConfig{ + TopData2DA: []GeneratedTopData2DAConfig{ + { + ID: "parts", + Source: "../topdata", + Output: "{paths.cache}/generated-assets/parts-2da", + IncludeDatasets: []string{"parts/**"}, + PackageRoot: "part", + Autogen: AutogenConsumerConfig{ + ID: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "trailing_numeric_suffix", GroupFrom: "first_path_segment"}, + }, + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil || !strings.Contains(err.Error(), "generated_assets.topdata_2da[0].source") { + t.Fatalf("expected generated topdata source validation error, got %v", err) + } +} + +func TestValidateLayoutRejectsInvalidGeneratedPartsRowsACBonusConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + proj := Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "testmod"}, + Paths: PathConfig{Assets: "assets"}, + Generated: GeneratedConfig{ + TopData2DA: []GeneratedTopData2DAConfig{ + { + ID: "parts", + Source: "topdata", + Output: "{paths.cache}/generated-assets/parts-2da", + IncludeDatasets: []string{"parts/**"}, + PackageRoot: "part", + Autogen: AutogenConsumerConfig{ + ID: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: AutogenDeriveConfig{Kind: "trailing_numeric_suffix", GroupFrom: "first_path_segment"}, + PartsRows: PartsRowsConfig{ + ACBonus: PartsRowsACBonusConfig{ + Default: PartsRowsACBonusPolicy{Strategy: "descending_row_id_sort_key", MaxRowID: 0, Divisor: -1, Format: "%d"}, + Datasets: map[string]PartsRowsACBonusPolicy{ + "chest": {Strategy: "fixed"}, + "robe": {Strategy: "unknown"}, + }, + }, + }, + }, + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected invalid parts_rows acbonus config to fail") + } + text := err.Error() + for _, want := range []string{ + "generated_assets.topdata_2da[0].autogen.parts_rows.acbonus.default.max_row_id", + "generated_assets.topdata_2da[0].autogen.parts_rows.acbonus.default.divisor", + "generated_assets.topdata_2da[0].autogen.parts_rows.acbonus.default.format", + "generated_assets.topdata_2da[0].autogen.parts_rows.acbonus.datasets.chest.value", + "generated_assets.topdata_2da[0].autogen.parts_rows.acbonus.datasets.robe.strategy", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected validation error containing %q, got %v", want, err) + } + } +} + +func TestValidateLayoutRejectsDuplicateHAKNames(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + HAKs: []HAKConfig{ + {Name: "core", Priority: 1, Include: []string{"core/**"}}, + {Name: "CORE", Priority: 2, Include: []string{"other/**"}}, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected duplicate hak validation error") + } + if !strings.Contains(err.Error(), "duplicated") { + t.Fatalf("expected duplicate validation error, got %v", err) + } +} + +func TestValidateLayoutRejectsAmbiguousMusicPrefixes(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Assets: "assets", Build: "build"}, + Music: MusicConfig{ + Prefixes: map[string]string{ + "envi/music": "mus_", + "/envi/music": "mus2_", + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected music prefix validation error") + } + if !strings.Contains(err.Error(), "normalize to the same path") { + t.Fatalf("expected ambiguous prefix validation error, got %v", err) + } +} + +func TestScanAllowsMissingAssetsDir(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Assets: "assets", Build: "build"}, + }, + } + + if err := proj.Scan(); err != nil { + t.Fatalf("Scan returned error: %v", err) + } + if len(proj.Inventory.AssetFiles) != 0 { + t.Fatalf("expected no asset files, got %#v", proj.Inventory.AssetFiles) + } +} + +func mkdirAll(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } +} + +func TestCloneWithHAKNamesFiltersConfiguredHAKs(t *testing.T) { + proj := &Project{ + Root: "/tmp/test", + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Assets: "assets", Build: "build"}, + HAKs: []HAKConfig{ + {Name: "sow_over", Priority: 1, Include: []string{"over/**"}}, + {Name: "sow_core", Priority: 2, Include: []string{"core/**"}}, + {Name: "sow_item", Priority: 3, Include: []string{"item/**"}}, + }, + }, + } + + filtered, err := proj.CloneWithHAKNames([]string{"sow_core", "sow_item"}) + if err != nil { + t.Fatalf("CloneWithHAKNames returned error: %v", err) + } + + if filtered == proj { + t.Fatalf("expected filtered clone, got original pointer") + } + + got := make([]string, 0, len(filtered.Config.HAKs)) + for _, hak := range filtered.Config.HAKs { + got = append(got, hak.Name) + } + want := []string{"sow_core", "sow_item"} + if !slices.Equal(got, want) { + t.Fatalf("unexpected filtered haks: got %v want %v", got, want) + } + + if len(proj.Config.HAKs) != 3 { + t.Fatalf("original project config should remain unchanged, got %d haks", len(proj.Config.HAKs)) + } +} + +func TestCloneWithHAKNamesRejectsUnknownHAKs(t *testing.T) { + proj := &Project{ + Config: Config{ + HAKs: []HAKConfig{ + {Name: "sow_core", Priority: 1, Include: []string{"core/**"}}, + }, + }, + } + + if _, err := proj.CloneWithHAKNames([]string{"missing_hak"}); err == nil { + t.Fatal("expected error for unknown hak name") + } +} + +func loadAndValidate(root string) error { + proj, err := Load(root) + if err != nil { + return err + } + return proj.ValidateLayout() +} + +func writeProjectFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(strings.TrimPrefix(content, "\n")), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} diff --git a/internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md b/internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md new file mode 100644 index 0000000..cb812a5 --- /dev/null +++ b/internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md @@ -0,0 +1,155 @@ +# AI Agent Contract: Auto-Include Existing Part Models in 2DA Generation + +## Status Snapshot + +Current state as of 2026-05-17: + +- Superseded for current Shadows Over Westgate builds. +- The module no longer consumes a released parts manifest. Parts data is owned by + `assets/`, generated as HAK-side 2DA assets from `assets/topdata/data/parts` + plus a direct scan of `assets/content/part/**/*.mdl`, and packaged in + `sow_part`. +- This file remains as historical context for the legacy manifest-backed + consumer path. + +## Objective + +The native 2DA generation pipeline must automatically include existing part models already present in the **`sow-assets` repository** when building final parts tables. + +This behavior is required for parity with the current asset set and must not depend on manually authored entries for models that already exist in repository assets. + +--- + +## Source of Truth + +The superseded manifest-backed native builder obtained existing model +information from a released artifact published by the **`sow-assets` repository**. + +Resolution rules: + +- if `topdata.assets` is configured to a local path, use that exact path +- otherwise derive the **`sow-assets` repository** from the module repo's Gitea origin + and fetch only the configured released manifest + +Normal builds must not clone or scan the `sow-assets` repo tree. They may cache the +downloaded manifest locally for reuse, but the source of truth remains the released +manifest on Gitea. + +Manifest-backed discovery is derived from: + +```text +sow-assets/assets/part/**/*.mdl +``` + +The manifest generator may also accept the repo's legacy `assets/parts/` layout when +resolving the physical paths, but the native builder itself must consume the manifest, +not the repo tree. + +--- + +## Required Scope + +Process existing `.mdl` files for these parts table categories only: + +- `belt` +- `bicep` +- `chest` +- `foot` +- `forearm` +- `leg` +- `neck` +- `pelvis` +- `robe` +- `shin` +- `shoulder` + +Dataset mapping rules: + +- `parts/legs` maps to asset category `leg` +- `parts/hand` remains unsupported and is not auto-generated + +The scan must recurse through subdirectories under each applicable category path. + +--- + +## Required Behavior + +### 1. Manifest-backed discovery + +For each supported part category, read the released manifest and discover all existing +numeric model IDs listed for that category. + +### 2. Trailing numeric suffix extraction + +The released manifest must be generated using trailing-numeric-suffix extraction from +the source `.mdl` filenames. Native topdata consumes the resulting numeric IDs only. + +### 3. Row generation + +If a trailing numeric suffix is present: + +- use that numeric suffix as the output row ID +- emit a row for that part model in the final generated 2DA + +### 4. Default emitted values + +For rows generated from discovered existing models, set: + +- `COSTMODIFIER = 0` +- `ACBONUS = 0.00` + +Rows without discovered existing models may be intentionally emitted as dense +2DA null rows when configured. In that mode, missing rows must have +`COSTMODIFIER = ****` and `ACBONUS = ****` in the final generated 2DA. + +### 5. Override precedence + +If file-based overrides exist, they must take precedence over discovered defaults. + +That means: + +- manifest discovery provides the baseline row presence and default values +- override data may replace or augment those values +- override data wins in any conflict + +--- + +## Non-Negotiable Requirements + +- The implementation must read model presence from the **released parts manifest**, + not from assumption, hardcoded lists, or legacy output. +- Existing repository models with trailing numeric suffixes must be reflected in generated output even if they are not otherwise explicitly authored. +- Override application is mandatory and must win over auto-discovered defaults. +- This is replacement behavior for the native pipeline, not an optional enhancement. + +--- + +## Acceptance Criteria + +The superseded implementation was correct only if all of the following were true: + +1. The builder fetches the configured released manifest artifact for part models. +2. Only the supported part categories are considered. +3. Manifest IDs derived from trailing numeric suffixes generate corresponding 2DA rows. +4. The numeric suffix is used as the row ID. +5. Auto-generated rows default to: + - `COSTMODIFIER = 0` + - `ACBONUS = 0.00` + +6. Authored base rows remain exposed. Authored `ACBONUS = ****` rows remain + null unless model discovery activates the row. +7. Dense gaps without authored rows or discovered models are emitted as null + rows with `COSTMODIFIER = ****` and `ACBONUS = ****`. +8. File overrides are applied after discovery and take precedence. +9. The final output includes all eligible existing models present in the manifest. +10. Discovery does not require a sibling local `sow-assets` checkout or a Git clone. + +--- + +## Implementation Directive + +When generating native parts 2DAs, first scan the project-resolved assets repo for +supported part models, collect existing supported part models by category, extract +trailing numeric suffixes from filenames, emit rows using those suffixes as row IDs +with default `COSTMODIFIER = 0` and `ACBONUS = 0.00`, then apply overrides with +override values taking precedence. diff --git a/internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md b/internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md new file mode 100644 index 0000000..f4e956d --- /dev/null +++ b/internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md @@ -0,0 +1,76 @@ +# Class Feat Global Injection Contract + +## Status Snapshot + +Current state as of 2026-06-05: + +- Preferred class feat injection policy is authored in + `topdata/data/classes/feats/global.json`. +- `global.json` rows flow through the ordinary class feat expansion pipeline, + including successor expansion, masterfeat expansion, class-skill filtering, + reference resolution, deduplication, and final 2DA emission. +- Legacy `topdata.class_feat_injections` YAML still loads only as a + compatibility path when no applicable class feat `global.json` exists. +- Toolkit hardcoded defaults are compatibility-only and must not be the normal + source of project policy. + +## Objective + +Native class feat generation must produce deterministic, legacy-equivalent +effective output while keeping project policy in topdata authoring files. + +## Requirements + +- For each class skill, inject the configured skill-focus masterfeat rows from + `classes/feats/global.json` using `FeatIndex.filter = "classskills"`. +- Inject `feat:literate` only when the class table does not already include + `feat:illiterate`. +- Inject the shared combat/menu feat rows declared in the manifest unless a + class already authors the equivalent `FeatIndex`. +- Explicit class feat rows take precedence over injected rows. +- Injections must be deterministic for identical inputs. + +## Manifest Shape + +```json +{ + "position": "prepend", + "injections": [ + { + "row": { + "FeatIndex": { "id": "feat:literate" }, + "GrantedOnLevel": 1, + "List": 3, + "OnMenu": 0 + }, + "unless_present": [ + { "field": "FeatIndex", "id": "feat:illiterate" } + ] + }, + { + "row": { + "FeatIndex": { + "id": "masterfeats:skill_focus", + "filter": "classskills" + }, + "GrantedOnLevel": -1, + "List": 1, + "OnMenu": 0 + } + } + ] +} +``` + +`position` defaults to `append`. The module uses `prepend` for class feat +injections to keep generated policy rows before per-class authored rows. + +## Acceptance Criteria + +- Class feat output contains the same effective shared feats as the prior YAML + policy. +- Literacy injection is skipped for illiterate classes. +- Skill-focus masterfeat rows expand only for class skills. +- YAML compatibility continues to work for consumers that have not migrated. +- A project with `classes/feats/global.json` does not also receive YAML or + hardcoded default class feat injections. diff --git a/internal/topdata/FAMILY_EXPANSION_CONTRACT.md b/internal/topdata/FAMILY_EXPANSION_CONTRACT.md new file mode 100644 index 0000000..3eb6fcd --- /dev/null +++ b/internal/topdata/FAMILY_EXPANSION_CONTRACT.md @@ -0,0 +1,119 @@ +# Generic Family Expansion Contract + +## Status Snapshot + +Current state as of 2026-05-13: + +- Implemented and actively used by native feat generation. +- `family_expansion.go`, `native.go`, and `topdata_test.go` cover the current + underscore-family interpretation, generated family metadata, and reapplication + behavior. +- Remaining work is mostly consumer expansion: use the same primitive in more + datasets only when the authored data actually benefits from it. + +Topdata now treats underscore expansion as a structural rule: + +- `parent_child` means `child` is an expansion of `parent` +- underscores are for family structure, not simulated spaces +- existing canonical keys still win when lock or TLK state already established them + +This is a global interpretation rule, not a wiki-only convention. + +## Identity + +- `weaponspecialization_club` + - parent: `weaponspecialization` + - child: `club` +- `gnome_rock` + - parent: `gnome` + - child: `rock` +- `toughness_10` + - parent: `toughness` + - child: `10` + +Standalone keys without an underscore are treated as a parent identity with no child. + +## Generated Family Files + +Canonical generated families declare: + +```json +{ + "family": "weapon_focus", + "family_key": "weaponfocus", + "template": "masterfeats:weaponfocus", + "name_prefix": "Weapon Focus", + "label_prefix": "FEAT_WEAPON_FOCUS", + "constant_prefix": "FEAT_WEAPON_FOCUS", + "legacy_family_keys": ["epic_weapon_focus"], + "default_fields": { + "DESCRIPTION": { + "tlk": { + "key": "masterfeats:weaponfocus.description", + "text": "..." + } + }, + "MINATTACKBONUS": "1", + "TOOLSCATEGORIES": "1" + }, + "apply_after_modules": true, + "identity_source": "child_source_value", + "auto_prereq_fields": { + "PREREQFEAT1": "weaponfocus" + }, + "child_source": { + "dataset": "baseitems", + "column": "WeaponFocusFeat" + } +} +``` + +Rules: + +- `family` is an authored label and is not a hardcoded switch key +- `family_key` is the structural parent identity used in generated child keys +- `template` is optional for the primitive in general, but required by current + masterfeat-backed `feat` families +- `name_prefix`, `label_prefix`, and `constant_prefix` define how new child rows are + named when they are not already authored +- `template_fields` lists template-backed fields copied from the referenced template row + when authored; it is optional if the family owns its shared values directly +- `default_fields` applies authored shared/default values to every generated family row + and can be used to move family-wide shared values out of the template row +- `apply_after_modules: true` re-applies the generated family after normal module files, + so authored family-shared values remain authoritative even for legacy explicitly + authored child rows +- `child_ref_field` binds the generated child row back to the source row key, e.g. + `REQSKILL` +- `identity_source: "child_source_value"` tells the builder to reuse IDs/keys from the + source column when that column already carries feat identity +- `legacy_family_keys` declares previous generated-family prefixes that should donate + existing locked row IDs to the current family when a family is renamed +- `auto_prereq_fields` binds prerequisite fields to other authored families by + `family_key`, without code changes +- `allow_existing_only` limits expansion to already-authored child rows when a family is + intentionally partial +- `child_source.dataset` identifies the canonical dataset driving expansion +- `child_source.column` is used when expansion is gated by a non-null source field +- `child_source.predicate` is used when expansion depends on a named rule such as + accessibility + +## Row Metadata + +Generated rows can carry: + +```json +{ + "meta": { + "family": { + "parent": "weaponfocus", + "child": "club", + "source": "baseitems:club", + "template": "masterfeats:weaponfocus" + } + } +} +``` + +This metadata is builder-owned and does not affect emitted 2DA columns. It preserves +family structure for later phases without enabling wiki generation yet. diff --git a/internal/topdata/FEAT_GENERATED_FAMILIES_CONTRACT.md b/internal/topdata/FEAT_GENERATED_FAMILIES_CONTRACT.md new file mode 100644 index 0000000..6c5f574 --- /dev/null +++ b/internal/topdata/FEAT_GENERATED_FAMILIES_CONTRACT.md @@ -0,0 +1,174 @@ +# `feat` Generated Families Contract + +## Status Snapshot + +Current state as of 2026-05-13: + +- Implemented for the current native `feat` pipeline. +- Native tests cover generated-family expansion, family-owned shared defaults, + masterfeat-backed inheritance, and lock/TLK identity preservation. +- The one explicitly deferred item in this contract is still current: + `feat.2da` remains native-built but excluded from `compare-topdata` + reference-catalog coverage through `compare_reference: false`. + +Canonical authored paths: + +- `topdata/data/feat/base.json` +- `topdata/data/feat/lock.json` +- `topdata/data/feat/generated/skill_focus.json` +- `topdata/data/feat/generated/greater_skill_focus.json` +- `topdata/data/feat/generated/weapon_focus.json` +- `topdata/data/feat/generated/weapon_specialization.json` +- `topdata/data/feat/generated/greater_weapon_focus.json` +- `topdata/data/feat/generated/greater_weapon_specialization.json` +- `topdata/data/feat/generated/improved_critical.json` +- `topdata/data/feat/generated/overwhelming_critical.json` +- `topdata/data/feat/modules/activecombat/core.json` +- `topdata/data/feat/modules/activecombat/specialattacks.json` +- `topdata/data/feat/modules/class/core.json` +- `topdata/data/feat/modules/combat/core.json` +- `topdata/data/feat/modules/defensive/core.json` +- `topdata/data/feat/modules/magical/core.json` +- `topdata/data/feat/modules/other/core.json` +- `topdata/data/feat/modules/proficiency/core.json` +- `topdata/data/feat/modules/racial/core.json` +- `topdata/data/feat/modules/removedandhidden/core.json` +- `topdata/data/feat/modules/skillfeat/affinity.json` +- `topdata/data/feat/modules/skillfeat/focus.json` +- `topdata/data/feat/modules/skillfeat/greaterfocus.json` +- `topdata/data/feat/modules/skillfeat/other.json` + +`skill_affinity` is no longer authored as a generated feat file. It is synthesized from +canonical `racialtypes` race grants. + +Compact family-expansion shape: + +```json +{ + "family": "skill_focus", + "family_key": "skillfocus", + "template": "masterfeats:skillfocus", + "name_prefix": "Skill Focus", + "label_prefix": "FEAT_SKILL_FOCUS", + "constant_prefix": "FEAT_SKILL_FOCUS", + "default_fields": { + "DESCRIPTION": { + "tlk": { + "key": "masterfeats:skillfocus.description", + "text": "..." + } + }, + "CRValue": "0.5", + "ReqSkillMinRanks": "1", + "TOOLSCATEGORIES": "6" + }, + "apply_after_modules": true, + "child_ref_field": "REQSKILL", + "child_source": { + "dataset": "skills", + "predicate": "accessible" + }, + "title_style": { + "child_case": "lower", + "child_parenthetical": "comma" + }, + "overrides": { + "skills:craft_alchemy": { + "ICON": "ife_foc_alchm" + } + } +} +``` + +Weapon family shape: + +```json +{ + "family": "weapon_focus", + "family_key": "weaponfocus", + "template": "masterfeats:weaponfocus", + "name_prefix": "Weapon Focus", + "label_prefix": "FEAT_WEAPON_FOCUS", + "constant_prefix": "FEAT_WEAPON_FOCUS", + "legacy_family_keys": ["epic_weapon_focus"], + "default_fields": { + "DESCRIPTION": { + "tlk": { + "key": "masterfeats:weaponfocus.description", + "text": "..." + } + }, + "MINATTACKBONUS": "1", + "TOOLSCATEGORIES": "1" + }, + "apply_after_modules": true, + "identity_source": "child_source_value", + "child_source": { + "dataset": "baseitems", + "column": "WeaponFocusFeat" + }, + "title_style": { + "child_case": "lower", + "child_parenthetical": "preserve" + }, + "overrides": { + "baseitems:heavymace": { + "ICON": "ife_wepfoc_Lma" + } + } +} +``` + +Rules: + +- any generated feat file with family-expansion fields is treated as an authored + family-expansion definition +- `family` is descriptive only; the builder does not hardcode family names +- family behavior is driven by authored fields such as `template_fields`, + `default_fields`, `apply_after_modules`, `identity_source`, `child_ref_field`, + `auto_prereq_fields`, and optional `title_style` +- `default_fields` is a family-wide shared layer; it applies to existing and newly + generated child rows before per-child overrides are merged +- `apply_after_modules: true` makes the generated family the final authoritative shared + layer for legacy explicitly authored child rows, so shared baselines can live in the + generated family file instead of `masterfeats` overrides +- `legacy_family_keys` declares old generated-family prefixes whose existing row IDs + should be inherited by the current family; this is how renamed families such as + `greater_skill_focus` retaining vanilla `epic_skill_focus` rows preserve hardcoded + engine behavior without hardcoding the rename in toolkit logic +- family-expansion files must declare `template`, `family_key`, and `child_source.dataset` +- `child_source.column` is used when expansion is gated by a non-null source field +- `child_source.predicate: "accessible"` currently means `HideFromLevelUp != 1` +- `title_style` only changes generated child display text while composing titles; + generated feat keys, TLK keys, row IDs, family metadata, and source references + remain tied to generated family identity +- generated family title text is authoritative for family-managed existing rows + as well as newly allocated rows, so in-game TLK names and generated wiki titles + are derived from the same normalized `FEAT` text +- `title_style.child_case` defaults to `preserve`; `lower` lowercases the resolved + child display text after parenthetical formatting +- `title_style.child_parenthetical` defaults to `preserve`; `comma` flattens one + child parenthetical suffix such as `Knowledge (local)` into + `Knowledge, local` before the child text is wrapped by the family title +- unsupported `title_style` values fail generated-family validation instead of + silently changing title behavior +- skill focus families intentionally do not set `allow_existing_only`, so any new + accessible skill automatically receives matching focus rows +- compact `overrides` are keyed by source dataset key: + - `skills:*` for skill families + - `baseitems:*` for weapon families +- generated rows carry `meta.family` so the underscore family rule is preserved even + before wiki generation exists +- emitted feat keys must prefer existing canonical lock/TLK identities over newly + derived child spellings +- manual and irregular feat content continues to live under `feat/modules/` +- generated families and module-authored feats are merged into the same final `feat.2da` + pipeline and share inheritance, override, TLK, and family-metadata behavior + +Current rollout policy: + +- `feat.2da` still builds natively +- `feat.2da` is still excluded from `compare-topdata` output-catalog self-check coverage + via `compare_reference: false` +- manual and irregular feat families remain authored directly in modules rather than + compact generated-family files diff --git a/internal/topdata/INHERITANCE_CONTRACT.md b/internal/topdata/INHERITANCE_CONTRACT.md new file mode 100644 index 0000000..3cf958d --- /dev/null +++ b/internal/topdata/INHERITANCE_CONTRACT.md @@ -0,0 +1,78 @@ +# Topdata Inheritance Contract + +## Status Snapshot + +Current state as of 2026-05-13: + +- Implemented. +- `native.go` supports both row-sugar inheritance and generic object + inheritance, including cycle detection and missing-target failures. +- Validation and build regression coverage live in `topdata_test.go`. + +Native topdata now supports two inheritance forms: + +## 1. Row Sugar Compatibility + +This is the existing narrow row helper: + +```json +{ + "PARENT": { "id": "custom:parent" }, + "inherit": { + "from": "PARENT", + "fields": ["VALUE", "ICON"] + } +} +``` + +It remains supported for row-local field copying. + +## 2. Generic Object Inheritance + +This is the new reusable primitive: + +```json +{ + "inherit": { + "ref": "masterfeats:skillfocus" + }, + "LABEL": "FEAT_SKILL_FOCUS_ATHLETICS" +} +``` + +Or for a nested object: + +```json +{ + "DETAILS": { + "inherit": { + "ref": "custom:parent", + "field": "DETAILS" + }, + "meta": { + "cost": "9" + } + } +} +``` + +## Rules + +- `inherit.ref` must use stable `dataset:key` identity. +- `inherit.field` is optional and selects an object-valued field on the target row. +- Generic inheritance can appear on any authored object, not only top-level rows. +- Merge precedence is `local overrides inherited`. +- Scalars replace inherited values. +- Arrays replace inherited values. +- Objects merge recursively unless the local object is an atomic topdata value object such as: + - TLK payloads + - row refs + - table refs +- Cycles fail the build. +- Missing targets fail the build. + +## Current Intended Consumer + +`feat` is the first dataset expected to use this primitive heavily, especially for +borrowing shared properties from `masterfeats` while keeping `feat`'s own dataset +contract and row model. diff --git a/internal/topdata/MASTERFEATS_CONTRACT.md b/internal/topdata/MASTERFEATS_CONTRACT.md new file mode 100644 index 0000000..b7204b1 --- /dev/null +++ b/internal/topdata/MASTERFEATS_CONTRACT.md @@ -0,0 +1,82 @@ +# `masterfeats` Canonical Contract + +## Status Snapshot + +Current state as of 2026-05-13: + +- Implemented as a first-class native dataset. +- The repository contains `topdata/data/masterfeats/`, migration/import support, + validator checks, and native output coverage for `masterfeats.2da`. +- Remaining work is only downstream adoption and regression maintenance as other + native datasets continue to consume `masterfeats` more heavily. + +`masterfeats` is a stable native canonical family consumed directly by `feat` +generation, class-feat expansion, and inheritance/ref resolution. + +Current canonical scope: + +- dataset root: `topdata/data/masterfeats/` +- required files: + - `base.json` + - `lock.json` +- generated output: + - `masterfeats.2da` + +## `base.json` + +Canonical shape: + +```json +{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "..."], + "rows": [ + { + "id": 0, + "key": "masterfeats:weaponfocus", + "LABEL": "WeaponFocus", + "STRREF": "6490", + "DESCRIPTION": "436" + } + ] +} +``` + +Rules: + +- `output` may be omitted; native dataset discovery deterministically derives + `masterfeats.2da` from `topdata/data/masterfeats/base.json` +- if `output` is present, it must be exactly `masterfeats.2da` +- `columns` must include `LABEL`, `STRREF`, and `DESCRIPTION` +- every canonical row must have a non-empty `key` +- row keys must start with `masterfeats:` +- inline TLK payloads are allowed in `STRREF` and `DESCRIPTION` +- numeric/text references are both allowed where the native TLK compiler already supports them + +## `lock.json` + +Canonical shape: + +```json +{ + "masterfeats:weaponfocus": 0 +} +``` + +Rules: + +- every key must start with `masterfeats:` +- every value must be numeric +- the lockfile remains the canonical stable id source for authored keys + +## Module contributions + +This family continues to support the shared canonical module shapes already used by native +datasets: + +- `columns` +- `entries` +- `overrides` + +Those shapes are validated generically in `topdata.go`; this contract only adds the +family-specific guarantees for `masterfeats`. diff --git a/internal/topdata/appearance_migrate.go b/internal/topdata/appearance_migrate.go new file mode 100644 index 0000000..1983194 --- /dev/null +++ b/internal/topdata/appearance_migrate.go @@ -0,0 +1,96 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacyAppearance(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "appearance") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "appearance") + targetBasePath := filepath.Join(targetDir, "base.json") + targetLockPath := filepath.Join(targetDir, "lock.json") + targetModulesDir := filepath.Join(targetDir, "modules") + moduleNames := []string{ + "cotblreaver.json", + "crawlingclaw.json", + "halfogre.json", + "zombieknight.json", + } + targetModulePaths := make([]string, 0, len(moduleNames)) + for _, name := range moduleNames { + targetModulePaths = append(targetModulePaths, filepath.Join(targetModulesDir, name)) + } + if fileExists(targetBasePath) && fileExists(targetLockPath) { + allModulesPresent := true + for _, path := range targetModulePaths { + if !fileExists(path) { + allModulesPresent = false + break + } + } + if allModulesPresent { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + baseObj["output"] = "appearance.2da" + + lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) + if err != nil { + return 0, err + } + + moduleObjs := make([]map[string]any, 0, len(moduleNames)) + for _, name := range moduleNames { + obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", name)) + if err != nil { + return 0, err + } + moduleObjs = append(moduleObjs, obj) + } + + if err := os.MkdirAll(targetModulesDir, 0o755); err != nil { + return 0, err + } + + writes := []struct { + path string + obj map[string]any + }{ + {path: targetBasePath, obj: baseObj}, + {path: targetLockPath, obj: lockObj}, + } + for i, path := range targetModulePaths { + writes = append(writes, struct { + path string + obj map[string]any + }{path: path, obj: moduleObjs[i]}) + } + + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + return len(writes), nil +} diff --git a/internal/topdata/armor_migrate.go b/internal/topdata/armor_migrate.go new file mode 100644 index 0000000..0bdfbdd --- /dev/null +++ b/internal/topdata/armor_migrate.go @@ -0,0 +1,160 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" +) + +func importLegacyArmor(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "armor") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "armor") + targetPath := filepath.Join(targetDir, "armor.json") + if _, err := os.Stat(targetPath); err == nil { + obj, err := loadJSONObject(targetPath) + if err == nil && countLegacyTLKRefsInValue(obj) == 0 { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + rows, err := mergeArmorRows(baseObj, filepath.Join(legacyDir, "modules")) + if err != nil { + return 0, err + } + + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(targetDir, "base.json")); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(targetDir, "lock.json")); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(targetDir, "modules")); err != nil { + return 0, err + } + + out := map[string]any{ + "output": "armor.2da", + "columns": baseObj["columns"], + "rows": rows, + } + raw, err := json.MarshalIndent(out, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(targetPath, raw, 0o644); err != nil { + return 0, err + } + return 1, nil +} + +func mergeArmorRows(baseObj map[string]any, modulesDir string) ([]any, error) { + rawRows, ok := baseObj["rows"].([]any) + if !ok { + return nil, nil + } + + rows := make([]map[string]any, 0, len(rawRows)) + byID := map[int]map[string]any{} + for _, raw := range rawRows { + row, ok := deepCopyValue(raw).(map[string]any) + if !ok { + continue + } + delete(row, "key") + rawID, ok := row["id"] + if !ok { + continue + } + id, err := asInt(rawID) + if err != nil { + return nil, err + } + row["id"] = id + rows = append(rows, row) + byID[id] = row + } + + modulePaths, err := collectModulePaths(modulesDir) + if err != nil { + return nil, err + } + for _, path := range modulePaths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + if entries, ok := obj["entries"].(map[string]any); ok { + for _, raw := range entries { + row, ok := deepCopyValue(raw).(map[string]any) + if !ok { + continue + } + delete(row, "key") + rawID, ok := row["id"] + if !ok { + continue + } + id, err := asInt(rawID) + if err != nil { + return nil, err + } + row["id"] = id + rows = append(rows, row) + byID[id] = row + } + } + if overrides, ok := obj["overrides"].([]any); ok { + for _, raw := range overrides { + override, ok := raw.(map[string]any) + if !ok { + continue + } + rawID, ok := override["id"] + if !ok { + continue + } + id, err := asInt(rawID) + if err != nil { + return nil, err + } + row := byID[id] + if row == nil { + continue + } + for key, value := range override { + if key == "id" || key == "key" || key == "_tlk" || isMetadataField(key) { + continue + } + row[key] = deepCopyValue(value) + } + } + } + } + + sort.Slice(rows, func(i, j int) bool { + return rows[i]["id"].(int) < rows[j]["id"].(int) + }) + out := make([]any, 0, len(rows)) + for _, row := range rows { + out = append(out, row) + } + return out, nil +} diff --git a/internal/topdata/assets_repo.go b/internal/topdata/assets_repo.go new file mode 100644 index 0000000..c6e7aa2 --- /dev/null +++ b/internal/topdata/assets_repo.go @@ -0,0 +1,46 @@ +package topdata + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func resolvePartsAssetsOverrideDir(p *project.Project) (string, error) { + spec := strings.TrimSpace(p.Config.TopData.Assets) + if spec == "" { + return "", nil + } + if looksLikeGitRepoSpec(spec) { + return "", fmt.Errorf("topdata.assets no longer supports git repo specs for parts discovery; use a local path override or the default released manifest") + } + path := p.TopDataAssetsDir() + if path == "" { + return "", nil + } + if _, err := os.Stat(path); err != nil { + return "", fmt.Errorf("configured topdata.assets path %s is not accessible: %w", path, err) + } + return path, nil +} + +func looksLikeGitRepoSpec(value string) bool { + return strings.Contains(value, "://") || + strings.HasPrefix(value, "git@") || + strings.HasSuffix(value, ".git") +} + +func gitOutput(dir string, args ...string) (string, error) { + cmd := exec.Command("git", args...) + if dir != "" { + cmd.Dir = dir + } + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("%v: %s", args, strings.TrimSpace(string(output))) + } + return strings.TrimSpace(string(output)), nil +} diff --git a/internal/topdata/autogen.go b/internal/topdata/autogen.go new file mode 100644 index 0000000..8af0265 --- /dev/null +++ b/internal/topdata/autogen.go @@ -0,0 +1,1132 @@ +package topdata + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +const autogenManifestCacheMaxAge = time.Hour + +type autogenManifest struct { + ID string `json:"id"` + Repo string `json:"repo"` + Ref string `json:"ref"` + GeneratedAt string `json:"generated_at"` + HeadVisualeffects *project.HeadVisualeffectsConfig `json:"accessory_visualeffects,omitempty"` + Entries []autogenManifestEntry `json:"entries"` +} + +type autogenManifestEntry struct { + Source string `json:"source"` + Group string `json:"group,omitempty"` + Subgroup string `json:"subgroup,omitempty"` + Category string `json:"category,omitempty"` + ModelStem string `json:"model_stem,omitempty"` + RowID int `json:"row_id,omitempty"` +} + +type autogenManifestAssetMetadata struct { + DownloadURL string + UpdatedAt time.Time +} + +type ProducedAutogenManifest struct { + ID string + AssetName string + OutputPath string + Payload []byte +} + +func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDataset, progress func(string)) ([]nativeCollectedDataset, error) { + if len(p.Config.Autogen.Consumers) == 0 { + return collected, nil + } + if progress == nil { + progress = func(string) {} + } + + result := append([]nativeCollectedDataset(nil), collected...) + for _, consumer := range p.Config.Autogen.Consumers { + if !autogenConsumerTargetsCollectedDataset(result, consumer) { + continue + } + manifest, err := resolveAutogenConsumerManifest(p, consumer, progress) + if err != nil { + if consumer.Optional && isIgnorableOptionalAutogenError(err) { + if progress != nil { + progress(fmt.Sprintf("Skipping optional autogen consumer %s: %v", consumer.ID, err)) + } + continue + } + return nil, err + } + entries := manifest.Entries + if len(entries) == 0 { + continue + } + switch consumer.Mode { + case "parts_rows": + result = augmentWithAutogeneratedParts(result, autogenPartsInventory(entries)) + case "accessory_visualeffects": + result, err = augmentWithAutogeneratedHeadVisualeffects(result, entries, consumer, manifest.HeadVisualeffects) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unsupported autogen consumer mode %q", consumer.Mode) + } + } + return result, nil +} + +func autogenConsumerTargetsCollectedDataset(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig) bool { + for _, dataset := range collected { + switch consumer.Mode { + case "parts_rows": + if isAutogenEligiblePartsDataset(dataset) { + return true + } + case "accessory_visualeffects": + if dataset.Dataset.Name == "visualeffects" { + return true + } + case "cachedmodels_rows": + if dataset.Dataset.Name == "cachedmodels" { + return true + } + } + } + return false +} + +func isIgnorableOptionalAutogenError(err error) bool { + if err == nil { + return false + } + message := err.Error() + return strings.Contains(message, "HTTP 404") || + strings.Contains(message, "not found") || + strings.Contains(message, "entries is empty") +} + +func resolveAutogenConsumerManifest(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) (*autogenManifest, error) { + overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer) + if err != nil { + return nil, err + } + if overrideRoot != "" { + if progress != nil { + progress(fmt.Sprintf("Scanning local autogen override for %s from %s...", consumer.ID, overrideRoot)) + } + entries, err := scanLocalAutogenEntries(overrideRoot, consumer) + if err != nil { + return nil, err + } + return &autogenManifest{ID: consumer.Producer, Entries: entries}, nil + } + return resolveReleasedAutogenManifest(p, consumer, progress) +} + +func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.AutogenConsumerConfig) (string, error) { + root := strings.TrimSpace(consumer.LocalOverrideRoot) + if root != "" { + if filepath.IsAbs(root) { + return root, nil + } + return filepath.Join(p.Root, root), nil + } + + spec := strings.TrimSpace(p.Config.TopData.Assets) + if spec == "" { + return "", nil + } + if looksLikeGitRepoSpec(spec) { + return "", fmt.Errorf("topdata.assets no longer supports git repo specs for autogen discovery; use a local path override or the default released manifest") + } + path := p.TopDataAssetsDir() + if path == "" { + return "", nil + } + if _, err := os.Stat(path); err != nil { + return "", fmt.Errorf("configured topdata.assets path %s is not accessible: %w", path, err) + } + candidate := filepath.Join(path, consumer.Root) + info, err := os.Stat(candidate) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", fmt.Errorf("configured topdata.assets override root %s is not accessible: %w", candidate, err) + } + if !info.IsDir() { + return "", fmt.Errorf("configured topdata.assets override root %s is not a directory", candidate) + } + return path, nil +} + +func scanLocalAutogenEntries(overrideRoot string, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) { + scanRoot := filepath.Join(overrideRoot, consumer.Root) + return scanConfiguredAutogenEntries(scanRoot, consumer.Include, consumer.Derive) +} + +func scanConfiguredAutogenEntries(scanRoot string, include []string, derive project.AutogenDeriveConfig) ([]autogenManifestEntry, error) { + info, err := os.Stat(scanRoot) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("stat autogen override root %s: %w", scanRoot, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("autogen override root %s is not a directory", scanRoot) + } + + var entries []autogenManifestEntry + err = filepath.Walk(scanRoot, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + if info.IsDir() { + return nil + } + rel, err := filepath.Rel(scanRoot, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if !matchesAutogenInclude(rel, include) { + return nil + } + entry, ok := deriveAutogenManifestEntry(rel, derive) + if !ok { + return nil + } + entries = append(entries, entry) + return nil + }) + if err != nil { + return nil, fmt.Errorf("scan autogen override %s: %w", scanRoot, err) + } + + slices.SortFunc(entries, func(a, b autogenManifestEntry) int { + return strings.Compare(a.Source, b.Source) + }) + return entries, nil +} + +func ProduceAutogenManifests(p *project.Project) ([]ProducedAutogenManifest, error) { + if len(p.Config.Autogen.Producers) == 0 { + return nil, nil + } + + assetsRoot := strings.TrimSpace(p.AssetsDir()) + if assetsRoot == "" { + return nil, fmt.Errorf("autogen producers require paths.assets to be configured") + } + info, err := os.Stat(assetsRoot) + if err != nil { + return nil, fmt.Errorf("stat assets root %s: %w", assetsRoot, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("assets root %s is not a directory", assetsRoot) + } + + repo, ref := autogenManifestSourceMetadata(p.Root) + results := make([]ProducedAutogenManifest, 0, len(p.Config.Autogen.Producers)) + for _, producer := range p.Config.Autogen.Producers { + scanRoot := filepath.Join(assetsRoot, producer.Root) + entries, err := scanConfiguredAutogenEntries(scanRoot, producer.Include, producer.Derive) + if err != nil { + return nil, fmt.Errorf("scan autogen producer %s: %w", producer.ID, err) + } + if len(entries) == 0 { + return nil, fmt.Errorf("autogen producer %s discovered no matching entries under %s", producer.ID, scanRoot) + } + payload, err := formatAutogenManifest(p.Root, producer, repo, ref, entries) + if err != nil { + return nil, fmt.Errorf("format autogen producer %s manifest: %w", producer.ID, err) + } + payload = append(payload, '\n') + results = append(results, ProducedAutogenManifest{ + ID: producer.ID, + AssetName: producer.Manifest.AssetName, + OutputPath: filepath.Join(p.BuildDir(), producer.Manifest.AssetName), + Payload: payload, + }) + } + return results, nil +} + +func autogenManifestSourceMetadata(root string) (repo string, ref string) { + if remote, err := gitOutput(root, "remote", "get-url", "origin"); err == nil { + if spec, ok := parseRepoSpec(strings.TrimSpace(remote)); ok { + repo = spec.Owner + "/" + spec.Repo + } + } + if head, err := gitOutput(root, "rev-parse", "HEAD"); err == nil { + ref = strings.TrimSpace(head) + } + return repo, ref +} + +func matchesAutogenInclude(rel string, include []string) bool { + rel = filepath.ToSlash(rel) + for _, pattern := range include { + pattern = filepath.ToSlash(strings.TrimSpace(pattern)) + if pattern == "" { + continue + } + switch { + case pattern == "**/*.mdl": + if strings.HasSuffix(strings.ToLower(rel), ".mdl") { + return true + } + case strings.HasSuffix(pattern, "/**/*.mdl"): + prefix := strings.TrimSuffix(pattern, "/**/*.mdl") + if strings.HasPrefix(rel, prefix+"/") && strings.HasSuffix(strings.ToLower(rel), ".mdl") { + return true + } + case strings.HasSuffix(pattern, "/*.mdl"): + prefix := strings.TrimSuffix(pattern, "/*.mdl") + dir := filepath.ToSlash(filepath.Dir(rel)) + if dir == prefix && strings.HasSuffix(strings.ToLower(rel), ".mdl") { + return true + } + case rel == pattern: + return true + } + } + return false +} + +func deriveAutogenManifestEntry(rel string, derive project.AutogenDeriveConfig) (autogenManifestEntry, bool) { + rel = filepath.ToSlash(rel) + stem := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)) + entry := autogenManifestEntry{ + Source: rel, + ModelStem: stem, + } + parts := strings.Split(rel, "/") + if derive.GroupFrom == "first_path_segment" && len(parts) > 1 { + entry.Group = parts[0] + if len(parts) > 2 { + entry.Subgroup = strings.Join(parts[1:len(parts)-1], "/") + entry.Category = parts[len(parts)-2] + } + } + + switch derive.Kind { + case "trailing_numeric_suffix": + rowID, ok := extractTrailingNumericSuffix(stem) + if !ok { + return autogenManifestEntry{}, false + } + entry.RowID = rowID + case "model_stem": + default: + return autogenManifestEntry{}, false + } + + return entry, true +} + +func resolveReleasedAutogenManifest(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) (*autogenManifest, error) { + cachePath := p.AutogenCachePath(consumer.Manifest.CacheName) + if strings.TrimSpace(os.Getenv(p.AutogenRefreshEnv())) == "" { + manifest, fresh, err := readFreshAutogenManifestCache(cachePath, p.AutogenCacheMaxAge()) + if err != nil { + if progress != nil { + progress(fmt.Sprintf("Ignoring cached autogen manifest %s: %v", consumer.Manifest.CacheName, err)) + } + } else if fresh { + if current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, manifest); err != nil { + if progress != nil { + progress(fmt.Sprintf("Using cached released autogen manifest %s (remote check failed: %v)...", cachePath, err)) + } + return manifest, nil + } else if current { + if progress != nil { + progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath)) + } + return manifest, nil + } else if progress != nil { + progress(fmt.Sprintf("Refreshing released autogen manifest %s; remote asset changed at %s.", consumer.Manifest.CacheName, remoteTime.Format(time.RFC3339))) + } + } + } + + spec, err := deriveSowAssetsRepoSpec(p) + if err != nil { + return nil, err + } + manifestURL, err := resolveAutogenManifestAssetURL(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName) + if err != nil { + if consumer.Mode == "parts_rows" { + return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1)) + } + return nil, err + } + if progress != nil { + progress(fmt.Sprintf("Fetching released autogen manifest from %s...", manifestURL)) + } + manifest, err := fetchAutogenManifest(manifestURL) + if err != nil { + if consumer.Mode == "parts_rows" { + return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1)) + } + return nil, err + } + if manifest.ID != "" && manifest.ID != consumer.Producer { + return nil, fmt.Errorf("autogen manifest %s declared id %q, expected %q", consumer.Manifest.AssetName, manifest.ID, consumer.Producer) + } + if err := writeAutogenManifestCache(cachePath, manifest); err != nil { + return nil, err + } + return manifest, nil +} + +func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) { + manifest, err := resolveReleasedAutogenManifest(p, consumer, progress) + if err != nil { + return nil, err + } + return manifest.Entries, nil +} + +func releasedAutogenManifestCacheCurrent(p *project.Project, consumer project.AutogenConsumerConfig, cachePath string, manifest *autogenManifest) (bool, time.Time, error) { + spec, err := deriveSowAssetsRepoSpec(p) + if err != nil { + return false, time.Time{}, err + } + asset, err := resolveAutogenManifestAssetMetadata(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName) + if err != nil { + return false, time.Time{}, err + } + if asset.UpdatedAt.IsZero() { + return true, time.Time{}, nil + } + return !asset.UpdatedAt.After(localAutogenManifestComparableTime(cachePath, manifest)), asset.UpdatedAt, nil +} + +func localAutogenManifestComparableTime(cachePath string, manifest *autogenManifest) time.Time { + best := time.Time{} + if info, err := os.Stat(cachePath); err == nil { + best = info.ModTime() + } + if generatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(manifest.GeneratedAt)); err == nil && generatedAt.After(best) { + best = generatedAt + } + return best +} + +func newestReleasedAutogenManifestInput(p *project.Project, consumer project.AutogenConsumerConfig, now time.Time, cachePath string) (time.Time, string, error) { + if strings.TrimSpace(os.Getenv(p.AutogenRefreshEnv())) != "" { + return now, cachePath, nil + } + + info, err := os.Stat(cachePath) + if err != nil { + if os.IsNotExist(err) { + return now, cachePath, nil + } + return time.Time{}, "", fmt.Errorf("stat autogen manifest cache %s: %w", cachePath, err) + } + if autogenManifestCacheMaxAge > 0 && now.Sub(info.ModTime()) > autogenManifestCacheMaxAge { + return now, cachePath, nil + } + + raw, err := os.ReadFile(cachePath) + if err != nil { + return info.ModTime(), cachePath, nil + } + var manifest autogenManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return info.ModTime(), cachePath, nil + } + + current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, &manifest) + if err != nil || current || remoteTime.IsZero() { + return info.ModTime(), cachePath, nil + } + return remoteTime, consumer.Manifest.AssetName, nil +} + +func resolveAutogenManifestAssetURL(spec giteaRepoSpec, releaseTag, assetName string) (string, error) { + asset, err := resolveAutogenManifestAssetMetadata(spec, releaseTag, assetName) + if err != nil { + return "", err + } + return asset.DownloadURL, nil +} + +func resolveAutogenManifestAssetMetadata(spec giteaRepoSpec, releaseTag, assetName string) (autogenManifestAssetMetadata, error) { + apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), releasePathEscape(spec.Owner), releasePathEscape(spec.Repo), releasePathEscape(releaseTag)) + var release struct { + Assets []struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + DownloadURL string `json:"download_url"` + UpdatedAt string `json:"updated_at"` + } `json:"assets"` + } + if err := fetchJSON(apiURL, &release); err != nil { + return autogenManifestAssetMetadata{}, fmt.Errorf("fetch autogen manifest release metadata: %w", err) + } + for _, asset := range release.Assets { + if asset.Name != assetName { + continue + } + metadata := autogenManifestAssetMetadata{} + if updatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(asset.UpdatedAt)); err == nil { + metadata.UpdatedAt = updatedAt + } + if asset.DownloadURL != "" { + metadata.DownloadURL = asset.DownloadURL + return metadata, nil + } + if asset.BrowserDownloadURL != "" { + metadata.DownloadURL = asset.BrowserDownloadURL + return metadata, nil + } + } + return autogenManifestAssetMetadata{}, fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag) +} + +func releasePathEscape(value string) string { + replacer := strings.NewReplacer("/", "%2F") + return replacer.Replace(value) +} + +func fetchAutogenManifest(manifestURL string) (*autogenManifest, error) { + var manifest autogenManifest + if err := fetchJSON(manifestURL, &manifest); err != nil { + return nil, fmt.Errorf("download autogen manifest: %w", err) + } + if len(manifest.Entries) == 0 { + var legacy partsManifest + if err := fetchJSON(manifestURL, &legacy); err == nil && len(legacy.Categories) > 0 { + return legacyPartsManifestToAutogen(&legacy), nil + } + } + if len(manifest.Entries) == 0 { + return nil, fmt.Errorf("download autogen manifest: entries is empty") + } + return &manifest, nil +} + +func readFreshAutogenManifestCache(path string, maxAge time.Duration) (*autogenManifest, bool, error) { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil, false, nil + } + return nil, false, err + } + if maxAge > 0 && time.Since(info.ModTime()) > maxAge { + return nil, false, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, false, err + } + var manifest autogenManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, false, err + } + if len(manifest.Entries) == 0 { + var legacy partsManifest + if err := json.Unmarshal(raw, &legacy); err == nil && len(legacy.Categories) > 0 { + return legacyPartsManifestToAutogen(&legacy), true, nil + } + } + if len(manifest.Entries) == 0 { + return nil, false, fmt.Errorf("entries is empty") + } + return &manifest, true, nil +} + +func legacyPartsManifestToAutogen(legacy *partsManifest) *autogenManifest { + entries := make([]autogenManifestEntry, 0) + for _, category := range supportedPartCategories { + for _, rowID := range legacy.Categories[category] { + entries = append(entries, autogenManifestEntry{ + Group: category, + RowID: rowID, + }) + } + } + slices.SortFunc(entries, func(a, b autogenManifestEntry) int { + if cmp := strings.Compare(a.Group, b.Group); cmp != 0 { + return cmp + } + return a.RowID - b.RowID + }) + return &autogenManifest{ + ID: "parts", + Repo: legacy.Repo, + Ref: legacy.Ref, + GeneratedAt: legacy.GeneratedAt, + Entries: entries, + } +} + +func writeAutogenManifestCache(path string, manifest *autogenManifest) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create autogen manifest cache dir: %w", err) + } + payload, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return fmt.Errorf("marshal autogen manifest cache: %w", err) + } + payload = append(payload, '\n') + current, err := os.ReadFile(path) + if err == nil && bytes.Equal(current, payload) { + now := time.Now() + if err := os.Chtimes(path, now, now); err != nil { + return fmt.Errorf("refresh autogen manifest cache timestamp: %w", err) + } + return nil + } + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("read autogen manifest cache: %w", err) + } + tmpPath := path + ".tmp" + if err := os.WriteFile(tmpPath, payload, 0o644); err != nil { + return fmt.Errorf("write autogen manifest cache: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("replace autogen manifest cache: %w", err) + } + return nil +} + +func autogenPartsInventory(entries []autogenManifestEntry) map[string]map[int]struct{} { + result := make(map[string]map[int]struct{}, len(supportedPartCategories)) + for _, category := range supportedPartCategories { + result[category] = map[int]struct{}{} + } + for _, entry := range entries { + if !isSupportedPartCategory(entry.Group) || entry.RowID <= 0 { + continue + } + result[entry.Group][entry.RowID] = struct{}{} + } + return result +} + +func augmentWithAutogeneratedCachedModels(collected []nativeCollectedDataset, entries []autogenManifestEntry) ([]nativeCollectedDataset, error) { + if len(entries) == 0 { + return collected, nil + } + + result := append([]nativeCollectedDataset(nil), collected...) + for i, dataset := range result { + if dataset.Dataset.Name != "cachedmodels" { + continue + } + if !slices.Contains(dataset.Columns, "Model") { + return nil, fmt.Errorf("cachedmodels dataset must include Model column for cachedmodels_rows autogen") + } + + rows := make([]map[string]any, len(dataset.Rows), len(dataset.Rows)+len(entries)) + seenModels := make(map[string]struct{}, len(dataset.Rows)+len(entries)) + nextID := 0 + for index, row := range dataset.Rows { + cloned := cloneRowMap(row) + rows[index] = cloned + if rowID, ok := cloned["id"].(int); ok && rowID >= nextID { + nextID = rowID + 1 + } + if model, ok := cloned["Model"].(string); ok { + model = strings.TrimSpace(model) + if model != "" && model != nullValue { + seenModels[strings.ToLower(model)] = struct{}{} + } + } + } + + for _, entry := range entries { + model := strings.TrimSpace(entry.ModelStem) + if model == "" { + continue + } + key := strings.ToLower(model) + if _, exists := seenModels[key]; exists { + continue + } + rows = append(rows, map[string]any{ + "id": nextID, + "Model": model, + }) + seenModels[key] = struct{}{} + nextID++ + } + + result[i].Rows = rows + } + return result, nil +} + +func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDataset, entries []autogenManifestEntry, consumer project.AutogenConsumerConfig, manifestPolicy *project.HeadVisualeffectsConfig) ([]nativeCollectedDataset, error) { + if len(entries) == 0 { + return collected, nil + } + + policy := resolveHeadVisualeffectsPolicy(consumer, manifestPolicy) + result := append([]nativeCollectedDataset(nil), collected...) + for i, dataset := range result { + if dataset.Dataset.Name != "visualeffects" { + continue + } + + rows := make([]map[string]any, len(dataset.Rows)) + rowByKey := make(map[string]map[string]any, len(dataset.Rows)) + historicalLockData := make(map[string]int, len(dataset.LockData)) + for key, rowID := range dataset.LockData { + historicalLockData[key] = rowID + } + if dataset.Dataset.LockPath != "" { + if existingLockData, err := loadLockfile(dataset.Dataset.LockPath); err == nil { + for key, rowID := range existingLockData { + if _, ok := historicalLockData[key]; !ok { + historicalLockData[key] = rowID + } + } + } + } + usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(historicalLockData)) + lockData := make(map[string]int, len(dataset.LockData)) + for _, rowID := range historicalLockData { + usedIDs[rowID] = struct{}{} + } + for key, rowID := range dataset.LockData { + lockData[key] = rowID + } + for index, row := range dataset.Rows { + cloned := cloneRowMap(row) + rows[index] = cloned + if key, ok := cloned["key"].(string); ok && key != "" { + rowByKey[key] = cloned + } + if rowID, ok := cloned["id"].(int); ok { + usedIDs[rowID] = struct{}{} + } + } + + nextID := nextAvailableAutogenID(usedIDs) + allocateNextID := func() int { + rowID := nextID + usedIDs[rowID] = struct{}{} + nextID = nextAvailableAutogenID(usedIDs) + return rowID + } + + for _, entry := range entries { + key, label, modelStem, groupPolicy, ok := headVisualeffectIdentity(dataset.Dataset.Name, entry, policy) + if !ok { + continue + } + if existing, exists := rowByKey[key]; exists { + applyDiscoveredHeadVisualeffectDefaults(existing, dataset.Columns, modelStem, label, policy, groupPolicy) + continue + } + + rowID, ok := lockData[key] + if !ok { + if preservedRowID, preserved := historicalLockData[key]; preserved { + rowID = preservedRowID + } else { + rowID = allocateNextID() + } + lockData[key] = rowID + } + newRow := createDefaultHeadVisualeffectRow(dataset.Columns, rowID, key, label, modelStem, policy, groupPolicy) + rows = append(rows, newRow) + rowByKey[key] = newRow + } + + slices.SortFunc(rows, func(a, b map[string]any) int { + return a["id"].(int) - b["id"].(int) + }) + result[i].Rows = rows + result[i].LockData = lockData + result[i].LockModified = !lockDataEqual(dataset.LockData, lockData) + } + return result, nil +} + +func nextAvailableAutogenID(used map[int]struct{}) int { + for rowID := 0; ; rowID++ { + if _, ok := used[rowID]; !ok { + return rowID + } + } +} + +type headVisualeffectsPolicy struct { + Groups map[string]headVisualeffectsGroupPolicy + GroupTokenSource string + CategoryFrom string + Delimiter string + Case string + StripModelPrefixes []string + KeyFormat string + LabelFormat string + ModelColumn string + RowDefaults map[string]string +} + +type headVisualeffectsGroupPolicy struct { + Prefix string + ModelColumns []string + RowDefaults map[string]string +} + +func resolveHeadVisualeffectsPolicy(consumer project.AutogenConsumerConfig, manifestPolicy *project.HeadVisualeffectsConfig) headVisualeffectsPolicy { + policy := defaultHeadVisualeffectsPolicy() + if manifestPolicy != nil { + applyHeadVisualeffectsConfig(&policy, *manifestPolicy) + } + applyHeadVisualeffectsConfig(&policy, consumer.HeadVisualeffects) + return policy +} + +func applyHeadVisualeffectsConfig(policy *headVisualeffectsPolicy, cfg project.HeadVisualeffectsConfig) { + for group, groupCfg := range cfg.Groups { + group = strings.TrimSpace(group) + prefix := strings.TrimSpace(groupCfg.Prefix) + if group == "" { + continue + } + groupPolicy := policy.Groups[group] + if prefix != "" { + groupPolicy.Prefix = prefix + } + if columns := normalizeHeadVisualeffectsModelColumns(groupCfg.ModelColumn, groupCfg.ModelColumns); len(columns) > 0 { + groupPolicy.ModelColumns = columns + } + if groupPolicy.RowDefaults == nil { + groupPolicy.RowDefaults = map[string]string{} + } + mergeStringMap(groupPolicy.RowDefaults, normalizeHeadVisualeffectsRowDefaults(groupCfg.RowDefaults)) + policy.Groups[group] = groupPolicy + } + if strings.TrimSpace(cfg.GroupTokenSource) != "" { + policy.GroupTokenSource = strings.TrimSpace(cfg.GroupTokenSource) + } + if strings.TrimSpace(cfg.CategoryFrom) != "" { + policy.CategoryFrom = strings.TrimSpace(cfg.CategoryFrom) + } + if cfg.Delimiter != "" { + policy.Delimiter = cfg.Delimiter + } + if strings.TrimSpace(cfg.Case) != "" { + policy.Case = strings.TrimSpace(cfg.Case) + } + if len(cfg.StripModelPrefixes) > 0 { + policy.StripModelPrefixes = append([]string(nil), cfg.StripModelPrefixes...) + } + if strings.TrimSpace(cfg.KeyFormat) != "" { + policy.KeyFormat = strings.TrimSpace(cfg.KeyFormat) + } + if strings.TrimSpace(cfg.LabelFormat) != "" { + policy.LabelFormat = strings.TrimSpace(cfg.LabelFormat) + } + if strings.TrimSpace(cfg.ModelColumn) != "" { + policy.ModelColumn = strings.TrimSpace(cfg.ModelColumn) + } + mergeStringMap(policy.RowDefaults, normalizeHeadVisualeffectsRowDefaults(cfg.RowDefaults)) +} + +func defaultHeadVisualeffectsPolicy() headVisualeffectsPolicy { + return headVisualeffectsPolicy{ + Groups: map[string]headVisualeffectsGroupPolicy{ + "chest_accessories": {}, + "head_accessories": {}, + "head_decorations": {}, + "head_features": {}, + }, + GroupTokenSource: "folder_name", + CategoryFrom: "immediate_parent", + Delimiter: "/", + Case: "preserve", + StripModelPrefixes: []string{"hfx_", "tfx_", "vfx_", "cfx_"}, + KeyFormat: "{dataset}:{group}{delimiter}{category_segment}{stem}", + LabelFormat: "{group}{delimiter}{category_segment}{stem}", + ModelColumn: "Imp_HeadCon_Node", + RowDefaults: map[string]string{ + "Type_FD": "D", + "OrientWithGround": "0", + "OrientWithObject": "1", + }, + } +} + +func normalizeHeadVisualeffectsModelColumns(modelColumn string, modelColumns []string) []string { + var result []string + if column := strings.TrimSpace(modelColumn); column != "" { + result = append(result, column) + } + for _, column := range modelColumns { + column = strings.TrimSpace(column) + if column == "" { + continue + } + if !slices.Contains(result, column) { + result = append(result, column) + } + } + return result +} + +func normalizeHeadVisualeffectsRowDefaults(defaults map[string]string) map[string]string { + result := map[string]string{} + for column, value := range defaults { + column = strings.TrimSpace(column) + if column == "" { + continue + } + result[column] = value + } + return result +} + +func mergeStringMap(target map[string]string, source map[string]string) { + for key, value := range source { + target[key] = value + } +} + +func headVisualeffectIdentity(dataset string, entry autogenManifestEntry, policy headVisualeffectsPolicy) (string, string, string, headVisualeffectsGroupPolicy, bool) { + entry = normalizeHeadVisualeffectEntry(entry) + group, ok := policy.Groups[entry.Group] + if !ok || strings.TrimSpace(entry.ModelStem) == "" { + return "", "", "", headVisualeffectsGroupPolicy{}, false + } + modelStem := strings.TrimSpace(entry.ModelStem) + stem := stripHeadVisualeffectModelPrefix(modelStem, policy) + stem = strings.TrimSpace(stem) + if stem == "" { + return "", "", "", headVisualeffectsGroupPolicy{}, false + } + category := headVisualeffectCategory(entry, policy) + groupToken := headVisualeffectGroupToken(entry.Group, group, policy) + values := map[string]string{ + "dataset": dataset, + "group": applyHeadVisualeffectCase(groupToken, policy), + "group_raw": entry.Group, + "prefix": applyHeadVisualeffectCase(group.Prefix, policy), + "category": applyHeadVisualeffectCase(category, policy), + "category_upper": strings.ToUpper(category), + "category_segment": headVisualeffectDelimitedSegment(applyHeadVisualeffectCase(category, policy), policy.Delimiter), + "category_segment_upper": headVisualeffectDelimitedSegment(strings.ToUpper(category), policy.Delimiter), + "subgroup": applyHeadVisualeffectCase(entry.Subgroup, policy), + "stem": applyHeadVisualeffectCase(stem, policy), + "stem_upper": strings.ToUpper(stem), + "model_stem": modelStem, + "delimiter": policy.Delimiter, + } + key := expandHeadVisualeffectsFormat(policy.KeyFormat, values) + label := expandHeadVisualeffectsFormat(policy.LabelFormat, values) + if strings.TrimSpace(key) == "" || strings.TrimSpace(label) == "" { + return "", "", "", headVisualeffectsGroupPolicy{}, false + } + return key, label, modelStem, group, true +} + +func normalizeHeadVisualeffectEntry(entry autogenManifestEntry) autogenManifestEntry { + entry.Group = strings.TrimSpace(entry.Group) + if strings.TrimSpace(entry.Group) == "" && strings.TrimSpace(entry.Source) != "" { + parts := strings.Split(filepath.ToSlash(entry.Source), "/") + if len(parts) > 1 { + entry.Group = parts[0] + } + } + if strings.TrimSpace(entry.Subgroup) == "" && strings.TrimSpace(entry.Source) != "" { + parts := strings.Split(filepath.ToSlash(entry.Source), "/") + if len(parts) > 2 { + entry.Subgroup = strings.Join(parts[1:len(parts)-1], "/") + } + } + if strings.TrimSpace(entry.Category) == "" && strings.TrimSpace(entry.Source) != "" { + parts := strings.Split(filepath.ToSlash(entry.Source), "/") + if len(parts) > 2 { + entry.Category = parts[len(parts)-2] + } + } + return entry +} + +func stripHeadVisualeffectModelPrefix(modelStem string, policy headVisualeffectsPolicy) string { + stem := modelStem + for _, prefix := range policy.StripModelPrefixes { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + continue + } + stem = strings.TrimPrefix(stem, prefix) + } + return stem +} + +func headVisualeffectGroupToken(groupName string, group headVisualeffectsGroupPolicy, policy headVisualeffectsPolicy) string { + if policy.GroupTokenSource == "folder_name" { + return groupName + } + if strings.TrimSpace(group.Prefix) != "" { + return group.Prefix + } + return groupName +} + +func headVisualeffectCategory(entry autogenManifestEntry, policy headVisualeffectsPolicy) string { + switch policy.CategoryFrom { + case "immediate_parent": + return strings.TrimSpace(entry.Category) + case "full_subgroup": + return strings.TrimSpace(entry.Subgroup) + default: + return "" + } +} + +func headVisualeffectDelimitedSegment(value, delimiter string) string { + if strings.TrimSpace(value) == "" { + return "" + } + return value + delimiter +} + +func applyHeadVisualeffectCase(value string, policy headVisualeffectsPolicy) string { + switch policy.Case { + case "lower": + return strings.ToLower(value) + case "upper": + return strings.ToUpper(value) + default: + return value + } +} + +func expandHeadVisualeffectsFormat(format string, values map[string]string) string { + result := format + for key, value := range values { + result = strings.ReplaceAll(result, "{"+key+"}", value) + } + return result +} + +func createDefaultHeadVisualeffectRow(columns []string, rowID int, key, label, modelStem string, policy headVisualeffectsPolicy, group headVisualeffectsGroupPolicy) map[string]any { + row := map[string]any{ + "id": rowID, + "key": key, + } + for _, column := range columns { + row[column] = nullValue + } + row["Label"] = label + for column, value := range policy.RowDefaults { + row[column] = value + } + for column, value := range group.RowDefaults { + row[column] = value + } + for _, column := range headVisualeffectModelColumns(policy, group) { + row[column] = modelStem + } + return row +} + +func applyDiscoveredHeadVisualeffectDefaults(row map[string]any, columns []string, modelStem, label string, policy headVisualeffectsPolicy, group headVisualeffectsGroupPolicy) { + if isNullLikeValue(row["Label"]) { + row["Label"] = label + } + for column, value := range policy.RowDefaults { + if isNullLikeValue(row[column]) { + row[column] = value + } + } + for column, value := range group.RowDefaults { + if isNullLikeValue(row[column]) { + row[column] = value + } + } + for _, column := range headVisualeffectModelColumns(policy, group) { + if isNullLikeValue(row[column]) { + row[column] = modelStem + } + } + for _, column := range columns { + if _, ok := row[column]; !ok { + row[column] = nullValue + } + } +} + +func headVisualeffectModelColumns(policy headVisualeffectsPolicy, group headVisualeffectsGroupPolicy) []string { + if len(group.ModelColumns) > 0 { + return group.ModelColumns + } + if strings.TrimSpace(policy.ModelColumn) == "" { + return nil + } + return []string{policy.ModelColumn} +} + +func formatAutogenManifest(root string, producer project.AutogenProducerConfig, repo, ref string, entries []autogenManifestEntry) ([]byte, error) { + manifest := autogenManifest{ + ID: producer.ID, + Repo: repo, + Ref: ref, + GeneratedAt: time.Now().UTC().Format(time.RFC3339), + Entries: entries, + } + if autogenHeadVisualeffectsConfigConfigured(producer.HeadVisualeffects) { + cfg := producer.HeadVisualeffects + manifest.HeadVisualeffects = &cfg + } + return json.MarshalIndent(manifest, "", " ") +} + +func autogenHeadVisualeffectsConfigConfigured(cfg project.HeadVisualeffectsConfig) bool { + return len(cfg.Groups) > 0 || + strings.TrimSpace(cfg.GroupTokenSource) != "" || + strings.TrimSpace(cfg.CategoryFrom) != "" || + cfg.Delimiter != "" || + strings.TrimSpace(cfg.Case) != "" || + len(cfg.StripModelPrefixes) > 0 || + strings.TrimSpace(cfg.KeyFormat) != "" || + strings.TrimSpace(cfg.LabelFormat) != "" || + strings.TrimSpace(cfg.ModelColumn) != "" || + len(cfg.RowDefaults) > 0 +} + +func parseAutogenManifestEntryList(raw []byte) ([]autogenManifestEntry, error) { + var manifest autogenManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, err + } + return manifest.Entries, nil +} + +func normalizeAutogenNumericValue(value any) string { + switch typed := value.(type) { + case string: + return typed + case int: + return strconv.Itoa(typed) + case float64: + return strconv.Itoa(int(typed)) + default: + return fmt.Sprintf("%v", value) + } +} diff --git a/internal/topdata/base_dialog_migrate.go b/internal/topdata/base_dialog_migrate.go new file mode 100644 index 0000000..40ca883 --- /dev/null +++ b/internal/topdata/base_dialog_migrate.go @@ -0,0 +1,41 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacyBaseDialog(referenceBuilderDir, sourceDir string) (int, error) { + legacyPath := filepath.Join(referenceBuilderDir, "tlk", "base.json") + if _, err := os.Stat(legacyPath); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetPath := filepath.Join(sourceDir, "base_dialog.json") + if fileExists(targetPath) { + current, err := loadJSONObject(targetPath) + if err == nil { + if entries, ok := current["entries"].(map[string]any); ok && len(entries) > 0 { + return 0, nil + } + } + } + + obj, err := loadJSONObject(legacyPath) + if err != nil { + return 0, err + } + raw, err := json.MarshalIndent(obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(targetPath, raw, 0o644); err != nil { + return 0, err + } + return 1, nil +} diff --git a/internal/topdata/baseitems_migrate.go b/internal/topdata/baseitems_migrate.go new file mode 100644 index 0000000..e048b71 --- /dev/null +++ b/internal/topdata/baseitems_migrate.go @@ -0,0 +1,105 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacyBaseitems(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "baseitems") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "baseitems") + targetBasePath := filepath.Join(targetDir, "base.json") + targetLockPath := filepath.Join(targetDir, "lock.json") + targetModulesDir := filepath.Join(targetDir, "modules") + moduleNames := []string{ + "blunderbuss.json", + "coin.json", + "estoc.json", + "heavymace.json", + "maulandfalchion.json", + "ovr_baseitems.json", + "ovr_cloak255.json", + "ovr_helmet255.json", + "shortspear.json", + } + targetModulePaths := make([]string, 0, len(moduleNames)) + for _, name := range moduleNames { + targetModulePaths = append(targetModulePaths, filepath.Join(targetModulesDir, name)) + } + if fileExists(targetBasePath) && fileExists(targetLockPath) { + allModulesPresent := true + for _, path := range targetModulePaths { + if !fileExists(path) { + allModulesPresent = false + break + } + } + if allModulesPresent { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + baseObj["output"] = "baseitems.2da" + canonicalizeWikiMetadataDocument(baseObj) + + lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) + if err != nil { + return 0, err + } + + moduleObjs := make([]map[string]any, 0, len(moduleNames)) + for _, name := range moduleNames { + obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", name)) + if err != nil { + return 0, err + } + canonicalizeWikiMetadataDocument(obj) + moduleObjs = append(moduleObjs, obj) + } + + if err := os.MkdirAll(targetModulesDir, 0o755); err != nil { + return 0, err + } + + writes := []struct { + path string + obj map[string]any + }{ + {path: targetBasePath, obj: baseObj}, + {path: targetLockPath, obj: lockObj}, + } + for index, path := range targetModulePaths { + writes = append(writes, struct { + path string + obj map[string]any + }{ + path: path, + obj: moduleObjs[index], + }) + } + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + return len(writes), nil +} diff --git a/internal/topdata/class_spellbooks.go b/internal/topdata/class_spellbooks.go new file mode 100644 index 0000000..790a0a2 --- /dev/null +++ b/internal/topdata/class_spellbooks.go @@ -0,0 +1,373 @@ +package topdata + +import ( + "fmt" + "path/filepath" + "slices" + "strconv" + "strings" +) + +type classSpellbookSpec struct { + Path string + Key string + Class string + Levels map[int][]string +} + +type classSpellbookPlan struct { + Spec classSpellbookSpec + Column string + SpellLevels map[string]int +} + +func isClassSpellbookPath(path string) bool { + slashed := filepath.ToSlash(path) + return strings.Contains(slashed, "/data/classes/spellbooks/") && strings.HasSuffix(strings.ToLower(slashed), ".json") +} + +func validateClassSpellbookShape(path string, obj map[string]any, report *ValidationReport) { + if _, ok := obj["class"]; !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: "class spellbook requires class"}) + } + if _, ok := obj["levels"]; !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: "class spellbook requires levels"}) + } +} + +func validateClassSpellbooks(dataDir string, report *ValidationReport) { + collected, err := collectSpellbookValidationDatasets(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataDir, + Message: fmt.Sprintf("validate class spellbooks: %v", err), + }) + return + } + _, diagnostics := classSpellbookPlans(dataDir, collected) + report.Diagnostics = append(report.Diagnostics, diagnostics...) + if len(diagnostics) == 0 { + warnSpellModulesForManagedSpellbookColumns(dataDir, collected, report) + } +} + +func collectSpellbookValidationDatasets(dataDir string) ([]nativeCollectedDataset, error) { + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return nil, err + } + out := make([]nativeCollectedDataset, 0, 2) + for _, dataset := range datasets { + if dataset.Name != "classes/core" && dataset.Name != "spells" { + continue + } + collected, err := collectNativeDataset(dataset) + if err != nil { + return nil, err + } + out = append(out, collected) + } + return out, nil +} + +func applyClassSpellbooks(sourceDir string, collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) { + dataDir := filepath.Join(sourceDir, "data") + plans, diagnostics := classSpellbookPlans(dataDir, collected) + if len(diagnostics) > 0 { + return nil, fmt.Errorf("class spellbook validation failed: %s", diagnosticsTextForError(diagnostics)) + } + if len(plans) == 0 { + return collected, nil + } + + out := make([]nativeCollectedDataset, len(collected)) + copy(out, collected) + spellsIndex := -1 + for index := range out { + if out[index].Dataset.Name == "spells" { + spellsIndex = index + break + } + } + if spellsIndex == -1 { + return nil, fmt.Errorf("class spellbooks require canonical dataset %q", "spells") + } + + spells := out[spellsIndex] + columns := append([]string(nil), spells.Columns...) + rows := make([]map[string]any, 0, len(spells.Rows)) + for _, row := range spells.Rows { + rows = append(rows, cloneRowMap(row)) + } + rowByKey := rowsByKey(rows) + + for _, plan := range plans { + column := plan.Column + if _, ok := canonicalColumn(columns, column); !ok { + columns = append(columns, column) + ensureRowsExposeColumns(rows, columns) + } + for _, row := range rows { + row[column] = nullValue + } + for spellKey, level := range plan.SpellLevels { + row := rowByKey[spellKey] + if row == nil { + return nil, fmt.Errorf("%s: unknown spell %q", plan.Spec.Path, spellKey) + } + row[column] = level + } + } + + spells.Columns = columns + spells.Rows = rows + out[spellsIndex] = spells + return out, nil +} + +func classSpellbookPlans(dataDir string, collected []nativeCollectedDataset) ([]classSpellbookPlan, []Diagnostic) { + specs, diagnostics := loadClassSpellbookSpecs(filepath.Join(dataDir, "classes", "spellbooks")) + if len(specs) == 0 { + return nil, diagnostics + } + classes := collectedDatasetByName(collected, "classes/core") + spells := collectedDatasetByName(collected, "spells") + if classes == nil { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: dataDir, Message: `class spellbooks require canonical dataset "classes/core"`}) + return nil, diagnostics + } + if spells == nil { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: dataDir, Message: `class spellbooks require canonical dataset "spells"`}) + return nil, diagnostics + } + + classRows := rowsByKey(classes.Rows) + spellRows := rowsByKey(spells.Rows) + seenColumns := map[string]string{} + plans := make([]classSpellbookPlan, 0, len(specs)) + for _, spec := range specs { + classRow := classRows[spec.Class] + if classRow == nil { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: spec.Path, Message: fmt.Sprintf("unknown class %q", spec.Class)}) + } + column := "" + if classRow != nil { + column, _ = classRow["SpellTableColumn"].(string) + column = strings.TrimSpace(column) + if column == "" || column == nullValue { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: spec.Path, Message: fmt.Sprintf("class %q has no SpellTableColumn", spec.Class)}) + } + } + if column != "" && column != nullValue { + folded := strings.ToLower(column) + if previous, ok := seenColumns[folded]; ok { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: spec.Path, Message: fmt.Sprintf("SpellTableColumn %q is already managed by %s", column, previous)}) + } else { + seenColumns[folded] = spec.Path + } + } + + spellLevels := map[string]int{} + for _, level := range sortedIntKeys(spec.Levels) { + for _, spellKey := range spec.Levels[level] { + if spellRows[spellKey] == nil { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: spec.Path, Message: fmt.Sprintf("unknown spell %q", spellKey)}) + continue + } + if previous, ok := spellLevels[spellKey]; ok { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: spec.Path, Message: fmt.Sprintf("spell %q appears at both level %d and level %d", spellKey, previous, level)}) + continue + } + spellLevels[spellKey] = level + } + } + + if column != "" && column != nullValue { + plans = append(plans, classSpellbookPlan{Spec: spec, Column: column, SpellLevels: spellLevels}) + } + } + return plans, diagnostics +} + +func warnSpellModulesForManagedSpellbookColumns(dataDir string, collected []nativeCollectedDataset, report *ValidationReport) { + plans, diagnostics := classSpellbookPlans(dataDir, collected) + if len(diagnostics) > 0 || len(plans) == 0 { + return + } + managed := map[string]classSpellbookPlan{} + for _, plan := range plans { + managed[strings.ToLower(plan.Column)] = plan + } + paths, err := collectModulePaths(filepath.Join(dataDir, "spells", "modules")) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{Severity: SeverityError, Path: dataDir, Message: fmt.Sprintf("collect spell modules for spellbook warnings: %v", err)}) + return + } + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + continue + } + warnSpellbookManagedColumnsInValue(dataDir, path, obj["entries"], managed, report) + warnSpellbookManagedColumnsInValue(dataDir, path, obj["overrides"], managed, report) + warnSpellbookManagedColumnsInValue(dataDir, path, obj["rows"], managed, report) + } +} + +func warnSpellbookManagedColumnsInValue(dataDir, path string, value any, managed map[string]classSpellbookPlan, report *ValidationReport) { + switch typed := value.(type) { + case map[string]any: + for _, key := range sortedKeys(typed) { + row, ok := typed[key].(map[string]any) + if !ok { + continue + } + warnSpellbookManagedColumnsInRow(dataDir, path, row, managed, report) + } + case []any: + for _, raw := range typed { + row, ok := raw.(map[string]any) + if !ok { + continue + } + warnSpellbookManagedColumnsInRow(dataDir, path, row, managed, report) + } + } +} + +func warnSpellbookManagedColumnsInRow(dataDir, path string, row map[string]any, managed map[string]classSpellbookPlan, report *ValidationReport) { + for field, value := range row { + plan, ok := managed[strings.ToLower(field)] + if !ok || isNullLikeValue(value) { + continue + } + spellbookPath := displayTopdataPath(dataDir, plan.Spec.Path) + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: path, + Message: fmt.Sprintf( + "spell module authors class spell column %q managed by class spellbook %s; use %s instead", + plan.Column, + spellbookPath, + spellbookPath, + ), + }) + } +} + +func displayTopdataPath(dataDir, path string) string { + topdataDir := filepath.Dir(dataDir) + rel, err := filepath.Rel(filepath.Dir(topdataDir), path) + if err != nil { + return filepath.ToSlash(path) + } + return filepath.ToSlash(rel) +} + +func loadClassSpellbookSpecs(dir string) ([]classSpellbookSpec, []Diagnostic) { + paths, err := collectModulePaths(dir) + if err != nil { + return nil, []Diagnostic{{Severity: SeverityError, Path: dir, Message: err.Error()}} + } + specs := make([]classSpellbookSpec, 0, len(paths)) + diagnostics := []Diagnostic{} + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: err.Error()}) + continue + } + spec, specDiagnostics := parseClassSpellbookSpec(path, obj) + diagnostics = append(diagnostics, specDiagnostics...) + if spec.Class != "" && spec.Levels != nil { + specs = append(specs, spec) + } + } + return specs, diagnostics +} + +func parseClassSpellbookSpec(path string, obj map[string]any) (classSpellbookSpec, []Diagnostic) { + diagnostics := []Diagnostic{} + spec := classSpellbookSpec{Path: path, Levels: map[int][]string{}} + if key, ok := obj["key"].(string); ok { + spec.Key = strings.TrimSpace(key) + } + classKey, ok := obj["class"].(string) + if !ok || strings.TrimSpace(classKey) == "" { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: "class spellbook requires class"}) + } else { + spec.Class = strings.TrimSpace(classKey) + if !strings.HasPrefix(spec.Class, "classes:") { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: "class must use a classes: reference"}) + } + } + rawLevels, ok := obj["levels"].(map[string]any) + if !ok { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: "class spellbook levels must be an object"}) + return spec, diagnostics + } + for _, rawLevel := range sortedKeys(rawLevels) { + level, err := strconv.Atoi(rawLevel) + if err != nil || level < 0 { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: fmt.Sprintf("level key %q must be a nonnegative integer", rawLevel)}) + continue + } + rawList, ok := rawLevels[rawLevel].([]any) + if !ok { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: fmt.Sprintf("level %d must be an array", level)}) + continue + } + for index, rawSpell := range rawList { + spellKey, ok := rawSpell.(string) + if !ok || strings.TrimSpace(spellKey) == "" { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: fmt.Sprintf("level %d spell %d must be a spells: reference", level, index)}) + continue + } + spellKey = strings.TrimSpace(spellKey) + if !strings.HasPrefix(spellKey, "spells:") { + diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: fmt.Sprintf("level %d spell %d must use a spells: reference", level, index)}) + continue + } + spec.Levels[level] = append(spec.Levels[level], spellKey) + } + } + return spec, diagnostics +} + +func collectedDatasetByName(collected []nativeCollectedDataset, name string) *nativeCollectedDataset { + for index := range collected { + if collected[index].Dataset.Name == name { + return &collected[index] + } + } + return nil +} + +func rowsByKey(rows []map[string]any) map[string]map[string]any { + out := make(map[string]map[string]any, len(rows)) + for _, row := range rows { + key, _ := row["key"].(string) + if key != "" { + out[key] = row + } + } + return out +} + +func sortedIntKeys(values map[int][]string) []int { + keys := make([]int, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +func diagnosticsTextForError(diags []Diagnostic) string { + messages := make([]string, 0, len(diags)) + for _, diag := range diags { + messages = append(messages, diag.Message) + } + return strings.Join(messages, "; ") +} diff --git a/internal/topdata/classes_migrate.go b/internal/topdata/classes_migrate.go new file mode 100644 index 0000000..4443ab5 --- /dev/null +++ b/internal/topdata/classes_migrate.go @@ -0,0 +1,226 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" +) + +var legacyClassesPlainFamilies = []string{"feats", "skills", "savthr", "bfeat", "pres"} + +func importLegacyClasses(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + legacyRoot := filepath.Join(referenceBuilderDir, "data", "classes") + if _, err := os.Stat(legacyRoot); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetRoot := filepath.Join(dataDir, "classes") + if canonicalClassesPresent(targetRoot) { + return 0, nil + } + + coreCollected, plainCollected, err := collectLegacyClassesDatasets(legacyRoot) + if err != nil { + return 0, err + } + + tableKeyByOutput := map[string]string{} + for _, dataset := range plainCollected { + if dataset.TableKey == "" { + continue + } + registerLegacyClassesTableKey(tableKeyByOutput, dataset.TableKey, dataset.Dataset.OutputName) + } + + coreRows := make([]map[string]any, 0, len(coreCollected.Rows)) + for _, rawRow := range coreCollected.Rows { + row, ok := deepCopyValue(rawRow).(map[string]any) + if !ok { + continue + } + if legacyTLK != nil { + if _, _, err := inlineLegacyTLKValue(row, legacyTLK); err != nil { + return 0, err + } + } + rewriteLegacyClassesCoreRow(row, tableKeyByOutput) + coreRows = append(coreRows, row) + } + + if err := removePathIfExists(targetRoot); err != nil { + return 0, err + } + if err := os.MkdirAll(filepath.Join(targetRoot, "core"), 0o755); err != nil { + return 0, err + } + + writes := []struct { + path string + obj map[string]any + }{ + { + path: filepath.Join(targetRoot, "core", "base.json"), + obj: map[string]any{ + "output": coreCollected.Dataset.OutputName, + "columns": stringSliceToAny(coreCollected.Columns), + "rows": rowsToAny(coreRows), + }, + }, + { + path: filepath.Join(targetRoot, "core", "lock.json"), + obj: anyMapInt(coreCollected.LockData), + }, + } + + for _, collected := range plainCollected { + obj := map[string]any{ + "output": collected.Dataset.OutputName, + "columns": stringSliceToAny(collected.Columns), + "rows": rowsToAny(collected.Rows), + } + if strings.TrimSpace(collected.TableKey) != "" { + obj["key"] = collected.TableKey + } + writes = append(writes, struct { + path string + obj map[string]any + }{ + path: filepath.Join(dataDir, collected.Dataset.Name+".json"), + obj: obj, + }) + } + + for _, write := range writes { + if err := os.MkdirAll(filepath.Dir(write.path), 0o755); err != nil { + return 0, err + } + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + + return len(writes), nil +} + +func canonicalClassesPresent(targetRoot string) bool { + if !fileExists(filepath.Join(targetRoot, "core", "base.json")) || !fileExists(filepath.Join(targetRoot, "core", "lock.json")) { + return false + } + for _, family := range legacyClassesPlainFamilies { + ok, err := hasJSONFiles(filepath.Join(targetRoot, family)) + if err != nil || !ok { + return false + } + } + return true +} + +func collectLegacyClassesDatasets(legacyRoot string) (nativeCollectedDataset, []nativeCollectedDataset, error) { + coreCollected, err := collectBaseDataset(nativeDataset{ + Name: "classes/core", + BasePath: filepath.Join(legacyRoot, "core", "base.json"), + LockPath: filepath.Join(legacyRoot, "core", "lock.json"), + ModulesDir: filepath.Join(legacyRoot, "core", "modules"), + OutputName: "classes.2da", + Spec: specForDataset("classes"), + }) + if err != nil { + return nativeCollectedDataset{}, nil, err + } + coreCollected.Dataset.OutputName = "classes.2da" + + plainCollected := make([]nativeCollectedDataset, 0) + for _, family := range legacyClassesPlainFamilies { + familyDir := filepath.Join(legacyRoot, family) + entries, err := os.ReadDir(familyDir) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nativeCollectedDataset{}, nil, err + } + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" || strings.HasPrefix(entry.Name(), ".") || entry.Name() == "lock.json" { + continue + } + filePath := filepath.Join(familyDir, entry.Name()) + tableData, err := loadJSONObject(filePath) + if err != nil { + return nativeCollectedDataset{}, nil, err + } + outputName, _ := tableData["output"].(string) + if strings.TrimSpace(outputName) == "" { + outputName = strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())) + ".2da" + } + collected, err := collectPlainDataset(nativeDataset{ + Name: filepath.ToSlash(filepath.Join("classes", family, strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())))), + BasePath: filePath, + LockPath: filepath.Join(familyDir, "lock.json"), + OutputName: outputName, + Spec: specForDataset(filepath.ToSlash(filepath.Join("classes", family, strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name()))))), + }) + if err != nil { + return nativeCollectedDataset{}, nil, err + } + plainCollected = append(plainCollected, collected) + } + } + + slices.SortFunc(plainCollected, func(a, b nativeCollectedDataset) int { + return strings.Compare(a.Dataset.Name, b.Dataset.Name) + }) + return coreCollected, plainCollected, nil +} + +func registerLegacyClassesTableKey(tableKeyByOutput map[string]string, tableKey, outputName string) { + trimmedOutput := strings.TrimSpace(outputName) + if trimmedOutput != "" { + tableKeyByOutput[trimmedOutput] = tableKey + } + trimmedStem := strings.TrimSpace(outputStem(outputName)) + if trimmedStem != "" { + tableKeyByOutput[trimmedStem] = tableKey + } +} + +func rewriteLegacyClassesCoreRow(row map[string]any, tableKeyByOutput map[string]string) { + for _, field := range []string{"FeatsTable", "SavingThrowTable", "SkillsTable", "BonusFeatsTable", "PreReqTable"} { + tableKey := legacyClassesTableKeyForValue(row[field], tableKeyByOutput) + if tableKey == "" { + continue + } + row[field] = map[string]any{"table": tableKey} + } +} + +func legacyClassesTableKeyForValue(value any, tableKeyByOutput map[string]string) string { + switch typed := value.(type) { + case map[string]any: + tableKey, _ := typed["table"].(string) + return strings.TrimSpace(tableKey) + case string: + trimmed := strings.TrimSpace(typed) + if trimmed == "" || trimmed == nullValue { + return "" + } + if trimmed != strings.ToLower(trimmed) { + return "" + } + if tableKey, ok := tableKeyByOutput[trimmed]; ok { + return tableKey + } + return "" + default: + return "" + } +} diff --git a/internal/topdata/cloakmodel_migrate.go b/internal/topdata/cloakmodel_migrate.go new file mode 100644 index 0000000..d26bdbf --- /dev/null +++ b/internal/topdata/cloakmodel_migrate.go @@ -0,0 +1,6 @@ +package topdata + +func importLegacyCloakmodel(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "cloakmodel", "cloakmodel.2da", nil) +} diff --git a/internal/topdata/convert.go b/internal/topdata/convert.go new file mode 100644 index 0000000..a2e3de4 --- /dev/null +++ b/internal/topdata/convert.go @@ -0,0 +1,888 @@ +package topdata + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "gopkg.in/yaml.v3" +) + +type convertOptions struct { + Namespace string + KeyFields []string + CollisionMode string + BaseDialog string + Type string + Name string + FilenamePrefixes map[string]string +} + +func RunConvertCommand(args []string, stdout io.Writer) error { + if len(args) == 0 || args[0] == "--help" || args[0] == "-h" { + printConvertUsage(stdout) + return nil + } + switch args[0] { + case "2da-to-json": + if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") { + _, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-json [flags] ") + return nil + } + opts, input, output, err := parseConvertArgs(args[1:], false, "suffix") + if err != nil { + return err + } + data, err := parse2DAFile(input) + if err != nil { + return err + } + result, err := convert2DAToJSON(data, output, opts) + if err != nil { + return err + } + return writeJSONFile(output, result) + case "2da-to-module": + if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") { + _, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-module [--namespace ] [--name ] [--type entries|override] [flags] [output.json]") + return nil + } + opts, input, output, err := parseConvertArgs(args[1:], true, "error") + if err != nil { + return err + } + data, err := parse2DAFile(input) + if err != nil { + return err + } + if opts.Type == "override" { + result := map[string]any{"overrides": toOverridesRows(data.Rows)} + if err := writeJSONFile(output, result); err != nil { + return err + } + _, _ = fmt.Fprintf(stdout, "converted overrides: %d\n", len(result["overrides"].([]map[string]any))) + return nil + } + result, err := convert2DAToModule(data, output, opts) + if err != nil { + return err + } + if err := writeJSONFile(output, result); err != nil { + return err + } + _, _ = fmt.Fprintf(stdout, "converted entries: %d\n", len(result["entries"].(map[string]any))) + return nil + case "json-to-2da": + if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") { + _, _ = fmt.Fprintln(stdout, "usage: convert-topdata json-to-2da ") + return nil + } + if len(args) != 3 { + return errors.New("usage: convert-topdata json-to-2da ") + } + data, err := readCanonicalJSON(args[1]) + if err != nil { + return err + } + return write2DAFile(data, args[2]) + default: + return fmt.Errorf("unknown convert-topdata subcommand %q", args[0]) + } +} + +func printConvertUsage(stdout io.Writer) { + _, _ = fmt.Fprintln(stdout, "usage: convert-topdata <2da-to-json|2da-to-module|json-to-2da> ...") + _, _ = fmt.Fprintln(stdout, "") + _, _ = fmt.Fprintln(stdout, "subcommands:") + _, _ = fmt.Fprintln(stdout, " 2da-to-json Convert a 2DA file into canonical JSON rows") + _, _ = fmt.Fprintln(stdout, " 2da-to-module Convert a 2DA file into module entries or override JSON") + _, _ = fmt.Fprintln(stdout, " json-to-2da Convert canonical JSON rows into a 2DA file") + _, _ = fmt.Fprintln(stdout, "") + _, _ = fmt.Fprintln(stdout, "2da-to-module notes:") + _, _ = fmt.Fprintln(stdout, " - --namespace is optional when topdata/templates/config.yaml declares it for the input table") + _, _ = fmt.Fprintln(stdout, " - --name controls inferred output naming: __.json") + _, _ = fmt.Fprintln(stdout, " - topdata/templates/config.yaml module_output.filename_prefixes can set entries/override prefixes") + _, _ = fmt.Fprintln(stdout, " - flags accept both --flag value and --flag=value forms") + _, _ = fmt.Fprintln(stdout, " - unkeyed rows now fail conversion instead of being skipped") +} + +type parsed2DA struct { + Columns []string `json:"columns"` + Rows []map[string]any `json:"rows"` +} + +func parseConvertArgs(args []string, allowFormat bool, defaultCollision string) (convertOptions, string, string, error) { + opts := convertOptions{ + CollisionMode: defaultCollision, + Type: "entries", + } + positional := make([]string, 0, 2) + for index := 0; index < len(args); index++ { + arg := args[index] + switch arg { + case "--namespace": + index++ + if index >= len(args) { + return opts, "", "", errors.New("--namespace requires a value") + } + opts.Namespace = args[index] + case "--key-field": + index++ + if index >= len(args) { + return opts, "", "", errors.New("--key-field requires a value") + } + opts.KeyFields = append(opts.KeyFields, args[index]) + case "--name": + index++ + if index >= len(args) { + return opts, "", "", errors.New("--name requires a value") + } + opts.Name = args[index] + case "--collision": + index++ + if index >= len(args) { + return opts, "", "", errors.New("--collision requires a value") + } + opts.CollisionMode = args[index] + case "--base-dialog": + index++ + if index >= len(args) { + return opts, "", "", errors.New("--base-dialog requires a value") + } + opts.BaseDialog = args[index] + case "--format", "--type": + if !allowFormat { + return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), arg) + } + index++ + if index >= len(args) { + return opts, "", "", fmt.Errorf("%s requires a value", arg) + } + opts.Type = args[index] + default: + if value, ok := parseInlineFlagValue(arg, "--namespace"); ok { + opts.Namespace = value + continue + } + if value, ok := parseInlineFlagValue(arg, "--key-field"); ok { + opts.KeyFields = append(opts.KeyFields, value) + continue + } + if value, ok := parseInlineFlagValue(arg, "--name"); ok { + opts.Name = value + continue + } + if value, ok := parseInlineFlagValue(arg, "--collision"); ok { + opts.CollisionMode = value + continue + } + if value, ok := parseInlineFlagValue(arg, "--base-dialog"); ok { + opts.BaseDialog = value + continue + } + if value, ok := parseInlineFlagValue(arg, "--format"); ok { + if !allowFormat { + return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), "--format") + } + opts.Type = value + continue + } + if value, ok := parseInlineFlagValue(arg, "--type"); ok { + if !allowFormat { + return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), "--type") + } + opts.Type = value + continue + } + if strings.HasPrefix(arg, "--") { + return opts, "", "", fmt.Errorf("unknown flag %q", arg) + } + positional = append(positional, arg) + } + } + if allowFormat { + if len(positional) < 1 || len(positional) > 2 { + return opts, "", "", errors.New("usage: convert-topdata 2da-to-module [--namespace ] [--name ] [--type entries|override] [flags] [output.json]") + } + } else if len(positional) != 2 { + return opts, "", "", errors.New("usage: convert-topdata 2da-to-json [flags] ") + } + if opts.CollisionMode != "suffix" && opts.CollisionMode != "error" { + return opts, "", "", fmt.Errorf("unsupported collision mode %q", opts.CollisionMode) + } + if allowFormat { + output := "" + if len(positional) == 2 { + output = positional[1] + } + ctx, err := resolveConvertContext(positional[0]) + if err != nil { + return opts, "", "", err + } + if ctx.Project != nil { + if outputCtx, err := resolveConvertOutputContext(ctx.Project, output); err != nil { + return opts, "", "", err + } else if ctx.Template.Namespace == "" && len(ctx.Template.KeyFields) == 0 { + ctx.Template = outputCtx + } + } + if strings.TrimSpace(opts.Namespace) == "" { + opts.Namespace = ctx.Template.Namespace + } + if len(opts.KeyFields) == 0 && len(ctx.Template.KeyFields) > 0 { + opts.KeyFields = append(opts.KeyFields, ctx.Template.KeyFields...) + } + if len(opts.FilenamePrefixes) == 0 && len(ctx.Config.ModuleOutput.FilenamePrefixes) > 0 { + opts.FilenamePrefixes = cloneStringMap(ctx.Config.ModuleOutput.FilenamePrefixes) + } + if strings.TrimSpace(opts.Namespace) == "" { + return opts, "", "", errors.New("2da-to-module requires --namespace or a topdata/templates/config.yaml entry for the input table") + } + switch opts.Type { + case "entry": + opts.Type = "entries" + case "overrides": + opts.Type = "override" + } + if opts.Type != "entries" && opts.Type != "override" { + return opts, "", "", fmt.Errorf("unsupported module type %q", opts.Type) + } + if output == "" { + resolved, err := defaultModuleOutputPath(positional[0], opts, ctx.Project) + if err != nil { + return opts, "", "", err + } + output = resolved + } + return opts, positional[0], output, nil + } + ctx, err := resolveConvertContext(positional[0]) + if err != nil { + return opts, "", "", err + } + if ctx.Project != nil { + if outputCtx, err := resolveConvertOutputContext(ctx.Project, positional[1]); err != nil { + return opts, "", "", err + } else if ctx.Template.Namespace == "" && len(ctx.Template.KeyFields) == 0 { + ctx.Template = outputCtx + } + } + if strings.TrimSpace(opts.Namespace) == "" { + opts.Namespace = ctx.Template.Namespace + } + if len(opts.KeyFields) == 0 && len(ctx.Template.KeyFields) > 0 { + opts.KeyFields = append(opts.KeyFields, ctx.Template.KeyFields...) + } + return opts, positional[0], positional[1], nil +} + +func parseInlineFlagValue(arg, name string) (string, bool) { + prefix := name + "=" + if !strings.HasPrefix(arg, prefix) { + return "", false + } + return strings.TrimSpace(arg[len(prefix):]), true +} + +func positionalSafe(args []string) string { + if len(args) == 0 { + return "command" + } + return args[0] +} + +func parse2DAFile(path string) (parsed2DA, error) { + raw, err := os.ReadFile(path) + if err != nil { + return parsed2DA{}, fmt.Errorf("read %s: %w", path, err) + } + lines := strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") + trimmed := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.TrimSpace(line) + if line != "" { + trimmed = append(trimmed, line) + } + } + if len(trimmed) < 2 || !strings.HasPrefix(trimmed[0], "2DA") { + return parsed2DA{}, fmt.Errorf("%s: invalid 2DA file", path) + } + columns := split2DALine(trimmed[1]) + if len(columns) == 0 { + return parsed2DA{}, fmt.Errorf("%s: missing 2DA columns", path) + } + rows := make([]map[string]any, 0, max(0, len(trimmed)-2)) + seenIDs := map[int]struct{}{} + for _, line := range trimmed[2:] { + fields := split2DALine(line) + if len(fields) == 0 { + continue + } + rowID, err := strconv.Atoi(fields[0]) + if err != nil { + return parsed2DA{}, fmt.Errorf("%s: invalid 2DA row id %q", path, fields[0]) + } + if len(fields)-1 > len(columns) { + return parsed2DA{}, fmt.Errorf("%s: row %d has %d values but only %d columns are declared", path, rowID, len(fields)-1, len(columns)) + } + if _, ok := seenIDs[rowID]; ok { + return parsed2DA{}, fmt.Errorf("%s: duplicate 2DA row id %d", path, rowID) + } + seenIDs[rowID] = struct{}{} + row := map[string]any{"id": rowID} + for index, column := range columns { + if index+1 < len(fields) { + row[column] = smartConvertScalar(fields[index+1]) + continue + } + row[column] = nullValue + } + rows = append(rows, row) + } + return parsed2DA{Columns: columns, Rows: rows}, nil +} + +func split2DALine(line string) []string { + fields := make([]string, 0) + var current strings.Builder + inQuotes := false + for _, ch := range line { + switch { + case ch == '"': + inQuotes = !inQuotes + case !inQuotes && (ch == '\t' || ch == ' '): + if current.Len() > 0 { + fields = append(fields, current.String()) + current.Reset() + } + default: + current.WriteRune(ch) + } + } + if current.Len() > 0 { + fields = append(fields, current.String()) + } + return fields +} + +func smartConvertScalar(value string) any { + if value == "" { + return "" + } + if parsed, err := strconv.Atoi(value); err == nil { + return parsed + } + return value +} + +func convert2DAToJSON(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) { + rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts) + if err != nil { + return nil, err + } + outRows := make([]map[string]any, 0, len(rows)) + for _, row := range rows { + ordered := map[string]any{"id": row["id"]} + if key, ok := row["key"].(string); ok && key != "" { + ordered["key"] = key + } + for _, column := range data.Columns { + ordered[column] = row[column] + } + outRows = append(outRows, ordered) + } + return map[string]any{"columns": data.Columns, "rows": outRows}, nil +} + +func convert2DAToModule(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) { + rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts) + if err != nil { + return nil, err + } + entries := map[string]any{} + missing := []string{} + for _, row := range rows { + key, _ := row["key"].(string) + if key == "" { + missing = append(missing, strconv.Itoa(row["id"].(int))) + continue + } + entry := map[string]any{} + for _, column := range data.Columns { + value := row[column] + if format2DAValue(value) == nullValue { + continue + } + entry[column] = value + } + entries[key] = entry + } + if len(missing) > 0 { + return nil, fmt.Errorf("unable to generate keys for row ids %s; declare --key-field or configure key_fields in topdata/templates/config.yaml", strings.Join(missing, ", ")) + } + return map[string]any{"entries": entries}, nil +} + +func toOverridesRows(rows []map[string]any) []map[string]any { + out := make([]map[string]any, 0, len(rows)) + for _, row := range rows { + override := map[string]any{"id": row["id"]} + keys := make([]string, 0, len(row)) + for key := range row { + if key == "id" || key == "key" { + continue + } + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if format2DAValue(row[key]) == nullValue { + continue + } + override[key] = row[key] + } + out = append(out, override) + } + return out +} + +func assignConvertedRowKeys(rows []map[string]any, outputPath string, opts convertOptions) ([]map[string]any, error) { + namespace := strings.TrimSpace(opts.Namespace) + out := make([]map[string]any, 0, len(rows)) + used := map[string]struct{}{} + rowIDsByKey := map[string]int{} + for _, row := range rows { + cloned := cloneRowMap(row) + if namespace == "" { + out = append(out, cloned) + continue + } + candidates := convertedRowKeyCandidates(cloned, opts.KeyFields) + if len(candidates) == 0 { + out = append(out, cloned) + continue + } + key := "" + for _, candidate := range candidates { + candidateKey := namespace + ":" + candidate + if _, ok := used[candidateKey]; ok { + continue + } + key = candidateKey + break + } + if key == "" { + key = namespace + ":" + candidates[0] + if opts.CollisionMode == "error" { + return nil, fmt.Errorf("duplicate generated key %q for row %v (already used by row %d)", key, cloned["id"], rowIDsByKey[key]) + } + key = fmt.Sprintf("%s_%v", key, cloned["id"]) + } + used[key] = struct{}{} + rowID, _ := asInt(cloned["id"]) + rowIDsByKey[key] = rowID + cloned["key"] = key + out = append(out, cloned) + } + return out, nil +} + +var convertKeyWhitespace = regexp.MustCompile(`\s+`) +var convertKeyCleaner = regexp.MustCompile(`[^a-z0-9_]`) + +func convertedRowKeyCandidates(row map[string]any, preferred []string) []string { + fields := preferred + if len(fields) == 0 { + fields = []string{"LABEL", "Label", "Name"} + } else { + for _, field := range fields { + value, ok := row[field] + if !ok { + return nil + } + text := strings.TrimSpace(format2DAValue(value)) + if text == "" || text == nullValue { + return nil + } + } + } + values := make([]string, 0, len(fields)) + seen := map[string]struct{}{} + candidates := make([]string, 0, len(fields)) + for _, field := range fields { + value, ok := row[field] + if !ok { + continue + } + text := strings.TrimSpace(format2DAValue(value)) + if text == "" || text == nullValue { + continue + } + normalized := normalizeConvertedKeyText(text) + if normalized == "" { + continue + } + values = append(values, normalized) + joined := strings.Join(values, "_") + if _, ok := seen[joined]; ok { + continue + } + seen[joined] = struct{}{} + candidates = append(candidates, joined) + } + return candidates +} + +func normalizeConvertedKeyText(text string) string { + text = strings.TrimSpace(strings.ToLower(text)) + text = convertKeyWhitespace.ReplaceAllString(text, "_") + text = convertKeyCleaner.ReplaceAllString(text, "") + text = strings.Trim(text, "_") + return text +} + +type convertTemplateTable struct { + Namespace string `json:"namespace" yaml:"namespace"` + KeyFields []string `json:"key_fields" yaml:"key_fields"` +} + +type convertTemplateConfig struct { + ModuleOutput convertModuleOutputConfig `json:"module_output" yaml:"module_output"` + Tables map[string]convertTemplateTable `json:"tables" yaml:"tables"` +} + +type convertModuleOutputConfig struct { + FilenamePrefixes map[string]string `json:"filename_prefixes" yaml:"filename_prefixes"` +} + +type convertContext struct { + Project *project.Project + Config convertTemplateConfig + Template convertTemplateTable +} + +func resolveConvertContext(input string) (convertContext, error) { + cwd, err := os.Getwd() + if err != nil { + return convertContext{}, err + } + root, err := project.FindRoot(cwd) + if err != nil { + return convertContext{}, nil + } + p, err := project.Load(root) + if err != nil { + return convertContext{}, err + } + if !p.HasTopData() { + return convertContext{Project: p}, nil + } + config, err := loadConvertTemplateConfig(p) + if err != nil { + return convertContext{}, err + } + table, err := templateTableConfig(config, p, input) + if err != nil { + return convertContext{}, err + } + return convertContext{ + Project: p, + Config: config, + Template: table, + }, nil +} + +func loadTemplateTableConfig(p *project.Project, input string) (convertTemplateTable, error) { + config, err := loadConvertTemplateConfig(p) + if err != nil { + return convertTemplateTable{}, err + } + return templateTableConfig(config, p, input) +} + +func templateTableConfig(config convertTemplateConfig, p *project.Project, input string) (convertTemplateTable, error) { + if len(config.Tables) == 0 { + return convertTemplateTable{}, nil + } + templatesDir := filepath.Join(p.TopDataSourceDir(), "templates") + inputAbs, err := filepath.Abs(input) + if err != nil { + return convertTemplateTable{}, err + } + templatesAbs, err := filepath.Abs(templatesDir) + if err != nil { + return convertTemplateTable{}, err + } + rel, err := filepath.Rel(templatesAbs, inputAbs) + if err != nil || strings.HasPrefix(rel, "..") || rel == "." { + return convertTemplateTable{}, nil + } + + rel = filepath.ToSlash(rel) + base := filepath.Base(rel) + stem := strings.TrimSuffix(base, filepath.Ext(base)) + for _, key := range []string{rel, base, stem} { + if table, ok := config.Tables[key]; ok { + return table, nil + } + } + return convertTemplateTable{}, nil +} + +func loadConvertTemplateConfig(p *project.Project) (convertTemplateConfig, error) { + templatesDir := filepath.Join(p.TopDataSourceDir(), "templates") + for _, candidate := range []string{"config.yaml", "config.yml", "config.json"} { + configPath := filepath.Join(templatesDir, candidate) + raw, err := os.ReadFile(configPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + return convertTemplateConfig{}, fmt.Errorf("read %s: %w", configPath, err) + } + + var config convertTemplateConfig + switch filepath.Ext(configPath) { + case ".yaml", ".yml": + if err := yaml.Unmarshal(raw, &config); err != nil { + return convertTemplateConfig{}, fmt.Errorf("parse %s: %w", configPath, err) + } + default: + if err := json.Unmarshal(raw, &config); err != nil { + return convertTemplateConfig{}, fmt.Errorf("parse %s: %w", configPath, err) + } + } + if err := validateConvertTemplateConfig(config); err != nil { + return convertTemplateConfig{}, fmt.Errorf("validate %s: %w", configPath, err) + } + return config, nil + } + return convertTemplateConfig{}, nil +} + +func validateConvertTemplateConfig(config convertTemplateConfig) error { + for moduleType, prefix := range config.ModuleOutput.FilenamePrefixes { + switch moduleType { + case "entries", "override": + default: + return fmt.Errorf("unsupported module_output.filename_prefixes key %q; expected entries or override", moduleType) + } + if normalizeModuleFilenamePrefix(prefix) == "" { + return fmt.Errorf("module_output.filename_prefixes.%s must contain at least one filename-safe character", moduleType) + } + } + return nil +} + +func resolveConvertOutputContext(p *project.Project, output string) (convertTemplateTable, error) { + if p == nil || !p.HasTopData() || strings.TrimSpace(output) == "" { + return convertTemplateTable{}, nil + } + config, err := loadConvertTemplateConfig(p) + if err != nil { + return convertTemplateTable{}, err + } + if len(config.Tables) == 0 { + return convertTemplateTable{}, nil + } + dataDir := filepath.Join(p.TopDataSourceDir(), "data") + dataAbs, err := filepath.Abs(dataDir) + if err != nil { + return convertTemplateTable{}, err + } + outputAbs, err := filepath.Abs(output) + if err != nil { + return convertTemplateTable{}, err + } + rel, err := filepath.Rel(dataAbs, outputAbs) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + return convertTemplateTable{}, nil + } + rel = filepath.ToSlash(rel) + + bestMatch := "" + bestTable := convertTemplateTable{} + for _, table := range config.Tables { + namespacePath := namespaceToPath(table.Namespace) + if namespacePath == "" { + continue + } + if rel != namespacePath && !strings.HasPrefix(rel, namespacePath+"/") { + continue + } + if len(namespacePath) <= len(bestMatch) { + continue + } + bestMatch = namespacePath + bestTable = table + } + return bestTable, nil +} + +func namespaceToPath(namespace string) string { + namespace = strings.TrimSpace(strings.ToLower(namespace)) + if namespace == "" { + return "" + } + namespace = strings.ReplaceAll(namespace, "\\", "/") + namespace = strings.ReplaceAll(namespace, ":", "/") + namespace = convertKeyWhitespace.ReplaceAllString(namespace, "_") + namespace = strings.Trim(namespace, "/") + return namespace +} + +func defaultModuleOutputPath(input string, opts convertOptions, p *project.Project) (string, error) { + if p == nil { + return "", errors.New("2da-to-module requires an explicit output path when not run inside a project root") + } + if !p.HasTopData() { + return "", errors.New("2da-to-module requires topdata to be configured when inferring an output path") + } + filename, err := defaultModuleFileName(input, opts, p) + if err != nil { + return "", err + } + modulesDir := filepath.Join(p.TopDataSourceDir(), "data", filepath.FromSlash(opts.Namespace), "modules") + return filepath.Join(modulesDir, filename), nil +} + +func defaultModuleFileName(input string, opts convertOptions, p *project.Project) (string, error) { + if strings.TrimSpace(opts.Name) != "" { + name := normalizeConvertedKeyText(opts.Name) + if name == "" { + return "", fmt.Errorf("invalid --name %q", opts.Name) + } + namespace := normalizeModuleFilenameNamespace(opts.Namespace) + prefix := moduleFilenamePrefix(opts) + return fmt.Sprintf("%s_%s_%s.json", prefix, namespace, name), nil + } + if isTemplateInput(p, input) { + return "", errors.New("2da-to-module requires --name when converting from topdata/templates without an explicit output path") + } + return strings.TrimSuffix(filepath.Base(input), filepath.Ext(input)) + ".json", nil +} + +func moduleFilenamePrefix(opts convertOptions) string { + moduleType := strings.TrimSpace(opts.Type) + if moduleType == "entry" { + moduleType = "entries" + } + if moduleType == "overrides" { + moduleType = "override" + } + if configured := strings.TrimSpace(opts.FilenamePrefixes[moduleType]); configured != "" { + if prefix := normalizeModuleFilenamePrefix(configured); prefix != "" { + return prefix + } + } + return "add" +} + +func normalizeModuleFilenamePrefix(prefix string) string { + prefix = strings.TrimSpace(strings.ToLower(prefix)) + prefix = strings.ReplaceAll(prefix, "/", "_") + prefix = strings.ReplaceAll(prefix, "\\", "_") + prefix = convertKeyWhitespace.ReplaceAllString(prefix, "_") + prefix = convertKeyCleaner.ReplaceAllString(prefix, "_") + prefix = strings.Trim(prefix, "_") + return prefix +} + +func normalizeModuleFilenameNamespace(namespace string) string { + namespace = strings.TrimSpace(strings.ToLower(namespace)) + namespace = strings.ReplaceAll(namespace, "/", "_") + namespace = strings.ReplaceAll(namespace, "\\", "_") + namespace = convertKeyWhitespace.ReplaceAllString(namespace, "_") + namespace = convertKeyCleaner.ReplaceAllString(namespace, "_") + namespace = strings.Trim(namespace, "_") + if namespace == "" { + return "module" + } + return namespace +} + +func isTemplateInput(p *project.Project, input string) bool { + if p == nil || !p.HasTopData() { + return false + } + templatesAbs, err := filepath.Abs(filepath.Join(p.TopDataSourceDir(), "templates")) + if err != nil { + return false + } + inputAbs, err := filepath.Abs(input) + if err != nil { + return false + } + rel, err := filepath.Rel(templatesAbs, inputAbs) + if err != nil { + return false + } + return rel != "." && !strings.HasPrefix(rel, "..") +} + +func readCanonicalJSON(path string) (parsed2DA, error) { + raw, err := os.ReadFile(path) + if err != nil { + return parsed2DA{}, err + } + var payload struct { + Columns []string `json:"columns"` + Rows []map[string]any `json:"rows"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return parsed2DA{}, err + } + return parsed2DA{Columns: payload.Columns, Rows: payload.Rows}, nil +} + +func writeJSONFile(path string, payload any) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." { + return err + } + raw, err := json.MarshalIndent(payload, "", " ") + if err != nil { + return err + } + raw = append(raw, '\n') + return os.WriteFile(path, raw, 0o644) +} + +func write2DAFile(data parsed2DA, path string) error { + rows := make([]map[string]any, 0, len(data.Rows)) + for _, row := range data.Rows { + cloned := cloneRowMap(row) + if rowID, err := asInt(cloned["id"]); err == nil { + cloned["id"] = rowID + } + rows = append(rows, cloned) + } + sort.Slice(rows, func(i, j int) bool { + left, _ := asInt(rows[i]["id"]) + right, _ := asInt(rows[j]["id"]) + return left < right + }) + table := map[string]any{ + "columns": data.Columns, + "rows": rows, + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." { + return err + } + return write2DA(table, path, false, -1) +} + +func cloneStringMap(in map[string]string) map[string]string { + out := make(map[string]string, len(in)) + for key, value := range in { + out[key] = value + } + return out +} diff --git a/internal/topdata/convert_test.go b/internal/topdata/convert_test.go new file mode 100644 index 0000000..eb21a42 --- /dev/null +++ b/internal/topdata/convert_test.go @@ -0,0 +1,611 @@ +package topdata + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunConvertCommandInfersTemplateNamespaceAndOutput(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test + resref: test +paths: + source: src + assets: assets + build: build +topdata: + source: topdata + build: build/topdata +`) + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance", "modules")) + writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), ` +tables: + template_appearance: + namespace: appearance + key_fields: + - LABEL +`) + writeFile(t, filepath.Join(root, "topdata", "templates", "template_appearance.2da"), "2DA V2.0\n\nLABEL STRING_REF NAME\n960 \"Zombie, Ash, Hot\" **** Zombie_Ash_Hot\n961 \"Zombie, Ash, Done\" **** Zombie_Ash_Done\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + var stdout bytes.Buffer + if err := RunConvertCommand([]string{ + "2da-to-module", + "--name", "ashzombies", + "topdata/templates/template_appearance.2da", + }, &stdout); err != nil { + t.Fatalf("RunConvertCommand failed: %v", err) + } + + outputPath := filepath.Join(root, "topdata", "data", "appearance", "modules", "add_appearance_ashzombies.json") + raw, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read output: %v", err) + } + text := string(raw) + if !strings.Contains(text, `"appearance:zombie_ash_hot"`) || !strings.Contains(text, `"appearance:zombie_ash_done"`) { + t.Fatalf("unexpected output:\n%s", text) + } + if !strings.Contains(stdout.String(), "converted entries: 2") { + t.Fatalf("unexpected stdout: %s", stdout.String()) + } +} + +func TestRunConvertCommandInfersOverrideOutputPrefixFromTemplateConfig(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test + resref: test +paths: + source: src + assets: assets + build: build +topdata: + source: topdata + build: build/topdata +`) + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), ` +module_output: + filename_prefixes: + entries: add + override: ovr +tables: + template_appearance: + namespace: appearance + key_fields: + - LABEL +`) + writeFile(t, filepath.Join(root, "topdata", "templates", "template_appearance.2da"), "2DA V2.0\n\nLABEL STRING_REF NAME\n960 \"Zombie, Ash, Hot\" **** Zombie_Ash_Hot\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + var stdout bytes.Buffer + if err := RunConvertCommand([]string{ + "2da-to-module", + "--type", "override", + "--name", "ashzombies", + "topdata/templates/template_appearance.2da", + }, &stdout); err != nil { + t.Fatalf("RunConvertCommand failed: %v", err) + } + + outputPath := filepath.Join(root, "topdata", "data", "appearance", "modules", "ovr_appearance_ashzombies.json") + raw, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read output: %v", err) + } + if !strings.Contains(string(raw), `"overrides"`) { + t.Fatalf("expected override payload, got:\n%s", string(raw)) + } + if _, err := os.Stat(filepath.Join(root, "topdata", "data", "appearance", "modules", "add_appearance_ashzombies.json")); !os.IsNotExist(err) { + t.Fatalf("expected add-prefixed output to be absent, stat err: %v", err) + } + if !strings.Contains(stdout.String(), "converted overrides: 1") { + t.Fatalf("unexpected stdout: %s", stdout.String()) + } +} + +func TestRunConvertCommand2DAToJSONInfersTemplateKeys(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test + resref: test +paths: + source: src + assets: assets + build: build +topdata: + source: topdata + build: build/topdata +`) + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), ` +tables: + template_appearance: + namespace: appearance + key_fields: + - LABEL +`) + inputPath := filepath.Join(root, "topdata", "templates", "template_appearance.2da") + writeFile(t, inputPath, "2DA V2.0\n\nLABEL STRING_REF NAME\n960 \"Zombie, Ash, Hot\" **** Zombie_Ash_Hot\n961 \"Zombie, Ash, Done\" **** Zombie_Ash_Done\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + outputPath := filepath.Join(root, "out", "appearance.json") + if err := RunConvertCommand([]string{ + "2da-to-json", + "topdata/templates/template_appearance.2da", + outputPath, + }, &bytes.Buffer{}); err != nil { + t.Fatalf("RunConvertCommand failed: %v", err) + } + + raw, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read output: %v", err) + } + var payload struct { + Rows []map[string]any `json:"rows"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + t.Fatalf("unmarshal output: %v", err) + } + if got := payload.Rows[0]["key"]; got != "appearance:zombie_ash_hot" { + t.Fatalf("expected first key to be inferred from template config, got %#v", got) + } + if got := payload.Rows[1]["key"]; got != "appearance:zombie_ash_done" { + t.Fatalf("expected second key to be inferred from template config, got %#v", got) + } +} + +func TestRunConvertCommand2DAToJSONInfersOutputDatasetKeysFromConfig(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test + resref: test +paths: + source: src + assets: assets + build: build +topdata: + source: topdata + build: build/topdata +`) + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "soundset")) + writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), ` +tables: + template_soundset: + namespace: soundset + key_fields: + - RESREF + - LABEL +`) + inputPath := filepath.Join(root, "scratch_soundset.2da") + writeFile(t, inputPath, "2DA V2.0\n\nLABEL RESREF STRREF GENDER TYPE\n0 \"Bandit Male\" nw_bandit 100 1 4\n1 \"Bandit Female\" nw_bandit 101 2 4\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + outputPath := filepath.Join(root, "topdata", "data", "soundset", "base.json") + if err := RunConvertCommand([]string{ + "2da-to-json", + "scratch_soundset.2da", + outputPath, + }, &bytes.Buffer{}); err != nil { + t.Fatalf("RunConvertCommand failed: %v", err) + } + + raw, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read output: %v", err) + } + var payload struct { + Rows []map[string]any `json:"rows"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + t.Fatalf("unmarshal output: %v", err) + } + if got := payload.Rows[0]["key"]; got != "soundset:nw_bandit" { + t.Fatalf("expected first soundset key, got %#v", got) + } + if got := payload.Rows[1]["key"]; got != "soundset:nw_bandit_bandit_female" { + t.Fatalf("expected second soundset key to use config disambiguation, got %#v", got) + } +} + +func TestRunConvertCommand2DAToJSONInfersAmbientTemplateKeysFromResource(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test + resref: test +paths: + source: src + assets: assets + build: build +topdata: + source: topdata + build: build/topdata +`) + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), ` +tables: + template_ambientmusic: + namespace: ambientmusic + key_fields: + - Resource +`) + inputPath := filepath.Join(root, "topdata", "templates", "template_ambientmusic.2da") + writeFile(t, inputPath, "2DA V2.0\n\nDescription DisplayName Resource Stinger1 Stinger2 Stinger3\n0 61901 **** **** **** **** ****\n1 61842 **** mus_ruralday1 **** **** ****\n2 61843 **** mus_ruralday2 **** **** ****\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + outputPath := filepath.Join(root, "out", "ambientmusic.json") + if err := RunConvertCommand([]string{ + "2da-to-json", + "topdata/templates/template_ambientmusic.2da", + outputPath, + }, &bytes.Buffer{}); err != nil { + t.Fatalf("RunConvertCommand failed: %v", err) + } + + raw, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read output: %v", err) + } + var payload struct { + Rows []map[string]any `json:"rows"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + t.Fatalf("unmarshal output: %v", err) + } + if _, ok := payload.Rows[0]["key"]; ok { + t.Fatalf("expected null Resource row to remain unkeyed, got %#v", payload.Rows[0]["key"]) + } + if got := payload.Rows[1]["key"]; got != "ambientmusic:mus_ruralday1" { + t.Fatalf("expected first populated resource row key, got %#v", got) + } + if got := payload.Rows[2]["key"]; got != "ambientmusic:mus_ruralday2" { + t.Fatalf("expected second populated resource row key, got %#v", got) + } +} + +func TestRunConvertCommand2DAToJSONSuffixesDuplicateGeneratedKeys(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test + resref: test +paths: + source: src + assets: assets + build: build +topdata: + source: topdata + build: build/topdata +`) + inputPath := filepath.Join(root, "dup.2da") + writeFile(t, inputPath, "2DA V2.0\n\nResource\n1 cmp_reserved\n2 cmp_reserved\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + outputPath := filepath.Join(root, "out", "ambientmusic.json") + if err := RunConvertCommand([]string{ + "2da-to-json", + "--namespace", "ambientmusic", + "--key-field", "Resource", + inputPath, + outputPath, + }, &bytes.Buffer{}); err != nil { + t.Fatalf("RunConvertCommand failed: %v", err) + } + + raw, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read output: %v", err) + } + var payload struct { + Rows []map[string]any `json:"rows"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + t.Fatalf("unmarshal output: %v", err) + } + if got := payload.Rows[0]["key"]; got != "ambientmusic:cmp_reserved" { + t.Fatalf("expected first duplicate key to remain unsuffixed, got %#v", got) + } + if got := payload.Rows[1]["key"]; got != "ambientmusic:cmp_reserved_2" { + t.Fatalf("expected second duplicate key to gain row-id suffix, got %#v", got) + } +} + +func TestParseConvertArgsSupportsEqualsSyntax(t *testing.T) { + opts, input, output, err := parseConvertArgs([]string{ + "--namespace=environment", + "--name=projectq", + "--type=entries", + "--collision=suffix", + "--key-field=Label", + "input.2da", + "output.json", + }, true, "error") + if err != nil { + t.Fatalf("parseConvertArgs failed: %v", err) + } + if opts.Namespace != "environment" { + t.Fatalf("expected namespace from equals syntax, got %q", opts.Namespace) + } + if opts.Name != "projectq" { + t.Fatalf("expected name from equals syntax, got %q", opts.Name) + } + if opts.Type != "entries" { + t.Fatalf("expected type from equals syntax, got %q", opts.Type) + } + if opts.CollisionMode != "suffix" { + t.Fatalf("expected collision from equals syntax, got %q", opts.CollisionMode) + } + if len(opts.KeyFields) != 1 || opts.KeyFields[0] != "Label" { + t.Fatalf("expected key field from equals syntax, got %#v", opts.KeyFields) + } + if input != "input.2da" || output != "output.json" { + t.Fatalf("unexpected positional parse result: input=%q output=%q", input, output) + } +} + +func TestConvert2DAToJSONSkipsConfiguredKeysWhenAnyKeyFieldIsNull(t *testing.T) { + result, err := convert2DAToJSON(parsed2DA{ + Columns: []string{"LABEL", "RESREF", "STRREF"}, + Rows: []map[string]any{ + {"id": 309, "LABEL": "Unused", "RESREF": "unused", "STRREF": 1}, + {"id": 310, "LABEL": nullValue, "RESREF": "unused", "STRREF": 2}, + }, + }, "topdata/data/soundset/base.json", convertOptions{ + Namespace: "soundset", + KeyFields: []string{"RESREF", "LABEL"}, + CollisionMode: "error", + }) + if err != nil { + t.Fatalf("convert2DAToJSON failed: %v", err) + } + + rows := result["rows"].([]map[string]any) + if got := rows[0]["key"]; got != "soundset:unused" { + t.Fatalf("expected first row key to use the primary configured key field, got %#v", got) + } + if _, ok := rows[1]["key"]; ok { + t.Fatalf("expected second row to remain unkeyed when one configured key field is null, got %#v", rows[1]["key"]) + } +} + +func TestRunConvertCommandFailsOnUnkeyedRows(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": {"name": "Test", "resref": "test"}, + "paths": {"source": "src", "assets": "assets", "build": "build"}, + "topdata": {"source": "topdata", "build": "build/topdata"} +}`+"\n") + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), ` +tables: + template_appearance: + namespace: appearance + key_fields: + - STRING_REF +`) + writeFile(t, filepath.Join(root, "topdata", "templates", "template_appearance.2da"), "2DA V2.0\n\nLABEL STRING_REF NAME\n960 \"Zombie, Ash, Hot\" **** Zombie_Ash_Hot\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + err = RunConvertCommand([]string{ + "2da-to-module", + "--name", "ashzombies", + "topdata/templates/template_appearance.2da", + }, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "unable to generate keys for row ids 960") { + t.Fatalf("expected unkeyed row failure, got %v", err) + } +} + +func TestRunConvertCommandFallsBackToLegacyTemplateJSONConfig(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test + resref: test +paths: + source: src + assets: assets + build: build +topdata: + source: topdata + build: build/topdata +`) + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + writeFile(t, filepath.Join(root, "topdata", "templates", "config.json"), `{ + "tables": { + "template_appearance": { + "namespace": "appearance", + "key_fields": ["LABEL"] + } + } +}`+"\n") + inputPath := filepath.Join(root, "topdata", "templates", "template_appearance.2da") + writeFile(t, inputPath, "2DA V2.0\n\nLABEL\n960 Zombie_Ash_Hot\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + outputPath := filepath.Join(root, "out", "appearance.json") + if err := RunConvertCommand([]string{ + "2da-to-json", + "topdata/templates/template_appearance.2da", + outputPath, + }, &bytes.Buffer{}); err != nil { + t.Fatalf("RunConvertCommand failed: %v", err) + } + + raw, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read output: %v", err) + } + if !strings.Contains(string(raw), `"appearance:zombie_ash_hot"`) { + t.Fatalf("expected legacy JSON template config fallback to be honored, got %s", string(raw)) + } +} + +func TestRunConvertCommandRejectsUnknownTemplateFilenamePrefixType(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test + resref: test +paths: + source: src + assets: assets + build: build +topdata: + source: topdata + build: build/topdata +`) + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), ` +module_output: + filename_prefixes: + additions: add +tables: + template_appearance: + namespace: appearance + key_fields: + - LABEL +`) + writeFile(t, filepath.Join(root, "topdata", "templates", "template_appearance.2da"), "2DA V2.0\n\nLABEL\n960 Zombie_Ash_Hot\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + err = RunConvertCommand([]string{ + "2da-to-module", + "--name", "ashzombies", + "topdata/templates/template_appearance.2da", + }, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), `unsupported module_output.filename_prefixes key "additions"`) { + t.Fatalf("expected invalid filename prefix type error, got %v", err) + } +} + +func TestConvert2DAToModuleFailsOnDuplicateGeneratedKeysByDefault(t *testing.T) { + _, err := convert2DAToModule(parsed2DA{ + Columns: []string{"LABEL"}, + Rows: []map[string]any{ + {"id": 1, "LABEL": "Zombie"}, + {"id": 2, "LABEL": "Zombie"}, + }, + }, "topdata/data/appearance/modules/add_appearance_ashzombies.json", convertOptions{ + Namespace: "appearance", + CollisionMode: "error", + }) + if err == nil || !strings.Contains(err.Error(), `duplicate generated key "appearance:zombie"`) { + t.Fatalf("expected duplicate generated key error, got %v", err) + } +} + +func TestParse2DAFileRejectsRowsWithTooManyFields(t *testing.T) { + root := t.TempDir() + inputPath := filepath.Join(root, "bad.2da") + writeFile(t, inputPath, "2DA V2.0\n\nLABEL NAME\n0 one two three\n") + + _, err := parse2DAFile(inputPath) + if err == nil || !strings.Contains(err.Error(), "has 3 values but only 2 columns are declared") { + t.Fatalf("expected row width error, got %v", err) + } +} diff --git a/internal/topdata/creaturespeed_migrate.go b/internal/topdata/creaturespeed_migrate.go new file mode 100644 index 0000000..7771a4c --- /dev/null +++ b/internal/topdata/creaturespeed_migrate.go @@ -0,0 +1,145 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacyCreaturespeed(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "creaturespeed") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "creaturespeed") + targetPath := filepath.Join(targetDir, "creaturespeed.json") + if _, err := os.Stat(targetPath); err == nil { + obj, err := loadJSONObject(targetPath) + if err == nil && countLegacyTLKRefsInValue(obj) == 0 { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + rows, err := mergeCreaturespeedRows(baseObj, filepath.Join(legacyDir, "modules")) + if err != nil { + return 0, err + } + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return 0, err + } + + if err := removePathIfExists(filepath.Join(targetDir, "base.json")); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(targetDir, "lock.json")); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(targetDir, "modules")); err != nil { + return 0, err + } + + out := map[string]any{ + "output": "creaturespeed.2da", + "columns": baseObj["columns"], + "rows": rows, + } + + raw, err := json.MarshalIndent(out, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(targetPath, raw, 0o644); err != nil { + return 0, err + } + return 1, nil +} + +func mergeCreaturespeedRows(baseObj map[string]any, modulesDir string) ([]any, error) { + rawRows, ok := baseObj["rows"].([]any) + if !ok { + return nil, nil + } + rows := make([]map[string]any, 0, len(rawRows)) + byID := map[int]map[string]any{} + for _, raw := range rawRows { + row, ok := deepCopyValue(raw).(map[string]any) + if !ok { + continue + } + delete(row, "key") + rows = append(rows, row) + if rawID, ok := row["id"]; ok { + id, err := asInt(rawID) + if err != nil { + return nil, err + } + byID[id] = row + } + } + + modulePaths, err := collectModulePaths(modulesDir) + if err != nil { + return nil, err + } + for _, path := range modulePaths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + overrideList, ok := obj["overrides"].([]any) + if !ok { + continue + } + for _, raw := range overrideList { + override, ok := raw.(map[string]any) + if !ok { + continue + } + rawID, ok := override["id"] + if !ok { + continue + } + id, err := asInt(rawID) + if err != nil { + return nil, err + } + row := byID[id] + if row == nil { + continue + } + for key, value := range override { + if key == "id" || key == "key" || key == "_tlk" { + continue + } + row[key] = deepCopyValue(value) + } + } + } + + out := make([]any, 0, len(rows)) + for _, row := range rows { + out = append(out, row) + } + return out, nil +} + +func removePathIfExists(path string) error { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + return os.RemoveAll(path) +} diff --git a/internal/topdata/damagetypes_migrate.go b/internal/topdata/damagetypes_migrate.go new file mode 100644 index 0000000..d3c70cb --- /dev/null +++ b/internal/topdata/damagetypes_migrate.go @@ -0,0 +1,288 @@ +package topdata + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +func importLegacyDamagetypes(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyRoot := filepath.Join(referenceBuilderDir, "data", "damagetypes") + if _, err := os.Stat(legacyRoot); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "damagetypes", "registry") + targetPath := filepath.Join(targetDir, "types.json") + if _, err := os.Stat(targetPath); err == nil { + obj, err := loadJSONObject(targetPath) + if err == nil && countLegacyTLKRefsInValue(obj) == 0 { + return 0, nil + } + } + + coreBase, err := loadJSONObject(filepath.Join(legacyRoot, "core", "base.json")) + if err != nil { + return 0, err + } + groupBase, err := loadJSONObject(filepath.Join(legacyRoot, "groups", "base.json")) + if err != nil { + return 0, err + } + hitvisualBase, err := loadJSONObject(filepath.Join(legacyRoot, "hitvisual", "base.json")) + if err != nil { + return 0, err + } + coreModule, err := loadJSONObject(filepath.Join(legacyRoot, "core", "modules", "add_eos.json")) + if err != nil { + return 0, err + } + coreLock, err := loadLockfile(filepath.Join(legacyRoot, "core", "lock.json")) + if err != nil { + return 0, err + } + groupModule, err := loadJSONObject(filepath.Join(legacyRoot, "groups", "modules", "add_eos.json")) + if err != nil { + return 0, err + } + groupLock, err := loadLockfile(filepath.Join(legacyRoot, "groups", "lock.json")) + if err != nil { + return 0, err + } + hitvisualModule, err := loadJSONObject(filepath.Join(legacyRoot, "hitvisual", "modules", "add_eos.json")) + if err != nil { + return 0, err + } + hitvisualLock, err := loadLockfile(filepath.Join(legacyRoot, "hitvisual", "lock.json")) + if err != nil { + return 0, err + } + + rows, lockData, err := mergeLegacyDamagetypes(coreBase, groupBase, hitvisualBase, coreModule, coreLock, groupModule, groupLock, hitvisualModule, hitvisualLock) + if err != nil { + return 0, err + } + + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(dataDir, "damagetypes", "core")); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(dataDir, "damagetypes", "groups")); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(dataDir, "damagetypes", "hitvisual")); err != nil { + return 0, err + } + + typesOut := map[string]any{"rows": rows} + raw, err := json.MarshalIndent(typesOut, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(targetPath, raw, 0o644); err != nil { + return 0, err + } + if err := saveLockfile(filepath.Join(targetDir, damagetypesRegistryLock), lockData); err != nil { + return 0, err + } + return 2, nil +} + +func mergeLegacyDamagetypes(coreBase, groupBase, hitvisualBase, coreModule map[string]any, coreLock map[string]int, groupModule map[string]any, groupLock map[string]int, hitvisualModule map[string]any, hitvisualLock map[string]int) ([]map[string]any, map[string]int, error) { + coreRows, err := coerceRowMapSlice(coreBase["rows"]) + if err != nil { + return nil, nil, err + } + groupRows, err := coerceRowMapSlice(groupBase["rows"]) + if err != nil { + return nil, nil, err + } + hitRows, err := coerceRowMapSlice(hitvisualBase["rows"]) + if err != nil { + return nil, nil, err + } + groupByID := map[int]map[string]any{} + for _, row := range groupRows { + id, err := asInt(row["id"]) + if err != nil { + return nil, nil, err + } + groupByID[id] = row + } + hitByID := map[int]map[string]any{} + for _, row := range hitRows { + id, err := asInt(row["id"]) + if err != nil { + return nil, nil, err + } + hitByID[id] = row + } + + outRows := make([]map[string]any, 0, len(coreRows)) + lockData := map[string]int{} + for _, core := range coreRows { + id, err := asInt(core["id"]) + if err != nil { + return nil, nil, err + } + groupID, err := asInt(core["DamageTypeGroup"]) + if err != nil { + return nil, nil, err + } + group := groupByID[groupID] + if group == nil { + return nil, nil, fmt.Errorf("core id %d references missing group %d", id, groupID) + } + hit := hitByID[id] + if hit == nil { + return nil, nil, fmt.Errorf("core id %d is missing hitvisual row", id) + } + key := "damagetype:" + normalizeDamagetypeKey(core["Label"]) + lockData[key] = id + outRows = append(outRows, map[string]any{ + "key": key, + "id": id, + "label": deepCopyValue(core["Label"]), + "charsheet_strref": deepCopyValue(core["CharsheetStrref"]), + "damage_type_group": strconvString(groupID), + "damage_ranged_projectile": deepCopyValue(core["DamageRangedProjectile"]), + "group_label": deepCopyValue(group["Label"]), + "feedback_strref": deepCopyValue(group["FeedbackStrref"]), + "color_r": deepCopyValue(group["ColorR"]), + "color_g": deepCopyValue(group["ColorG"]), + "color_b": deepCopyValue(group["ColorB"]), + "visual_effect_id": deepCopyValue(hit["VisualEffectID"]), + "ranged_effect_id": deepCopyValue(hit["RangedEffectID"]), + }) + } + + coreEntries, err := coerceEntriesMap(coreModule["entries"]) + if err != nil { + return nil, nil, err + } + groupEntries, err := coerceEntriesMap(groupModule["entries"]) + if err != nil { + return nil, nil, err + } + hitEntries, err := coerceEntriesMap(hitvisualModule["entries"]) + if err != nil { + return nil, nil, err + } + + keys := make([]string, 0, len(coreEntries)) + for key := range coreEntries { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + a := strings.TrimPrefix(keys[i], "damagetypes:") + b := strings.TrimPrefix(keys[j], "damagetypes:") + return a < b + }) + for _, coreKey := range keys { + core := coreEntries[coreKey] + suffix := strings.TrimPrefix(coreKey, "damagetypes:") + group := groupEntries["damagetypegroups:"+suffix] + if group == nil { + return nil, nil, fmt.Errorf("%s: missing matching group entry", coreKey) + } + hit := hitEntries["damagehitvisual:"+suffix] + if hit == nil { + return nil, nil, fmt.Errorf("%s: missing matching hitvisual entry", coreKey) + } + id, ok := coreLock[coreKey] + if !ok { + return nil, nil, fmt.Errorf("%s: missing core lock id", coreKey) + } + groupKey := "damagetypegroups:" + suffix + groupID, ok := groupLock[groupKey] + if !ok { + return nil, nil, fmt.Errorf("%s: missing group lock id", groupKey) + } + hitKey := "damagehitvisual:" + suffix + hitID, ok := hitvisualLock[hitKey] + if !ok { + return nil, nil, fmt.Errorf("%s: missing hitvisual lock id", hitKey) + } + if hitID != id { + return nil, nil, fmt.Errorf("%s: core id %d does not match hitvisual id %d", suffix, id, hitID) + } + key := "damagetype:" + suffix + lockData[key] = id + outRows = append(outRows, map[string]any{ + "key": key, + "id": id, + "label": deepCopyValue(core["Label"]), + "charsheet_strref": deepCopyValue(core["CharsheetStrref"]), + "damage_type_group": strconvString(groupID), + "damage_ranged_projectile": deepCopyValue(core["DamageRangedProjectile"]), + "group_label": deepCopyValue(group["Label"]), + "feedback_strref": deepCopyValue(group["FeedbackStrref"]), + "color_r": deepCopyValue(group["ColorR"]), + "color_g": deepCopyValue(group["ColorG"]), + "color_b": deepCopyValue(group["ColorB"]), + "visual_effect_id": deepCopyValue(hit["VisualEffectID"]), + "ranged_effect_id": deepCopyValue(hit["RangedEffectID"]), + }) + } + + sort.Slice(outRows, func(i, j int) bool { + return outRows[i]["id"].(int) < outRows[j]["id"].(int) + }) + return outRows, lockData, nil +} + +func coerceRowMapSlice(value any) ([]map[string]any, error) { + rawRows, ok := value.([]any) + if !ok { + return nil, fmt.Errorf("rows must be an array") + } + rows := make([]map[string]any, 0, len(rawRows)) + for _, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("row must be an object") + } + rows = append(rows, row) + } + return rows, nil +} + +func coerceEntriesMap(value any) (map[string]map[string]any, error) { + rawEntries, ok := value.(map[string]any) + if !ok { + return nil, fmt.Errorf("entries must be an object") + } + entries := make(map[string]map[string]any, len(rawEntries)) + for key, raw := range rawEntries { + row, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("entry %s must be an object", key) + } + entries[key] = row + } + return entries, nil +} + +func normalizeDamagetypeKey(value any) string { + text := strings.TrimSpace(strings.ToLower(format2DAValue(value))) + text = strings.TrimSuffix(text, "damage") + text = strings.ReplaceAll(text, " ", "") + text = strings.ReplaceAll(text, "_", "") + text = strings.ReplaceAll(text, "-", "") + return text +} + +func strconvString(v int) string { + return fmt.Sprintf("%d", v) +} diff --git a/internal/topdata/damagetypes_registry.go b/internal/topdata/damagetypes_registry.go new file mode 100644 index 0000000..69b9be3 --- /dev/null +++ b/internal/topdata/damagetypes_registry.go @@ -0,0 +1,243 @@ +package topdata + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" +) + +const ( + damagetypesRegistryDirName = "damagetypes/registry" + damagetypesRegistryLock = "lock.json" +) + +type damagetypesRegistry struct { + RootPath string + LockPath string + LockData map[string]int + Types []map[string]any +} + +func collectGeneratedRegistryDatasets(dataDir string) ([]nativeCollectedDataset, error) { + itempropsDatasets, err := collectItempropsRegistryDatasets(dataDir) + if err != nil { + return nil, err + } + damagetypeDatasets, err := collectDamagetypesRegistryDatasets(dataDir) + if err != nil { + return nil, err + } + racialtypesDatasets, err := collectRacialtypesRegistryDatasets(dataDir) + if err != nil { + return nil, err + } + out := make([]nativeCollectedDataset, 0, len(itempropsDatasets)+len(damagetypeDatasets)+len(racialtypesDatasets)) + out = append(out, itempropsDatasets...) + out = append(out, damagetypeDatasets...) + out = append(out, racialtypesDatasets...) + return out, nil +} + +func loadDamagetypesRegistry(dataDir string) (*damagetypesRegistry, error) { + root := filepath.Join(dataDir, filepath.FromSlash(damagetypesRegistryDirName)) + info, err := os.Stat(root) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("damagetypes registry root must be a directory: %s", root) + } + + lockData, err := loadLockfile(filepath.Join(root, damagetypesRegistryLock)) + if err != nil { + return nil, err + } + types, err := loadRegistryRows(filepath.Join(root, "types.json")) + if err != nil { + return nil, err + } + return &damagetypesRegistry{ + RootPath: root, + LockPath: filepath.Join(root, damagetypesRegistryLock), + LockData: lockData, + Types: types, + }, nil +} + +func collectDamagetypesRegistryDatasets(dataDir string) ([]nativeCollectedDataset, error) { + registry, err := loadDamagetypesRegistry(dataDir) + if err != nil { + return nil, err + } + if registry == nil { + return nil, nil + } + + typeIDs, lockModified, err := assignDamagetypeRegistryIDs(registry.Types, registry.LockData) + if err != nil { + return nil, err + } + if lockModified { + if err := saveLockfile(registry.LockPath, registry.LockData); err != nil { + return nil, err + } + } + + typeRowsSorted := slices.Clone(registry.Types) + slices.SortFunc(typeRowsSorted, func(a, b map[string]any) int { + return typeIDs[stringField(a, "key")] - typeIDs[stringField(b, "key")] + }) + + coreRows := make([]map[string]any, 0, len(typeRowsSorted)) + hitvisualRows := make([]map[string]any, 0, len(typeRowsSorted)) + groupRowsByID := map[int]map[string]any{} + groupOrder := make([]int, 0) + for _, row := range typeRowsSorted { + key := stringField(row, "key") + rowID := typeIDs[key] + + groupID, err := registryIntField(row, "damage_type_group") + if err != nil { + return nil, fmt.Errorf("%s: %w", key, err) + } + coreRows = append(coreRows, map[string]any{ + "id": rowID, + "key": key, + "Label": registryStringField(row, "label"), + "CharsheetStrref": deepCopyValue(row["charsheet_strref"]), + "DamageTypeGroup": strconv.Itoa(groupID), + "DamageRangedProjectile": registryNullableField(row, "damage_ranged_projectile"), + }) + hitvisualRows = append(hitvisualRows, map[string]any{ + "id": rowID, + "key": strings.Replace(key, "damagetype:", "damagehitvisual:", 1), + "Label": registryStringField(row, "label"), + "VisualEffectID": registryNullableField(row, "visual_effect_id"), + "RangedEffectID": registryNullableField(row, "ranged_effect_id"), + }) + + if _, ok := groupRowsByID[groupID]; !ok { + groupRowsByID[groupID] = map[string]any{ + "id": groupID, + "key": strings.Replace(key, "damagetype:", "damagetypegroup:", 1), + "Label": registryStringField(row, "group_label"), + "FeedbackStrref": deepCopyValue(row["feedback_strref"]), + "ColorR": registryNullableField(row, "color_r"), + "ColorG": registryNullableField(row, "color_g"), + "ColorB": registryNullableField(row, "color_b"), + } + groupOrder = append(groupOrder, groupID) + continue + } + existing := groupRowsByID[groupID] + for field, candidate := range map[string]any{ + "Label": registryStringField(row, "group_label"), + "FeedbackStrref": deepCopyValue(row["feedback_strref"]), + "ColorR": registryNullableField(row, "color_r"), + "ColorG": registryNullableField(row, "color_g"), + "ColorB": registryNullableField(row, "color_b"), + } { + if format2DAValue(existing[field]) != format2DAValue(candidate) { + return nil, fmt.Errorf("%s: conflicting group projection for DamageTypeGroup %d field %s", key, groupID, field) + } + } + } + slices.Sort(groupOrder) + groupRows := make([]map[string]any, 0, len(groupOrder)) + for _, id := range groupOrder { + groupRows = append(groupRows, groupRowsByID[id]) + } + + return []nativeCollectedDataset{ + newGeneratedDataset("damagetypes/registry/core", "damagetypes.2da", []string{"Label", "CharsheetStrref", "DamageTypeGroup", "DamageRangedProjectile"}, coreRows), + newGeneratedDataset("damagetypes/registry/groups", "damagetypegroups.2da", []string{"Label", "FeedbackStrref", "ColorR", "ColorG", "ColorB"}, groupRows), + newGeneratedDataset("damagetypes/registry/hitvisual", "damagehitvisual.2da", []string{"Label", "VisualEffectID", "RangedEffectID"}, hitvisualRows), + }, nil +} + +func assignDamagetypeRegistryIDs(rows []map[string]any, lockData map[string]int) (map[string]int, bool, error) { + ids := map[string]int{} + used := map[int]struct{}{} + for key, id := range lockData { + if strings.HasPrefix(key, "damagetype:") { + used[id] = struct{}{} + } + } + modified := false + for _, row := range rows { + key := stringField(row, "key") + if key == "" { + return nil, false, fmt.Errorf("registry row is missing key") + } + if !strings.HasPrefix(key, "damagetype:") { + return nil, false, fmt.Errorf("registry key %q must use prefix %q", key, "damagetype:") + } + if rawID, ok := row["id"]; ok { + id, err := asInt(rawID) + if err != nil { + return nil, false, fmt.Errorf("%s: invalid id: %w", key, err) + } + if existing, ok := lockData[key]; ok && existing != id { + return nil, false, fmt.Errorf("%s: explicit id %d does not match lock id %d", key, id, existing) + } + lockData[key] = id + ids[key] = id + used[id] = struct{}{} + modified = true + continue + } + if id, ok := lockData[key]; ok { + ids[key] = id + used[id] = struct{}{} + } + } + nextID := nextAvailableID(used) + for _, row := range rows { + key := stringField(row, "key") + if _, ok := ids[key]; ok { + continue + } + lockData[key] = nextID + ids[key] = nextID + used[nextID] = struct{}{} + nextID = nextAvailableID(used) + modified = true + } + return ids, modified, nil +} + +func registryStringField(row map[string]any, field string) string { + text := stringField(row, field) + if text == "" { + return nullValue + } + return text +} + +func registryNullableField(row map[string]any, field string) any { + value, ok := row[field] + if !ok { + return nullValue + } + switch typed := value.(type) { + case string: + if strings.TrimSpace(typed) == "" { + return nullValue + } + } + return deepCopyValue(value) +} + +func registryIntField(row map[string]any, field string) (int, error) { + value, ok := row[field] + if !ok { + return 0, fmt.Errorf("missing %s", field) + } + return asInt(value) +} diff --git a/internal/topdata/doortypes_migrate.go b/internal/topdata/doortypes_migrate.go new file mode 100644 index 0000000..234d868 --- /dev/null +++ b/internal/topdata/doortypes_migrate.go @@ -0,0 +1,99 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacyDoortypes(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "doortypes") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "doortypes") + targetBasePath := filepath.Join(targetDir, "base.json") + targetLockPath := filepath.Join(targetDir, "lock.json") + targetModulesDir := filepath.Join(targetDir, "modules") + moduleNames := []string{ + "add_pld01.json", + "add_sic11.json", + "add_tapr.json", + "add_tdm01.json", + "add_tdx01.json", + "add_tei01.json", + "add_tfm01.json", + } + targetModulePaths := make([]string, 0, len(moduleNames)) + for _, name := range moduleNames { + targetModulePaths = append(targetModulePaths, filepath.Join(targetModulesDir, name)) + } + if fileExists(targetBasePath) && fileExists(targetLockPath) { + allModulesPresent := true + for _, path := range targetModulePaths { + if !fileExists(path) { + allModulesPresent = false + break + } + } + if allModulesPresent { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + baseObj["output"] = "doortypes.2da" + + lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) + if err != nil { + return 0, err + } + + moduleObjs := make([]map[string]any, 0, len(moduleNames)) + for _, name := range moduleNames { + obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", name)) + if err != nil { + return 0, err + } + moduleObjs = append(moduleObjs, obj) + } + + if err := os.MkdirAll(targetModulesDir, 0o755); err != nil { + return 0, err + } + + writes := []struct { + path string + obj map[string]any + }{ + {path: targetBasePath, obj: baseObj}, + {path: targetLockPath, obj: lockObj}, + } + for i, path := range targetModulePaths { + writes = append(writes, struct { + path string + obj map[string]any + }{path: path, obj: moduleObjs[i]}) + } + + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + return len(writes), nil +} diff --git a/internal/topdata/editorconfig_format.go b/internal/topdata/editorconfig_format.go new file mode 100644 index 0000000..530e030 --- /dev/null +++ b/internal/topdata/editorconfig_format.go @@ -0,0 +1,246 @@ +package topdata + +import ( + "bufio" + "bytes" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +type jsonFileFormatting struct { + Indent string + LineEnding string + FinalNewline bool +} + +type editorConfigDocument struct { + Dir string + Root bool + Sections []editorConfigSection +} + +type editorConfigSection struct { + Pattern string + Properties map[string]string +} + +func defaultJSONFileFormatting() jsonFileFormatting { + return jsonFileFormatting{ + Indent: " ", + LineEnding: "\n", + FinalNewline: true, + } +} + +func resolveJSONFileFormatting(path string) (jsonFileFormatting, error) { + formatting := defaultJSONFileFormatting() + documents, err := editorConfigDocumentsForPath(path) + if err != nil { + return jsonFileFormatting{}, err + } + for _, document := range documents { + relative, err := filepath.Rel(document.Dir, path) + if err != nil { + return jsonFileFormatting{}, fmt.Errorf("resolve editorconfig path for %s relative to %s: %w", path, document.Dir, err) + } + relative = filepath.ToSlash(relative) + if strings.HasPrefix(relative, "../") || relative == ".." { + continue + } + for _, section := range document.Sections { + matches, err := editorConfigPatternMatches(section.Pattern, relative) + if err != nil { + return jsonFileFormatting{}, fmt.Errorf("%s/.editorconfig section [%s]: %w", document.Dir, section.Pattern, err) + } + if !matches { + continue + } + next, err := applyEditorConfigProperties(formatting, section.Properties) + if err != nil { + return jsonFileFormatting{}, fmt.Errorf("%s/.editorconfig section [%s]: %w", document.Dir, section.Pattern, err) + } + formatting = next + } + } + return formatting, nil +} + +func editorConfigDocumentsForPath(path string) ([]editorConfigDocument, error) { + dir := filepath.Dir(path) + documents := []editorConfigDocument{} + for { + configPath := filepath.Join(dir, ".editorconfig") + raw, err := os.ReadFile(configPath) + if err == nil { + document, err := parseEditorConfig(configPath, raw) + if err != nil { + return nil, err + } + document.Dir = dir + documents = append(documents, document) + if document.Root { + break + } + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("read %s: %w", configPath, err) + } + + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + for left, right := 0, len(documents)-1; left < right; left, right = left+1, right-1 { + documents[left], documents[right] = documents[right], documents[left] + } + return documents, nil +} + +func parseEditorConfig(path string, raw []byte) (editorConfigDocument, error) { + document := editorConfigDocument{} + scanner := bufio.NewScanner(bytes.NewReader(raw)) + var current *editorConfigSection + for lineNumber := 1; scanner.Scan(); lineNumber++ { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") { + continue + } + if strings.HasPrefix(line, "[") { + end := strings.LastIndex(line, "]") + if end < 0 { + return editorConfigDocument{}, fmt.Errorf("%s:%d: malformed section header", path, lineNumber) + } + section := editorConfigSection{ + Pattern: strings.TrimSpace(line[1:end]), + Properties: map[string]string{}, + } + document.Sections = append(document.Sections, section) + current = &document.Sections[len(document.Sections)-1] + continue + } + + index := strings.IndexAny(line, "=:") + if index < 0 { + return editorConfigDocument{}, fmt.Errorf("%s:%d: expected key/value property", path, lineNumber) + } + key := strings.ToLower(strings.TrimSpace(line[:index])) + value := strings.TrimSpace(line[index+1:]) + if current == nil { + if key == "root" { + document.Root = strings.EqualFold(value, "true") + } + continue + } + current.Properties[key] = value + } + if err := scanner.Err(); err != nil { + return editorConfigDocument{}, fmt.Errorf("scan %s: %w", path, err) + } + return document, nil +} + +func applyEditorConfigProperties(formatting jsonFileFormatting, properties map[string]string) (jsonFileFormatting, error) { + style := strings.ToLower(strings.TrimSpace(properties["indent_style"])) + sizeText := strings.ToLower(strings.TrimSpace(properties["indent_size"])) + if style == "tab" { + formatting.Indent = "\t" + } else if style != "" && style != "space" && style != "unset" { + return jsonFileFormatting{}, fmt.Errorf("unsupported indent_style %q", properties["indent_style"]) + } + if sizeText != "" && sizeText != "unset" && sizeText != "tab" { + size, err := strconv.Atoi(sizeText) + if err != nil || size < 1 { + return jsonFileFormatting{}, fmt.Errorf("indent_size must be a positive integer, got %q", properties["indent_size"]) + } + if style != "tab" { + formatting.Indent = strings.Repeat(" ", size) + } + } + + switch value := strings.ToLower(strings.TrimSpace(properties["end_of_line"])); value { + case "", "unset": + case "lf": + formatting.LineEnding = "\n" + case "crlf": + formatting.LineEnding = "\r\n" + case "cr": + formatting.LineEnding = "\r" + default: + return jsonFileFormatting{}, fmt.Errorf("unsupported end_of_line %q", properties["end_of_line"]) + } + + switch value := strings.ToLower(strings.TrimSpace(properties["insert_final_newline"])); value { + case "", "unset": + case "true": + formatting.FinalNewline = true + case "false": + formatting.FinalNewline = false + default: + return jsonFileFormatting{}, fmt.Errorf("insert_final_newline must be true or false, got %q", properties["insert_final_newline"]) + } + return formatting, nil +} + +func editorConfigPatternMatches(pattern, relativePath string) (bool, error) { + pattern = filepath.ToSlash(strings.TrimSpace(pattern)) + pattern = strings.TrimPrefix(pattern, "/") + if pattern == "" { + return false, nil + } + if !strings.Contains(pattern, "/") { + relativePath = pathBase(relativePath) + } + expression, err := editorConfigGlobRegexp(pattern) + if err != nil { + return false, err + } + return regexp.MatchString(expression, relativePath) +} + +func editorConfigGlobRegexp(pattern string) (string, error) { + var builder strings.Builder + builder.WriteByte('^') + for index := 0; index < len(pattern); { + char := pattern[index] + switch char { + case '*': + if index+1 < len(pattern) && pattern[index+1] == '*' { + if index+2 < len(pattern) && pattern[index+2] == '/' { + builder.WriteString("(?:.*/)?") + index += 3 + } else { + builder.WriteString(".*") + index += 2 + } + continue + } + builder.WriteString("[^/]*") + case '?': + builder.WriteString("[^/]") + case '.', '+', '(', ')', '|', '^', '$', '{', '}', '[', ']', '\\': + builder.WriteByte('\\') + builder.WriteByte(char) + default: + builder.WriteByte(char) + } + index++ + } + builder.WriteByte('$') + expression := builder.String() + if _, err := regexp.Compile(expression); err != nil { + return "", err + } + return expression, nil +} + +func pathBase(path string) string { + if index := strings.LastIndex(path, "/"); index >= 0 { + return path[index+1:] + } + return path +} diff --git a/internal/topdata/expansion_native.go b/internal/topdata/expansion_native.go new file mode 100644 index 0000000..1637b54 --- /dev/null +++ b/internal/topdata/expansion_native.go @@ -0,0 +1,284 @@ +package topdata + +import ( + "fmt" + "slices" + "strings" +) + +func normalizeMetadataKey(key string) string { + replacer := strings.NewReplacer("-", "_", " ", "_") + return strings.ToLower(replacer.Replace(strings.TrimSpace(key))) +} + +func parseRowMetadata(raw any) (map[string]any, error) { + if raw == nil { + return nil, nil + } + obj, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("meta must be an object") + } + if len(obj) == 0 { + return map[string]any{}, nil + } + + meta := make(map[string]any, len(obj)) + for key, value := range obj { + switch normalizeMetadataKey(key) { + case "family": + family, err := parseFamilyMetadata(value) + if err != nil { + return nil, fmt.Errorf("meta.family: %w", err) + } + meta["family"] = family + case "wiki": + wiki, err := parseWikiMetadata(value) + if err != nil { + return nil, fmt.Errorf("meta.wiki: %w", err) + } + if len(wiki) > 0 { + meta["wiki"] = wiki + } + default: + return nil, fmt.Errorf("unknown metadata key %q", key) + } + } + return meta, nil +} + +func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) { + datasetIndexMap := make(map[string]int) + for i, ds := range collected { + datasetIndexMap[ds.Dataset.Name] = i + datasetIndexMap[ds.Dataset.OutputName] = i + } + + rowsWithExpansion := []struct { + datasetName string + row map[string]any + }{} + + for _, ds := range collected { + for _, row := range ds.Rows { + if hasExpansionData(row) { + rowsWithExpansion = append(rowsWithExpansion, struct { + datasetName string + row map[string]any + }{ds.Dataset.Name, row}) + } + } + } + + if len(rowsWithExpansion) == 0 { + return collected, nil + } + + modifiedTargets := map[int]struct{}{} + for _, src := range rowsWithExpansion { + expansion, _ := extractExpansionData(src.row) + for targetDatasetName, targetRows := range expansion.Data { + targetIndex, ok := datasetIndexMap[targetDatasetName] + if !ok { + outputName := targetDatasetName + if !strings.HasSuffix(outputName, ".2da") { + outputName = outputName + ".2da" + } + targetIndex, ok = datasetIndexMap[outputName] + } + if !ok { + return nil, fmt.Errorf("expansion targets unknown dataset %q", targetDatasetName) + } + targetDS := collected[targetIndex] + originalLockData, _ := loadLockfile(targetDS.Dataset.LockPath) + if originalLockData == nil { + originalLockData = map[string]int{} + } + lockModified := false + usedIDs := map[int]struct{}{} + usedKeys := map[string]struct{}{} + for _, rowID := range targetDS.LockData { + usedIDs[rowID] = struct{}{} + } + for _, row := range targetDS.Rows { + if id, ok := row["id"].(int); ok { + usedIDs[id] = struct{}{} + } + if key, ok := row["key"].(string); ok { + usedKeys[key] = struct{}{} + } + } + nextID := nextAvailableIDAtLeast(usedIDs, targetDS.Dataset.RowGenerationMinRow) + + for _, targetRow := range targetRows { + var rowID int + var hasID bool + if rawID, ok := targetRow["id"]; ok { + switch typed := rawID.(type) { + case int: + rowID = typed + hasID = true + case float64: + rowID = int(typed) + hasID = true + case string: + parsed, err := asInt(typed) + if err == nil { + rowID = parsed + hasID = true + } + } + } + + key, _ := targetRow["key"].(string) + if strings.TrimSpace(key) == "" { + return nil, fmt.Errorf("expansion into %s: injected rows must specify key", targetDatasetName) + } + + if key != "" { + if existingID, exists := targetDS.LockData[key]; exists { + rowID = existingID + hasID = true + } else if existingID, exists := originalLockData[key]; exists && targetDS.Dataset.RowGeneration != "first_null_row" { + rowID = existingID + hasID = true + targetDS.LockData[key] = existingID + lockModified = true + targetDS.LockModified = true + } + if _, seen := usedKeys[key]; seen { + continue + } + usedKeys[key] = struct{}{} + } + + if !hasID { + rowID = nextID + usedIDs[rowID] = struct{}{} + nextID = nextAvailableIDAtLeast(usedIDs, targetDS.Dataset.RowGenerationMinRow) + } else { + if _, exists := usedIDs[rowID]; exists && key == "" { + return nil, fmt.Errorf("expansion into %s: row id %d already exists", targetDatasetName, rowID) + } + usedIDs[rowID] = struct{}{} + } + + if key != "" { + if _, exists := targetDS.LockData[key]; !exists { + targetDS.LockData[key] = rowID + lockModified = true + targetDS.LockAdded++ + targetDS.LockModified = true + } + } + + newRow := map[string]any{ + "id": rowID, + } + for k, v := range targetRow { + if k != "id" { + newRow[k] = v + } + } + if key != "" { + newRow["key"] = key + } + targetDS.Rows = append(targetDS.Rows, newRow) + } + + if lockModified { + modifiedTargets[targetIndex] = struct{}{} + } + + slices.SortFunc(targetDS.Rows, func(a, b map[string]any) int { + return a["id"].(int) - b["id"].(int) + }) + collected[targetIndex] = targetDS + } + + if err := applyExpansionValueToRow(src.row, expansion.Value); err != nil { + return nil, err + } + } + + for targetIndex := range modifiedTargets { + targetDS := collected[targetIndex] + referencedKeys, err := collectReferencedLockKeys(nativeDatasetSourceDir(targetDS.Dataset)) + if err != nil { + return nil, fmt.Errorf("dataset %s: collect referenced keys: %w", targetDS.Dataset.Name, err) + } + if pruned, updated := pruneLockDataToActiveRows(targetDS.LockData, targetDS.Rows, referencedKeys, targetDS.RetiredKeys); pruned > 0 || updated > 0 { + targetDS.LockPruned += pruned + targetDS.LockModified = true + collected[targetIndex] = targetDS + } + } + + return collected, nil +} + +type expansionSpec struct { + Value string + Data map[string][]map[string]any +} + +func hasExpansionData(row map[string]any) bool { + for _, value := range row { + if isExpansionValue(value) { + return true + } + } + return false +} + +func isExpansionValue(value any) bool { + obj, ok := value.(map[string]any) + if !ok { + return false + } + _, hasValue := obj["value"] + _, hasData := obj["data"] + return hasValue && hasData +} + +func extractExpansionData(row map[string]any) (expansionSpec, bool) { + for _, value := range row { + if isExpansionValue(value) { + obj := value.(map[string]any) + valueStr, _ := obj["value"].(string) + dataMap, _ := obj["data"].(map[string]any) + + result := expansionSpec{ + Value: valueStr, + Data: make(map[string][]map[string]any), + } + + for datasetName, rawRows := range dataMap { + switch typed := rawRows.(type) { + case []any: + rows := make([]map[string]any, 0, len(typed)) + for _, r := range typed { + if rowObj, ok := r.(map[string]any); ok { + rows = append(rows, rowObj) + } + } + result.Data[datasetName] = rows + case map[string]any: + result.Data[datasetName] = []map[string]any{typed} + } + } + + return result, true + } + } + return expansionSpec{}, false +} + +func applyExpansionValueToRow(row map[string]any, value string) error { + for field, oldValue := range row { + if isExpansionValue(oldValue) { + row[field] = value + } + } + return nil +} diff --git a/internal/topdata/family_expansion.go b/internal/topdata/family_expansion.go new file mode 100644 index 0000000..4e2cf1d --- /dev/null +++ b/internal/topdata/family_expansion.go @@ -0,0 +1,345 @@ +package topdata + +import ( + "fmt" + "strings" +) + +type familyIdentity struct { + Parent string + Child string +} + +type familyExpansionSource struct { + Dataset string + Column string + Predicate string +} + +type familyExpansionTitleStyle struct { + ChildCase string + ChildParenthetical string +} + +type familyExpansionSpec struct { + Family string + FamilyKey string + Template string + ChildSource familyExpansionSource + NamePrefix string + LabelPrefix string + ConstantPrefix string + TemplateFields []string + DefaultFields map[string]any + ApplyAfterModules bool + ChildRefField string + IdentitySource string + AllowExistingOnly bool + AutoPrereqFields map[string]string + LegacyFamilyKeys []string + TitleStyle familyExpansionTitleStyle +} + +func splitFamilyExpansionIdentity(text string) familyIdentity { + text = strings.TrimSpace(text) + if text == "" { + return familyIdentity{} + } + if idx := strings.Index(text, "_"); idx > 0 && idx < len(text)-1 { + return familyIdentity{ + Parent: text[:idx], + Child: text[idx+1:], + } + } + return familyIdentity{Parent: text} +} + +func parseFamilyExpansionSource(raw any) (familyExpansionSource, error) { + if raw == nil { + return familyExpansionSource{}, fmt.Errorf("child_source is required") + } + obj, ok := raw.(map[string]any) + if !ok { + return familyExpansionSource{}, fmt.Errorf("child_source must be an object") + } + dataset, ok := obj["dataset"].(string) + if !ok || strings.TrimSpace(dataset) == "" { + return familyExpansionSource{}, fmt.Errorf("child_source.dataset must be a non-empty string") + } + source := familyExpansionSource{ + Dataset: strings.TrimSpace(dataset), + } + if column, ok := obj["column"].(string); ok && strings.TrimSpace(column) != "" { + source.Column = strings.TrimSpace(column) + } + if predicate, ok := obj["predicate"].(string); ok && strings.TrimSpace(predicate) != "" { + source.Predicate = strings.TrimSpace(predicate) + } + return source, nil +} + +func parseFamilyExpansionSpec(path string, obj map[string]any) (familyExpansionSpec, error) { + family, ok := obj["family"].(string) + if !ok || strings.TrimSpace(family) == "" { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: family must be a non-empty string", path) + } + familyKey, ok := obj["family_key"].(string) + if !ok || strings.TrimSpace(familyKey) == "" { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: family_key must be a non-empty string", path) + } + template, ok := obj["template"].(string) + if !ok || strings.TrimSpace(template) == "" { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: template must be a non-empty string", path) + } + source, err := parseFamilyExpansionSource(obj["child_source"]) + if err != nil { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err) + } + namePrefix, _ := optionalTrimmedString(obj, "name_prefix") + labelPrefix, _ := optionalTrimmedString(obj, "label_prefix") + constantPrefix, _ := optionalTrimmedString(obj, "constant_prefix") + templateFields, err := parseOptionalStringArray(obj["template_fields"], "template_fields") + if err != nil { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err) + } + defaultFields, err := parseOptionalObject(obj["default_fields"], "default_fields") + if err != nil { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err) + } + applyAfterModules, err := parseOptionalBoolField(obj["apply_after_modules"], "apply_after_modules") + if err != nil { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err) + } + childRefField, _ := optionalTrimmedString(obj, "child_ref_field") + identitySource, _ := optionalTrimmedString(obj, "identity_source") + switch identitySource { + case "", "child_source_value": + default: + return familyExpansionSpec{}, fmt.Errorf("generated file %s: identity_source must be empty or child_source_value", path) + } + autoPrereqFields, err := parseOptionalStringMap(obj["auto_prereq_fields"], "auto_prereq_fields") + if err != nil { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err) + } + legacyFamilyKeys, err := parseOptionalStringArray(obj["legacy_family_keys"], "legacy_family_keys") + if err != nil { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err) + } + if err := validateLegacyFamilyKeys(familyKey, legacyFamilyKeys); err != nil { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err) + } + titleStyle, err := parseFamilyExpansionTitleStyle(obj["title_style"]) + if err != nil { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err) + } + allowExistingOnly, err := parseOptionalBoolField(obj["allow_existing_only"], "allow_existing_only") + if err != nil { + return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err) + } + return familyExpansionSpec{ + Family: strings.TrimSpace(family), + FamilyKey: strings.TrimSpace(familyKey), + Template: strings.TrimSpace(template), + ChildSource: source, + NamePrefix: namePrefix, + LabelPrefix: labelPrefix, + ConstantPrefix: constantPrefix, + TemplateFields: templateFields, + DefaultFields: defaultFields, + ApplyAfterModules: applyAfterModules, + ChildRefField: childRefField, + IdentitySource: identitySource, + AllowExistingOnly: allowExistingOnly, + AutoPrereqFields: autoPrereqFields, + LegacyFamilyKeys: legacyFamilyKeys, + TitleStyle: titleStyle, + }, nil +} + +func parseFamilyExpansionTitleStyle(raw any) (familyExpansionTitleStyle, error) { + style := familyExpansionTitleStyle{ + ChildCase: "preserve", + ChildParenthetical: "preserve", + } + if raw == nil { + return style, nil + } + obj, ok := raw.(map[string]any) + if !ok { + return familyExpansionTitleStyle{}, fmt.Errorf("title_style must be an object") + } + if childCase, ok := obj["child_case"].(string); ok && strings.TrimSpace(childCase) != "" { + style.ChildCase = strings.TrimSpace(childCase) + } + switch style.ChildCase { + case "preserve", "lower": + default: + return familyExpansionTitleStyle{}, fmt.Errorf("title_style.child_case must be preserve or lower") + } + if childParenthetical, ok := obj["child_parenthetical"].(string); ok && strings.TrimSpace(childParenthetical) != "" { + style.ChildParenthetical = strings.TrimSpace(childParenthetical) + } + switch style.ChildParenthetical { + case "preserve", "comma": + default: + return familyExpansionTitleStyle{}, fmt.Errorf("title_style.child_parenthetical must be preserve or comma") + } + return style, nil +} + +func validateLegacyFamilyKeys(familyKey string, legacyFamilyKeys []string) error { + seen := map[string]string{} + normalizedFamilyKey := normalizeKeyIdentity(familyKey) + for _, legacyKey := range legacyFamilyKeys { + normalizedLegacyKey := normalizeKeyIdentity(legacyKey) + if normalizedLegacyKey == normalizedFamilyKey { + return fmt.Errorf("legacy_family_keys must not include family_key %q", familyKey) + } + if previous, ok := seen[normalizedLegacyKey]; ok { + return fmt.Errorf("legacy_family_keys contains duplicate-equivalent keys %q and %q", previous, legacyKey) + } + seen[normalizedLegacyKey] = legacyKey + } + return nil +} + +func isFamilyExpansionObject(obj map[string]any) bool { + _, hasFamilyKey := obj["family_key"] + _, hasTemplate := obj["template"] + _, hasChildSource := obj["child_source"] + _, hasNamePrefix := obj["name_prefix"] + _, hasLabelPrefix := obj["label_prefix"] + _, hasConstantPrefix := obj["constant_prefix"] + _, hasTemplateFields := obj["template_fields"] + _, hasDefaultFields := obj["default_fields"] + _, hasApplyAfterModules := obj["apply_after_modules"] + _, hasChildRefField := obj["child_ref_field"] + _, hasIdentitySource := obj["identity_source"] + _, hasAllowExistingOnly := obj["allow_existing_only"] + _, hasAutoPrereqFields := obj["auto_prereq_fields"] + _, hasLegacyFamilyKeys := obj["legacy_family_keys"] + _, hasTitleStyle := obj["title_style"] + return hasFamilyKey || hasTemplate || hasChildSource || hasNamePrefix || hasLabelPrefix || + hasConstantPrefix || hasTemplateFields || hasDefaultFields || hasApplyAfterModules || hasChildRefField || + hasIdentitySource || hasAllowExistingOnly || hasAutoPrereqFields || hasLegacyFamilyKeys || hasTitleStyle +} + +func optionalTrimmedString(obj map[string]any, field string) (string, bool) { + text, ok := obj[field].(string) + if !ok || strings.TrimSpace(text) == "" { + return "", false + } + return strings.TrimSpace(text), true +} + +func parseOptionalStringArray(raw any, field string) ([]string, error) { + if raw == nil { + return nil, nil + } + items, ok := raw.([]any) + if !ok { + return nil, fmt.Errorf("%s must be an array of strings", field) + } + out := make([]string, 0, len(items)) + for _, item := range items { + text, ok := item.(string) + if !ok || strings.TrimSpace(text) == "" { + return nil, fmt.Errorf("%s must contain only non-empty strings", field) + } + out = append(out, strings.TrimSpace(text)) + } + return out, nil +} + +func parseOptionalObject(raw any, field string) (map[string]any, error) { + if raw == nil { + return nil, nil + } + obj, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s must be an object", field) + } + return obj, nil +} + +func parseOptionalStringMap(raw any, field string) (map[string]string, error) { + if raw == nil { + return nil, nil + } + obj, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s must be an object of strings", field) + } + out := make(map[string]string, len(obj)) + for key, value := range obj { + text, ok := value.(string) + if !ok || strings.TrimSpace(text) == "" { + return nil, fmt.Errorf("%s must contain only non-empty string values", field) + } + out[key] = strings.TrimSpace(text) + } + return out, nil +} + +func parseOptionalBoolField(raw any, field string) (bool, error) { + if raw == nil { + return false, nil + } + value, ok := raw.(bool) + if !ok { + return false, fmt.Errorf("%s must be a boolean", field) + } + return value, nil +} + +func familyMetadata(parent, child, source, template string) map[string]any { + meta := map[string]any{ + "parent": strings.TrimSpace(parent), + } + if strings.TrimSpace(child) != "" { + meta["child"] = strings.TrimSpace(child) + } + if strings.TrimSpace(source) != "" { + meta["source"] = strings.TrimSpace(source) + } + if strings.TrimSpace(template) != "" { + meta["template"] = strings.TrimSpace(template) + } + return map[string]any{"family": meta} +} + +func parseFamilyMetadata(raw any) (map[string]any, error) { + obj, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("must be an object") + } + parent, ok := obj["parent"].(string) + if !ok || strings.TrimSpace(parent) == "" { + return nil, fmt.Errorf("parent must be a non-empty string") + } + meta := map[string]any{ + "parent": strings.TrimSpace(parent), + } + if child, ok := obj["child"].(string); ok && strings.TrimSpace(child) != "" { + meta["child"] = strings.TrimSpace(child) + } + if source, ok := obj["source"].(string); ok && strings.TrimSpace(source) != "" { + meta["source"] = strings.TrimSpace(source) + } + if template, ok := obj["template"].(string); ok && strings.TrimSpace(template) != "" { + meta["template"] = strings.TrimSpace(template) + } + return meta, nil +} + +func childTokenFromExpandedKey(key, parent string) string { + key = strings.TrimSpace(key) + key = strings.TrimPrefix(key, "feat:") + if strings.HasPrefix(key, parent+"_") { + return strings.TrimPrefix(key, parent+"_") + } + if strings.HasPrefix(key, parent) { + return strings.TrimPrefix(key, parent) + } + return "" +} diff --git a/internal/topdata/feat_migrate.go b/internal/topdata/feat_migrate.go new file mode 100644 index 0000000..b7b48f1 --- /dev/null +++ b/internal/topdata/feat_migrate.go @@ -0,0 +1,64 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacyFeat(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "feat") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + legacyBasePath := filepath.Join(legacyDir, "base.json") + if _, err := os.Stat(legacyBasePath); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "feat") + targetBasePath := filepath.Join(targetDir, "base.json") + var targetObj map[string]any + if _, err := os.Stat(targetBasePath); err == nil { + targetObj, err = loadJSONObject(targetBasePath) + if err != nil { + return 0, err + } + } else if !os.IsNotExist(err) { + return 0, err + } + if targetObj != nil { + if rows, ok := targetObj["rows"].([]any); ok && len(rows) > 0 { + return 0, nil + } + } + + baseObj, err := loadJSONObject(legacyBasePath) + if err != nil { + return 0, err + } + baseObj["output"] = "feat.2da" + baseObj["compare_reference"] = false + canonicalizeWikiMetadataDocument(baseObj) + + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return 0, err + } + raw, err := json.MarshalIndent(baseObj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(targetBasePath, raw, 0o644); err != nil { + return 0, err + } + return 1, nil +} diff --git a/internal/topdata/generated_assets.go b/internal/topdata/generated_assets.go new file mode 100644 index 0000000..26e1c1c --- /dev/null +++ b/internal/topdata/generated_assets.go @@ -0,0 +1,205 @@ +package topdata + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +type Generated2DAAsset struct { + Rel string + SourcePath string +} + +func BuildGenerated2DAAssets(p *project.Project, progress func(string)) ([]Generated2DAAsset, error) { + if progress == nil { + progress = func(string) {} + } + configs := p.EffectiveConfig().Generated.TopData2DA + if len(configs) == 0 { + return nil, nil + } + + results := make([]Generated2DAAsset, 0) + for _, cfg := range configs { + progress(fmt.Sprintf("Building generated 2DA assets for %s...", cfg.ID)) + generated, err := buildGenerated2DAAssetGroup(p, cfg, progress) + if err != nil { + return nil, err + } + results = append(results, generated...) + } + return results, nil +} + +func buildGenerated2DAAssetGroup(p *project.Project, cfg project.GeneratedTopData2DAConfig, progress func(string)) ([]Generated2DAAsset, error) { + sourceDir := filepath.Join(p.Root, filepath.FromSlash(strings.TrimSpace(cfg.Source))) + dataDir := filepath.Join(sourceDir, "data") + outputDir := filepath.Join(p.Root, filepath.FromSlash(strings.TrimSpace(cfg.Output))) + + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return nil, fmt.Errorf("discover generated topdata 2DA datasets %s: %w", cfg.ID, err) + } + selected := make([]nativeDataset, 0, len(datasets)) + for _, dataset := range datasets { + if generatedDatasetIncluded(dataset.Name, cfg.IncludeDatasets) { + selected = append(selected, dataset) + } + } + if len(selected) == 0 { + return nil, fmt.Errorf("generated topdata 2DA config %s matched no datasets under %s", cfg.ID, dataDir) + } + + collected := make([]nativeCollectedDataset, 0, len(selected)) + for _, dataset := range selected { + collectedDataset, err := collectNativeDataset(dataset) + if err != nil { + return nil, err + } + collected = append(collected, collectedDataset) + } + + consumer := cfg.Autogen + consumer.LocalOverrideRoot = p.AssetsDir() + collected, err = applyAutogenConsumersForGeneratedAssets(collected, consumer, progress) + if err != nil { + return nil, err + } + collected, err = normalizePartsRowsACBonus(collected, consumer.PartsRows) + if err != nil { + return nil, err + } + collected, err = applyPartOverridesWithConfig(sourceDir, collected, consumer.PartsRows) + if err != nil { + return nil, err + } + if err := rejectGenerated2DATLKValues(cfg.ID, collected); err != nil { + return nil, err + } + if _, _, err := saveNativeLockfiles(collected); err != nil { + return nil, fmt.Errorf("save generated topdata 2DA lockfiles %s: %w", cfg.ID, err) + } + + tableRegistry, err := newResolvedTableRegistry(collected) + if err != nil { + return nil, err + } + keyToID, rowByKey := generated2DAGlobalRows(collected) + + if err := os.RemoveAll(outputDir); err != nil { + return nil, fmt.Errorf("clean generated topdata 2DA output %s: %w", outputDir, err) + } + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return nil, fmt.Errorf("create generated topdata 2DA output %s: %w", outputDir, err) + } + + results := make([]Generated2DAAsset, 0, len(collected)) + for _, dataset := range collected { + compiled, err := resolveNativeDataset(dataset, keyToID, rowByKey, tableRegistry, nil, project.TopDataClassFeatInjectionConfig{}, nil) + if err != nil { + return nil, err + } + outputPath := filepath.Join(outputDir, dataset.Dataset.OutputName) + denseRows := dataset.Dataset.Kind == nativeDatasetBase || + (consumer.Mode == "parts_rows" && isPartsDataset(dataset.Dataset.Name)) + if err := write2DA(compiled, outputPath, denseRows, -1); err != nil { + return nil, err + } + results = append(results, Generated2DAAsset{ + Rel: filepath.ToSlash(filepath.Join(cfg.PackageRoot, dataset.Dataset.OutputName)), + SourcePath: outputPath, + }) + } + return results, nil +} + +func applyAutogenConsumersForGeneratedAssets(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig, progress func(string)) ([]nativeCollectedDataset, error) { + if strings.TrimSpace(consumer.Mode) == "" { + return collected, nil + } + if progress == nil { + progress = func(string) {} + } + if !autogenConsumerTargetsCollectedDataset(collected, consumer) { + return collected, nil + } + if progress != nil { + progress(fmt.Sprintf("Scanning local generated 2DA autogen input for %s from %s...", consumer.ID, consumer.LocalOverrideRoot)) + } + entries, err := scanLocalAutogenEntries(consumer.LocalOverrideRoot, consumer) + if err != nil { + return nil, err + } + switch consumer.Mode { + case "parts_rows": + return augmentWithAutogeneratedPartsWithConfig(collected, autogenPartsInventory(entries), consumer.PartsRows), nil + case "cachedmodels_rows": + return augmentWithAutogeneratedCachedModels(collected, entries) + default: + return nil, fmt.Errorf("unsupported generated topdata 2DA autogen mode %q", consumer.Mode) + } +} + +func generatedDatasetIncluded(name string, patterns []string) bool { + name = filepath.ToSlash(strings.TrimSpace(name)) + for _, pattern := range patterns { + pattern = filepath.ToSlash(strings.TrimSpace(pattern)) + if pattern == "" { + continue + } + if pattern == name { + return true + } + if strings.HasSuffix(pattern, "/**") { + prefix := strings.TrimSuffix(pattern, "/**") + if name == prefix || strings.HasPrefix(name, prefix+"/") { + return true + } + } + } + return false +} + +func generated2DAGlobalRows(collected []nativeCollectedDataset) (map[string]int, map[string]map[string]any) { + keyToID := map[string]int{} + rowByKey := map[string]map[string]any{} + for _, dataset := range collected { + for key, rowID := range dataset.LockData { + keyToID[key] = rowID + } + for _, row := range dataset.Rows { + key, _ := row["key"].(string) + if key == "" { + continue + } + if rowID, ok := row["id"].(int); ok { + keyToID[key] = rowID + rowByKey[key] = row + } + } + } + return keyToID, rowByKey +} + +func rejectGenerated2DATLKValues(configID string, collected []nativeCollectedDataset) error { + for _, dataset := range collected { + for _, row := range dataset.Rows { + for field, value := range row { + if field == "id" || field == "key" { + continue + } + allowBare := columnMatchesSpec(dataset.Dataset.Spec, field) + if _, ok, err := parseTLKPayload(value, allowBare); err != nil { + return fmt.Errorf("generated topdata 2DA config %s dataset %s field %s has invalid TLK payload: %w", configID, dataset.Dataset.Name, field, err) + } else if ok { + return fmt.Errorf("generated topdata 2DA config %s dataset %s field %s uses TLK-backed text; 2DA-only generated asset builds do not generate TLK output", configID, dataset.Dataset.Name, field) + } + } + } + } + return nil +} diff --git a/internal/topdata/genericdoors_migrate.go b/internal/topdata/genericdoors_migrate.go new file mode 100644 index 0000000..07f3c15 --- /dev/null +++ b/internal/topdata/genericdoors_migrate.go @@ -0,0 +1,6 @@ +package topdata + +func importLegacyGenericdoors(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "genericdoors", "genericdoors.2da", nil) +} diff --git a/internal/topdata/itemprops_registry.go b/internal/topdata/itemprops_registry.go new file mode 100644 index 0000000..d66bde9 --- /dev/null +++ b/internal/topdata/itemprops_registry.go @@ -0,0 +1,1214 @@ +package topdata + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" +) + +const ( + itempropsRegistryDirName = "itemprops/registry" + itempropsRegistryLock = "lock.json" +) + +var itempropsAvailabilityColumns = []string{ + "0_Melee", + "1_Ranged", + "2_Thrown", + "3_Staves", + "4_Rods", + "5_Ammo", + "6_Arm_Shld", + "7_Helm", + "8_Potions", + "9_Scrolls", + "10_Wands", + "11_Thieves", + "12_TrapKits", + "13_Hide", + "14_Claw", + "15_Misc_Uneq", + "16_Misc", + "17_No_Props", + "18_Containers", + "19_HealerKit", + "20_Torch", + "21_Glove", +} + +var itempropsAvailabilityAliases = map[string]string{ + "0_melee": "melee", + "1_ranged": "ranged", + "2_thrown": "thrown", + "3_staves": "staves", + "4_rods": "rods", + "5_ammo": "ammo", + "6_arm_shld": "arm_shld", + "7_helm": "helm", + "8_potions": "potions", + "9_scrolls": "scrolls", + "10_wands": "wands", + "11_thieves": "thieves", + "12_trapkits": "trapkits", + "13_hide": "hide", + "14_claw": "claw", + "15_misc_uneq": "misc_uneq", + "16_misc": "misc", + "17_no_props": "no_props", + "18_containers": "containers", + "19_healerkit": "healerkit", + "20_torch": "torch", + "21_glove": "glove", +} + +var itempropsAvailabilityToColumn = map[string]string{ + "melee": "0_Melee", + "ranged": "1_Ranged", + "thrown": "2_Thrown", + "staves": "3_Staves", + "rods": "4_Rods", + "ammo": "5_Ammo", + "arm_shld": "6_Arm_Shld", + "helm": "7_Helm", + "potions": "8_Potions", + "scrolls": "9_Scrolls", + "wands": "10_Wands", + "thieves": "11_Thieves", + "trapkits": "12_TrapKits", + "hide": "13_Hide", + "claw": "14_Claw", + "misc_uneq": "15_Misc_Uneq", + "misc": "16_Misc", + "no_props": "17_No_Props", + "containers": "18_Containers", + "healerkit": "19_HealerKit", + "torch": "20_Torch", + "glove": "21_Glove", +} + +type itempropsRegistry struct { + RootPath string + LockPath string + LockData map[string]int + Properties []map[string]any + CostModels []map[string]any + ParamModels []map[string]any + SubtypeModels []map[string]any +} + +func loadItempropsRegistry(dataDir string) (*itempropsRegistry, error) { + root := filepath.Join(dataDir, filepath.FromSlash(itempropsRegistryDirName)) + info, err := os.Stat(root) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("itemprops registry root must be a directory: %s", root) + } + + lockData, err := loadLockfile(filepath.Join(root, itempropsRegistryLock)) + if err != nil { + return nil, err + } + + properties, err := loadRegistryRows(filepath.Join(root, "properties.json")) + if err != nil { + return nil, err + } + costModels, err := loadRegistryRows(filepath.Join(root, "cost_models.json")) + if err != nil { + return nil, err + } + paramModels, err := loadRegistryRows(filepath.Join(root, "param_models.json")) + if err != nil { + return nil, err + } + subtypeModels, err := loadRegistryRows(filepath.Join(root, "subtype_models.json")) + if err != nil { + return nil, err + } + + return &itempropsRegistry{ + RootPath: root, + LockPath: filepath.Join(root, itempropsRegistryLock), + LockData: lockData, + Properties: properties, + CostModels: costModels, + ParamModels: paramModels, + SubtypeModels: subtypeModels, + }, nil +} + +func loadRegistryRows(path string) ([]map[string]any, error) { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + rawRows, ok := obj["rows"].([]any) + if !ok { + return nil, fmt.Errorf("%s: rows must be an array", path) + } + rows := make([]map[string]any, 0, len(rawRows)) + for index, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: row %d must be an object", path, index) + } + rows = append(rows, row) + } + return rows, nil +} + +func collectItempropsRegistryDatasets(dataDir string) ([]nativeCollectedDataset, error) { + registry, err := loadItempropsRegistry(dataDir) + if err != nil { + return nil, err + } + if registry == nil { + return nil, nil + } + + propertyIDs, lockModified, err := assignRegistryIDs(registry.Properties, registry.LockData, "itemprop:") + if err != nil { + return nil, err + } + costIDs, costModified, err := assignRegistryIDs(registry.CostModels, registry.LockData, "cost_models:") + if err != nil { + return nil, err + } + paramIDs, paramModified, err := assignRegistryIDs(registry.ParamModels, registry.LockData, "param_models:") + if err != nil { + return nil, err + } + lockModified = lockModified || costModified || paramModified + if lockModified { + if err := saveLockfile(registry.LockPath, registry.LockData); err != nil { + return nil, err + } + } + + subtypeByKey := map[string]map[string]any{} + for _, row := range registry.SubtypeModels { + key := stringField(row, "key") + if key != "" { + subtypeByKey[key] = row + } + } + costByKey := map[string]map[string]any{} + for _, row := range registry.CostModels { + key := stringField(row, "key") + if key != "" { + costByKey[key] = row + } + } + paramByKey := map[string]map[string]any{} + for _, row := range registry.ParamModels { + key := stringField(row, "key") + if key != "" { + paramByKey[key] = row + } + } + + datasets := make([]nativeCollectedDataset, 0) + itempropsRows := make([]map[string]any, 0, len(registry.Properties)) + itempropdefRows := make([]map[string]any, 0, len(registry.Properties)) + propertyRowsSorted := slices.Clone(registry.Properties) + slices.SortFunc(propertyRowsSorted, func(a, b map[string]any) int { + return propertyIDs[stringField(a, "key")] - propertyIDs[stringField(b, "key")] + }) + for _, property := range propertyRowsSorted { + key := stringField(property, "key") + rowID := propertyIDs[key] + + itempropsRow := map[string]any{ + "id": rowID, + "key": "itemprops:" + strings.TrimPrefix(key, "itemprop:"), + "StringRef": itempropsSetName(property), + "Label": itempropsSetLabel(property), + } + for _, column := range itempropsAvailabilityColumns { + itempropsRow[column] = nullValue + } + availability, err := canonicalAvailability(property) + if err != nil { + return nil, fmt.Errorf("%s: %w", key, err) + } + for _, group := range availability { + column := itempropsAvailabilityToColumn[group] + itempropsRow[column] = "1" + } + itempropsRows = append(itempropsRows, itempropsRow) + + subtypeResRef := nullValue + if subtypeKey, ok := refField(property["subtype"]); ok { + subtype, ok := subtypeByKey[subtypeKey] + if !ok { + return nil, fmt.Errorf("%s: unknown subtype ref %s", key, subtypeKey) + } + subtypeResRef = stringField(subtype, "table_resref") + if subtypeResRef == "" { + return nil, fmt.Errorf("%s: subtype model %s is missing table_resref", key, subtypeKey) + } + } + + costValue := nullValue + costRefID := nullValue + if rawCost, ok := property["cost"].(map[string]any); ok { + if value, ok := rawCost["value"]; ok { + costValue = formatRegistryScalar(value) + } + costRefKey, ok := refField(rawCost) + if ok { + id, ok := costIDs[costRefKey] + if !ok { + return nil, fmt.Errorf("%s: unknown cost ref %s", key, costRefKey) + } + costRefID = strconv.Itoa(id) + } + } + + paramRefID := nullValue + if paramKey, ok := refField(property["param"]); ok { + id, ok := paramIDs[paramKey] + if !ok { + return nil, fmt.Errorf("%s: unknown param ref %s", key, paramKey) + } + paramRefID = strconv.Itoa(id) + } + + itempropdefRows = append(itempropdefRows, map[string]any{ + "id": rowID, + "key": "itempropdef:" + strings.TrimPrefix(key, "itemprop:"), + "Name": deepCopyValue(property["name"]), + "Label": itempropsDefLabel(property), + "SubTypeResRef": subtypeResRef, + "Cost": costValue, + "CostTableResRef": costRefID, + "Param1ResRef": paramRefID, + "GameStrRef": deepCopyValue(property["property_text"]), + "Description": nullableRegistryField(property, "description"), + }) + } + + costRows := make([]map[string]any, 0, len(registry.CostModels)) + costRowsSorted := slices.Clone(registry.CostModels) + slices.SortFunc(costRowsSorted, func(a, b map[string]any) int { + return costIDs[stringField(a, "key")] - costIDs[stringField(b, "key")] + }) + for _, costModel := range costRowsSorted { + key := stringField(costModel, "key") + costRows = append(costRows, map[string]any{ + "id": costIDs[key], + "key": key, + "Name": stringField(costModel, "index_name"), + "Label": stringField(costModel, "label"), + "ClientLoad": formatRegistryScalar(costModel["client_load"]), + }) + } + + paramRows := make([]map[string]any, 0, len(registry.ParamModels)) + paramRowsSorted := slices.Clone(registry.ParamModels) + slices.SortFunc(paramRowsSorted, func(a, b map[string]any) int { + return paramIDs[stringField(a, "key")] - paramIDs[stringField(b, "key")] + }) + for _, paramModel := range paramRowsSorted { + key := stringField(paramModel, "key") + table, ok := paramModel["table"].(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: param model missing table", key) + } + paramRows = append(paramRows, map[string]any{ + "id": paramIDs[key], + "key": key, + "Name": nullableRegistryField(paramModel, "name"), + "Lable": stringField(paramModel, "label"), + "TableResRef": stringField(table, "resref"), + }) + } + + datasets = append(datasets, + newGeneratedDataset("itemprops/registry/itemprops", "itemprops.2da", itempropsAvailabilityColumnsWithMeta(), itempropsRows), + newGeneratedDataset("itemprops/registry/itempropdef", "itempropdef.2da", []string{"Name", "Label", "SubTypeResRef", "Cost", "CostTableResRef", "Param1ResRef", "GameStrRef", "Description"}, itempropdefRows), + newGeneratedDataset("itemprops/registry/costtable_index", "iprp_costtable.2da", []string{"Name", "Label", "ClientLoad"}, costRows), + newGeneratedDataset("itemprops/registry/paramtable_index", "iprp_paramtable.2da", []string{"Name", "Lable", "TableResRef"}, paramRows), + ) + + auxDatasets, err := collectItempropsAuxiliaryDatasets(registry.CostModels, "cost_models", "itemprops/registry/cost_models") + if err != nil { + return nil, err + } + datasets = append(datasets, auxDatasets...) + auxDatasets, err = collectItempropsAuxiliaryDatasets(registry.ParamModels, "param_models", "itemprops/registry/param_models") + if err != nil { + return nil, err + } + datasets = append(datasets, auxDatasets...) + auxDatasets, err = collectItempropsAuxiliaryDatasets(registry.SubtypeModels, "subtype_models", "itemprops/registry/subtype_models") + if err != nil { + return nil, err + } + datasets = append(datasets, auxDatasets...) + return datasets, nil +} + +func newGeneratedDataset(name, outputName string, columns []string, rows []map[string]any) nativeCollectedDataset { + return nativeCollectedDataset{ + Dataset: nativeDataset{ + Kind: nativeDatasetPlain, + Name: name, + OutputName: outputName, + Spec: specForDataset(name), + }, + Columns: columns, + Rows: rows, + LockData: map[string]int{}, + } +} + +func itempropsAvailabilityColumnsWithMeta() []string { + columns := append([]string{}, itempropsAvailabilityColumns...) + columns = append(columns, "StringRef", "Label") + return columns +} + +func collectItempropsAuxiliaryDatasets(models []map[string]any, keyPrefix, namePrefix string) ([]nativeCollectedDataset, error) { + out := make([]nativeCollectedDataset, 0) + for _, model := range models { + key := stringField(model, "key") + table, ok := model["table"].(map[string]any) + if !ok { + continue + } + if !strings.EqualFold(stringField(table, "ownership"), "owned") { + continue + } + outputName := stringField(table, "output") + if outputName == "" { + outputName = stringField(table, "resref") + ".2da" + } + rawColumns, ok := table["columns"].([]any) + if !ok { + return nil, fmt.Errorf("%s: owned table is missing columns", key) + } + columns := make([]string, 0, len(rawColumns)) + for _, rawColumn := range rawColumns { + column, ok := rawColumn.(string) + if !ok { + return nil, fmt.Errorf("%s: owned table columns must be strings", key) + } + columns = append(columns, column) + } + rawRows, ok := table["rows"].([]any) + if !ok { + return nil, fmt.Errorf("%s: owned table rows must be an array", key) + } + rows := make([]map[string]any, 0, len(rawRows)) + for index, rawRow := range rawRows { + row, ok := rawRow.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: owned table row %d must be an object", key, index) + } + canonical, err := canonicalizeBaseRow(nativeDataset{Name: keyPrefix + "/" + strings.TrimPrefix(key, keyPrefix+":"), Spec: specForDataset("itemprops")}, columns, row, index) + if err != nil { + return nil, err + } + rows = append(rows, canonical) + } + out = append(out, newGeneratedDataset(namePrefix+"/"+strings.TrimPrefix(key, keyPrefix+":"), outputName, columns, rows)) + } + return out, nil +} + +func assignRegistryIDs(rows []map[string]any, lockData map[string]int, requiredPrefix string) (map[string]int, bool, error) { + ids := map[string]int{} + used := map[int]struct{}{} + for key, id := range lockData { + if strings.HasPrefix(key, requiredPrefix) { + used[id] = struct{}{} + } + } + modified := false + for _, row := range rows { + key := stringField(row, "key") + if key == "" { + return nil, false, fmt.Errorf("registry row is missing key") + } + if !strings.HasPrefix(key, requiredPrefix) { + return nil, false, fmt.Errorf("registry key %q must use prefix %q", key, requiredPrefix) + } + if id, ok := lockData[key]; ok { + ids[key] = id + used[id] = struct{}{} + continue + } + } + nextID := nextAvailableID(used) + for _, row := range rows { + key := stringField(row, "key") + if _, ok := ids[key]; ok { + continue + } + lockData[key] = nextID + ids[key] = nextID + used[nextID] = struct{}{} + nextID = nextAvailableID(used) + modified = true + } + return ids, modified, nil +} + +func canonicalAvailability(row map[string]any) ([]string, error) { + rawAvailability, ok := row["availability"] + if !ok { + return nil, nil + } + rawList, ok := rawAvailability.([]any) + if !ok { + return nil, fmt.Errorf("availability must be an array") + } + out := make([]string, 0, len(rawList)) + seen := map[string]struct{}{} + for _, raw := range rawList { + text, ok := raw.(string) + if !ok { + return nil, fmt.Errorf("availability values must be strings") + } + normalized := normalizeAvailabilityName(text) + if _, ok := itempropsAvailabilityToColumn[normalized]; !ok { + return nil, fmt.Errorf("unknown availability group %q", text) + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + slices.Sort(out) + return out, nil +} + +func normalizeAvailabilityName(name string) string { + normalized := strings.TrimSpace(strings.ToLower(name)) + normalized = strings.ReplaceAll(normalized, " ", "_") + normalized = strings.ReplaceAll(normalized, "-", "_") + if alias, ok := itempropsAvailabilityAliases[normalized]; ok { + return alias + } + return normalized +} + +func refField(value any) (string, bool) { + if value == nil { + return "", false + } + refMap, ok := value.(map[string]any) + if !ok { + return "", false + } + rawRef, ok := refMap["ref"] + if !ok { + return "", false + } + text, ok := rawRef.(string) + return text, ok && strings.TrimSpace(text) != "" +} + +func stringField(obj map[string]any, field string) string { + raw, ok := obj[field] + if !ok || raw == nil { + return "" + } + switch typed := raw.(type) { + case string: + return typed + case float64: + if typed == float64(int64(typed)) { + return strconv.FormatInt(int64(typed), 10) + } + return strconv.FormatFloat(typed, 'f', -1, 64) + default: + return "" + } +} + +func nullableRegistryField(obj map[string]any, field string) any { + value, ok := obj[field] + if !ok || value == nil { + return nullValue + } + if text, ok := value.(string); ok && strings.TrimSpace(text) == "" { + return nullValue + } + return deepCopyValue(value) +} + +func itempropsDefLabel(obj map[string]any) string { + return stringField(obj, "label") +} + +func itempropsSetName(obj map[string]any) any { + if value, ok := obj["set_name"]; ok { + return deepCopyValue(value) + } + return deepCopyValue(obj["name"]) +} + +func itempropsSetLabel(obj map[string]any) string { + if label := stringField(obj, "set_label"); label != "" { + return label + } + return itempropsDefLabel(obj) +} + +func formatRegistryScalar(value any) string { + if value == nil { + return nullValue + } + switch typed := value.(type) { + case string: + if strings.TrimSpace(typed) == "" { + return nullValue + } + return typed + case float64: + if typed == float64(int64(typed)) { + return strconv.FormatInt(int64(typed), 10) + } + return strconv.FormatFloat(typed, 'f', -1, 64) + case int: + return strconv.Itoa(typed) + default: + return fmt.Sprint(typed) + } +} + +func deepCopyValue(value any) any { + raw, err := json.Marshal(value) + if err != nil { + return value + } + var out any + if err := json.Unmarshal(raw, &out); err != nil { + return value + } + return out +} + +func importLegacyItempropsRegistry(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + legacyDataDir := filepath.Join(referenceBuilderDir, "data") + registryDir := filepath.Join(dataDir, filepath.FromSlash(itempropsRegistryDirName)) + if _, err := os.Stat(filepath.Join(registryDir, "properties.json")); err == nil { + if refs, err := countLegacyTLKRefsInRegistry(registryDir); err == nil && refs == 0 { + return 0, nil + } + } + if _, err := os.Stat(filepath.Join(legacyDataDir, "itemprops")); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + defs, err := collectBaseDataset(nativeDataset{ + Name: "itemprops/defs", + BasePath: filepath.Join(legacyDataDir, "itemprops", "defs", "base.json"), + LockPath: filepath.Join(legacyDataDir, "itemprops", "defs", "lock.json"), + ModulesDir: filepath.Join(legacyDataDir, "itemprops", "defs", "modules"), + Spec: specForDataset("itemprops"), + }) + if err != nil { + return 0, err + } + sets, err := collectBaseDataset(nativeDataset{ + Name: "itemprops/sets", + BasePath: filepath.Join(legacyDataDir, "itemprops", "sets", "base.json"), + LockPath: filepath.Join(legacyDataDir, "itemprops", "sets", "lock.json"), + ModulesDir: filepath.Join(legacyDataDir, "itemprops", "sets", "modules"), + Spec: specForDataset("itemprops"), + }) + if err != nil { + return 0, err + } + costIndex, err := collectBaseDataset(nativeDataset{ + Name: "itemprops/costtables/index", + BasePath: filepath.Join(legacyDataDir, "itemprops", "costtables", "index", "base.json"), + LockPath: filepath.Join(legacyDataDir, "itemprops", "costtables", "index", "lock.json"), + ModulesDir: filepath.Join(legacyDataDir, "itemprops", "costtables", "index", "modules"), + Spec: specForDataset("itemprops"), + }) + if err != nil { + return 0, err + } + paramIndex, err := collectBaseDataset(nativeDataset{ + Name: "itemprops/paramtables/index", + BasePath: filepath.Join(legacyDataDir, "itemprops", "paramtables", "index", "base.json"), + LockPath: filepath.Join(legacyDataDir, "itemprops", "paramtables", "index", "lock.json"), + ModulesDir: filepath.Join(legacyDataDir, "itemprops", "paramtables", "index", "modules"), + Spec: specForDataset("itemprops"), + }) + if err != nil { + return 0, err + } + + ownedCostTables, err := collectLegacyOwnedItempropTables(filepath.Join(legacyDataDir, "itemprops", "costtables"), "index") + if err != nil { + return 0, err + } + ownedParamTables, err := collectLegacyOwnedItempropTables(filepath.Join(legacyDataDir, "itemprops", "paramtables"), "index") + if err != nil { + return 0, err + } + ownedSubtypeTables, err := collectLegacyOwnedItempropTables(filepath.Join(legacyDataDir, "itemprops", "subtypes"), "") + if err != nil && !os.IsNotExist(err) { + return 0, err + } + + setsByID := map[int]map[string]any{} + for _, row := range sets.Rows { + setsByID[row["id"].(int)] = row + } + costByID := map[int]map[string]any{} + for _, row := range costIndex.Rows { + costByID[row["id"].(int)] = row + } + paramByID := map[int]map[string]any{} + for _, row := range paramIndex.Rows { + paramByID[row["id"].(int)] = row + } + + registryLock := map[string]int{} + legacyKeyByID := map[string]string{} + if legacyTLK != nil { + for key, id := range legacyTLK.Lock { + legacyKeyByID[strconv.Itoa(id)] = key + } + } + properties := make([]map[string]any, 0, len(defs.Rows)) + subtypeRegistryByKey := map[string]map[string]any{} + for _, row := range defs.Rows { + rowID := row["id"].(int) + key := canonicalItempropPropertyKey(row) + registryLock[key] = rowID + + property := map[string]any{ + "key": key, + "label": row["Label"], + "name": deepCopyValue(row["Name"]), + "property_text": deepCopyValue(row["GameStrRef"]), + } + if description := nullableRegistryField(row, "Description"); description != nullValue { + property["description"] = description + } + if setRow, ok := setsByID[rowID]; ok { + property["availability"] = importLegacyAvailability(setRow) + if setName := nullableRegistryField(setRow, "StringRef"); setName != nullValue && setName != stringField(row, "Name") { + property["set_name"] = setName + } + if setLabel := stringField(setRow, "Label"); setLabel != "" && setLabel != stringField(row, "Label") { + property["set_label"] = setLabel + } + } else { + property["availability"] = []any{} + } + + if subtypeResRef := stringField(row, "SubTypeResRef"); subtypeResRef != "" && subtypeResRef != nullValue { + subtypeKey := canonicalModelKey("subtype_models", subtypeResRef) + property["subtype"] = map[string]any{"ref": subtypeKey} + if _, ok := subtypeRegistryByKey[subtypeKey]; !ok { + subtypeRegistryByKey[subtypeKey] = buildLegacySubtypeModel(subtypeKey, subtypeResRef, ownedSubtypeTables) + } + } + costValue := stringField(row, "Cost") + costID := stringField(row, "CostTableResRef") + if costValue != "" && costValue != nullValue || costID != "" && costID != nullValue { + cost := map[string]any{} + if costValue != "" && costValue != nullValue { + cost["value"] = costValue + } + if costID != "" && costID != nullValue { + id, err := strconv.Atoi(costID) + if err != nil { + return 0, err + } + costRow, ok := costByID[id] + if !ok { + return 0, fmt.Errorf("%s references missing cost table index %s", key, costID) + } + cost["ref"] = canonicalModelKey("cost_models", stringField(costRow, "Name")) + } + property["cost"] = cost + } + if paramID := stringField(row, "Param1ResRef"); paramID != "" && paramID != nullValue { + id, err := strconv.Atoi(paramID) + if err != nil { + return 0, err + } + paramRow, ok := paramByID[id] + if !ok { + return 0, fmt.Errorf("%s references missing param table index %s", key, paramID) + } + property["param"] = map[string]any{"ref": canonicalModelKey("param_models", stringField(paramRow, "TableResRef"))} + } + if legacyTLK != nil { + property["name"] = importLegacyStrrefValue(property["name"], key+".name", legacyTLK, legacyKeyByID) + property["property_text"] = importLegacyStrrefValue(property["property_text"], key+".property_text", legacyTLK, legacyKeyByID) + if description, ok := property["description"]; ok { + property["description"] = importLegacyStrrefValue(description, key+".description", legacyTLK, legacyKeyByID) + } + if updated, _, err := inlineLegacyTLKValue(property, legacyTLK); err != nil { + return 0, err + } else if updated { + // property updated in place + } + } + properties = append(properties, property) + } + + costModels := make([]map[string]any, 0, len(costIndex.Rows)) + for _, row := range costIndex.Rows { + key := canonicalModelKey("cost_models", stringField(row, "Name")) + registryLock[key] = row["id"].(int) + model := map[string]any{ + "key": key, + "index_name": row["Name"], + "label": row["Label"], + "client_load": row["ClientLoad"], + "table": buildLegacyOwnedModelTable(stringField(row, "Name"), ownedCostTables), + } + costModels = append(costModels, model) + } + + paramModels := make([]map[string]any, 0, len(paramIndex.Rows)) + for _, row := range paramIndex.Rows { + key := canonicalModelKey("param_models", stringField(row, "TableResRef")) + registryLock[key] = row["id"].(int) + model := map[string]any{ + "key": key, + "name": deepCopyValue(row["Name"]), + "label": row["Lable"], + "table": buildLegacyOwnedModelTable(stringField(row, "TableResRef"), ownedParamTables), + } + paramModels = append(paramModels, model) + } + + subtypeModels := make([]map[string]any, 0, len(subtypeRegistryByKey)) + for _, key := range sortedKeysAny(subtypeRegistryByKey) { + subtypeModels = append(subtypeModels, subtypeRegistryByKey[key]) + } + + writes := []struct { + path string + obj map[string]any + }{ + {filepath.Join(registryDir, "properties.json"), map[string]any{"rows": properties}}, + {filepath.Join(registryDir, "cost_models.json"), map[string]any{"rows": costModels}}, + {filepath.Join(registryDir, "param_models.json"), map[string]any{"rows": paramModels}}, + {filepath.Join(registryDir, "subtype_models.json"), map[string]any{"rows": subtypeModels}}, + {filepath.Join(registryDir, itempropsRegistryLock), anyMapInt(registryLock)}, + } + if err := os.MkdirAll(registryDir, 0o755); err != nil { + return 0, err + } + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + return len(writes), nil +} + +func countLegacyTLKRefsInRegistry(registryDir string) (int, error) { + paths, err := collectDataJSONPaths(registryDir) + if err != nil { + return 0, err + } + refs := 0 + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + return 0, err + } + refs += countLegacyTLKRefsInValue(obj) + } + return refs, nil +} + +func collectLegacyOwnedItempropTables(root, skipDir string) (map[string]map[string]any, error) { + result := map[string]map[string]any{} + entries, err := os.ReadDir(root) + if err != nil { + return nil, err + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if skipDir != "" && entry.Name() == skipDir { + continue + } + dir := filepath.Join(root, entry.Name()) + basePath := filepath.Join(dir, "base.json") + if _, err := os.Stat(basePath); err != nil { + continue + } + collected, err := collectBaseDataset(nativeDataset{ + Name: filepath.ToSlash(filepath.Join("itemprops", filepath.Base(root), entry.Name())), + BasePath: basePath, + LockPath: filepath.Join(dir, "lock.json"), + ModulesDir: filepath.Join(dir, "modules"), + Spec: specForDataset("itemprops"), + }) + if err != nil { + return nil, err + } + result[strings.ToLower(outputStem(collected.Dataset.OutputName))] = map[string]any{ + "ownership": "owned", + "resref": outputStem(collected.Dataset.OutputName), + "output": collected.Dataset.OutputName, + "columns": stringSliceToAny(collected.Columns), + "rows": rowsToAny(collected.Rows), + } + } + return result, nil +} + +func buildLegacySubtypeModel(key, resref string, owned map[string]map[string]any) map[string]any { + return map[string]any{ + "key": key, + "table_resref": resref, + "table": buildLegacyOwnedModelTable(resref, owned), + } +} + +func buildLegacyOwnedModelTable(resref string, owned map[string]map[string]any) map[string]any { + if table, ok := owned[strings.ToLower(resref)]; ok { + return table + } + return map[string]any{ + "ownership": "external", + "resref": resref, + } +} + +func canonicalItempropPropertyKey(row map[string]any) string { + if key := stringField(row, "key"); key != "" { + return "itemprop:" + strings.TrimPrefix(key, "itempropdef:") + } + return canonicalModelKey("itemprop", stringField(row, "Label")) +} + +func canonicalModelKey(prefix, source string) string { + normalized := strings.ToLower(strings.TrimSpace(source)) + normalized = strings.ReplaceAll(normalized, " ", "") + normalized = strings.ReplaceAll(normalized, "-", "") + normalized = strings.ReplaceAll(normalized, "_", "") + return prefix + ":" + normalized +} + +func importLegacyStrrefValue(value any, fallbackKey string, legacy *legacyTLKData, keyByID map[string]string) any { + if legacy == nil { + return value + } + if _, ok, _ := parseTLKPayload(value, true); ok { + return value + } + _ = keyByID + key := fallbackKey + entry, ok := legacy.Entries[key] + if !ok { + return value + } + payload := map[string]any{ + "key": key, + "text": entry.Text, + } + if entry.SoundResRef != "" { + payload["sound_resref"] = entry.SoundResRef + } + if entry.SoundLength != 0 { + payload["sound_length"] = entry.SoundLength + } + return map[string]any{"tlk": payload} +} + +func importLegacyAvailability(row map[string]any) []any { + out := make([]any, 0) + for _, column := range itempropsAvailabilityColumns { + value, ok := row[column] + if !ok { + continue + } + text := formatRegistryScalar(value) + if text != "1" { + continue + } + out = append(out, normalizeAvailabilityName(column)) + } + return out +} + +func anyMapInt(values map[string]int) map[string]any { + out := make(map[string]any, len(values)) + for key, value := range values { + out[key] = value + } + return out +} + +func stringSliceToAny(values []string) []any { + out := make([]any, 0, len(values)) + for _, value := range values { + out = append(out, value) + } + return out +} + +func rowsToAny(rows []map[string]any) []any { + out := make([]any, 0, len(rows)) + for _, row := range rows { + out = append(out, deepCopyValue(row)) + } + return out +} + +func sortedKeysAny(values map[string]map[string]any) []string { + out := make([]string, 0, len(values)) + for key := range values { + out = append(out, key) + } + slices.Sort(out) + return out +} + +func validateItempropsRegistryGraph(dataDir string, report *ValidationReport) { + registry, err := loadItempropsRegistry(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(dataDir, filepath.FromSlash(itempropsRegistryDirName)), + Message: err.Error(), + }) + return + } + if registry == nil { + return + } + + seenPropertyKeys := map[string]struct{}{} + seenDefLabels := map[string]string{} + seenSetLabels := map[string]string{} + seenCostKeys := map[string]struct{}{} + seenParamKeys := map[string]struct{}{} + seenSubtypeKeys := map[string]string{} + ownedOutputs := map[string]string{} + + for _, row := range registry.CostModels { + key := stringField(row, "key") + if key == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "cost_models.json"), + Message: "cost model row is missing key", + }) + continue + } + if _, ok := seenCostKeys[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "cost_models.json"), + Message: fmt.Sprintf("duplicate cost model key %q", key), + }) + } + seenCostKeys[key] = struct{}{} + validateOwnedRegistryTable(filepath.Join(registry.RootPath, "cost_models.json"), key, row, ownedOutputs, report) + } + for _, row := range registry.ParamModels { + key := stringField(row, "key") + if key == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "param_models.json"), + Message: "param model row is missing key", + }) + continue + } + if _, ok := seenParamKeys[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "param_models.json"), + Message: fmt.Sprintf("duplicate param model key %q", key), + }) + } + seenParamKeys[key] = struct{}{} + validateOwnedRegistryTable(filepath.Join(registry.RootPath, "param_models.json"), key, row, ownedOutputs, report) + } + for _, row := range registry.SubtypeModels { + key := stringField(row, "key") + if key == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "subtype_models.json"), + Message: "subtype model row is missing key", + }) + continue + } + resref := stringField(row, "table_resref") + if existing, ok := seenSubtypeKeys[key]; ok { + if !strings.EqualFold(existing, resref) { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "subtype_models.json"), + Message: fmt.Sprintf("duplicate subtype model key %q", key), + }) + } + continue + } + seenSubtypeKeys[key] = resref + validateOwnedRegistryTable(filepath.Join(registry.RootPath, "subtype_models.json"), key, row, ownedOutputs, report) + } + for _, row := range registry.Properties { + key := stringField(row, "key") + if key == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "properties.json"), + Message: "property row is missing key", + }) + continue + } + if _, ok := seenPropertyKeys[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "properties.json"), + Message: fmt.Sprintf("duplicate property key %q", key), + }) + } + seenPropertyKeys[key] = struct{}{} + defLabel := itempropsDefLabel(row) + if defLabel == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "properties.json"), + Message: fmt.Sprintf("%s is missing label", key), + }) + } else if owner, ok := seenDefLabels[strings.ToLower(defLabel)]; ok && owner != key { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "properties.json"), + Message: fmt.Sprintf("generated itempropdef label collision between %q and %q", owner, key), + }) + } else { + seenDefLabels[strings.ToLower(defLabel)] = key + } + setLabel := itempropsSetLabel(row) + if setLabel == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "properties.json"), + Message: fmt.Sprintf("%s is missing itemprops set label", key), + }) + } else if owner, ok := seenSetLabels[strings.ToLower(setLabel)]; ok && owner != key { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "properties.json"), + Message: fmt.Sprintf("generated itemprops set label collision between %q and %q", owner, key), + }) + } else { + seenSetLabels[strings.ToLower(setLabel)] = key + } + if _, err := canonicalAvailability(row); err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "properties.json"), + Message: fmt.Sprintf("%s: %v", key, err), + }) + } + if subtypeKey, ok := refField(row["subtype"]); ok { + if _, exists := seenSubtypeKeys[subtypeKey]; !exists { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "properties.json"), + Message: fmt.Sprintf("%s: unknown subtype ref %s", key, subtypeKey), + }) + } + } + if rawCost, ok := row["cost"].(map[string]any); ok { + if costKey, ok := refField(rawCost); ok { + if _, exists := seenCostKeys[costKey]; !exists { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "properties.json"), + Message: fmt.Sprintf("%s: unknown cost ref %s", key, costKey), + }) + } + } + } + if paramKey, ok := refField(row["param"]); ok { + if _, exists := seenParamKeys[paramKey]; !exists { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(registry.RootPath, "properties.json"), + Message: fmt.Sprintf("%s: unknown param ref %s", key, paramKey), + }) + } + } + } +} + +func validateOwnedRegistryTable(path, key string, row map[string]any, ownedOutputs map[string]string, report *ValidationReport) { + table, ok := row["table"].(map[string]any) + if !ok { + return + } + if !strings.EqualFold(stringField(table, "ownership"), "owned") { + return + } + resref := stringField(table, "resref") + if resref == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s: owned table is missing resref", key), + }) + return + } + outputName := stringField(table, "output") + if outputName == "" { + outputName = resref + ".2da" + } + if owner, ok := ownedOutputs[strings.ToLower(outputName)]; ok && owner != key { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("owned table output collision between %q and %q", owner, key), + }) + } else { + ownedOutputs[strings.ToLower(outputName)] = key + } + if _, ok := table["columns"].([]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s: owned table columns must be an array", key), + }) + } + if _, ok := table["rows"].([]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s: owned table rows must be an array", key), + }) + } +} diff --git a/internal/topdata/legacy_dataset_migrate.go b/internal/topdata/legacy_dataset_migrate.go new file mode 100644 index 0000000..0037cbf --- /dev/null +++ b/internal/topdata/legacy_dataset_migrate.go @@ -0,0 +1,176 @@ +package topdata + +import ( + "encoding/json" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +type legacyDatasetTransform func(relativePath string, obj map[string]any) + +func importLegacyDatasetMirror(referenceBuilderDir, dataDir, datasetName, outputName string, transform legacyDatasetTransform) (int, error) { + legacyDir := filepath.Join(referenceBuilderDir, "data", datasetName) + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, datasetName) + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + if outputName != "" { + baseObj["output"] = outputName + } + if transform != nil { + transform("base.json", baseObj) + } + + lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) + if err != nil { + return 0, err + } + if transform != nil { + transform("lock.json", lockObj) + } + + moduleRelPaths, err := collectJSONRelativePaths(filepath.Join(legacyDir, "modules")) + if err != nil { + return 0, err + } + + moduleObjs := make(map[string]map[string]any, len(moduleRelPaths)) + for _, relPath := range moduleRelPaths { + obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", relPath)) + if err != nil { + return 0, err + } + normalizedRel := filepath.ToSlash(relPath) + if transform != nil { + transform(filepath.ToSlash(filepath.Join("modules", normalizedRel)), obj) + } + moduleObjs[normalizedRel] = obj + } + + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return 0, err + } + + updated := 0 + for _, write := range []struct { + path string + obj map[string]any + }{ + {path: filepath.Join(targetDir, "base.json"), obj: baseObj}, + {path: filepath.Join(targetDir, "lock.json"), obj: lockObj}, + } { + changed, err := writeJSONObjectIfChanged(write.path, write.obj) + if err != nil { + return 0, err + } + if changed { + updated++ + } + } + + targetModulesDir := filepath.Join(targetDir, "modules") + if len(moduleObjs) > 0 { + if err := os.MkdirAll(targetModulesDir, 0o755); err != nil { + return 0, err + } + } + for relPath, obj := range moduleObjs { + targetPath := filepath.Join(targetModulesDir, filepath.FromSlash(relPath)) + if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { + return 0, err + } + changed, err := writeJSONObjectIfChanged(targetPath, obj) + if err != nil { + return 0, err + } + if changed { + updated++ + } + } + + staleModules, err := collectJSONRelativePaths(targetModulesDir) + if err != nil { + return 0, err + } + for _, relPath := range staleModules { + normalizedRel := filepath.ToSlash(relPath) + if _, ok := moduleObjs[normalizedRel]; ok { + continue + } + if err := os.Remove(filepath.Join(targetModulesDir, filepath.FromSlash(normalizedRel))); err != nil && !os.IsNotExist(err) { + return 0, err + } + updated++ + } + + return updated, nil +} + +func collectJSONRelativePaths(root string) ([]string, error) { + if _, err := os.Stat(root); err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + var relPaths []string + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if !strings.EqualFold(filepath.Ext(d.Name()), ".json") { + return nil + } + relPath, err := filepath.Rel(root, path) + if err != nil { + return err + } + relPaths = append(relPaths, filepath.ToSlash(relPath)) + return nil + }) + if err != nil { + return nil, err + } + sort.Strings(relPaths) + return relPaths, nil +} + +func writeJSONObjectIfChanged(path string, obj map[string]any) (bool, error) { + raw, err := json.MarshalIndent(obj, "", " ") + if err != nil { + return false, err + } + raw = append(raw, '\n') + + current, err := os.ReadFile(path) + if err == nil && string(current) == string(raw) { + return false, nil + } + if err != nil && !os.IsNotExist(err) { + return false, err + } + if err := os.WriteFile(path, raw, 0o644); err != nil { + return false, err + } + return true, nil +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/internal/topdata/loadscreens_migrate.go b/internal/topdata/loadscreens_migrate.go new file mode 100644 index 0000000..b0d6203 --- /dev/null +++ b/internal/topdata/loadscreens_migrate.go @@ -0,0 +1,6 @@ +package topdata + +func importLegacyLoadscreens(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "loadscreens", "loadscreens.2da", nil) +} diff --git a/internal/topdata/masterfeats_migrate.go b/internal/topdata/masterfeats_migrate.go new file mode 100644 index 0000000..76bbe82 --- /dev/null +++ b/internal/topdata/masterfeats_migrate.go @@ -0,0 +1,237 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" + "strconv" +) + +func importLegacyMasterfeats(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + legacyDir := filepath.Join(referenceBuilderDir, "data", "feat", "masterfeats") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "masterfeats") + targetPath := filepath.Join(targetDir, "base.json") + legacyPlainPath := filepath.Join(targetDir, "masterfeats.json") + if _, err := os.Stat(targetPath); err == nil { + obj, err := loadJSONObject(targetPath) + if err == nil && countLegacyTLKRefsInValue(obj) == 0 { + return 0, nil + } + } + + collected, err := importLegacyMasterfeatsDataset(legacyDir) + if err != nil { + return 0, err + } + referenceIDs, err := collectReferenceLockIDs(filepath.Join(referenceBuilderDir, "data")) + if err != nil { + return 0, err + } + + rows := make([]any, 0, len(collected.Rows)) + for _, row := range collected.Rows { + copyRow, ok := deepCopyValue(row).(map[string]any) + if !ok { + continue + } + if legacyTLK != nil { + if _, _, err := inlineLegacyTLKValue(copyRow, legacyTLK); err != nil { + return 0, err + } + } + copyRow = rewriteImportedIDRefs(copyRow, referenceIDs) + rows = append(rows, copyRow) + } + + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return 0, err + } + writes := []struct { + path string + obj map[string]any + }{ + { + path: targetPath, + obj: map[string]any{ + "output": collected.Dataset.OutputName, + "columns": stringSliceToAny(collected.Columns), + "rows": rows, + }, + }, + { + path: filepath.Join(targetDir, "lock.json"), + obj: anyMapInt(collected.LockData), + }, + } + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + updatedFiles := len(writes) + if err := os.Remove(legacyPlainPath); err == nil { + updatedFiles++ + } else if err != nil && !os.IsNotExist(err) { + return 0, err + } + return updatedFiles, nil +} + +func importLegacyMasterfeatsDataset(legacyDir string) (nativeCollectedDataset, error) { + dataset := nativeDataset{ + Name: "masterfeats", + BasePath: filepath.Join(legacyDir, "masterfeats.json"), + LockPath: filepath.Join(legacyDir, "lock.json"), + Spec: specForDataset("masterfeats"), + } + tableData, err := loadJSONObject(dataset.BasePath) + if err != nil { + return nativeCollectedDataset{}, err + } + columns, err := parseColumns(tableData, dataset.Name) + if err != nil { + return nativeCollectedDataset{}, err + } + rawRows, ok := tableData["rows"].([]any) + if !ok { + return nativeCollectedDataset{}, nil + } + lockData, err := loadLockfile(dataset.LockPath) + if err != nil { + return nativeCollectedDataset{}, err + } + usedIDs := map[int]struct{}{} + for _, rowID := range lockData { + usedIDs[rowID] = struct{}{} + } + + rows := make([]map[string]any, 0, len(rawRows)) + explicitID := make([]bool, 0, len(rawRows)) + for index, raw := range rawRows { + rowObj, ok := raw.(map[string]any) + if !ok { + continue + } + rowID := index + if value, ok := rowObj["id"]; ok { + parsed, err := asInt(value) + if err != nil { + return nativeCollectedDataset{}, err + } + rowID = parsed + } + row := map[string]any{"id": rowID} + for _, column := range columns { + row[column] = nullValue + } + for key, value := range rowObj { + switch key { + case "id": + case "key": + if keyText, ok := value.(string); ok && keyText != "" { + row["key"] = keyText + } + default: + columnName, ok := canonicalColumn(columns, key) + if !ok { + continue + } + row[columnName] = value + } + } + rows = append(rows, row) + _, hasExplicitID := rowObj["id"] + explicitID = append(explicitID, hasExplicitID) + if hasExplicitID { + usedIDs[rowID] = struct{}{} + } + } + + nextID := nextAvailableID(usedIDs) + lockModified := false + for index, row := range rows { + key, hasKey := row["key"].(string) + if hasKey && key != "" { + if lockedID, ok := lockData[key]; ok { + row["id"] = lockedID + continue + } + if explicitID[index] { + lockData[key] = row["id"].(int) + } else { + row["id"] = nextID + lockData[key] = nextID + usedIDs[nextID] = struct{}{} + nextID = nextAvailableID(usedIDs) + } + lockModified = true + continue + } + if !explicitID[index] { + row["id"] = index + } + } + if lockModified { + _ = lockModified + } + + return nativeCollectedDataset{ + Dataset: nativeDataset{ + Name: dataset.Name, + OutputName: "masterfeats.2da", + Columns: columns, + Spec: dataset.Spec, + }, + Columns: columns, + Rows: rows, + LockData: lockData, + }, nil +} + +func rewriteImportedIDRefs(value any, ids map[string]int) map[string]any { + row, ok := rewriteImportedIDRefsValue(value, ids).(map[string]any) + if !ok { + return map[string]any{} + } + return row +} + +func rewriteImportedIDRefsValue(value any, ids map[string]int) any { + switch typed := value.(type) { + case map[string]any: + if len(typed) == 1 { + if rawID, ok := typed["id"]; ok { + if key, ok := rawID.(string); ok { + if resolved, ok := ids[key]; ok { + return strconv.Itoa(resolved) + } + } + } + } + out := make(map[string]any, len(typed)) + for key, child := range typed { + out[key] = rewriteImportedIDRefsValue(child, ids) + } + return out + case []any: + out := make([]any, 0, len(typed)) + for _, child := range typed { + out = append(out, rewriteImportedIDRefsValue(child, ids)) + } + return out + default: + return value + } +} diff --git a/internal/topdata/metadata_wiki.go b/internal/topdata/metadata_wiki.go new file mode 100644 index 0000000..875fe5c --- /dev/null +++ b/internal/topdata/metadata_wiki.go @@ -0,0 +1,190 @@ +package topdata + +import ( + "fmt" + "strings" +) + +func isNullLike(value any) bool { + if value == nil { + return true + } + text, ok := value.(string) + if !ok { + return false + } + return strings.TrimSpace(text) == nullValue +} + +func normalizeWikiMetadataKey(key string) string { + switch normalizeMetadataKey(key) { + case "reqfeats", "req_feats": + return "reqfeats" + case "generate", "wikigenerate": + return "generate" + case "canonical", "wikicanonical": + return "canonical" + case "status", "wikistatus": + return "status" + case "progressionnotes", "progression_notes", "classprogressionnotes", "class_progression_notes": + return "progression_notes" + default: + return "" + } +} + +func legacyWikiMetadataField(key string) string { + switch normalizeMetadataKey(key) { + case "wikireqfeats": + return "reqfeats" + case "wikigenerate": + return "generate" + case "wikicanonical": + return "canonical" + case "wikistatus": + return "status" + default: + return "" + } +} + +func parseWikiMetadata(raw any) (map[string]any, error) { + obj, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("must be an object") + } + wiki := map[string]any{} + for key, value := range obj { + normalized := normalizeWikiMetadataKey(key) + if normalized == "" { + return nil, fmt.Errorf("unknown field %q", key) + } + if isNullLike(value) { + continue + } + wiki[normalized] = cloneAuthoringValue(value) + } + return wiki, nil +} + +func mergeMetadataMaps(base, overlay map[string]any) map[string]any { + if len(base) == 0 && len(overlay) == 0 { + return nil + } + out := map[string]any{} + for key, value := range base { + out[key] = cloneAuthoringValue(value) + } + for key, value := range overlay { + baseValue, hasBase := out[key] + baseMap, baseIsMap := baseValue.(map[string]any) + valueMap, valueIsMap := value.(map[string]any) + if hasBase && baseIsMap && valueIsMap { + out[key] = mergeMetadataMaps(baseMap, valueMap) + continue + } + out[key] = cloneAuthoringValue(value) + } + if len(out) == 0 { + return nil + } + return out +} + +func parseExistingMetadata(row map[string]any) (map[string]any, error) { + rawMeta, ok := lookupField(row, "meta") + if !ok { + return nil, nil + } + return parseRowMetadata(rawMeta) +} + +func setLegacyWikiMetadata(row map[string]any, field string, value any) error { + wikiKey := legacyWikiMetadataField(field) + if wikiKey == "" { + return nil + } + + meta, err := parseExistingMetadata(row) + if err != nil { + return err + } + if meta == nil { + meta = map[string]any{} + } + + wiki, _ := meta["wiki"].(map[string]any) + if wiki == nil { + wiki = map[string]any{} + } + + if isNullLike(value) { + delete(wiki, wikiKey) + } else { + wiki[wikiKey] = cloneAuthoringValue(value) + } + + if len(wiki) == 0 { + delete(meta, "wiki") + } else { + meta["wiki"] = wiki + } + if len(meta) == 0 { + delete(row, "meta") + } else { + row["meta"] = meta + } + return nil +} + +func canonicalizeWikiMetadataDocument(obj map[string]any) { + if rawColumns, ok := obj["columns"].([]any); ok { + filtered := make([]any, 0, len(rawColumns)) + for _, raw := range rawColumns { + column, ok := raw.(string) + if ok && isAuthoringOnlyField(column) { + continue + } + filtered = append(filtered, raw) + } + obj["columns"] = filtered + } + if rows, ok := obj["rows"].([]any); ok { + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + canonicalizeWikiMetadataRow(row) + } + } + if entries, ok := obj["entries"].(map[string]any); ok { + for _, raw := range entries { + row, ok := raw.(map[string]any) + if !ok { + continue + } + canonicalizeWikiMetadataRow(row) + } + } + if overrides, ok := obj["overrides"].([]any); ok { + for _, raw := range overrides { + row, ok := raw.(map[string]any) + if !ok { + continue + } + canonicalizeWikiMetadataRow(row) + } + } +} + +func canonicalizeWikiMetadataRow(row map[string]any) { + for _, field := range []string{"WIKIREQFEATS", "WIKIGENERATE", "WIKICANONICAL", "WIKISTATUS"} { + value, ok := row[field] + if !ok { + continue + } + _ = setLegacyWikiMetadata(row, field, value) + delete(row, field) + } +} diff --git a/internal/topdata/migrate.go b/internal/topdata/migrate.go new file mode 100644 index 0000000..2f61a08 --- /dev/null +++ b/internal/topdata/migrate.go @@ -0,0 +1,415 @@ +package topdata + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +type NormalizeResult struct { + StatePath string + ScannedFiles int + UpdatedFiles int + InlineValues int + RemainingFiles int + RemainingLegacyRefs int +} + +func NormalizeProject(p *project.Project) (NormalizeResult, error) { + if !p.HasTopData() { + return NormalizeResult{}, fmt.Errorf("topdata is not configured for this project") + } + + sourceDir := p.TopDataSourceDir() + dataDir := filepath.Join(sourceDir, "data") + migrationRoot := resolveMigrationInputRoot(sourceDir, p.TopDataReferenceBuilderDir()) + legacyTLK, err := loadLegacyTLK(filepath.Join(sourceDir, "tlk")) + if err != nil { + return NormalizeResult{}, err + } + if migrationRoot != "" { + migrationTLK, err := loadLegacyTLK(filepath.Join(migrationRoot, "tlk")) + if err != nil { + return NormalizeResult{}, err + } + legacyTLK = mergeLegacyTLK(migrationTLK, legacyTLK) + } + baseDialogImported, err := importLegacyBaseDialog(migrationRoot, sourceDir) + if err != nil { + return NormalizeResult{}, err + } + + itempropsImported, err := importLegacyItempropsRegistry(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + masterfeatsImported, err := importLegacyMasterfeats(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + featImported, err := importLegacyFeat(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + creaturespeedImported, err := importLegacyCreaturespeed(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + armorImported, err := importLegacyArmor(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + baseitemsImported, err := importLegacyBaseitems(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + damagetypesImported, err := importLegacyDamagetypes(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + skyboxesImported, err := importLegacySkyboxes(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + vfxPersistentImported, err := importLegacyVFXPersistent(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + loadscreensImported, err := importLegacyLoadscreens(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + progfxImported, err := importLegacyProgfx(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + genericdoorsImported, err := importLegacyGenericdoors(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + cloakmodelImported, err := importLegacyCloakmodel(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + repadjustImported, err := importLegacyRepadjust(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + wingmodelImported, err := importLegacyWingmodel(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + tailmodelImported, err := importLegacyTailmodel(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + doortypesImported, err := importLegacyDoortypes(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + visualeffectsImported, err := importLegacyVisualeffects(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + appearanceImported, err := importLegacyAppearance(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + placeablesImported, err := importLegacyPlaceables(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + rulesetImported, err := importLegacyRuleset(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + skillsImported, err := importLegacySkills(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + spellsImported, err := importLegacySpells(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + racialtypesImported, err := importLegacyRacialtypes(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + classesImported, err := importLegacyClasses(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + portraitsImported, err := importLegacyPortraits(migrationRoot, dataDir, legacyTLK) + if err != nil { + return NormalizeResult{}, err + } + + paths, err := collectDataJSONPaths(dataDir) + if err != nil { + return NormalizeResult{}, err + } + + result := NormalizeResult{ + StatePath: filepath.Join(sourceDir, tlkStateFile), + ScannedFiles: len(paths), + UpdatedFiles: baseDialogImported + itempropsImported + masterfeatsImported + featImported + creaturespeedImported + armorImported + baseitemsImported + damagetypesImported + skyboxesImported + vfxPersistentImported + loadscreensImported + progfxImported + genericdoorsImported + cloakmodelImported + repadjustImported + wingmodelImported + tailmodelImported + doortypesImported + visualeffectsImported + appearanceImported + placeablesImported + rulesetImported + skillsImported + spellsImported + racialtypesImported + classesImported + portraitsImported, + } + state, err := loadTLKState(result.StatePath) + if err != nil { + return result, err + } + if state.Entries == nil { + state.Entries = map[string]tlkStateMapping{} + } + if legacyTLK != nil && state.Language == "" { + state.Language = legacyTLK.Language + } + if legacyTLK != nil { + for key, id := range legacyTLK.Lock { + if _, ok := state.Entries[key]; !ok { + state.Entries[key] = tlkStateMapping{ID: id} + } + } + } + + if legacyTLK != nil { + for _, path := range paths { + updated, count, err := inlineLegacyTLKFile(path, legacyTLK) + if err != nil { + return result, err + } + if updated { + result.UpdatedFiles++ + result.InlineValues += count + } + } + } + + if err := saveTLKState(result.StatePath, state); err != nil { + return result, err + } + if err := removeLegacyTLKAuthoredInput(filepath.Join(sourceDir, "tlk")); err != nil { + return result, err + } + remainingFiles, remainingRefs, err := countLegacyTLKRefs(paths) + if err != nil { + return result, err + } + result.RemainingFiles = remainingFiles + result.RemainingLegacyRefs = remainingRefs + return result, nil +} + +func mergeLegacyTLK(base, override *legacyTLKData) *legacyTLKData { + if base == nil && override == nil { + return nil + } + merged := &legacyTLKData{ + Language: "en", + Entries: map[string]tlkEntryData{}, + Lock: map[string]int{}, + } + if base != nil { + if base.Language != "" { + merged.Language = base.Language + } + for key, entry := range base.Entries { + merged.Entries[key] = entry + } + for key, id := range base.Lock { + merged.Lock[key] = id + } + } + if override != nil { + if override.Language != "" { + merged.Language = override.Language + } + for key, entry := range override.Entries { + merged.Entries[key] = entry + } + for key, id := range override.Lock { + merged.Lock[key] = id + } + } + return merged +} + +func removeLegacyTLKAuthoredInput(tlkDir string) error { + modulesDir := filepath.Join(tlkDir, "modules") + if err := os.RemoveAll(modulesDir); err != nil { + return fmt.Errorf("remove %s: %w", modulesDir, err) + } + lockPath := filepath.Join(tlkDir, "lock.json") + if err := os.Remove(lockPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove %s: %w", lockPath, err) + } + gitkeepPath := filepath.Join(tlkDir, ".gitkeep") + if err := os.Remove(gitkeepPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove %s: %w", gitkeepPath, err) + } + if err := os.Remove(tlkDir); err != nil && !os.IsNotExist(err) { + if !strings.Contains(err.Error(), "directory not empty") { + return fmt.Errorf("remove %s: %w", tlkDir, err) + } + } + return nil +} + +func resolveMigrationInputRoot(sourceDir, referenceBuilderDir string) string { + if strings.TrimSpace(referenceBuilderDir) != "" { + return referenceBuilderDir + } + snapshotRoot := filepath.Join(sourceDir, "migration_snapshot") + if info, err := os.Stat(snapshotRoot); err == nil && info.IsDir() { + return snapshotRoot + } + return "" +} + +func inlineLegacyTLKFile(path string, legacy *legacyTLKData) (bool, int, error) { + obj, err := loadJSONObject(path) + if err != nil { + return false, 0, err + } + updated, count, err := inlineLegacyTLKValue(obj, legacy) + if err != nil { + return false, 0, err + } + if !updated { + return false, 0, nil + } + raw, err := json.MarshalIndent(obj, "", " ") + if err != nil { + return false, 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(path, raw, 0o644); err != nil { + return false, 0, err + } + return true, count, nil +} + +func inlineLegacyTLKValue(value any, legacy *legacyTLKData) (bool, int, error) { + switch typed := value.(type) { + case map[string]any: + if rawTLK, ok := typed["tlk"]; ok { + key, ok := rawTLK.(string) + if ok { + entry, ok := legacy.Entries[key] + if !ok { + return false, 0, nil + } + payload := map[string]any{ + "key": key, + "text": entry.Text, + } + if entry.SoundResRef != "" { + payload["sound_resref"] = entry.SoundResRef + } + if entry.SoundLength != 0 { + payload["sound_length"] = entry.SoundLength + } + typed["tlk"] = payload + return true, 1, nil + } + } + + updated := false + total := 0 + for key, child := range typed { + childUpdated, childCount, err := inlineLegacyTLKValue(child, legacy) + if err != nil { + return false, 0, err + } + if childUpdated { + typed[key] = child + updated = true + total += childCount + } + } + return updated, total, nil + case []any: + updated := false + total := 0 + for index, child := range typed { + childUpdated, childCount, err := inlineLegacyTLKValue(child, legacy) + if err != nil { + return false, 0, err + } + if childUpdated { + typed[index] = child + updated = true + total += childCount + } + } + return updated, total, nil + default: + return false, 0, nil + } +} + +func collectDataJSONPaths(dataDir string) ([]string, error) { + paths := make([]string, 0) + err := filepath.WalkDir(dataDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.HasSuffix(strings.ToLower(d.Name()), ".json") && d.Name() != "lock.json" { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return nil, err + } + return paths, nil +} + +func countLegacyTLKRefs(paths []string) (int, int, error) { + files := 0 + refs := 0 + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + return 0, 0, err + } + count := countLegacyTLKRefsInValue(obj) + if count == 0 { + continue + } + files++ + refs += count + } + return files, refs, nil +} + +func countLegacyTLKRefsInValue(value any) int { + switch typed := value.(type) { + case map[string]any: + total := 0 + if rawTLK, ok := typed["tlk"]; ok { + if _, ok := rawTLK.(string); ok { + total++ + } + } + for _, child := range typed { + total += countLegacyTLKRefsInValue(child) + } + return total + case []any: + total := 0 + for _, child := range typed { + total += countLegacyTLKRefsInValue(child) + } + return total + default: + return 0 + } +} diff --git a/internal/topdata/native.go b/internal/topdata/native.go new file mode 100644 index 0000000..bee7548 --- /dev/null +++ b/internal/topdata/native.go @@ -0,0 +1,6936 @@ +package topdata + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +const nullValue = "****" + +type nativeDataset struct { + Kind nativeDatasetKind + Name string + RootPath string + BasePath string + LockPath string + ModulesDir string + GeneratedDir string + GlobalPaths []string + OutputName string + Spec datasetTextSpec + Columns []string + ValueEncodings map[string]project.TopDataValueEncodingConfig + ValueDefaults map[string]any + RowGeneration string + RowGenerationMinRow int + CompareReference bool + HasGlobalInjections bool +} + +type nativeDatasetKind string + +const ( + nativeDatasetBase nativeDatasetKind = "base" + nativeDatasetPlain nativeDatasetKind = "plain" +) + +type nativeCollectedDataset struct { + Dataset nativeDataset + Columns []string + Rows []map[string]any + LockData map[string]int + RetiredKeys map[string]struct{} + TableKey string + DenseRowMax int + LockAdded int + LockPruned int + LockModified bool +} + +type nativeRowSource string + +const ( + nativeRowSourceBase nativeRowSource = "base" + nativeRowSourceEntries nativeRowSource = "entries" + nativeRowSourceOverride nativeRowSource = "overrides" +) + +type nativeRowIdentity struct { + ID int + Key string +} + +type resolvedTable struct { + Key string + DatasetName string + OutputName string + OutputStem string + Columns []string + Rows []map[string]any + LockData map[string]int + Kind nativeDatasetKind +} + +type resolvedTableRegistry struct { + stemByKey map[string]string + tableByKey map[string]resolvedTable +} + +type nativeSidecarCollector struct { + tables map[string][]map[string]any +} + +func newNativeSidecarCollector() *nativeSidecarCollector { + return &nativeSidecarCollector{tables: map[string][]map[string]any{}} +} + +func (c *nativeSidecarCollector) addExternalHexList(dataset nativeDataset, row map[string]any, field string, values []int, encoding project.TopDataValueEncodingConfig) (string, error) { + if c == nil { + return "", fmt.Errorf("external hex list sidecar collector is not available") + } + ref := sidecarRowReference(dataset.Name, row) + if ref == "" { + return "", fmt.Errorf("field %s: external hex list requires row id or key", field) + } + outputName := sidecarOutputName(dataset.OutputName, field) + rows := c.tables[outputName] + for _, value := range values { + parsed, err := parseConfiguredListValue(value, encoding) + if err != nil { + return "", err + } + rows = append(rows, map[string]any{ + "id": len(rows), + "Label": ref, + "Value": "0x" + fmt.Sprintf("%0*X", encoding.HexWidth, parsed), + }) + } + c.tables[outputName] = rows + return ref, nil +} + +func (c *nativeSidecarCollector) writeAll(outputDir string) error { + if c == nil || len(c.tables) == 0 { + return nil + } + for _, outputName := range sortedStringMapKeys(c.tables) { + data := map[string]any{ + "columns": []string{"Label", "Value"}, + "rows": c.tables[outputName], + } + if err := write2DA(data, filepath.Join(outputDir, outputName), false, -1); err != nil { + return err + } + } + return nil +} + +func sidecarOutputName(parentOutputName, field string) string { + parent := normalizeSidecarToken(outputStem(parentOutputName)) + acronym := sidecarTokenAcronym(field) + if acronym != "" { + candidate := parent + "_" + acronym + if len(candidate) <= 16 { + return candidate + ".2da" + } + } + stem := normalizeSidecarToken(parent + "_" + field) + if len(stem) <= 16 { + return stem + ".2da" + } + hash := stableShortHash(stem) + prefixLen := 16 - len(hash) - 1 + if prefixLen < 1 { + prefixLen = 1 + } + return stem[:prefixLen] + "_" + hash + ".2da" +} + +func sidecarTokenAcronym(value string) string { + var builder strings.Builder + previousWasLowerOrDigit := false + for _, r := range strings.TrimSpace(value) { + if r >= 'A' && r <= 'Z' { + if builder.Len() == 0 || previousWasLowerOrDigit { + builder.WriteByte(byte(r + ('a' - 'A'))) + } + previousWasLowerOrDigit = false + continue + } + if r >= 'a' && r <= 'z' { + if builder.Len() == 0 { + builder.WriteRune(r) + } + previousWasLowerOrDigit = true + continue + } + if r >= '0' && r <= '9' { + builder.WriteRune(r) + previousWasLowerOrDigit = true + continue + } + previousWasLowerOrDigit = false + } + return builder.String() +} + +func sidecarRowReference(datasetName string, row map[string]any) string { + if key, ok := row["key"].(string); ok && key != "" { + if index := strings.LastIndex(key, ":"); index >= 0 && index < len(key)-1 { + return normalizeSidecarToken(key[index+1:]) + } + return normalizeSidecarToken(key) + } + if id, ok := row["id"].(int); ok { + return fmt.Sprintf("%s_%d", normalizeSidecarToken(datasetName), id) + } + return "" +} + +func normalizeSidecarToken(value string) string { + trimmed := strings.ToLower(strings.TrimSpace(value)) + var builder strings.Builder + previousUnderscore := false + for _, r := range trimmed { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + builder.WriteRune(r) + previousUnderscore = false + continue + } + if !previousUnderscore { + builder.WriteByte('_') + previousUnderscore = true + } + } + return strings.Trim(builder.String(), "_") +} + +func stableShortHash(value string) string { + var hash uint32 = 2166136261 + for _, b := range []byte(value) { + hash ^= uint32(b) + hash *= 16777619 + } + return fmt.Sprintf("%04x", hash&0xffff) +} + +func newResolvedTableRegistry(collected []nativeCollectedDataset) (resolvedTableRegistry, error) { + registry := resolvedTableRegistry{ + stemByKey: map[string]string{}, + tableByKey: map[string]resolvedTable{}, + } + for _, collectedDataset := range collected { + if collectedDataset.TableKey == "" { + continue + } + key := collectedDataset.TableKey + if existing, ok := registry.tableByKey[key]; ok { + return resolvedTableRegistry{}, fmt.Errorf( + "duplicate table key %q for %s (%s) conflicts with %s (%s)", + key, + collectedDataset.Dataset.Name, + collectedDataset.Dataset.OutputName, + existing.DatasetName, + existing.OutputName, + ) + } + rows := make([]map[string]any, 0, len(collectedDataset.Rows)) + for _, row := range collectedDataset.Rows { + rows = append(rows, cloneRowMap(row)) + } + lockData := map[string]int{} + for lockKey, rowID := range collectedDataset.LockData { + lockData[lockKey] = rowID + } + registryTable := resolvedTable{ + Key: key, + DatasetName: collectedDataset.Dataset.Name, + OutputName: collectedDataset.Dataset.OutputName, + OutputStem: outputStem(collectedDataset.Dataset.OutputName), + Columns: append([]string(nil), collectedDataset.Columns...), + Rows: rows, + LockData: lockData, + Kind: collectedDataset.Dataset.Kind, + } + registry.stemByKey[key] = registryTable.OutputStem + registry.tableByKey[key] = registryTable + } + return registry, nil +} + +func (r resolvedTableRegistry) resolveTableByKey(key string) (resolvedTable, bool) { + table, ok := r.tableByKey[key] + return table, ok +} + +func cloneRowMap(row map[string]any) map[string]any { + if row == nil { + return nil + } + cloned := make(map[string]any, len(row)) + for key, value := range row { + cloned[key] = cloneAuthoringValue(value) + } + return cloned +} + +type legacyTLKData struct { + Language string + Entries map[string]tlkEntryData + Lock map[string]int +} + +func Build(p *project.Project, progress func(string)) (BuildResult, error) { + return BuildNative(p, progress) +} + +type NativeBuildOptions struct { + BuildWiki bool + Force bool +} + +type BuildWikiOptions struct { + Force bool +} + +func BuildWikiWithOptions(p *project.Project, opts BuildWikiOptions, progress func(string)) (wikiResult, error) { + if progress == nil { + progress = func(string) {} + } + if !p.HasTopData() { + return wikiResult{}, errors.New("topdata is not configured for this project") + } + + output2DA := compiled2DAOutputDir(p) + outputTLK := compiledTLKOutputDir(p) + if _, err := os.Stat(output2DA); os.IsNotExist(err) { + return wikiResult{}, fmt.Errorf("topdata build output missing: run build-topdata first") + } + if _, err := os.Stat(outputTLK); os.IsNotExist(err) { + return wikiResult{}, fmt.Errorf("topdata tlk output missing: run build-topdata first") + } + + nativeResult := BuildResult{ + Mode: "native", + Output2DADir: output2DA, + OutputTLKDir: outputTLK, + } + + wikiBuild, err := buildWiki(p, nativeResult, opts.Force, progress) + if err != nil { + return wikiResult{}, err + } + + return wikiBuild, nil +} + +func BuildNative(p *project.Project, progress func(string)) (BuildResult, error) { + return BuildNativeWithOptions(p, NativeBuildOptions{BuildWiki: true}, progress) +} + +func BuildNativeWithOptions(p *project.Project, opts NativeBuildOptions, progress func(string)) (BuildResult, error) { + if progress == nil { + progress = func(string) {} + } + if !p.HasTopData() { + return BuildResult{}, errors.New("topdata is not configured for this project") + } + + report := ValidateProject(p) + if report.HasErrors() { + summary := report.SummaryLines(3) + if len(summary) > 0 { + return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s): %s", report.ErrorCount(), summary[0]) + } + return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount()) + } + if warnings := report.WarningCount(); warnings > 0 { + progress(fmt.Sprintf("topdata validation warnings: %d", warnings)) + for _, line := range report.SummaryLines(5) { + if strings.HasPrefix(line, "warning:") { + progress(line) + } + } + } + var prebuiltWiki *wikiResult + if opts.BuildWiki { + if newestWikiSource, _, err := newestRelevantWikiSource(p); err == nil && !newestWikiSource.IsZero() { + statePath := wikiOutputStatePath(p) + outputDir := wikiOutputPagesDir(p) + if stateInfo, err := os.Stat(statePath); err == nil && !newestWikiSource.After(stateInfo.ModTime()) { + if outputInfo, err := os.Stat(outputDir); err == nil && outputInfo.IsDir() { + if state, err := loadWikiState(statePath); err == nil && wikiCachedPagesAreCurrent(outputDir, state) { + prebuiltWiki = &wikiResult{ + OutputDir: outputDir, + PageCount: state.Pages, + Status: wikiSkippedStatus, + Digest: state.Digest, + } + } + } + } + } + } + return buildNativeUnchecked(p, opts, progress, prebuiltWiki) +} + +func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress func(string), prebuiltWiki *wikiResult) (BuildResult, error) { + if progress == nil { + progress = func(string) {} + } + sourceDir := p.TopDataSourceDir() + dataDir := filepath.Join(sourceDir, "data") + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return BuildResult{}, err + } + datasets = applyTopDataValueEncodings(datasets, p.Config.TopData.ValueEncodings) + datasets = applyTopDataValueDefaults(datasets, p.Config.TopData.ValueDefaults) + datasets = applyTopDataRowGeneration(datasets, p.Config.TopData.RowGeneration) + registryDatasets, err := collectGeneratedRegistryDatasets(dataDir) + if err != nil { + return BuildResult{}, err + } + if len(datasets) == 0 && len(registryDatasets) == 0 { + return BuildResult{}, fmt.Errorf("no canonical topdata datasets were found in topdata/data") + } + + compiler, err := newTLKCompiler(sourceDir, nil) + if err != nil { + return BuildResult{}, err + } + + output2DA := compiled2DAOutputDir(p) + outputTLK := compiledTLKOutputDir(p) + progress("Preparing native topdata build outputs...") + if err := os.RemoveAll(output2DA); err != nil { + return BuildResult{}, fmt.Errorf("clean topdata 2da output dir: %w", err) + } + if err := os.MkdirAll(output2DA, 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create 2da output dir: %w", err) + } + if err := os.MkdirAll(outputTLK, 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create tlk output dir: %w", err) + } + + progress("Building native topdata datasets...") + collected := make([]nativeCollectedDataset, 0, len(datasets)+len(registryDatasets)) + globalKeyToID := map[string]int{} + globalRowByKey := map[string]map[string]any{} + for _, dataset := range datasets { + collectedDataset, err := collectNativeDataset(dataset) + if err != nil { + return BuildResult{}, err + } + collected = append(collected, collectedDataset) + for key, rowID := range collectedDataset.LockData { + globalKeyToID[key] = rowID + } + for _, row := range collectedDataset.Rows { + if key, ok := row["key"].(string); ok && key != "" { + globalKeyToID[key] = row["id"].(int) + globalRowByKey[key] = row + } + } + } + for _, collectedDataset := range registryDatasets { + collected = append(collected, collectedDataset) + for key, rowID := range collectedDataset.LockData { + globalKeyToID[key] = rowID + } + for _, row := range collectedDataset.Rows { + if key, ok := row["key"].(string); ok && key != "" { + globalKeyToID[key] = row["id"].(int) + globalRowByKey[key] = row + } + } + } + collected, err = applyAutogenConsumers(p, collected, progress) + if err != nil { + return BuildResult{}, err + } + collected, err = applyPartOverrides(sourceDir, collected) + if err != nil { + return BuildResult{}, err + } + collected, err = applyClassSpellbooks(sourceDir, collected) + if err != nil { + return BuildResult{}, err + } + collected, err = applyTopDataRowExtensions(collected, p.EffectiveConfig().TopData.RowExtensions) + if err != nil { + return BuildResult{}, err + } + + tableRegistry, err := newResolvedTableRegistry(collected) + if err != nil { + return BuildResult{}, err + } + collected, err = mergeExpansionData(collected) + if err != nil { + return BuildResult{}, err + } + lockAdded, lockPruned, err := saveNativeLockfiles(collected) + if err != nil { + return BuildResult{}, err + } + if lockAdded > 0 || lockPruned > 0 { + progress(fmt.Sprintf("Updated native lockfiles (%d new IDs, %d stale IDs pruned)", lockAdded, lockPruned)) + } + globalKeyToID = map[string]int{} + globalRowByKey = map[string]map[string]any{} + for _, collectedDataset := range collected { + for key, rowID := range collectedDataset.LockData { + globalKeyToID[key] = rowID + } + for _, row := range collectedDataset.Rows { + if key, ok := row["key"].(string); ok && key != "" { + globalKeyToID[key] = row["id"].(int) + globalRowByKey[key] = row + } + } + } + tableRegistry, err = newResolvedTableRegistry(collected) + if err != nil { + return BuildResult{}, err + } + supplementalKeyToID, err := collectSupplementalLockIDs(sourceDir, "") + if err != nil { + return BuildResult{}, err + } + for key, rowID := range supplementalKeyToID { + if _, ok := globalKeyToID[key]; ok { + continue + } + globalKeyToID[key] = rowID + } + + groupStats := nativeCompileGroupStats(collected) + currentGroup := "" + sidecars := newNativeSidecarCollector() + for _, dataset := range collected { + group := nativeCompileGroup(dataset.Dataset.Name) + if group != currentGroup { + currentGroup = group + stats := groupStats[group] + progress(fmt.Sprintf( + "Compiling %s datasets (%d output tables from %d source fragments)...", + group, + stats.OutputTables, + stats.SourceFragments, + )) + } + compiled, err := resolveNativeDataset(dataset, globalKeyToID, globalRowByKey, tableRegistry, compiler, p.EffectiveConfig().TopData.ClassFeatInjections, sidecars) + if err != nil { + return BuildResult{}, err + } + if err := write2DA(compiled, filepath.Join(output2DA, dataset.Dataset.OutputName), dataset.Dataset.Kind == nativeDatasetBase, dataset.DenseRowMax); err != nil { + return BuildResult{}, err + } + } + if err := sidecars.writeAll(output2DA); err != nil { + return BuildResult{}, err + } + + filesTLK, err := compiler.finish(outputTLK, filepath.Base(p.TopDataCompiledTLKPath())) + if err != nil { + return BuildResult{}, err + } + files2DA, err := countFiles(output2DA) + if err != nil { + return BuildResult{}, err + } + + wikiBuild := wikiResult{Status: wikiSkippedStatus} + if prebuiltWiki != nil { + wikiBuild = *prebuiltWiki + } else if opts.BuildWiki { + wikiBuild, err = buildWiki(p, BuildResult{ + Mode: "native", + Output2DADir: output2DA, + OutputTLKDir: outputTLK, + Files2DA: files2DA, + FilesTLK: filesTLK, + }, opts.Force, progress) + if err != nil { + return BuildResult{}, err + } + } + + return BuildResult{ + Mode: "native", + Output2DADir: output2DA, + OutputTLKDir: outputTLK, + Files2DA: files2DA, + FilesTLK: filesTLK, + WikiOutputDir: wikiBuild.OutputDir, + WikiPages: wikiBuild.PageCount, + WikiStatus: wikiBuild.Status, + }, nil +} + +func applyTopDataValueEncodings(datasets []nativeDataset, encodings []project.TopDataValueEncodingConfig) []nativeDataset { + if len(encodings) == 0 { + return datasets + } + out := append([]nativeDataset(nil), datasets...) + for index := range out { + for _, encoding := range encodings { + dataset := filepath.ToSlash(strings.TrimSpace(encoding.Dataset)) + if dataset != "*" { + continue + } + if out[index].ValueEncodings == nil { + out[index].ValueEncodings = map[string]project.TopDataValueEncodingConfig{} + } + out[index].ValueEncodings[strings.ToLower(strings.TrimSpace(encoding.Column))] = encoding + } + for _, encoding := range encodings { + if filepath.ToSlash(strings.TrimSpace(encoding.Dataset)) != out[index].Name { + continue + } + if out[index].ValueEncodings == nil { + out[index].ValueEncodings = map[string]project.TopDataValueEncodingConfig{} + } + out[index].ValueEncodings[strings.ToLower(strings.TrimSpace(encoding.Column))] = encoding + } + } + return out +} + +func applyTopDataValueDefaults(datasets []nativeDataset, defaults []project.TopDataValueDefaultConfig) []nativeDataset { + if len(defaults) == 0 { + return datasets + } + out := append([]nativeDataset(nil), datasets...) + for index := range out { + for _, valueDefault := range defaults { + if filepath.ToSlash(strings.TrimSpace(valueDefault.Dataset)) != out[index].Name { + continue + } + if out[index].ValueDefaults == nil { + out[index].ValueDefaults = map[string]any{} + } + out[index].ValueDefaults[strings.ToLower(strings.TrimSpace(valueDefault.Column))] = valueDefault.Value + } + } + return out +} + +func applyTopDataRowGeneration(datasets []nativeDataset, rules []project.TopDataRowGenerationConfig) []nativeDataset { + if len(rules) == 0 { + return datasets + } + out := append([]nativeDataset(nil), datasets...) + for index := range out { + for _, rule := range rules { + datasetName := strings.TrimSpace(rule.Dataset) + if datasetName == "" { + datasetName = strings.TrimSpace(rule.Namespace) + } + if filepath.ToSlash(datasetName) != out[index].Name { + continue + } + mode := strings.TrimSpace(rule.Mode) + if mode == "" { + mode = "after_base" + } + out[index].RowGeneration = mode + out[index].RowGenerationMinRow = rule.MinimumRow + } + } + return out +} + +func applyTopDataRowExtensions(collected []nativeCollectedDataset, rules []project.TopDataRowExtensionConfig) ([]nativeCollectedDataset, error) { + if len(rules) == 0 { + return collected, nil + } + out := append([]nativeCollectedDataset(nil), collected...) + for index := range out { + dataset := &out[index] + rule, ok, err := topDataRowExtensionForDataset(dataset.Dataset.Name, rules) + if err != nil { + return nil, err + } + if !ok { + continue + } + extended, err := extendCollectedRowsByRepeatingLastLevel(*dataset, rule) + if err != nil { + return nil, err + } + out[index] = extended + } + return out, nil +} + +func topDataRowExtensionForDataset(datasetName string, rules []project.TopDataRowExtensionConfig) (project.TopDataRowExtensionConfig, bool, error) { + name := filepath.ToSlash(strings.TrimSpace(datasetName)) + for _, rule := range rules { + pattern := filepath.ToSlash(strings.TrimSpace(rule.Dataset)) + matched, err := path.Match(pattern, name) + if err != nil { + return project.TopDataRowExtensionConfig{}, false, fmt.Errorf("topdata row extension pattern %q is invalid: %w", rule.Dataset, err) + } + if matched { + return rule, true, nil + } + } + return project.TopDataRowExtensionConfig{}, false, nil +} + +func extendCollectedRowsByRepeatingLastLevel(dataset nativeCollectedDataset, rule project.TopDataRowExtensionConfig) (nativeCollectedDataset, error) { + if dataset.Dataset.Kind != nativeDatasetPlain { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: topdata row extension supports only plain datasets", dataset.Dataset.Name) + } + if strings.TrimSpace(rule.Mode) != "repeat_last" { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: topdata row extension mode %q is not supported", dataset.Dataset.Name, rule.Mode) + } + levelColumn, ok := datasetColumnName(dataset.Columns, rule.LevelColumn) + if !ok { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: topdata row extension level_column %q is not present in columns", dataset.Dataset.Name, rule.LevelColumn) + } + if len(dataset.Rows) == 0 { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: topdata row extension requires at least one authored row", dataset.Dataset.Name) + } + + maxLevel := 0 + maxID := -1 + var reference map[string]any + usedIDs := map[int]struct{}{} + for index, row := range dataset.Rows { + rowID, err := asInt(row["id"]) + if err != nil { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d id must be numeric for topdata row extension", dataset.Dataset.Name, index) + } + usedIDs[rowID] = struct{}{} + if rowID > maxID { + maxID = rowID + } + rawLevel, ok := lookupField(row, levelColumn) + if !ok { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d missing level_column %q for topdata row extension", dataset.Dataset.Name, index, levelColumn) + } + level, err := asInt(rawLevel) + if err != nil { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d level_column %q must be numeric for topdata row extension", dataset.Dataset.Name, index, levelColumn) + } + if reference == nil || level > maxLevel { + maxLevel = level + reference = row + } + } + if rule.TargetLevel < maxLevel { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: topdata row extension target_level %d is below highest authored %s %d", dataset.Dataset.Name, rule.TargetLevel, levelColumn, maxLevel) + } + if rule.TargetLevel == maxLevel { + return dataset, nil + } + + rows := append([]map[string]any(nil), dataset.Rows...) + nextID := maxID + 1 + for level := maxLevel + 1; level <= rule.TargetLevel; level++ { + if _, exists := usedIDs[nextID]; exists { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: generated topdata row id %d already exists", dataset.Dataset.Name, nextID) + } + row := make(map[string]any, len(dataset.Columns)+1) + row["id"] = nextID + for _, column := range dataset.Columns { + if column == levelColumn { + row[column] = level + continue + } + if value, ok := lookupField(reference, column); ok { + row[column] = cloneAuthoringValue(value) + } + } + rows = append(rows, row) + usedIDs[nextID] = struct{}{} + nextID++ + } + slices.SortFunc(rows, func(a, b map[string]any) int { + left, _ := asInt(a["id"]) + right, _ := asInt(b["id"]) + return left - right + }) + dataset.Rows = rows + return dataset, nil +} + +func datasetColumnName(columns []string, name string) (string, bool) { + for _, column := range columns { + if strings.EqualFold(column, name) { + return column, true + } + } + return "", false +} + +func nativeCompileGroup(datasetName string) string { + head, _, ok := strings.Cut(datasetName, "/") + if !ok { + return datasetName + } + switch head { + case "itemprops", "racialtypes", "damagetypes", "parts", "classes": + return head + default: + return head + } +} + +func saveNativeLockfiles(collected []nativeCollectedDataset) (int, int, error) { + added := 0 + pruned := 0 + for _, dataset := range collected { + if dataset.Dataset.LockPath == "" { + continue + } + current, err := loadLockfile(dataset.Dataset.LockPath) + if err != nil { + return 0, 0, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err) + } + added += countAddedLockEntries(current, dataset.LockData) + pruned += countAddedLockEntries(dataset.LockData, current) + if err := saveLockfile(dataset.Dataset.LockPath, dataset.LockData); err != nil { + return 0, 0, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err) + } + } + return added, pruned, nil +} + +func countAddedLockEntries(before, after map[string]int) int { + count := 0 + for key, afterID := range after { + if beforeID, ok := before[key]; !ok || beforeID != afterID { + count++ + } + } + return count +} + +func lockDataEqual(a, b map[string]int) bool { + if len(a) != len(b) { + return false + } + for key, aID := range a { + if bID, ok := b[key]; !ok || bID != aID { + return false + } + } + return true +} + +type nativeCompileGroupStat struct { + OutputTables int + SourceFragments int +} + +func nativeCompileGroupStats(collected []nativeCollectedDataset) map[string]nativeCompileGroupStat { + stats := make(map[string]nativeCompileGroupStat) + for _, dataset := range collected { + group := nativeCompileGroup(dataset.Dataset.Name) + groupStat := stats[group] + groupStat.OutputTables++ + groupStat.SourceFragments += nativeDatasetSourceFragments(dataset.Dataset) + stats[group] = groupStat + } + return stats +} + +func nativeDatasetSourceFragments(dataset nativeDataset) int { + fragments := 1 + if dataset.Kind == nativeDatasetBase { + fragments += countModuleFiles(dataset.ModulesDir) + fragments += countModuleFiles(dataset.GeneratedDir) + } + return fragments +} + +func countModuleFiles(dir string) int { + if strings.TrimSpace(dir) == "" { + return 0 + } + paths, err := collectModulePaths(dir) + if err != nil { + return 0 + } + return len(paths) +} + +func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) { + var datasets []nativeDataset + appendPlainDataset := func(rootPath, filePath, relName string) error { + tableData, err := loadJSONObject(filePath) + if err != nil { + return err + } + globalPaths, err := collectGlobalJSONPaths(dataDir, rootPath) + if err != nil { + return err + } + lockPath := filepath.Join(rootPath, "lock.json") + if !plainDatasetNeedsLock(lockPath, tableData) { + lockPath = "" + } + outputName, _ := tableData["output"].(string) + if strings.TrimSpace(outputName) == "" { + outputName = nativeDatasetDefaultOutputName(filePath, nativeDatasetPlain) + } + datasets = append(datasets, nativeDataset{ + Kind: nativeDatasetPlain, + Name: filepath.ToSlash(relName), + RootPath: rootPath, + BasePath: filePath, + LockPath: lockPath, + GlobalPaths: globalPaths, + OutputName: outputName, + Spec: specForDataset(filepath.ToSlash(relName)), + CompareReference: optionalBool(tableData["compare_reference"], true), + }) + return nil + } + err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + return nil + } + if d.Name() == "modules" || d.Name() == "generated" { + return filepath.SkipDir + } + + rel, err := filepath.Rel(dataDir, path) + if err != nil { + return err + } + if rel == "." { + entries, err := os.ReadDir(path) + if err != nil { + return err + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if !strings.HasSuffix(strings.ToLower(name), ".json") || name == "lock.json" || name == "base.json" { + continue + } + if isGlobalJSONName(name) { + continue + } + filePath := filepath.Join(path, name) + if err := appendPlainDataset(path, filePath, strings.TrimSuffix(name, filepath.Ext(name))); err != nil { + return err + } + } + return nil + } + if filepath.ToSlash(rel) == "parts/overrides" { + return filepath.SkipDir + } + if filepath.ToSlash(rel) == "classes/spellbooks" { + return filepath.SkipDir + } + if filepath.ToSlash(rel) == itempropsRegistryDirName || filepath.ToSlash(rel) == damagetypesRegistryDirName || filepath.ToSlash(rel) == racialtypesRegistryDirName { + return filepath.SkipDir + } + + basePath := filepath.Join(path, "base.json") + if _, err := os.Stat(basePath); err == nil { + baseData, err := loadJSONObject(basePath) + if err != nil { + return err + } + outputName, _ := baseData["output"].(string) + if strings.TrimSpace(outputName) == "" { + outputName = nativeDatasetDefaultOutputName(path, nativeDatasetBase) + } + datasets = append(datasets, nativeDataset{ + Kind: nativeDatasetBase, + Name: filepath.ToSlash(rel), + RootPath: path, + BasePath: basePath, + LockPath: filepath.Join(path, "lock.json"), + ModulesDir: filepath.Join(path, "modules"), + GeneratedDir: filepath.Join(path, "generated"), + GlobalPaths: collectExistingGlobalJSONPath(path), + OutputName: outputName, + Spec: specForDataset(filepath.ToSlash(rel)), + CompareReference: optionalBool(baseData["compare_reference"], true), + }) + return nil + } + + entries, err := os.ReadDir(path) + if err != nil { + return err + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if strings.HasSuffix(strings.ToLower(name), ".json") && name != "lock.json" && !isGlobalJSONName(name) { + filePath := filepath.Join(path, name) + if err := appendPlainDataset(path, filePath, filepath.Join(rel, strings.TrimSuffix(name, filepath.Ext(name)))); err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return nil, err + } + + slices.SortFunc(datasets, func(a, b nativeDataset) int { + return strings.Compare(a.Name, b.Name) + }) + return datasets, nil +} + +func isGlobalJSONName(name string) bool { + return strings.EqualFold(strings.TrimSpace(name), "global.json") +} + +func collectExistingGlobalJSONPath(dir string) []string { + path := filepath.Join(dir, "global.json") + if _, err := os.Stat(path); err == nil { + return []string{path} + } + return nil +} + +func collectGlobalJSONPaths(dataDir, datasetDir string) ([]string, error) { + rel, err := filepath.Rel(dataDir, datasetDir) + if err != nil { + return nil, err + } + dirs := []string{dataDir} + if rel != "." { + current := dataDir + for _, segment := range strings.Split(filepath.ToSlash(rel), "/") { + if strings.TrimSpace(segment) == "" || segment == "." { + continue + } + current = filepath.Join(current, segment) + dirs = append(dirs, current) + } + } + paths := make([]string, 0, len(dirs)) + for _, dir := range dirs { + path := filepath.Join(dir, "global.json") + if _, err := os.Stat(path); err == nil { + paths = append(paths, path) + continue + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + } + return paths, nil +} + +func plainDatasetNeedsLock(lockPath string, tableData map[string]any) bool { + if _, err := os.Stat(lockPath); err == nil { + return true + } + rawRows, ok := tableData["rows"].([]any) + if !ok { + return false + } + for _, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + key, hasKey := row["key"].(string) + if hasKey && strings.TrimSpace(key) != "" { + if _, hasID := row["id"]; !hasID { + return true + } + } + } + return false +} + +func discoverNativeOutputCatalog(dataDir string) (map[string]string, error) { + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return nil, err + } + registryDatasets, err := collectGeneratedRegistryDatasets(dataDir) + if err != nil { + return nil, err + } + catalog := make(map[string]string, len(datasets)+len(registryDatasets)) + for _, dataset := range datasets { + if !dataset.CompareReference { + continue + } + catalog[dataset.OutputName] = dataset.Name + } + for _, dataset := range registryDatasets { + catalog[dataset.Dataset.OutputName] = dataset.Dataset.Name + } + return catalog, nil +} + +func missingNativeOutputs(projectOutputs, referenceOutputs map[string]string) []string { + missing := make([]string, 0) + for outputName, datasetName := range referenceOutputs { + if _, ok := projectOutputs[outputName]; ok { + continue + } + missing = append(missing, fmt.Sprintf("%s (%s)", outputName, datasetName)) + } + slices.Sort(missing) + return missing +} + +func collectNativeDataset(dataset nativeDataset) (nativeCollectedDataset, error) { + switch dataset.Kind { + case nativeDatasetBase: + return collectBaseDataset(dataset) + case nativeDatasetPlain: + return collectPlainDataset(dataset) + default: + return nativeCollectedDataset{}, fmt.Errorf("unsupported native dataset kind %q", dataset.Kind) + } +} + +func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { + baseData, err := loadJSONObject(dataset.BasePath) + if err != nil { + return nativeCollectedDataset{}, err + } + + columns, err := parseColumns(baseData, dataset.Name) + if err != nil { + return nativeCollectedDataset{}, err + } + modulePaths, err := collectModulePaths(dataset.ModulesDir) + if err != nil { + return nativeCollectedDataset{}, err + } + moduleData := make([]nativeGeneratedModule, 0, len(modulePaths)) + for _, path := range modulePaths { + obj, err := loadJSONObject(path) + if err != nil { + return nativeCollectedDataset{}, err + } + moduleData = append(moduleData, nativeGeneratedModule{ + Name: path, + Data: obj, + }) + } + globalData, err := loadGlobalModules(dataset.GlobalPaths) + if err != nil { + return nativeCollectedDataset{}, err + } + for _, module := range moduleData { + columns, err = extendColumns(columns, module.Data, dataset.Name, module.Name) + if err != nil { + return nativeCollectedDataset{}, err + } + } + for _, module := range globalData { + columns, err = extendColumns(columns, module.Data, dataset.Name, module.Name) + if err != nil { + return nativeCollectedDataset{}, err + } + } + rawBaseRows, ok := baseData["rows"].([]any) + if !ok { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: rows must be an array", dataset.Name) + } + baseBoundaryID := -1 + baseRowKeys := map[string]struct{}{} + baseRowKeyCounts := map[string]int{} + for index, raw := range rawBaseRows { + rowObj, ok := raw.(map[string]any) + if !ok { + continue + } + rowID := index + if rawID, ok := rowObj["id"]; ok { + parsed, err := asInt(rawID) + if err == nil { + rowID = parsed + } + } + baseBoundaryID = rowID + key, ok := rowObj["key"].(string) + if !ok || key == "" { + continue + } + baseRowKeys[key] = struct{}{} + baseRowKeyCounts[key]++ + } + + lockData, err := loadLockfile(dataset.LockPath) + if err != nil { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + + explicitModuleIDs := map[string]int{} + for _, module := range append(append([]nativeGeneratedModule(nil), moduleData...), globalData...) { + explicitIDs, err := collectExplicitModuleIDs(dataset.Name, module.Name, module.Data) + if err != nil { + return nativeCollectedDataset{}, err + } + for key, rowID := range explicitIDs { + explicitModuleIDs[key] = rowID + } + } + + rows := make([]map[string]any, 0, len(rawBaseRows)) + rowByID := map[int]map[string]any{} + rowByKey := map[string]map[string]any{} + usedIDs := map[int]struct{}{} + firstNullRowMode := dataset.RowGeneration == "first_null_row" + rowGenerationMinRow := dataset.RowGenerationMinRow + lockModified := false + lockAdded := 0 + lockPruned := 0 + retiredKeys := map[string]struct{}{} + rowIdentity := func(row map[string]any) (nativeRowIdentity, bool) { + rowID, ok := row["id"].(int) + if !ok { + return nativeRowIdentity{}, false + } + key, _ := row["key"].(string) + return nativeRowIdentity{ID: rowID, Key: key}, true + } + rowSourceByIdentity := map[nativeRowIdentity]nativeRowSource{} + sourceForIdentity := func(identity nativeRowIdentity, ok bool) nativeRowSource { + if !ok { + return nativeRowSourceBase + } + if source, ok := rowSourceByIdentity[identity]; ok { + return source + } + return nativeRowSourceBase + } + setRowSource := func(row map[string]any, source nativeRowSource) { + identity, ok := rowIdentity(row) + if !ok { + return + } + rowSourceByIdentity[identity] = source + } + moveRowSource := func(before nativeRowIdentity, beforeOK bool, row map[string]any, source nativeRowSource) { + after, afterOK := rowIdentity(row) + if beforeOK && (!afterOK || before != after) { + delete(rowSourceByIdentity, before) + } + if afterOK { + rowSourceByIdentity[after] = source + } + } + rowSource := func(row map[string]any) nativeRowSource { + identity, ok := rowIdentity(row) + return sourceForIdentity(identity, ok) + } + type pendingRowSourceMove struct { + Row map[string]any + Before nativeRowIdentity + BeforeOK bool + Source nativeRowSource + } + captureDisplacedRowSource := func(row map[string]any, expanded map[string]any) pendingRowSourceMove { + newKeyValue, hasKey := expanded["key"] + if !hasKey { + return pendingRowSourceMove{} + } + newKey, ok := newKeyValue.(string) + if !ok || newKey == "" || strings.TrimSpace(newKey) == nullValue { + return pendingRowSourceMove{} + } + oldKey, _ := row["key"].(string) + if oldKey == newKey { + return pendingRowSourceMove{} + } + conflicting, ok := rowByKey[newKey] + if !ok || conflicting == nil { + return pendingRowSourceMove{} + } + conflictingID, conflictingHasID := conflicting["id"].(int) + rowID, _ := row["id"].(int) + rowHasID := rowID != 0 || row["id"] != nil + if conflictingHasID && rowHasID && conflictingID == rowID { + return pendingRowSourceMove{} + } + identity, identityOK := rowIdentity(conflicting) + return pendingRowSourceMove{ + Row: conflicting, + Before: identity, + BeforeOK: identityOK, + Source: rowSource(conflicting), + } + } + moveCapturedRowSource := func(move pendingRowSourceMove) { + if move.Row == nil { + return + } + moveRowSource(move.Before, move.BeforeOK, move.Row, move.Source) + } + featFamilyAliases := generatedFamilyAliases{} + if dataset.Name == "feat" { + var err error + featFamilyAliases, err = collectGeneratedFamilyAliases(dataset.GeneratedDir) + if err != nil { + return nativeCollectedDataset{}, err + } + } + + if firstNullRowMode { + for key, rowID := range lockData { + if rowID >= rowGenerationMinRow && (rowID <= baseBoundaryID || rowGenerationMinRow > baseBoundaryID) { + continue + } + if _, ok := baseRowKeys[key]; ok { + continue + } + if explicitID, ok := explicitModuleIDs[key]; ok && explicitID == rowID { + continue + } + delete(lockData, key) + lockModified = true + lockPruned++ + } + } + + for key, rowID := range lockData { + if rowID <= baseBoundaryID { + if firstNullRowMode { + continue + } + if _, ok := baseRowKeys[key]; ok { + continue + } + if explicitID, ok := explicitModuleIDs[key]; ok && explicitID == rowID { + continue + } + delete(lockData, key) + lockModified = true + lockPruned++ + } + } + for _, rowID := range lockData { + usedIDs[rowID] = struct{}{} + } + for index, raw := range rawBaseRows { + rowObj, ok := raw.(map[string]any) + if !ok { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d must be an object", dataset.Name, index) + } + row, err := canonicalizeBaseRow(dataset, columns, rowObj, index) + if err != nil { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + rowID := row["id"].(int) + if firstNullRowMode && rowID >= rowGenerationMinRow && isNativeAllocationNullRow(row, columns) { + continue + } + rows = append(rows, row) + setRowSource(row, nativeRowSourceBase) + if key, ok := row["key"].(string); ok && key != "" { + if baseRowKeyCounts[key] > 1 { + if lockedID, ok := lockData[key]; !ok || lockedID != rowID { + lockData[key] = rowID + lockModified = true + } + } else if _, ok := lockData[key]; !ok { + lockData[key] = rowID + lockModified = true + lockAdded++ + } + } + rowByID[rowID] = row + if key, ok := row["key"].(string); ok && key != "" { + rowByKey[key] = row + } + usedIDs[rowID] = struct{}{} + } + if !firstNullRowMode { + for rowID := 0; rowID <= baseBoundaryID; rowID++ { + usedIDs[rowID] = struct{}{} + } + } + for _, rowID := range explicitModuleIDs { + usedIDs[rowID] = struct{}{} + } + + nextID := nextAvailableIDAtLeast(usedIDs, rowGenerationMinRow) + allocateNextID := func() int { + rowID := nextID + usedIDs[rowID] = struct{}{} + nextID = nextAvailableIDAtLeast(usedIDs, rowGenerationMinRow) + return rowID + } + lockedIDForKey := func(key string) (int, bool) { + lockedID, ok := lockData[key] + if !ok { + return 0, false + } + return lockedID, true + } + stripMismatchedLockedOverrideID := func(override map[string]any) (map[string]any, error) { + rawKey, hasKey := override["key"].(string) + if !hasKey || rawKey == "" { + return override, nil + } + lockedID, hasLockedID := lockData[rawKey] + if !hasLockedID { + return override, nil + } + rawID, hasID := override["id"] + if !hasID { + return override, nil + } + parsedID, err := asInt(rawID) + if err != nil { + return nil, fmt.Errorf("dataset %s: override id %v is not numeric", dataset.Name, rawID) + } + if parsedID == lockedID { + return override, nil + } + if dataset.Name == "feat" && generatedCanonicalLockCanMove(rawKey, parsedID, lockData, featFamilyAliases) { + delete(lockData, rawKey) + lockModified = true + return override, nil + } + return override, nil + } + validExplicitIDForNewKey := func(key string, rowID int) bool { + if key == "" { + return true + } + for lockedKey, lockedID := range lockData { + if lockedKey != key && lockedID == rowID { + if dataset.Name == "feat" && generatedAliasLockForKey(key, lockedKey, featFamilyAliases) { + delete(lockData, lockedKey) + lockModified = true + continue + } + return false + } + } + return true + } + applyOverrideFields := func(override map[string]any, row map[string]any) (bool, string, error) { + sourceBefore := rowSource(row) + identityBefore, identityBeforeOK := rowIdentity(row) + candidate := map[string]any{} + for field, value := range row { + candidate[field] = value + } + for field, value := range override { + if field == "id" || field == "key" || field == "_tlk" || field == "match" { + continue + } + if isMetadataField(field) { + meta, err := parseRowMetadata(value) + if err != nil { + return false, "", fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + if meta != nil { + existingMeta, _ := parseExistingMetadata(candidate) + candidate["meta"] = mergeMetadataMaps(existingMeta, meta) + } + continue + } + if isDeprecatedAuthoringOnlyField(dataset.Name, field) { + if err := setLegacyWikiMetadata(candidate, field, value); err != nil { + return false, "", fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + continue + } + columnName, ok := canonicalColumn(columns, field) + if !ok { + if isDeprecatedAuthoringOnlyField(dataset.Name, field) { + continue + } + return false, "", fmt.Errorf("dataset %s: override field %q is not a known column", dataset.Name, field) + } + candidate[columnName] = value + } + if rawKey, ok := override["key"]; ok { + candidate["key"] = rawKey + } + expanded, err := expandRowAuthoringSugar(dataset, columns, override, candidate) + if err != nil { + return false, "", err + } + displacedSource := captureDisplacedRowSource(row, expanded) + keyChanged, retiredKey, err := updateOverrideRowKey(dataset.Name, row, expanded, rowByKey, lockData) + if err != nil { + return false, "", err + } + for field, value := range expanded { + if field == "id" || field == "_tlk" { + continue + } + row[field] = value + } + moveRowSource(identityBefore, identityBeforeOK, row, sourceBefore) + moveCapturedRowSource(displacedSource) + return keyChanged, retiredKey, nil + } + applyModuleData := func(sourceLabel string, moduleData map[string]any, globalSource bool) error { + if rawEntries, ok := moduleData["entries"]; ok { + entries, ok := rawEntries.(map[string]any) + if !ok { + return fmt.Errorf("dataset %s: entries in %s must be an object", dataset.Name, sourceLabel) + } + for _, key := range sortedKeys(entries) { + rawEntry, ok := entries[key].(map[string]any) + if !ok { + return fmt.Errorf("dataset %s: entry %q in %s must be an object", dataset.Name, key, sourceLabel) + } + if existing, ok := rowByKey[key]; ok { + candidate := map[string]any{} + for field, value := range existing { + candidate[field] = value + } + for field, value := range rawEntry { + if field == "id" || field == "key" || field == "_tlk" { + continue + } + if isMetadataField(field) { + meta, err := parseRowMetadata(value) + if err != nil { + return fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + if meta != nil { + existingMeta, _ := parseExistingMetadata(candidate) + candidate["meta"] = mergeMetadataMaps(existingMeta, meta) + } + continue + } + if isDeprecatedAuthoringOnlyField(dataset.Name, field) { + if err := setLegacyWikiMetadata(candidate, field, value); err != nil { + return fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + continue + } + columnName, ok := canonicalColumn(columns, field) + if !ok { + return fmt.Errorf("dataset %s: entry field %q is not a known column", dataset.Name, field) + } + candidate[columnName] = value + } + expanded, err := expandRowAuthoringSugar(dataset, columns, rawEntry, candidate) + if err != nil { + return err + } + identityBefore, identityBeforeOK := rowIdentity(existing) + displacedSource := captureDisplacedRowSource(existing, expanded) + keyChanged, retiredKey, err := updateOverrideRowKey(dataset.Name, existing, expanded, rowByKey, lockData) + if err != nil { + return err + } + if keyChanged { + lockModified = true + } + if retiredKey != "" { + retiredKeys[retiredKey] = struct{}{} + } + for field, value := range expanded { + if field == "id" || field == "_tlk" { + continue + } + existing[field] = value + } + moveRowSource(identityBefore, identityBeforeOK, existing, nativeRowSourceEntries) + moveCapturedRowSource(displacedSource) + continue + } + rowID, ok := lockedIDForKey(key) + if !ok { + rowID = allocateNextID() + lockData[key] = rowID + lockModified = true + lockAdded++ + } + row, err := canonicalizeEntry(dataset, columns, rowID, key, rawEntry) + if err != nil { + return fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + rows = append(rows, row) + setRowSource(row, nativeRowSourceEntries) + rowByID[rowID] = row + rowByKey[key] = row + } + } + + if rawOverrides, ok := moduleData["overrides"]; ok { + overrideList, ok := rawOverrides.([]any) + if !ok { + return fmt.Errorf("dataset %s: overrides in %s must be an array", dataset.Name, sourceLabel) + } + for index, rawOverride := range overrideList { + override, ok := rawOverride.(map[string]any) + if !ok { + return fmt.Errorf("dataset %s: override %d in %s must be an object", dataset.Name, index, sourceLabel) + } + if rawMatch, hasMatch := override["match"]; hasMatch { + match, ok := rawMatch.(string) + if !ok || !strings.EqualFold(strings.TrimSpace(match), "all") { + return fmt.Errorf("dataset %s: override %d in %s has unsupported match value %v", dataset.Name, index, sourceLabel, rawMatch) + } + if !globalSource { + return fmt.Errorf("dataset %s: override %d in %s uses match, which is only supported in global.json", dataset.Name, index, sourceLabel) + } + for _, row := range rows { + keyChanged, retiredKey, err := applyOverrideFields(override, row) + if err != nil { + return err + } + if keyChanged { + lockModified = true + } + if retiredKey != "" { + retiredKeys[retiredKey] = struct{}{} + } + } + continue + } + override, err = stripMismatchedLockedOverrideID(override) + if err != nil { + return err + } + row, err := resolveOverrideTarget(dataset.Name, override, rowByID, rowByKey) + if err != nil { + rawKey, hasKey := override["key"].(string) + rowID := 0 + hasRowID := false + if rawID, hasID := override["id"]; hasID { + parsedID, parseErr := asInt(rawID) + if parseErr != nil { + return fmt.Errorf("dataset %s: override id %v is not numeric", dataset.Name, rawID) + } + rawKey, _ := override["key"].(string) + _, lockedIDExists := lockData[rawKey] + if !lockedIDExists { + if !validExplicitIDForNewKey(rawKey, parsedID) { + return fmt.Errorf("dataset %s: override key %q id %d collides with an existing lockfile id", dataset.Name, rawKey, parsedID) + } + rowID = parsedID + hasRowID = true + } + } + if hasKey && rawKey != "" { + if lockedID, hasLockedID := lockedIDForKey(rawKey); hasLockedID { + rowID = lockedID + hasRowID = true + } else { + if !hasRowID { + rowID = allocateNextID() + hasRowID = true + } + lockData[rawKey] = rowID + lockModified = true + lockAdded++ + } + } + if !hasRowID { + return err + } + usedIDs[rowID] = struct{}{} + row = map[string]any{"id": rowID} + if hasKey && rawKey != "" { + row["key"] = rawKey + } + rows = append(rows, row) + setRowSource(row, nativeRowSourceOverride) + rowByID[rowID] = row + if hasKey && rawKey != "" { + rowByKey[rawKey] = row + } + } + if overrideRequestsNullRow(override) { + sourceBefore := rowSource(row) + identityBefore, identityBeforeOK := rowIdentity(row) + retireRowIdentity(row, lockData, retiredKeys) + nullifyOverrideRow(row, columns, rowByKey) + moveRowSource(identityBefore, identityBeforeOK, row, sourceBefore) + continue + } + keyChanged, retiredKey, err := applyOverrideFields(override, row) + if err != nil { + return err + } + if keyChanged { + lockModified = true + } + if retiredKey != "" { + retiredKeys[retiredKey] = struct{}{} + } + } + } + return nil + } + applyGlobalDefaults := func(sourceLabel string, moduleData map[string]any) error { + rawDefaults, ok := moduleData["defaults"] + if !ok { + return nil + } + defaultList, ok := rawDefaults.([]any) + if !ok { + return fmt.Errorf("dataset %s: defaults in %s must be an array", dataset.Name, sourceLabel) + } + for index, rawDefault := range defaultList { + defaultRule, ok := rawDefault.(map[string]any) + if !ok { + return fmt.Errorf("dataset %s: default %d in %s must be an object", dataset.Name, index, sourceLabel) + } + rawValues, ok := defaultRule["values"] + if !ok { + return fmt.Errorf("dataset %s: default %d in %s must contain values", dataset.Name, index, sourceLabel) + } + values, ok := rawValues.(map[string]any) + if !ok { + return fmt.Errorf("dataset %s: default %d values in %s must be an object", dataset.Name, index, sourceLabel) + } + parsedValues := make([]parsedGlobalDefaultField, 0, len(values)) + for _, field := range sortedKeys(values) { + columnName, ok := canonicalColumn(columns, field) + if !ok { + return fmt.Errorf("dataset %s: default %d field %q in %s is not a known column", dataset.Name, index, field, sourceLabel) + } + value, err := parseGlobalDefaultValue(values[field]) + if err != nil { + return fmt.Errorf("dataset %s: default %d field %q in %s: %w", dataset.Name, index, field, sourceLabel, err) + } + parsedValues = append(parsedValues, parsedGlobalDefaultField{ + Field: field, + Column: columnName, + Value: value, + }) + } + for _, row := range rows { + matches, err := globalDefaultMatchesRow(defaultRule["match"], rowSource(row)) + if err != nil { + return fmt.Errorf("dataset %s: default %d in %s: %w", dataset.Name, index, sourceLabel, err) + } + if !matches { + continue + } + for _, parsedValue := range parsedValues { + if !isTopDataNullLike(row[parsedValue.Column]) { + continue + } + value, err := evaluateGlobalDefaultValue(parsedValue.Value, row) + if err != nil { + return fmt.Errorf("dataset %s: default %d field %q in %s: %w", dataset.Name, index, parsedValue.Field, sourceLabel, err) + } + row[parsedValue.Column] = value + } + } + } + return nil + } + + generatedModules, err := collectGeneratedModules(dataset, lockData) + if err != nil { + return nativeCollectedDataset{}, err + } + for _, module := range generatedModules { + updatedColumns, err := extendColumns(columns, module.Data, dataset.Name, module.Name) + if err != nil { + return nativeCollectedDataset{}, err + } + if len(updatedColumns) != len(columns) { + columns = updatedColumns + ensureRowsExposeColumns(rows, columns) + } + } + for _, module := range generatedModules { + if module.ApplyAfterModules { + continue + } + if err := applyModuleData(module.Name, module.Data, false); err != nil { + return nativeCollectedDataset{}, err + } + } + + for _, module := range moduleData { + if err := applyModuleData(module.Name, module.Data, false); err != nil { + return nativeCollectedDataset{}, err + } + } + for _, module := range generatedModules { + if !module.ApplyAfterModules { + continue + } + if err := applyModuleData(module.Name, module.Data, false); err != nil { + return nativeCollectedDataset{}, err + } + } + for _, module := range globalData { + if err := applyModuleData(module.Name, module.Data, true); err != nil { + return nativeCollectedDataset{}, err + } + if err := applyGlobalDefaults(module.Name, module.Data); err != nil { + return nativeCollectedDataset{}, err + } + } + referencedKeys, err := collectReferencedLockKeys(nativeDatasetSourceDir(dataset)) + if err != nil { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: collect referenced keys: %w", dataset.Name, err) + } + if pruned, updated := pruneLockDataToActiveRows(lockData, rows, referencedKeys, retiredKeys); pruned > 0 || updated > 0 { + lockModified = true + lockPruned += pruned + } + + rows = dedupeCollectedRows(rows) + slices.SortFunc(rows, func(a, b map[string]any) int { + return a["id"].(int) - b["id"].(int) + }) + return nativeCollectedDataset{ + Dataset: dataset, + Columns: columns, + Rows: rows, + LockData: lockData, + RetiredKeys: retiredKeys, + DenseRowMax: baseBoundaryID, + LockAdded: lockAdded, + LockPruned: lockPruned, + LockModified: lockModified, + }, nil +} + +func pruneLockDataToActiveRows(lockData map[string]int, rows []map[string]any, referencedKeys map[string]struct{}, retiredKeys map[string]struct{}) (int, int) { + if len(lockData) == 0 { + return 0, 0 + } + activeKeys := make(map[string]int, len(rows)) + for _, row := range rows { + if key, ok := row["key"].(string); ok && key != "" { + rowID, _ := row["id"].(int) + activeKeys[key] = rowID + } + } + updated := 0 + for key, rowID := range activeKeys { + if lockedID, ok := lockData[key]; !ok || lockedID != rowID { + lockData[key] = rowID + updated++ + } + } + pruned := 0 + for key := range lockData { + if _, ok := activeKeys[key]; ok { + continue + } + if _, retired := retiredKeys[key]; retired { + delete(lockData, key) + pruned++ + continue + } + if _, ok := referencedKeys[key]; ok { + continue + } + delete(lockData, key) + pruned++ + } + return pruned, updated +} + +func collectReferencedLockKeys(sourceDir string) (map[string]struct{}, error) { + dataDir := filepath.Join(sourceDir, "data") + referenced := map[string]struct{}{} + if _, err := os.Stat(dataDir); err != nil { + if errors.Is(err, os.ErrNotExist) { + return referenced, nil + } + return nil, err + } + err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if filepath.Base(path) == "lock.json" || !strings.EqualFold(filepath.Ext(path), ".json") { + return nil + } + obj, err := loadJSONObject(path) + if err != nil { + return err + } + collectReferencedLockKeysFromValue(obj, referenced) + return nil + }) + return referenced, err +} + +func nativeDatasetSourceDir(dataset nativeDataset) string { + dir := filepath.Clean(dataset.RootPath) + for { + if filepath.Base(dir) == "data" { + return filepath.Dir(dir) + } + parent := filepath.Dir(dir) + if parent == dir { + return filepath.Dir(filepath.Dir(dataset.RootPath)) + } + dir = parent + } +} + +func collectReferencedLockKeysFromValue(value any, referenced map[string]struct{}) { + collectReferencedLockKeysFromValueWithExpansion(value, referenced, false) +} + +func collectReferencedLockKeysFromValueWithExpansion(value any, referenced map[string]struct{}, inExpansionSubtree bool) { + switch typed := value.(type) { + case string: + if strings.Contains(typed, ":") { + referenced[typed] = struct{}{} + } + case []any: + for _, item := range typed { + collectReferencedLockKeysFromValueWithExpansion(item, referenced, inExpansionSubtree) + } + case map[string]any: + _, hasValue := typed["value"] + _, hasData := typed["data"] + isExpansion := hasValue && hasData + + for key, item := range typed { + if strings.Contains(key, ":") { + referenced[key] = struct{}{} + } + if key == "key" { + if inExpansionSubtree { + if s, ok := item.(string); ok && strings.Contains(s, ":") { + referenced[s] = struct{}{} + } + } + continue + } + collectReferencedLockKeysFromValueWithExpansion(item, referenced, inExpansionSubtree || isExpansion) + } + } +} + +func dedupeCollectedRows(rows []map[string]any) []map[string]any { + if len(rows) <= 1 { + return rows + } + seen := make(map[string]int, len(rows)) + out := make([]map[string]any, 0, len(rows)) + for _, row := range rows { + rowID, _ := row["id"].(int) + rowKey, _ := row["key"].(string) + if rowKey == "" { + out = append(out, row) + continue + } + identity := strconv.Itoa(rowID) + "\x00" + rowKey + if index, ok := seen[identity]; ok { + out[index] = row + continue + } + seen[identity] = len(out) + out = append(out, row) + } + return out +} + +type nativeGeneratedModule struct { + Name string + Data map[string]any + ApplyAfterModules bool +} + +func collectGeneratedModules(dataset nativeDataset, lockData map[string]int) ([]nativeGeneratedModule, error) { + if dataset.Name != "feat" { + return nil, nil + } + ctx, err := newFeatGeneratedContext(dataset, lockData) + if err != nil { + return nil, err + } + paths, err := collectModulePaths(dataset.GeneratedDir) + if err != nil { + return nil, err + } + type generatedSource struct { + path string + obj map[string]any + } + sources := make([]generatedSource, 0, len(paths)) + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + if isFamilyExpansionObject(obj) { + spec, err := parseFamilyExpansionSpec(path, obj) + if err != nil { + return nil, err + } + ctx.familySpecs[spec.FamilyKey] = spec + } + sources = append(sources, generatedSource{path: path, obj: obj}) + } + modules := make([]nativeGeneratedModule, 0, len(paths)) + for _, source := range sources { + applyAfterModules := false + if isFamilyExpansionObject(source.obj) { + spec, err := parseFamilyExpansionSpec(source.path, source.obj) + if err != nil { + return nil, err + } + applyAfterModules = spec.ApplyAfterModules + } + moduleData, err := buildFeatGeneratedModule(source.path, source.obj, ctx) + if err != nil { + return nil, err + } + modules = append(modules, nativeGeneratedModule{ + Name: source.path, + Data: moduleData, + ApplyAfterModules: applyAfterModules, + }) + } + racialtypesModule, err := buildRacialtypesSkillAffinityModule(ctx) + if err != nil { + return nil, err + } + if racialtypesModule != nil { + modules = append(modules, nativeGeneratedModule{ + Name: filepath.Join(dataset.RootPath, "generated", "skill_affinity"), + Data: racialtypesModule, + }) + } + return modules, nil +} + +type featGeneratedContext struct { + sourceDir string + dataDir string + lockData map[string]int + supplementalID map[string]int + baseDialog *legacyTLKData + tlkStateKeys map[string]struct{} + existingFeat map[string]struct{} + datasets map[string]nativeDataset + rowsByDataset map[string]map[string]map[string]any + familySpecs map[string]familyExpansionSpec +} + +type compactSourceOverride = map[string]map[string]any + +type affinityGrant struct { + featKey string + races []string +} + +func newFeatGeneratedContext(dataset nativeDataset, lockData map[string]int) (*featGeneratedContext, error) { + sourceDir := filepath.Dir(filepath.Dir(dataset.RootPath)) + dataDir := filepath.Join(sourceDir, "data") + allDatasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return nil, err + } + datasetMap := make(map[string]nativeDataset, len(allDatasets)) + for _, item := range allDatasets { + datasetMap[item.Name] = item + } + supplementalID, err := collectSupplementalLockIDs(sourceDir, "") + if err != nil { + return nil, err + } + baseDialog, err := loadBaseDialogData(filepath.Join(sourceDir, "base_dialog.json")) + if err != nil { + return nil, err + } + tlkState, err := loadTLKState(filepath.Join(sourceDir, tlkStateFile)) + if err != nil { + return nil, err + } + tlkStateKeys := make(map[string]struct{}, len(tlkState.Entries)) + for key := range tlkState.Entries { + tlkStateKeys[key] = struct{}{} + } + lockCopy := make(map[string]int, len(lockData)) + for key, value := range lockData { + lockCopy[key] = value + } + existingFeat, err := collectExistingFeatKeys(filepath.Join(dataDir, "feat")) + if err != nil { + return nil, err + } + retiredFeatKeys, err := collectRetiredFeatKeys(filepath.Join(dataDir, "feat")) + if err != nil { + return nil, err + } + for key, retiredID := range retiredFeatKeys { + if lockedID, ok := lockCopy[key]; ok && lockedID == retiredID { + delete(lockCopy, key) + } + delete(existingFeat, key) + } + return &featGeneratedContext{ + sourceDir: sourceDir, + dataDir: dataDir, + lockData: lockCopy, + supplementalID: supplementalID, + baseDialog: baseDialog, + tlkStateKeys: tlkStateKeys, + existingFeat: existingFeat, + datasets: datasetMap, + rowsByDataset: map[string]map[string]map[string]any{}, + familySpecs: map[string]familyExpansionSpec{}, + }, nil +} + +func collectExistingFeatKeys(featDir string) (map[string]struct{}, error) { + keys := map[string]struct{}{} + baseObj, err := loadJSONObject(filepath.Join(featDir, "base.json")) + if err != nil { + return nil, err + } + if rawRows, ok := baseObj["rows"].([]any); ok { + for _, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + if key, ok := row["key"].(string); ok && key != "" { + keys[key] = struct{}{} + } + } + } + modulePaths, err := collectModulePaths(filepath.Join(featDir, "modules")) + if err != nil { + return nil, err + } + for _, path := range modulePaths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + if rawEntries, ok := obj["entries"].(map[string]any); ok { + for _, key := range sortedKeys(rawEntries) { + if key != "" { + keys[key] = struct{}{} + } + } + } + if rawOverrides, ok := obj["overrides"].([]any); ok { + for _, raw := range rawOverrides { + override, ok := raw.(map[string]any) + if !ok { + continue + } + if key, ok := override["key"].(string); ok && key != "" { + keys[key] = struct{}{} + } + } + } + } + return keys, nil +} + +func (c *featGeneratedContext) featKeyExists(key string) bool { + _, ok := c.existingFeat[key] + return ok +} + +func collectRetiredFeatKeys(featDir string) (map[string]int, error) { + baseObj, err := loadJSONObject(filepath.Join(featDir, "base.json")) + if err != nil { + return nil, err + } + rowIDToKey := map[int]string{} + rowKeyToID := map[string]int{} + usedIDs := map[int]struct{}{} + if rawRows, ok := baseObj["rows"].([]any); ok { + for index, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + rowID := index + if rawID, ok := row["id"]; ok { + parsed, err := asInt(rawID) + if err == nil { + rowID = parsed + } + } + usedIDs[rowID] = struct{}{} + if key, ok := row["key"].(string); ok && key != "" { + rowIDToKey[rowID] = key + rowKeyToID[key] = rowID + } + } + } + nextID := nextAvailableID(usedIDs) + allocateNextID := func() int { + rowID := nextID + usedIDs[rowID] = struct{}{} + nextID = nextAvailableID(usedIDs) + return rowID + } + retired := map[string]int{} + modulePaths, err := collectModulePaths(filepath.Join(featDir, "modules")) + if err != nil { + return nil, err + } + for _, path := range modulePaths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + if rawEntries, ok := obj["entries"].(map[string]any); ok { + for _, key := range sortedKeys(rawEntries) { + rawEntry := rawEntries[key] + entry, ok := rawEntry.(map[string]any) + if !ok || key == "" { + continue + } + if _, exists := rowKeyToID[key]; exists { + continue + } + rowID := 0 + if rawID, ok := entry["id"]; ok { + parsed, err := asInt(rawID) + if err != nil { + return nil, fmt.Errorf("entry %q in %s has non-numeric id %v", key, path, rawID) + } + rowID = parsed + usedIDs[rowID] = struct{}{} + nextID = nextAvailableID(usedIDs) + } else { + rowID = allocateNextID() + } + rowIDToKey[rowID] = key + rowKeyToID[key] = rowID + } + } + rawOverrides, ok := obj["overrides"].([]any) + if !ok { + continue + } + for index, rawOverride := range rawOverrides { + override, ok := rawOverride.(map[string]any) + if !ok { + return nil, fmt.Errorf("override %d in %s must be an object", index, path) + } + rowID := 0 + hasRowID := false + if rawID, ok := override["id"]; ok { + parsed, err := asInt(rawID) + if err != nil { + return nil, fmt.Errorf("override %d in %s has non-numeric id %v", index, path, rawID) + } + rowID = parsed + hasRowID = true + } else if key, ok := override["key"].(string); ok && key != "" { + if mappedID, exists := rowKeyToID[key]; exists { + rowID = mappedID + hasRowID = true + } + } + if !hasRowID { + continue + } + currentKey := rowIDToKey[rowID] + if overrideRequestsNullRow(override) { + if currentKey != "" { + retired[currentKey] = rowID + delete(rowKeyToID, currentKey) + delete(rowIDToKey, rowID) + } + continue + } + if rawKey, present := override["key"]; present { + if rawKey == nil { + if currentKey != "" { + retired[currentKey] = rowID + delete(rowKeyToID, currentKey) + delete(rowIDToKey, rowID) + } + continue + } + newKey, ok := rawKey.(string) + if !ok || newKey == "" || strings.TrimSpace(newKey) == nullValue { + if currentKey != "" { + retired[currentKey] = rowID + delete(rowKeyToID, currentKey) + delete(rowIDToKey, rowID) + } + continue + } + if currentKey != "" && currentKey != newKey { + retired[currentKey] = rowID + delete(rowKeyToID, currentKey) + } + if previousID, exists := rowKeyToID[newKey]; exists && previousID != rowID { + delete(rowIDToKey, previousID) + } + rowKeyToID[newKey] = rowID + rowIDToKey[rowID] = newKey + } + } + } + return retired, nil +} + +func (c *featGeneratedContext) familyHasExistingRows(familyKey string) bool { + prefix := "feat:" + familyKey + "_" + for key := range c.existingFeat { + if strings.HasPrefix(key, prefix) { + return true + } + } + return false +} + +func (c *featGeneratedContext) datasetRows(name string) (map[string]map[string]any, error) { + if rows, ok := c.rowsByDataset[name]; ok { + return rows, nil + } + dataset, ok := c.datasets[name] + if !ok { + return nil, fmt.Errorf("generated feat family requires canonical dataset %q", name) + } + collected, err := collectNativeDataset(dataset) + if err != nil { + return nil, err + } + rows := make(map[string]map[string]any, len(collected.Rows)) + for _, row := range collected.Rows { + if key, ok := row["key"].(string); ok && key != "" { + rows[key] = row + } + } + c.rowsByDataset[name] = rows + return rows, nil +} + +func (c *featGeneratedContext) featID(key string, explicit any) (int, bool, error) { + if explicit != nil && explicit != nullValue { + parsed, err := featSourceID(explicit) + if err != nil { + return 0, false, err + } + if parsed > 0 { + return parsed, true, nil + } + } + if value, ok := c.lockData[key]; ok && value >= 0 { + return value, true, nil + } + if value, ok := c.supplementalID[key]; ok && value > 0 { + return value, true, nil + } + return 0, false, nil +} + +func (c *featGeneratedContext) preferredGeneratedFeatKey(familyKey, sourceSlug string) string { + exact := "feat:" + familyKey + "_" + sourceSlug + target := normalizeKeyIdentity(sourceSlug) + for _, candidates := range []map[string]int{c.lockData, c.supplementalID} { + matches := make([]string, 0) + for key, value := range candidates { + if value < 0 { + continue + } + if !strings.HasPrefix(key, "feat:"+familyKey+"_") { + continue + } + suffix := strings.TrimPrefix(key, "feat:"+familyKey+"_") + if normalizeKeyIdentity(suffix) == target { + matches = append(matches, key) + } + } + if best, ok := preferredExpandedKey(matches); ok { + return best + } + } + for _, candidates := range []map[string]int{c.lockData, c.supplementalID} { + if value, ok := candidates[exact]; ok && value > 0 { + return exact + } + } + stateMatches := make([]string, 0) + for key := range c.tlkStateKeys { + if !strings.HasPrefix(key, "feat:"+familyKey+"_") { + continue + } + suffix := strings.TrimPrefix(key, "feat:"+familyKey+"_") + suffix = strings.TrimSuffix(strings.TrimSuffix(suffix, ".name"), ".description") + if normalizeKeyIdentity(suffix) == target { + stateMatches = append(stateMatches, "feat:"+familyKey+"_"+suffix) + } + } + if best, ok := preferredExpandedKey(stateMatches); ok { + return best + } + return exact +} + +func preferredExpandedKey(matches []string) (string, bool) { + if len(matches) == 0 { + return "", false + } + best := matches[0] + for _, candidate := range matches[1:] { + best = betterExpandedKey(best, candidate) + } + return best, true +} + +func betterExpandedKey(a, b string) string { + aUnderscores := strings.Count(a, "_") + bUnderscores := strings.Count(b, "_") + if aUnderscores != bUnderscores { + if aUnderscores < bUnderscores { + return a + } + return b + } + if len(a) != len(b) { + if len(a) < len(b) { + return a + } + return b + } + if a <= b { + return a + } + return b +} + +func collectExplicitModuleIDs(datasetName, sourceLabel string, moduleData map[string]any) (map[string]int, error) { + ids := map[string]int{} + if rawEntries, ok := moduleData["entries"]; ok { + entries, ok := rawEntries.(map[string]any) + if !ok { + return nil, fmt.Errorf("dataset %s: entries in %s must be an object", datasetName, sourceLabel) + } + for key, rawEntry := range entries { + entry, ok := rawEntry.(map[string]any) + if !ok { + return nil, fmt.Errorf("dataset %s: entry %q in %s must be an object", datasetName, key, sourceLabel) + } + if rawID, ok := entry["id"]; ok { + rowID, err := asInt(rawID) + if err != nil { + return nil, fmt.Errorf("dataset %s: entry %q id %v is not numeric", datasetName, key, rawID) + } + ids[key] = rowID + } + } + } + if rawOverrides, ok := moduleData["overrides"]; ok { + switch overrides := rawOverrides.(type) { + case []any: + for index, rawOverride := range overrides { + override, ok := rawOverride.(map[string]any) + if !ok { + return nil, fmt.Errorf("dataset %s: override %d in %s must be an object", datasetName, index, sourceLabel) + } + if rawID, ok := override["id"]; ok { + rowID, err := asInt(rawID) + if err != nil { + return nil, fmt.Errorf("dataset %s: override id %v is not numeric", datasetName, rawID) + } + if key, ok := override["key"].(string); ok && key != "" { + ids[key] = rowID + } + } + } + case map[string]any: + for key, rawOverride := range overrides { + override, ok := rawOverride.(map[string]any) + if !ok { + return nil, fmt.Errorf("dataset %s: override %q in %s must be an object", datasetName, key, sourceLabel) + } + if rawID, ok := override["id"]; ok { + rowID, err := asInt(rawID) + if err != nil { + return nil, fmt.Errorf("dataset %s: override id %v is not numeric", datasetName, rawID) + } + ids[key] = rowID + } + } + default: + return nil, fmt.Errorf("dataset %s: overrides in %s must be an array or object", datasetName, sourceLabel) + } + } + return ids, nil +} + +func buildFeatGeneratedModule(path string, obj map[string]any, ctx *featGeneratedContext) (map[string]any, error) { + family, _ := obj["family"].(string) + if strings.TrimSpace(family) == "" { + return nil, fmt.Errorf("generated feat file %s is missing family", path) + } + if isFamilyExpansionObject(obj) { + return buildFamilyExpansionGeneratedModule(path, obj, ctx) + } + switch family { + case "special_attacks": + return buildLegacyFeatGeneratedModule(path, obj) + default: + return buildLegacyFeatGeneratedModule(path, obj) + } +} + +func buildLegacyFeatGeneratedModule(path string, obj map[string]any) (map[string]any, error) { + module := map[string]any{} + if rawEntries, ok := obj["entries"]; ok { + entries, ok := rawEntries.(map[string]any) + if !ok { + return nil, fmt.Errorf("generated feat file %s entries must be an object", path) + } + module["entries"] = entries + } + if rawOverrides, ok := obj["overrides"]; ok { + overrides, ok := rawOverrides.([]any) + if !ok { + return nil, fmt.Errorf("generated feat file %s overrides must be an array", path) + } + module["overrides"] = overrides + } + if len(module) == 0 { + return nil, fmt.Errorf("generated feat file %s must contain entries and/or overrides", path) + } + return module, nil +} + +func buildFamilyExpansionGeneratedModule(path string, obj map[string]any, ctx *featGeneratedContext) (map[string]any, error) { + spec, err := parseFamilyExpansionSpec(path, obj) + if err != nil { + return nil, err + } + overrides, err := parseCompactSourceOverrides(obj["overrides"]) + if err != nil { + return nil, fmt.Errorf("generated feat file %s: %w", path, err) + } + sourceRows, err := ctx.datasetRows(spec.ChildSource.Dataset) + if err != nil { + return nil, err + } + overrideList := make([]any, 0) + familyAllowlist := spec.AllowExistingOnly && ctx.familyHasExistingRows(spec.FamilyKey) + for _, sourceKey := range sortedStringMapKeys(sourceRows) { + row := sourceRows[sourceKey] + include, err := familyExpansionSourceEligible(spec, row) + if err != nil { + return nil, fmt.Errorf("generated feat file %s: %w", path, err) + } + if !include { + continue + } + slug := strings.TrimPrefix(sourceKey, spec.ChildSource.Dataset+":") + featKey, rowID, hasID, err := ctx.resolveGeneratedFeatIdentity(spec, slug, row) + if err != nil { + return nil, fmt.Errorf("generated feat file %s: %w", path, err) + } + childToken := childTokenFromExpandedKey(featKey, spec.FamilyKey) + displayName := ctx.displayNameForGeneratedSource(spec.ChildSource.Dataset, row) + titleChildName := applyFamilyExpansionTitleStyle(displayName, spec.TitleStyle) + labelSuffix := upperSnake(displayName) + override := map[string]any{ + "key": featKey, + "MASTERFEAT": map[string]any{"id": spec.Template}, + "meta": familyMetadata(spec.FamilyKey, childToken, sourceKey, spec.Template), + } + if hasID { + override["id"] = rowID + } + if familyAllowlist && !ctx.featKeyExists(featKey) { + continue + } + if spec.ChildRefField != "" { + override[spec.ChildRefField] = map[string]any{"id": sourceKey} + } + for field, prereqFamily := range spec.AutoPrereqFields { + if prereqKey, ok := generatedFamilyPrereqKey(ctx, row, slug, prereqFamily); ok { + override[field] = map[string]any{"id": prereqKey} + } + } + if spec.NamePrefix != "" { + override["FEAT"] = map[string]any{ + "tlk": map[string]any{ + "key": featKey + ".name", + "text": fmt.Sprintf("%s (%s)", spec.NamePrefix, titleChildName), + }, + } + } + if !ctx.featKeyExists(featKey) { + if spec.LabelPrefix != "" { + override["LABEL"] = spec.LabelPrefix + "_" + labelSuffix + } + for _, field := range spec.TemplateFields { + override[field] = map[string]any{"field": field, "ref": spec.Template} + } + if spec.ConstantPrefix != "" { + override["Constant"] = spec.ConstantPrefix + "_" + labelSuffix + } + } + for field, value := range spec.DefaultFields { + override[field] = value + } + mergeGeneratedOverride(override, overrides[sourceKey]) + overrideList = append(overrideList, override) + } + return map[string]any{"overrides": overrideList}, nil +} + +func buildRacialtypesSkillAffinityModule(ctx *featGeneratedContext) (map[string]any, error) { + racesDir := filepath.Join(ctx.dataDir, "racialtypes", "registry", "races") + entries, err := os.ReadDir(racesDir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, err + } + grants := map[string]*affinityGrant{} + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + continue + } + path := filepath.Join(racesDir, entry.Name()) + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + raceName := displayNameForRaceFile(obj) + rawFeats, ok := obj["feats"].([]any) + if !ok { + continue + } + for _, rawFeat := range rawFeats { + featObj, ok := rawFeat.(map[string]any) + if !ok { + continue + } + featKey, ok := extractRaceFeatKey(featObj) + if !ok || !isSkillAffinityFeatKey(featKey) { + continue + } + item := grants[featKey] + if item == nil { + item = &affinityGrant{featKey: featKey} + grants[featKey] = item + } + item.races = append(item.races, raceName) + } + } + if len(grants) == 0 { + return nil, nil + } + skillRows, err := ctx.datasetRows("skills") + if err != nil { + return nil, err + } + overrideList := make([]any, 0, len(grants)) + for _, featKey := range sortedGrantKeys(grants) { + grant := grants[featKey] + skillKey, bonus, err := parseSkillAffinityKey(featKey) + if err != nil { + return nil, err + } + skillKey = resolveAffinitySkillKey(skillRows, skillKey) + skillRow, ok := skillRows[skillKey] + skillName := affinitySkillDisplayName(featKey, skillKey) + if ok { + skillName = displayNameForSkill(skillRow) + } + rowID, hasID, err := ctx.featID(featKey, nil) + if err != nil { + return nil, err + } + races := dedupeAndSort(grant.races) + title := affinityTitleFromKey(featKey, skillName) + description := affinityDescription(skillName, bonus, races) + label := affinityLabel(featKey) + parent := "skillaffinity" + if bonus == 1 { + parent = "partialskillaffinity" + } + child := childTokenFromExpandedKey(featKey, parent) + override := map[string]any{ + "key": featKey, + "LABEL": label, + "FEAT": map[string]any{ + "tlk": map[string]any{ + "key": featKey + ".name", + "text": title, + }, + }, + "DESCRIPTION": map[string]any{ + "tlk": map[string]any{ + "key": featKey + ".description", + "text": description, + }, + }, + "ICON": "ife_racial", + "ALLCLASSESCANUSE": "0", + "TOOLSCATEGORIES": "6", + "PreReqEpic": "0", + "ReqAction": "1", + "Constant": label, + "meta": familyMetadata(parent, child, skillKey, ""), + } + if hasID { + override["id"] = rowID + } + overrideList = append(overrideList, override) + } + return map[string]any{"overrides": overrideList}, nil +} + +func affinitySkillDisplayName(featKey, skillKey string) string { + slug := strings.TrimPrefix(skillKey, "skills:") + if slug == "" { + slug = strings.TrimPrefix(strings.TrimPrefix(featKey, "feat:skillaffinity"), "feat:partialskillaffinity") + } + parts := strings.Fields(strings.ReplaceAll(strings.ReplaceAll(slug, "_", " "), "-", " ")) + for i, part := range parts { + if part == "" { + continue + } + parts[i] = strings.ToUpper(part[:1]) + part[1:] + } + return strings.Join(parts, " ") +} + +func requiredStringField(obj map[string]any, field string) (string, error) { + raw, ok := obj[field] + if !ok { + return "", fmt.Errorf("%s is required", field) + } + text, ok := raw.(string) + if !ok || strings.TrimSpace(text) == "" { + return "", fmt.Errorf("%s must be a non-empty string", field) + } + return strings.TrimSpace(text), nil +} + +func parseCompactSourceOverrides(raw any) (compactSourceOverride, error) { + if raw == nil { + return compactSourceOverride{}, nil + } + obj, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("overrides must be an object keyed by source dataset key") + } + result := make(compactSourceOverride, len(obj)) + for key, value := range obj { + item, ok := value.(map[string]any) + if !ok { + return nil, fmt.Errorf("override %q must be an object", key) + } + result[key] = item + } + return result, nil +} + +func sortedStringMapKeys[T any](items map[string]T) []string { + keys := make([]string, 0, len(items)) + for key := range items { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +func mergeGeneratedOverride(dst map[string]any, src map[string]any) { + if len(src) == 0 { + return + } + if name, ok := src["name"].(string); ok && strings.TrimSpace(name) != "" { + key, _ := dst["key"].(string) + dst["FEAT"] = map[string]any{ + "tlk": map[string]any{ + "key": key + ".name", + "text": strings.TrimSpace(name), + }, + } + } + if description, ok := src["description_text"].(string); ok && strings.TrimSpace(description) != "" { + key, _ := dst["key"].(string) + dst["DESCRIPTION"] = map[string]any{ + "tlk": map[string]any{ + "key": key + ".description", + "text": strings.TrimSpace(description), + }, + } + } + if icon, ok := src["icon"]; ok { + dst["ICON"] = icon + } + for field, value := range src { + switch field { + case "name", "description_text", "icon": + continue + default: + dst[field] = value + } + } +} + +func isHiddenSkill(row map[string]any) bool { + value, ok := lookupField(row, "HideFromLevelUp") + if !ok { + return false + } + switch typed := value.(type) { + case string: + return strings.TrimSpace(typed) == "1" + case int: + return typed == 1 + case float64: + return typed == 1 + default: + return false + } +} + +func displayNameForSkill(row map[string]any) string { + if text, ok := displayTextFromValue(row["Name"]); ok { + return text + } + if label, ok := row["Label"].(string); ok && strings.TrimSpace(label) != "" { + return normalizeDisplayName(strings.TrimSpace(label)) + } + if key, ok := row["key"].(string); ok { + return displayNameFromSlug(strings.TrimPrefix(key, "skills:")) + } + return "Unknown Skill" +} + +func displayNameForBaseitem(row map[string]any) string { + if text, ok := displayTextFromValue(row["Name"]); ok { + return text + } + if label, ok := row["label"].(string); ok && strings.TrimSpace(label) != "" { + return normalizeDisplayName(strings.TrimSpace(label)) + } + if key, ok := row["key"].(string); ok { + return displayNameFromSlug(strings.TrimPrefix(key, "baseitems:")) + } + return "Unknown Weapon" +} + +func (c *featGeneratedContext) displayNameForGeneratedSource(dataset string, row map[string]any) string { + if text, ok := c.displayTextFromGeneratedValue(row["Name"]); ok { + return text + } + switch dataset { + case "skills": + return displayNameForSkill(row) + case "baseitems": + return displayNameForBaseitem(row) + default: + if text, ok := displayTextFromValue(row["Name"]); ok { + return text + } + if label, ok := row["Label"].(string); ok && strings.TrimSpace(label) != "" { + return normalizeDisplayName(strings.TrimSpace(label)) + } + if label, ok := row["label"].(string); ok && strings.TrimSpace(label) != "" { + return normalizeDisplayName(strings.TrimSpace(label)) + } + if key, ok := row["key"].(string); ok { + return displayNameFromSlug(strings.TrimPrefix(key, dataset+":")) + } + return "Unknown Entry" + } +} + +func (c *featGeneratedContext) displayTextFromGeneratedValue(value any) (string, bool) { + if text, ok := displayTextFromValue(value); ok { + return text, true + } + if c == nil || c.baseDialog == nil { + return "", false + } + key := "" + switch typed := value.(type) { + case string: + if !isNumericText(typed) { + return "", false + } + key = strings.TrimSpace(typed) + case int: + key = strconv.Itoa(typed) + case float64: + if typed != float64(int(typed)) { + return "", false + } + key = strconv.Itoa(int(typed)) + default: + return "", false + } + entry, ok := c.baseDialog.Entries[key] + if !ok || strings.TrimSpace(entry.Text) == "" { + return "", false + } + return normalizeDisplayName(strings.TrimSpace(entry.Text)), true +} + +func applyFamilyExpansionTitleStyle(displayName string, style familyExpansionTitleStyle) string { + if style.ChildParenthetical == "comma" { + displayName = flattenGeneratedTitleChildParenthetical(displayName) + } + if style.ChildCase == "lower" { + displayName = strings.ToLower(displayName) + } + return displayName +} + +func flattenGeneratedTitleChildParenthetical(text string) string { + text = strings.TrimSpace(text) + if !strings.HasSuffix(text, ")") || strings.Count(text, "(") != 1 || strings.Count(text, ")") != 1 { + return text + } + open := strings.LastIndex(text, "(") + if open <= 0 || open >= len(text)-2 { + return text + } + prefix := strings.TrimSpace(text[:open]) + suffix := strings.TrimSpace(strings.TrimSuffix(text[open+1:], ")")) + if prefix == "" || suffix == "" { + return text + } + return prefix + ", " + suffix +} + +func displayNameForRaceFile(obj map[string]any) string { + core, ok := obj["core"].(map[string]any) + if ok { + if text, ok := displayTextFromValue(core["Name"]); ok { + return text + } + if label, ok := core["Label"].(string); ok && strings.TrimSpace(label) != "" { + return normalizeDisplayName(strings.TrimSpace(label)) + } + } + if key, ok := obj["key"].(string); ok { + return displayNameFromSlug(strings.TrimPrefix(key, "racialtypes:")) + } + return "Unknown Race" +} + +func displayTextFromValue(value any) (string, bool) { + switch typed := value.(type) { + case string: + if strings.TrimSpace(typed) == "" || isNumericText(typed) || typed == nullValue { + return "", false + } + return normalizeDisplayName(strings.TrimSpace(typed)), true + case map[string]any: + if tlk, ok := typed["tlk"].(map[string]any); ok { + if text, ok := tlk["text"].(string); ok && strings.TrimSpace(text) != "" { + return strings.TrimSpace(text), true + } + } + } + return "", false +} + +func isNumericText(text string) bool { + if strings.TrimSpace(text) == "" { + return false + } + for _, r := range text { + if r < '0' || r > '9' { + return false + } + } + return true +} + +func displayNameFromSlug(slug string) string { + text := strings.TrimSpace(slug) + text = strings.TrimPrefix(text, "knowledge") + if strings.HasPrefix(slug, "knowledge") { + remainder := strings.TrimPrefix(slug, "knowledge") + if remainder == "" { + return "Knowledge" + } + return "Knowledge (" + normalizeDisplayName(remainder) + ")" + } + if strings.HasPrefix(slug, "craft_") { + remainder := strings.TrimPrefix(slug, "craft_") + return "Craft (" + normalizeDisplayName(remainder) + ")" + } + return normalizeDisplayName(text) +} + +func normalizeDisplayName(text string) string { + text = strings.ReplaceAll(text, "_", " ") + text = splitCamelText(text) + parts := strings.Fields(text) + for index, part := range parts { + lower := strings.ToLower(part) + switch lower { + case "of", "and", "vs": + parts[index] = lower + default: + runes := []rune(lower) + if len(runes) == 0 { + continue + } + runes[0] = []rune(strings.ToUpper(string(runes[0])))[0] + parts[index] = string(runes) + } + } + return strings.Join(parts, " ") +} + +func splitCamelText(text string) string { + var out []rune + var prev rune + for index, r := range text { + if index > 0 && isCamelBoundary(prev, r) { + out = append(out, ' ') + } + out = append(out, r) + prev = r + } + return string(out) +} + +func isCamelBoundary(prev, current rune) bool { + return prev >= 'a' && prev <= 'z' && current >= 'A' && current <= 'Z' +} + +func upperSnake(text string) string { + normalized := normalizeDisplayName(text) + normalized = strings.ReplaceAll(normalized, "(", " ") + normalized = strings.ReplaceAll(normalized, ")", " ") + normalized = strings.ReplaceAll(normalized, "-", " ") + normalized = strings.Join(strings.Fields(normalized), "_") + return strings.ToUpper(normalized) +} + +func isNullishValue(value any) bool { + switch typed := value.(type) { + case nil: + return true + case string: + return strings.TrimSpace(typed) == "" || typed == nullValue + default: + return false + } +} + +func featSourceID(value any) (int, error) { + if isNullishValue(value) { + return 0, nil + } + switch typed := value.(type) { + case int: + return typed, nil + case float64: + return int(typed), nil + case string: + return asInt(typed) + default: + return 0, fmt.Errorf("cannot derive feat id from %T", value) + } +} + +func (c *featGeneratedContext) resolveGeneratedFeatIdentityBySource(familyKey, slug string, rawSource any) (string, int, bool, error) { + if ref, ok := rawSource.(string); ok && strings.HasPrefix(ref, "feat:") { + return c.resolveGeneratedFeatIdentityFromRef(familyKey, ref) + } + if rawSource != nil && rawSource != nullValue { + if parsedID, err := featSourceID(rawSource); err == nil && parsedID > 0 { + if generatedKey, ok := c.generatedFeatKeyForID(familyKey, parsedID); ok { + return generatedKey, parsedID, true, nil + } + return c.preferredGeneratedFeatKey(familyKey, slug), parsedID, true, nil + } + } + if rawMap, ok := rawSource.(map[string]any); ok { + if ref, ok := rawMap["id"].(string); ok && strings.HasPrefix(ref, "feat:") { + return c.resolveGeneratedFeatIdentityFromRef(familyKey, ref) + } + } + featKey := c.preferredGeneratedFeatKey(familyKey, slug) + rowID, hasID, err := c.featID(featKey, rawSource) + return featKey, rowID, hasID, err +} + +func (c *featGeneratedContext) resolveGeneratedFeatIdentityFromRef(familyKey, ref string) (string, int, bool, error) { + if generatedKey, rowID, ok := c.generatedFeatKeyFromCanonicalAlias(ref); ok { + return generatedKey, rowID, true, nil + } + if rowID, hasID, err := c.featID(ref, nil); err != nil { + return "", 0, false, err + } else if hasID { + if generatedKey, ok := c.generatedFeatKeyForID(familyKey, rowID); ok { + return generatedKey, rowID, true, nil + } + } + rowID, hasID, err := c.featID(ref, nil) + return ref, rowID, hasID, err +} + +func (c *featGeneratedContext) resolveGeneratedFeatIdentity(spec familyExpansionSpec, slug string, sourceRow map[string]any) (string, int, bool, error) { + if spec.IdentitySource == "child_source_value" { + rawSource, ok := lookupField(sourceRow, spec.ChildSource.Column) + if !ok { + rawSource = nil + } + return c.resolveGeneratedFeatIdentityBySource(spec.FamilyKey, slug, rawSource) + } + featKey := c.preferredGeneratedFeatKey(spec.FamilyKey, slug) + if aliasKey, aliasID, ok := c.generatedFeatKeyFromSameFamilyChildAlias(spec.FamilyKey, slug); ok { + return aliasKey, aliasID, true, nil + } + if legacyKey, legacyID, ok := c.generatedFeatKeyFromLegacyAlias(spec.FamilyKey, slug); ok { + return legacyKey, legacyID, true, nil + } + rowID, hasID, err := c.featID(featKey, nil) + return featKey, rowID, hasID, err +} + +func familyExpansionSourceEligible(spec familyExpansionSpec, row map[string]any) (bool, error) { + if spec.ChildSource.Column != "" { + rawValue, ok := lookupField(row, spec.ChildSource.Column) + if !ok || isNullishValue(rawValue) { + return false, nil + } + } + switch spec.ChildSource.Predicate { + case "": + return true, nil + case "accessible": + return !isHiddenSkill(row), nil + default: + return false, fmt.Errorf("unsupported child_source.predicate %q", spec.ChildSource.Predicate) + } +} + +func generatedFamilyPrereqKey(ctx *featGeneratedContext, sourceRow map[string]any, slug, familyKey string) (string, bool) { + if familyKey == "" { + return "", false + } + spec, ok := ctx.familySpecs[familyKey] + if !ok { + return "", false + } + featKey, _, _, err := ctx.resolveGeneratedFeatIdentity(spec, slug, sourceRow) + if err != nil { + return "", false + } + return featKey, true +} + +type generatedFamilyAliases map[string][]string + +func collectGeneratedFamilyAliases(generatedDir string) (generatedFamilyAliases, error) { + aliases := generatedFamilyAliases{} + if generatedDir == "" { + return aliases, nil + } + paths, err := collectModulePaths(generatedDir) + if err != nil { + return nil, err + } + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + if !isFamilyExpansionObject(obj) { + continue + } + spec, err := parseFamilyExpansionSpec(path, obj) + if err != nil { + return nil, err + } + if len(spec.LegacyFamilyKeys) > 0 { + aliases[spec.FamilyKey] = spec.LegacyFamilyKeys + } + } + return aliases, nil +} + +func configuredGeneratedFamilyLegacyAliases(familyKey string, aliases generatedFamilyAliases) []string { + normalized := normalizeKeyIdentity(familyKey) + for configuredFamilyKey, configured := range aliases { + if normalizeKeyIdentity(configuredFamilyKey) == normalized && len(configured) > 0 { + return configured + } + } + return compatibilityGeneratedFamilyLegacyAliases(familyKey) +} + +func compatibilityGeneratedFamilyLegacyAliases(familyKey string) []string { + switch normalizeKeyIdentity(familyKey) { + case "skillfocus": + return []string{"skillfocus"} + case "greaterskillfocus": + return []string{"epicskillfocus"} + case "greaterweaponfocus": + return []string{"epicweaponfocus"} + case "greaterweaponspecialization": + return []string{"epicweaponspecialization"} + case "overwhelmingcritical": + return []string{"epicoverwhelmingcritical"} + default: + return nil + } +} + +func knownGeneratedFamilyKeys(aliases generatedFamilyAliases) []string { + seen := map[string]struct{}{} + keys := make([]string, 0, 32) + add := func(key string) { + key = strings.TrimSpace(key) + if key == "" { + return + } + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + keys = append(keys, key) + } + for _, key := range []string{ + "greater_weapon_specialization", + "epic_weapon_specialization", + "greaterweaponspecialization", + "epicweaponspecialization", + "overwhelming_critical", + "epic_overwhelming_critical", + "overwhelmingcritical", + "epicoverwhelmingcritical", + "greater_weapon_focus", + "epic_weapon_focus", + "greaterweaponfocus", + "epicweaponfocus", + "greater_skill_focus", + "epic_skill_focus", + "greaterskillfocus", + "epicskillfocus", + "weapon_specialization", + "weaponspecialization", + "improved_critical", + "improvedcritical", + "weapon_of_choice", + "weaponofchoice", + "weapon_focus", + "weaponfocus", + "skill_focus", + "skillfocus", + } { + add(key) + } + for familyKey, legacyKeys := range aliases { + add(familyKey) + for _, legacyKey := range legacyKeys { + add(legacyKey) + } + } + slices.SortFunc(keys, func(a, b string) int { + if len(a) == len(b) { + return strings.Compare(a, b) + } + return len(b) - len(a) + }) + return keys +} + +func splitKnownGeneratedFamilyIdentity(text string, aliases generatedFamilyAliases) familyIdentity { + text = strings.TrimPrefix(strings.TrimSpace(text), "feat:") + for _, parent := range knownGeneratedFamilyKeys(aliases) { + if text == parent { + return familyIdentity{Parent: parent} + } + prefix := parent + "_" + if strings.HasPrefix(text, prefix) { + return familyIdentity{ + Parent: parent, + Child: strings.TrimPrefix(text, prefix), + } + } + } + return splitFamilyExpansionIdentity(text) +} + +func generatedAliasLockForKey(canonicalKey, lockedKey string, aliases generatedFamilyAliases) bool { + canonicalKey = strings.TrimPrefix(strings.TrimSpace(canonicalKey), "feat:") + lockedKey = strings.TrimPrefix(strings.TrimSpace(lockedKey), "feat:") + if canonicalKey == "" || lockedKey == "" { + return false + } + identity := splitKnownGeneratedFamilyIdentity(canonicalKey, aliases) + if identity.Parent == "" || identity.Child == "" { + return false + } + legacyChildren := legacyFamilyChildAliases(identity.Parent, identity.Child) + parentPrefix := identity.Parent + "_" + if strings.HasPrefix(lockedKey, parentPrefix) { + lockedChild := strings.TrimPrefix(lockedKey, parentPrefix) + for _, legacyChild := range legacyChildren { + if normalizeKeyIdentity(lockedChild) == normalizeKeyIdentity(legacyChild) { + return true + } + } + } + for _, alias := range configuredGeneratedFamilyLegacyAliases(identity.Parent, aliases) { + prefix := alias + "_" + if !strings.HasPrefix(lockedKey, prefix) { + continue + } + lockedChild := strings.TrimPrefix(lockedKey, prefix) + for _, legacyChild := range legacyChildren { + if normalizeKeyIdentity(lockedChild) == normalizeKeyIdentity(legacyChild) { + return true + } + } + } + return false +} + +func generatedCanonicalLockCanMove(canonicalKey string, rowID int, lockData map[string]int, aliases generatedFamilyAliases) bool { + if rowID <= 0 { + return false + } + for lockedKey, lockedID := range lockData { + if lockedID == rowID && generatedAliasLockForKey(canonicalKey, lockedKey, aliases) { + return true + } + } + return false +} + +func (c *featGeneratedContext) generatedFamilyAliases() generatedFamilyAliases { + aliases := generatedFamilyAliases{} + for familyKey, spec := range c.familySpecs { + if len(spec.LegacyFamilyKeys) > 0 { + aliases[familyKey] = spec.LegacyFamilyKeys + } + } + return aliases +} + +func (c *featGeneratedContext) generatedFamilyLegacyAliases(familyKey string) []string { + normalized := normalizeKeyIdentity(familyKey) + for specFamilyKey, spec := range c.familySpecs { + if normalizeKeyIdentity(specFamilyKey) == normalized && len(spec.LegacyFamilyKeys) > 0 { + return spec.LegacyFamilyKeys + } + } + return compatibilityGeneratedFamilyLegacyAliases(familyKey) +} + +func (c *featGeneratedContext) generatedFeatKeyForID(familyKey string, rowID int) (string, bool) { + if rowID <= 0 { + return "", false + } + for _, candidates := range []map[string]int{c.lockData, c.supplementalID} { + matches := make([]string, 0) + prefix := "feat:" + familyKey + "_" + for key, candidateID := range candidates { + if candidateID == rowID && strings.HasPrefix(key, prefix) { + matches = append(matches, key) + } + } + if best, ok := preferredExpandedKey(matches); ok { + return best, true + } + } + for _, alias := range c.generatedFamilyLegacyAliases(familyKey) { + for _, candidates := range []map[string]int{c.lockData, c.supplementalID} { + matches := make([]string, 0) + prefix := "feat:" + alias + "_" + for key, candidateID := range candidates { + if candidateID != rowID || !strings.HasPrefix(key, prefix) { + continue + } + child := strings.TrimPrefix(key, prefix) + matches = append(matches, "feat:"+familyKey+"_"+child) + } + if best, ok := preferredExpandedKey(matches); ok { + return best, true + } + } + } + return "", false +} + +func (c *featGeneratedContext) generatedFeatKeyFromSameFamilyChildAlias(familyKey, sourceSlug string) (string, int, bool) { + targets := legacyFamilyChildAliases(familyKey, sourceSlug) + canonicalKey := c.preferredGeneratedFeatKey(familyKey, sourceSlug) + prefix := "feat:" + familyKey + "_" + normalizedSource := normalizeKeyIdentity(sourceSlug) + for _, candidates := range []map[string]int{c.lockData, c.supplementalID} { + for _, target := range targets { + if normalizeKeyIdentity(target) == normalizedSource { + continue + } + for key, rowID := range candidates { + if rowID <= 0 || !strings.HasPrefix(key, prefix) { + continue + } + child := strings.TrimPrefix(key, prefix) + if normalizeKeyIdentity(child) == normalizeKeyIdentity(target) { + return canonicalKey, rowID, true + } + } + } + } + return "", 0, false +} + +func (c *featGeneratedContext) generatedFeatKeyFromLegacyAlias(familyKey, sourceSlug string) (string, int, bool) { + targets := legacyFamilyChildAliases(familyKey, sourceSlug) + canonicalKey := c.preferredGeneratedFeatKey(familyKey, sourceSlug) + for _, alias := range c.generatedFamilyLegacyAliases(familyKey) { + for _, candidates := range []map[string]int{c.lockData, c.supplementalID} { + prefix := "feat:" + alias + "_" + for _, target := range targets { + for key, rowID := range candidates { + if rowID <= 0 || !strings.HasPrefix(key, prefix) { + continue + } + child := strings.TrimPrefix(key, prefix) + if normalizeKeyIdentity(child) == normalizeKeyIdentity(target) { + return canonicalKey, rowID, true + } + } + } + } + } + return "", 0, false +} + +func legacyFamilyChildAliases(familyKey, sourceSlug string) []string { + aliases := []string{} + normalizedFamilyKey := normalizeKeyIdentity(familyKey) + if normalizedFamilyKey != "greaterskillfocus" && normalizedFamilyKey != "skillfocus" { + return []string{sourceSlug} + } + switch normalizeKeyIdentity(sourceSlug) { + case "acrobatics": + aliases = append(aliases, "tumble") + case "animalhandling": + aliases = append(aliases, "animalempathy") + case "craftarmorsmithing": + aliases = append(aliases, "craftarmor") + case "crafttinkering": + aliases = append(aliases, "settrap") + case "craftweaponsmithing": + aliases = append(aliases, "craftweapon") + case "craftwoodworking": + aliases = append(aliases, "crafttrap") + case "disabledevice": + aliases = append(aliases, "disabletrap") + case "influence": + aliases = append(aliases, "persuade") + case "sleightofhand": + aliases = append(aliases, "pickpocket") + } + aliases = append(aliases, sourceSlug) + return aliases +} + +func (c *featGeneratedContext) generatedFeatKeyFromCanonicalAlias(canonicalKey string) (string, int, bool) { + stripped := strings.TrimPrefix(strings.TrimSpace(canonicalKey), "feat:") + identity := splitKnownGeneratedFamilyIdentity(stripped, c.generatedFamilyAliases()) + if identity.Parent == "" || identity.Child == "" { + return "", 0, false + } + legacyChildren := legacyFamilyChildAliases(identity.Parent, identity.Child) + for _, alias := range c.generatedFamilyLegacyAliases(identity.Parent) { + for _, legacyChild := range legacyChildren { + if _, rowID, ok := c.featKeyForFamilyChild(alias, legacyChild); ok { + return "feat:" + identity.Parent + "_" + identity.Child, rowID, true + } + } + } + return "", 0, false +} + +func (c *featGeneratedContext) featKeyForFamilyChild(familyKey, child string) (string, int, bool) { + key := "feat:" + familyKey + "_" + child + for _, candidates := range []map[string]int{c.lockData, c.supplementalID} { + if rowID, ok := candidates[key]; ok && rowID > 0 { + return key, rowID, true + } + } + return "", 0, false +} + +func (c *featGeneratedContext) preferredFeatKeyForID(familyKey string, rowID int) (string, bool) { + prefix := "feat:" + familyKey + type candidateBuckets struct { + familyExisting []string + otherExisting []string + familyLegacy []string + otherLegacy []string + familyOther []string + otherOther []string + } + + buckets := candidateBuckets{} + appendMatches := func(candidates map[string]int, legacy bool) { + for key, candidateID := range candidates { + if candidateID != rowID { + continue + } + inFamily := strings.HasPrefix(key, prefix) + exists := c.featKeyExists(key) + switch { + case exists && inFamily: + buckets.familyExisting = append(buckets.familyExisting, key) + case exists: + buckets.otherExisting = append(buckets.otherExisting, key) + case legacy && inFamily: + buckets.familyLegacy = append(buckets.familyLegacy, key) + case legacy: + buckets.otherLegacy = append(buckets.otherLegacy, key) + case inFamily: + buckets.familyOther = append(buckets.familyOther, key) + default: + buckets.otherOther = append(buckets.otherOther, key) + } + } + } + + appendMatches(c.lockData, false) + appendMatches(c.supplementalID, true) + + for _, group := range [][]string{ + buckets.familyExisting, + buckets.otherExisting, + buckets.otherLegacy, + buckets.familyLegacy, + buckets.familyOther, + buckets.otherOther, + } { + if best, ok := preferredExpandedKey(group); ok { + return best, true + } + } + return "", false +} + +func extractRaceFeatKey(obj map[string]any) (string, bool) { + value, ok := obj["FeatIndex"].(map[string]any) + if !ok { + return "", false + } + key, ok := value["id"].(string) + if !ok || !strings.HasPrefix(key, "feat:") { + return "", false + } + return key, true +} + +func isSkillAffinityFeatKey(key string) bool { + return strings.HasPrefix(key, "feat:skillaffinity") || strings.HasPrefix(key, "feat:partialskillaffinity") +} + +func parseSkillAffinityKey(featKey string) (string, int, error) { + switch { + case strings.HasPrefix(featKey, "feat:partialskillaffinity"): + return "skills:" + strings.TrimPrefix(featKey, "feat:partialskillaffinity"), 1, nil + case strings.HasPrefix(featKey, "feat:skillaffinity"): + return "skills:" + strings.TrimPrefix(featKey, "feat:skillaffinity"), 2, nil + default: + return "", 0, fmt.Errorf("unsupported skill affinity key %q", featKey) + } +} + +func resolveAffinitySkillKey(skillRows map[string]map[string]any, skillKey string) string { + if _, ok := skillRows[skillKey]; ok { + return skillKey + } + target := normalizeKeyIdentity(strings.TrimPrefix(skillKey, "skills:")) + for _, key := range sortedStringMapKeys(skillRows) { + if normalizeKeyIdentity(strings.TrimPrefix(key, "skills:")) == target { + return key + } + } + return skillKey +} + +func normalizeKeyIdentity(text string) string { + text = strings.ToLower(strings.TrimSpace(text)) + text = strings.ReplaceAll(text, "_", "") + text = strings.ReplaceAll(text, " ", "") + return text +} + +func resolveCanonicalDatasetKey(key string, keyToID map[string]int) string { + key = strings.TrimSpace(key) + if key == "" { + return key + } + if _, ok := keyToID[key]; ok { + return key + } + prefix, suffix, ok := strings.Cut(key, ":") + if !ok || prefix == "" || suffix == "" { + return key + } + normalizedSuffix := normalizeKeyIdentity(suffix) + match := "" + for candidate := range keyToID { + candidatePrefix, candidateSuffix, ok := strings.Cut(candidate, ":") + if !ok || candidatePrefix != prefix { + continue + } + if normalizeKeyIdentity(candidateSuffix) != normalizedSuffix { + continue + } + if match == "" || candidate < match { + match = candidate + } + } + if match != "" { + return match + } + return key +} + +func affinityTitleFromKey(featKey, skillName string) string { + if strings.HasPrefix(featKey, "feat:partialskillaffinity") { + return fmt.Sprintf("Partial Skill Affinity (%s)", skillName) + } + return fmt.Sprintf("Skill Affinity (%s)", skillName) +} + +func affinityDescription(skillName string, bonus int, races []string) string { + return fmt.Sprintf("Gain a +%d racial bonus to %s checks. Prerequisites: %s.", bonus, skillName, strings.Join(races, ", ")) +} + +func affinityLabel(featKey string) string { + skillKey, _, err := parseSkillAffinityKey(featKey) + if err != nil { + return "FEAT_UNKNOWN_SKILL_AFFINITY" + } + skillSlug := strings.TrimPrefix(skillKey, "skills:") + if strings.HasPrefix(featKey, "feat:partialskillaffinity") { + return "FEAT_PARTIAL_SKILL_AFFINITY_" + upperSnake(displayNameFromSlug(skillSlug)) + } + return "FEAT_SKILL_AFFINITY_" + upperSnake(displayNameFromSlug(skillSlug)) +} + +func dedupeAndSort(items []string) []string { + seen := map[string]struct{}{} + out := make([]string, 0, len(items)) + for _, item := range items { + item = strings.TrimSpace(item) + if item == "" { + continue + } + if _, ok := seen[item]; ok { + continue + } + seen[item] = struct{}{} + out = append(out, item) + } + slices.Sort(out) + return out +} + +func sortedGrantKeys(items map[string]*affinityGrant) []string { + keys := make([]string, 0, len(items)) + for key := range items { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +func collectPlainDataset(dataset nativeDataset) (nativeCollectedDataset, error) { + tableData, err := loadJSONObject(dataset.BasePath) + if err != nil { + return nativeCollectedDataset{}, err + } + globalData, err := loadGlobalModules(dataset.GlobalPaths) + if err != nil { + return nativeCollectedDataset{}, err + } + for _, module := range globalData { + if _, ok := module.Data["defaults"]; ok { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: defaults in %s are only supported for base datasets", dataset.Name, module.Name) + } + } + + columns, err := parseColumns(tableData, dataset.Name) + if err != nil { + return nativeCollectedDataset{}, err + } + for _, module := range globalData { + columns, err = extendColumns(columns, module.Data, dataset.Name, module.Name) + if err != nil { + return nativeCollectedDataset{}, err + } + } + rawRows, ok := tableData["rows"].([]any) + if !ok { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: rows must be an array", dataset.Name) + } + + lockData := map[string]int{} + if dataset.LockPath != "" { + var err error + lockData, err = loadLockfile(dataset.LockPath) + if err != nil { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + } + usedIDs := map[int]struct{}{} + for _, rowID := range lockData { + usedIDs[rowID] = struct{}{} + } + + rows := make([]map[string]any, 0, len(rawRows)) + explicitID := make([]bool, 0, len(rawRows)) + for index, raw := range rawRows { + rowObj, ok := raw.(map[string]any) + if !ok { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d must be an object", dataset.Name, index) + } + row, err := canonicalizeBaseRow(dataset, columns, rowObj, index) + if err != nil { + return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + rows = append(rows, row) + _, hasExplicitID := rowObj["id"] + explicitID = append(explicitID, hasExplicitID) + if hasExplicitID { + usedIDs[row["id"].(int)] = struct{}{} + } + } + + rows, explicitID, dataset.HasGlobalInjections, err = applyPlainDatasetGlobalInjections(dataset, columns, rows, explicitID, globalData) + if err != nil { + return nativeCollectedDataset{}, err + } + + nextID := nextAvailableID(usedIDs) + for index, row := range rows { + key, hasKey := row["key"].(string) + if hasKey && key != "" { + if lockedID, ok := lockData[key]; ok { + row["id"] = lockedID + continue + } + if explicitID[index] { + lockData[key] = row["id"].(int) + } else { + row["id"] = nextID + lockData[key] = nextID + usedIDs[nextID] = struct{}{} + nextID = nextAvailableID(usedIDs) + } + continue + } + if !explicitID[index] { + row["id"] = index + } + } + + slices.SortFunc(rows, func(a, b map[string]any) int { + return a["id"].(int) - b["id"].(int) + }) + tableKey, _ := tableData["key"].(string) + return nativeCollectedDataset{ + Dataset: dataset, + Columns: columns, + Rows: rows, + LockData: lockData, + TableKey: tableKey, + }, nil +} + +func loadGlobalModules(paths []string) ([]nativeGeneratedModule, error) { + modules := make([]nativeGeneratedModule, 0, len(paths)) + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + modules = append(modules, nativeGeneratedModule{Name: path, Data: obj}) + } + return modules, nil +} + +func applyPlainDatasetGlobalInjections(dataset nativeDataset, columns []string, rows []map[string]any, explicitID []bool, modules []nativeGeneratedModule) ([]map[string]any, []bool, bool, error) { + if len(modules) == 0 { + return rows, explicitID, false, nil + } + prefixRows := []map[string]any{} + prefixExplicit := []bool{} + suffixRows := []map[string]any{} + suffixExplicit := []bool{} + hasInjections := false + + currentRows := func() []map[string]any { + out := make([]map[string]any, 0, len(prefixRows)+len(rows)+len(suffixRows)) + out = append(out, prefixRows...) + out = append(out, rows...) + out = append(out, suffixRows...) + return out + } + + for _, module := range modules { + rawInjections, ok := module.Data["injections"] + if !ok { + continue + } + injections, ok := rawInjections.([]any) + if !ok { + return nil, nil, false, fmt.Errorf("dataset %s: injections in %s must be an array", dataset.Name, module.Name) + } + position, err := globalManifestPosition(module) + if err != nil { + return nil, nil, false, fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + for index, rawInjection := range injections { + injection, ok := rawInjection.(map[string]any) + if !ok { + return nil, nil, false, fmt.Errorf("dataset %s: injection %d in %s must be an object", dataset.Name, index, module.Name) + } + rawRow, ok := injection["row"].(map[string]any) + if !ok { + return nil, nil, false, fmt.Errorf("dataset %s: injection %d in %s must contain row object", dataset.Name, index, module.Name) + } + presentRows := currentRows() + matches, err := globalInjectionConditionsMatch(injection, presentRows) + if err != nil { + return nil, nil, false, fmt.Errorf("dataset %s: injection %d in %s: %w", dataset.Name, index, module.Name, err) + } + if !matches { + continue + } + if globalRowIdentityPresent(rawRow, presentRows) { + continue + } + row, err := canonicalizeBaseRow(dataset, columns, rawRow, len(presentRows)) + if err != nil { + return nil, nil, false, fmt.Errorf("dataset %s: injection %d in %s: %w", dataset.Name, index, module.Name, err) + } + explicit := false + if _, ok := rawRow["id"]; ok { + explicit = true + } + if position == "prepend" { + prefixRows = append(prefixRows, row) + prefixExplicit = append(prefixExplicit, explicit) + } else { + suffixRows = append(suffixRows, row) + suffixExplicit = append(suffixExplicit, explicit) + } + hasInjections = true + } + } + + outRows := make([]map[string]any, 0, len(prefixRows)+len(rows)+len(suffixRows)) + outExplicit := make([]bool, 0, len(prefixExplicit)+len(explicitID)+len(suffixExplicit)) + outRows = append(outRows, prefixRows...) + outExplicit = append(outExplicit, prefixExplicit...) + outRows = append(outRows, rows...) + outExplicit = append(outExplicit, explicitID...) + outRows = append(outRows, suffixRows...) + outExplicit = append(outExplicit, suffixExplicit...) + return outRows, outExplicit, hasInjections, nil +} + +func globalManifestPosition(module nativeGeneratedModule) (string, error) { + raw, ok := module.Data["position"] + if !ok || raw == nil { + return "append", nil + } + position, ok := raw.(string) + if !ok { + return "", fmt.Errorf("position in %s must be a string", module.Name) + } + switch strings.ToLower(strings.TrimSpace(position)) { + case "", "append": + return "append", nil + case "prepend": + return "prepend", nil + default: + return "", fmt.Errorf("position in %s must be append or prepend", module.Name) + } +} + +func globalInjectionConditionsMatch(injection map[string]any, rows []map[string]any) (bool, error) { + if rawRequired, ok := injection["require_present"]; ok { + matches, err := globalConditionListMatches("require_present", rawRequired, rows, true) + if err != nil || !matches { + return matches, err + } + } + if rawBlocked, ok := injection["unless_present"]; ok { + matches, err := globalConditionListMatches("unless_present", rawBlocked, rows, false) + if err != nil || !matches { + return matches, err + } + } + return true, nil +} + +func globalConditionListMatches(name string, raw any, rows []map[string]any, required bool) (bool, error) { + conditions, ok := raw.([]any) + if !ok { + return false, fmt.Errorf("%s must be an array", name) + } + for index, rawCondition := range conditions { + condition, ok := rawCondition.(map[string]any) + if !ok { + return false, fmt.Errorf("%s[%d] must be an object", name, index) + } + field, _ := condition["field"].(string) + id, _ := condition["id"].(string) + if strings.TrimSpace(field) == "" { + return false, fmt.Errorf("%s[%d].field is required", name, index) + } + if strings.TrimSpace(id) == "" { + return false, fmt.Errorf("%s[%d].id is required", name, index) + } + found := globalReferencePresent(rows, field, id) + if required && !found { + return false, nil + } + if !required && found { + return false, nil + } + } + return true, nil +} + +func globalRowIdentityPresent(row map[string]any, rows []map[string]any) bool { + if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" { + normalized := normalizeGlobalReferenceID(key) + for _, existing := range rows { + if existingKey, ok := existing["key"].(string); ok && normalizeGlobalReferenceID(existingKey) == normalized { + return true + } + } + return false + } + for field, value := range row { + ref, ok := value.(map[string]any) + if !ok { + continue + } + id, _ := ref["id"].(string) + if strings.TrimSpace(id) == "" { + continue + } + if globalReferencePresent(rows, field, id) { + return true + } + } + return false +} + +func globalReferencePresent(rows []map[string]any, field, id string) bool { + field = strings.TrimSpace(field) + id = normalizeGlobalReferenceID(id) + if field == "" || id == "" { + return false + } + for _, row := range rows { + value, ok := row[field] + if !ok { + if canonical, exists := canonicalColumn(sortedStringMapKeys(row), field); exists { + value = row[canonical] + ok = true + } + } + if !ok { + continue + } + ref, ok := value.(map[string]any) + if !ok { + continue + } + existingID, _ := ref["id"].(string) + if normalizeGlobalReferenceID(existingID) == id { + return true + } + } + return false +} + +func normalizeGlobalReferenceID(value string) string { + trimmed := strings.ToLower(strings.TrimSpace(value)) + if trimmed == "" { + return "" + } + parts := strings.SplitN(trimmed, ":", 2) + if len(parts) != 2 { + return strings.ReplaceAll(trimmed, "_", "") + } + return parts[0] + ":" + strings.ReplaceAll(parts[1], "_", "") +} + +func resolveNativeDataset(dataset nativeCollectedDataset, keyToID map[string]int, globalRowByKey map[string]map[string]any, tableRegistry resolvedTableRegistry, compiler *tlkCompiler, classFeatInjections project.TopDataClassFeatInjectionConfig, sidecars *nativeSidecarCollector) (map[string]any, error) { + rows := dataset.Rows + if strings.HasPrefix(filepath.ToSlash(dataset.Dataset.Name), "classes/feats/") { + classKey := "classes:" + dataset.Dataset.Name[strings.LastIndex(dataset.Dataset.Name, "/")+1:] + featSuccessors := buildFeatSuccessorsIndex(globalRowByKey, keyToID) + classSkills := buildClassSkillsIndex(tableRegistry, classKey) + expanded, err := expandClassesFeatRows(rows, keyToID, globalRowByKey, featSuccessors, classSkills, globalRowByKey, classKey, classFeatInjections, !dataset.Dataset.HasGlobalInjections) + if err != nil { + return nil, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err) + } + rows = expanded + } + + dataset.Dataset.Columns = dataset.Columns + resolver := valueResolver{ + dataset: dataset.Dataset, + keyToID: keyToID, + rowByKey: globalRowByKey, + tableRegistry: tableRegistry, + compiler: compiler, + sidecars: sidecars, + cache: map[string]map[string]tlkCompiledValue{}, + } + + resolvedRows := make([]map[string]any, 0, len(rows)) + for _, row := range rows { + resolved, err := resolver.resolveRow(row, nil) + if err != nil { + return nil, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err) + } + resolvedRows = append(resolvedRows, resolved) + } + return map[string]any{ + "columns": dataset.Columns, + "rows": resolvedRows, + }, nil +} + +var ( + defaultClassFeatGlobalRules = []project.TopDataClassFeatGlobalRule{ + {Feat: "feat:combatexpertise", List: "0", GrantedOnLevel: "-1", OnMenu: "1"}, + {Feat: "feat:powerattack", List: "0", GrantedOnLevel: "-1", OnMenu: "1"}, + {Feat: "feat:specialattacks", List: "3", GrantedOnLevel: "1", OnMenu: "1"}, + {Feat: "feat:throw", List: "3", GrantedOnLevel: "1", OnMenu: "1"}, + {Feat: "feat:grapple", List: "3", GrantedOnLevel: "1", OnMenu: "1"}, + {Feat: "feat:offensivefighting", List: "3", GrantedOnLevel: "1", OnMenu: "1"}, + {Feat: "feat:defensivefighting", List: "3", GrantedOnLevel: "1", OnMenu: "1"}, + {Feat: "feat:horsemenu", List: "3", GrantedOnLevel: "1", OnMenu: "1"}, + } + defaultClassFeatClassSkillMasterfeatRules = []project.TopDataClassFeatMasterfeatRule{ + {Masterfeat: "masterfeats:skill_focus", List: "1", GrantedOnLevel: "-1", OnMenu: "0"}, + {Masterfeat: "masterfeats:greater_skill_focus", List: "1", GrantedOnLevel: "12", OnMenu: "0"}, + } +) + +func expandClassesFeatRows(rows []map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, featSuccessors map[string]string, classSkills map[string]bool, allRowByKey map[string]map[string]any, classKey string, classFeatInjections project.TopDataClassFeatInjectionConfig, useConfiguredInjections bool) ([]map[string]any, error) { + + globalRules, classSkillRules := []project.TopDataClassFeatGlobalRule{}, []project.TopDataClassFeatMasterfeatRule{} + if useConfiguredInjections { + globalRules, classSkillRules = effectiveClassFeatInjectionRules(classFeatInjections) + } + injected := make([]map[string]any, 0, len(globalRules)+len(classSkillRules)) + presentRefIDs := make(map[string]struct{}, len(rows)) + for _, row := range rows { + featRef, ok := row["FeatIndex"].(map[string]any) + if !ok { + continue + } + refID, _ := featRef["id"].(string) + if refID == "" { + continue + } + presentRefIDs[refID] = struct{}{} + if strings.HasPrefix(refID, "feat:") || strings.HasPrefix(refID, "masterfeats:") { + presentRefIDs[resolveCanonicalDatasetKey(refID, keyToID)] = struct{}{} + } + } + + for _, rule := range globalRules { + featKey := resolveCanonicalDatasetKey(strings.TrimSpace(rule.Feat), keyToID) + if _, exists := presentRefIDs[featKey]; exists { + continue + } + if !classFeatInjectionConditionsMatch(rule, presentRefIDs, keyToID) { + continue + } + if _, exists := allRowByKey[featKey]; !exists { + continue + } + cloned := classFeatGlobalRuleRow(rule, featKey) + if label := lookupReferencedFeatLabel(featKey, allRowByKey); label != "" { + cloned["FeatLabel"] = label + } + injected = append(injected, cloned) + presentRefIDs[featKey] = struct{}{} + } + + hasClassSkills := classSkills != nil && len(classSkills) > 0 + for _, rule := range classSkillRules { + if !hasClassSkills { + continue + } + refID := strings.TrimSpace(rule.Masterfeat) + canonicalRefID := resolveCanonicalDatasetKey(refID, keyToID) + if _, exists := presentRefIDs[canonicalRefID]; exists { + continue + } + row := classFeatMasterfeatRuleRow(rule) + rowItems, err := expandClassesFeatRow(row, keyToID, rowByKey, featSuccessors, classSkills, classKey) + if err != nil { + return nil, err + } + if len(rowItems) > 0 { + injected = append(injected, rowItems...) + presentRefIDs[refID] = struct{}{} + presentRefIDs[canonicalRefID] = struct{}{} + } + } + + expanded := make([]map[string]any, 0, len(rows)) + for _, row := range rows { + rowItems, err := expandClassesFeatRow(row, keyToID, rowByKey, featSuccessors, classSkills, classKey) + if err != nil { + return nil, err + } + expanded = append(expanded, rowItems...) + } + + combined := make([]map[string]any, 0, len(injected)+len(expanded)) + combined = append(combined, injected...) + combined = append(combined, expanded...) + + for index, row := range combined { + row["id"] = index + } + return combined, nil +} + +func effectiveClassFeatInjectionRules(config project.TopDataClassFeatInjectionConfig) ([]project.TopDataClassFeatGlobalRule, []project.TopDataClassFeatMasterfeatRule) { + if len(config.GlobalFeats) == 0 && len(config.ClassSkillMasterfeats) == 0 { + return defaultClassFeatGlobalRules, defaultClassFeatClassSkillMasterfeatRules + } + return config.GlobalFeats, config.ClassSkillMasterfeats +} + +func classFeatInjectionConditionsMatch(rule project.TopDataClassFeatGlobalRule, presentRefIDs map[string]struct{}, keyToID map[string]int) bool { + for _, required := range rule.RequirePresent { + refID := resolveCanonicalDatasetKey(strings.TrimSpace(required), keyToID) + if _, exists := presentRefIDs[refID]; !exists { + return false + } + } + for _, blocked := range rule.UnlessPresent { + refID := resolveCanonicalDatasetKey(strings.TrimSpace(blocked), keyToID) + if _, exists := presentRefIDs[refID]; exists { + return false + } + } + return true +} + +func classFeatGlobalRuleRow(rule project.TopDataClassFeatGlobalRule, featKey string) map[string]any { + return map[string]any{ + "FeatIndex": map[string]any{"id": featKey}, + "List": strings.TrimSpace(rule.List), + "GrantedOnLevel": strings.TrimSpace(rule.GrantedOnLevel), + "OnMenu": strings.TrimSpace(rule.OnMenu), + } +} + +func classFeatMasterfeatRuleRow(rule project.TopDataClassFeatMasterfeatRule) map[string]any { + return map[string]any{ + "FeatIndex": map[string]any{"id": strings.TrimSpace(rule.Masterfeat), "filter": "classskills"}, + "List": strings.TrimSpace(rule.List), + "GrantedOnLevel": strings.TrimSpace(rule.GrantedOnLevel), + "OnMenu": strings.TrimSpace(rule.OnMenu), + } +} + +func expandClassesFeatRow(row map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, featSuccessors map[string]string, classSkills map[string]bool, classKey string) ([]map[string]any, error) { + featRef, ok := row["FeatIndex"].(map[string]any) + if !ok { + return []map[string]any{row}, nil + } + refKey, _ := featRef["id"].(string) + filterMode, _ := featRef["filter"].(string) + + switch { + case strings.HasPrefix(refKey, "feat:"): + if filterMode == "successors" { + return expandFeatSuccessors(refKey, row, featSuccessors, rowByKey) + } + cloned := deepCopyValue(row).(map[string]any) + if label := lookupReferencedFeatLabel(refKey, rowByKey); label != "" && isNullLikeValue(cloned["FeatLabel"]) { + cloned["FeatLabel"] = label + } + return []map[string]any{cloned}, nil + case strings.HasPrefix(refKey, "masterfeats:"): + masterfeatKey := resolveCanonicalDatasetKey(refKey, keyToID) + children, err := expandMasterfeatChildren(masterfeatKey, row, keyToID, rowByKey, classKey) + if err != nil { + return nil, err + } + if filterMode == "classskills" { + if classSkills == nil || len(classSkills) == 0 { + return []map[string]any{}, nil + } + filtered := make([]map[string]any, 0, len(children)) + for _, child := range children { + featRef, ok := child["FeatIndex"].(map[string]any) + if !ok { + continue + } + featKey, _ := featRef["id"].(string) + featRow := rowByKey[featKey] + if featRow == nil { + continue + } + reqSkillKey := resolveReqSkill(featRow, keyToID) + if reqSkillKey == "" { + continue + } + if classSkills[reqSkillKey] { + filtered = append(filtered, child) + } + } + if len(filtered) == 0 { + return []map[string]any{}, nil + } + return filtered, nil + } + if len(children) == 0 { + return []map[string]any{row}, nil + } + return children, nil + default: + return []map[string]any{row}, nil + } +} + +func expandMasterfeatChildren(masterfeatKey string, row map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, classKey string) ([]map[string]any, error) { + masterfeatKey = resolveCanonicalDatasetKey(masterfeatKey, keyToID) + if _, ok := keyToID[masterfeatKey]; !ok { + return nil, fmt.Errorf("unknown masterfeat reference %q", masterfeatKey) + } + type match struct { + key string + id int + } + matches := make([]match, 0) + for key, featRow := range rowByKey { + if !strings.HasPrefix(key, "feat:") { + continue + } + if !rowReferencesMasterfeat(featRow, masterfeatKey, keyToID) { + continue + } + if !classFeatExpansionCanUseFeat(featRow, classKey, keyToID) { + continue + } + featID, ok := keyToID[key] + if !ok { + continue + } + matches = append(matches, match{key: key, id: featID}) + } + slices.SortFunc(matches, func(a, b match) int { + return a.id - b.id + }) + expanded := make([]map[string]any, 0, len(matches)) + for _, item := range matches { + cloned := deepCopyValue(row).(map[string]any) + cloned["FeatIndex"] = map[string]any{"id": item.key} + if label := lookupReferencedFeatLabel(item.key, rowByKey); label != "" { + cloned["FeatLabel"] = label + } + expanded = append(expanded, cloned) + } + return expanded, nil +} + +func classFeatExpansionCanUseFeat(featRow map[string]any, classKey string, keyToID map[string]int) bool { + value, ok := lookupField(featRow, "ALLCLASSESCANUSE") + if !ok || isNullishValue(value) { + return true + } + if strings.TrimSpace(fmt.Sprint(value)) != "0" { + return true + } + if classKey == "" { + return false + } + minLevelClass, ok := lookupField(featRow, "MinLevelClass") + if !ok || isNullishValue(minLevelClass) { + return false + } + return rowRefersToKey(minLevelClass, classKey, keyToID) +} + +func rowRefersToKey(value any, targetKey string, keyToID map[string]int) bool { + targetKey = resolveCanonicalDatasetKey(targetKey, keyToID) + targetID, hasTargetID := keyToID[targetKey] + switch typed := value.(type) { + case string: + text := resolveCanonicalDatasetKey(strings.TrimSpace(typed), keyToID) + if text == "" || text == nullValue || text == "****" { + return false + } + if text == targetKey { + return true + } + if hasTargetID { + if id, err := strconv.Atoi(text); err == nil { + return id == targetID + } + } + return false + case int: + return hasTargetID && typed == targetID + case float64: + return hasTargetID && int(typed) == targetID + case map[string]any: + if id, ok := typed["id"].(string); ok { + return rowRefersToKey(id, targetKey, keyToID) + } + if key, ok := typed["key"].(string); ok { + return rowRefersToKey(key, targetKey, keyToID) + } + return false + default: + return false + } +} + +func rowReferencesMasterfeat(row map[string]any, masterfeatKey string, keyToID map[string]int) bool { + masterfeatKey = resolveCanonicalDatasetKey(masterfeatKey, keyToID) + masterID, ok := keyToID[masterfeatKey] + if !ok { + return false + } + value, ok := row["MASTERFEAT"] + if !ok { + return false + } + switch typed := value.(type) { + case string: + return typed == strconv.Itoa(masterID) + case int: + return typed == masterID + case float64: + return int(typed) == masterID + case map[string]any: + if id, ok := typed["id"].(string); ok { + id = resolveCanonicalDatasetKey(id, keyToID) + if id == masterfeatKey { + return true + } + if refID, ok := keyToID[id]; ok { + return refID == masterID + } + return id == "masterfeats:"+strconv.Itoa(masterID) + } + return false + default: + return false + } +} + +func lookupReferencedFeatLabel(featKey string, rowByKey map[string]map[string]any) string { + row, ok := rowByKey[featKey] + if !ok { + return "" + } + label, _ := row["LABEL"].(string) + return strings.TrimSpace(label) +} + +func isNullLikeValue(value any) bool { + switch typed := value.(type) { + case nil: + return true + case string: + trimmed := strings.TrimSpace(typed) + return trimmed == "" || trimmed == nullValue + default: + return false + } +} + +func buildFeatSuccessorsIndex(rowByKey map[string]map[string]any, keyToID map[string]int) map[string]string { + successors := make(map[string]string) + for key, row := range rowByKey { + if !strings.HasPrefix(key, "feat:") { + continue + } + successorKey := resolveFeatSuccessor(row, keyToID) + if successorKey != "" { + successors[key] = successorKey + } + } + return successors +} + +func resolveFeatSuccessor(row map[string]any, keyToID map[string]int) string { + value, ok := row["SUCCESSOR"] + if !ok { + return "" + } + switch typed := value.(type) { + case map[string]any: + if idValue, ok := typed["id"]; ok { + switch idTyped := idValue.(type) { + case string: + trimmed := strings.TrimSpace(idTyped) + if trimmed == "" || trimmed == nullValue || trimmed == "****" { + return "" + } + if id, err := strconv.Atoi(trimmed); err == nil { + for k, v := range keyToID { + if v == id && strings.HasPrefix(k, "feat:") { + return k + } + } + return "" + } + if strings.HasPrefix(trimmed, "feat:") { + return trimmed + } + return "feat:" + trimmed + case int: + for k, v := range keyToID { + if v == idTyped && strings.HasPrefix(k, "feat:") { + return k + } + } + return "" + case float64: + targetID := int(idTyped) + for k, v := range keyToID { + if v == targetID && strings.HasPrefix(k, "feat:") { + return k + } + } + return "" + } + } + return "" + case string: + trimmed := strings.TrimSpace(typed) + if trimmed == "" || trimmed == nullValue || trimmed == "****" { + return "" + } + if id, err := strconv.Atoi(trimmed); err == nil { + for k, v := range keyToID { + if v == id && strings.HasPrefix(k, "feat:") { + return k + } + } + return "" + } + if strings.HasPrefix(trimmed, "feat:") { + return trimmed + } + return "feat:" + trimmed + case int: + for k, v := range keyToID { + if v == typed && strings.HasPrefix(k, "feat:") { + return k + } + } + return "" + case float64: + targetID := int(typed) + for k, v := range keyToID { + if v == targetID && strings.HasPrefix(k, "feat:") { + return k + } + } + return "" + default: + return "" + } +} + +func buildClassSkillsIndex(tableRegistry resolvedTableRegistry, classKey string) map[string]bool { + skillDatasetKey := classKey + if !strings.HasPrefix(skillDatasetKey, "classes/skills:") { + skillDatasetKey = "classes/skills:" + strings.TrimPrefix(classKey, "classes:") + } + + table, ok := tableRegistry.resolveTableByKey(skillDatasetKey) + if !ok { + return nil + } + + skills := make(map[string]bool) + for _, rowMap := range table.Rows { + classSkill, _ := rowMap["ClassSkill"].(string) + if classSkill != "1" { + continue + } + skillIndex, ok := rowMap["SkillIndex"].(map[string]any) + if !ok { + continue + } + skillKey, _ := skillIndex["id"].(string) + if skillKey != "" { + skills[skillKey] = true + } + } + + if len(skills) == 0 { + return nil + } + return skills +} + +func expandFeatSuccessors(rootFeatKey string, row map[string]any, featSuccessors map[string]string, rowByKey map[string]map[string]any) ([]map[string]any, error) { + current := rootFeatKey + visited := make(map[string]bool) + var results []map[string]any + for { + if current == "" { + break + } + if visited[current] { + return nil, fmt.Errorf("successor chain cycles at %q", current) + } + visited[current] = true + if rowByKey[current] == nil { + break + } + cloned := deepCopyValue(row).(map[string]any) + cloned["FeatIndex"] = map[string]any{"id": current} + if label := lookupReferencedFeatLabel(current, rowByKey); label != "" { + cloned["FeatLabel"] = label + } + results = append(results, cloned) + current = featSuccessors[current] + } + if len(results) == 0 { + return []map[string]any{row}, nil + } + return results, nil +} + +func expandMasterfeatChildrenFiltered(masterfeatKey string, row map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, classSkills map[string]bool) ([]map[string]any, error) { + if classSkills == nil || len(classSkills) == 0 { + return []map[string]any{}, nil + } + children, err := expandMasterfeatChildren(masterfeatKey, row, keyToID, rowByKey, "") + if err != nil { + return nil, err + } + var filtered []map[string]any + for _, child := range children { + featRef, ok := child["FeatIndex"].(map[string]any) + if !ok { + continue + } + featKey, _ := featRef["id"].(string) + featRow := rowByKey[featKey] + if featRow == nil { + continue + } + reqSkillKey := resolveReqSkill(featRow, keyToID) + if reqSkillKey == "" { + continue + } + if classSkills[reqSkillKey] { + filtered = append(filtered, child) + } + } + return filtered, nil +} + +func extractClassKeyFromDatasetName(rowByKey map[string]map[string]any) string { + for key := range rowByKey { + if strings.HasPrefix(key, "classes:") { + return key + } + } + return "" +} + +func resolveReqSkill(row map[string]any, keyToID map[string]int) string { + value, ok := row["REQSKILL"] + if !ok { + return "" + } + switch typed := value.(type) { + case string: + trimmed := strings.TrimSpace(typed) + if trimmed == "" || trimmed == nullValue || trimmed == "****" { + return "" + } + if strings.HasPrefix(trimmed, "skills:") { + return trimmed + } + if resolved := lookupKeyByIDPrefix(keyToID, "skills:", trimmed); resolved != "" { + return resolved + } + return "skills:" + trimmed + case map[string]any: + if id, ok := typed["id"].(string); ok { + if id == "" || id == nullValue || id == "****" { + return "" + } + if strings.HasPrefix(id, "skills:") { + return id + } + if resolved := lookupKeyByIDPrefix(keyToID, "skills:", id); resolved != "" { + return resolved + } + return "skills:" + id + } + return "" + default: + return "" + } +} + +func lookupKeyByIDPrefix(keyToID map[string]int, prefix string, rawID string) string { + idValue, err := strconv.Atoi(strings.TrimSpace(rawID)) + if err != nil { + return "" + } + for key, existingID := range keyToID { + if existingID == idValue && strings.HasPrefix(key, prefix) { + return key + } + } + return "" +} + +type valueResolver struct { + dataset nativeDataset + keyToID map[string]int + rowByKey map[string]map[string]any + tableRegistry resolvedTableRegistry + compiler *tlkCompiler + sidecars *nativeSidecarCollector + cache map[string]map[string]tlkCompiledValue +} + +func (r *valueResolver) resolveTableByKey(key string) (resolvedTable, bool) { + return r.tableRegistry.resolveTableByKey(key) +} + +func (r *valueResolver) resolveRow(row map[string]any, stack []string) (map[string]any, error) { + effectiveRow, err := r.resolveInheritedObject(row, rowIdentity(r.dataset.Name, row), row, stack) + if err != nil { + return nil, err + } + out := map[string]any{ + "id": effectiveRow["id"], + } + for _, column := range rowColumns(effectiveRow, r.dataset.Columns) { + resolved, err := r.resolveField(effectiveRow, column, stack) + if err != nil { + return nil, err + } + out[column] = resolved.Value + } + return out, nil +} + +func (r *valueResolver) resolveField(row map[string]any, field string, stack []string) (tlkCompiledValue, error) { + rowKey := rowCacheIdentity(r.dataset.Name, row) + if r.cache[rowKey] == nil { + r.cache[rowKey] = map[string]tlkCompiledValue{} + } + if cached, ok := r.cache[rowKey][field]; ok { + return cached, nil + } + + stackKey := rowKey + "." + field + for _, existing := range stack { + if existing == stackKey { + return tlkCompiledValue{}, fmt.Errorf("cyclic row/field reference detected at %s", stackKey) + } + } + + value, ok := lookupField(row, field) + if !ok { + if defaultValue, hasDefault := r.valueDefaultForField(field); hasDefault { + value = defaultValue + ok = true + } + } + if ok && isNullLike(value) { + if defaultValue, hasDefault := r.valueDefaultForField(field); hasDefault { + value = defaultValue + } + } + if !ok { + resolved := tlkCompiledValue{Value: nullValue} + r.cache[rowKey][field] = resolved + return resolved, nil + } + resolved, err := r.resolveValue(row, field, value, append(stack, stackKey)) + if err != nil { + return tlkCompiledValue{}, err + } + r.cache[rowKey][field] = resolved + return resolved, nil +} + +func (r *valueResolver) resolveValue(row map[string]any, field string, value any, stack []string) (tlkCompiledValue, error) { + if typed, ok := value.(map[string]any); ok { + merged, err := r.resolveInheritedObject(row, rowCacheIdentity(r.dataset.Name, row)+"."+field, typed, stack) + if err != nil { + return tlkCompiledValue{}, err + } + value = merged + } + if literal, ok := r.resolveLiteralFeatObject(row, field, value); ok { + return tlkCompiledValue{Value: literal}, nil + } + allowBare := columnMatchesSpec(r.dataset.Spec, field) + if payload, ok, err := parseTLKPayload(value, allowBare); err != nil { + return tlkCompiledValue{}, err + } else if ok { + return r.resolveTLKPayload(row, field, payload, stack) + } + if encoding, ok := r.valueEncodingForField(field); ok { + switch strings.TrimSpace(encoding.Mode) { + case "alignment_hex_list": + encoded, err := encodeConfiguredAlignmentHexList(value, encoding) + if err != nil { + return tlkCompiledValue{}, fmt.Errorf("field %s: %w", field, err) + } + return tlkCompiledValue{Value: encoded}, nil + case "alignment_set_mask": + encoded, err := encodeConfiguredAlignmentSetMask(value, encoding) + if err != nil { + return tlkCompiledValue{}, fmt.Errorf("field %s: %w", field, err) + } + return tlkCompiledValue{Value: encoded}, nil + } + } + + switch typed := value.(type) { + case map[string]any: + if encoding, ok := r.valueEncodingForField(field); ok { + mode := strings.TrimSpace(encoding.Mode) + _, hasList := typed["list"] + _, hasAllExcept := typed["all_except"] + _, hasIDs := typed["ids"] + if (mode == "packed_hex_list" || mode == "external_hex_list") && (hasList || hasAllExcept) { + value, err := r.encodeConfiguredValueList(row, field, typed, encoding) + if err != nil { + return tlkCompiledValue{}, fmt.Errorf("field %s: %w", field, err) + } + return tlkCompiledValue{Value: value}, nil + } + if mode == "id_hex_list" && hasIDs { + value, err := r.encodeConfiguredIDHexList(typed, encoding) + if err != nil { + return tlkCompiledValue{}, fmt.Errorf("field %s: %w", field, err) + } + return tlkCompiledValue{Value: value}, nil + } + } + if _, hasAlignments := typed["alignments"]; hasAlignments { + encoded, err := encodeConfiguredAlignmentSetMask(typed, defaultExplicitAlignmentSetMaskEncoding()) + if err != nil { + return tlkCompiledValue{}, fmt.Errorf("field %s: %w", field, err) + } + return tlkCompiledValue{Value: encoded}, nil + } + if _, hasIDs := typed["ids"]; hasIDs { + value, err := r.encodeConfiguredIDHexList(typed, defaultExplicitIDHexListEncoding()) + if err != nil { + return tlkCompiledValue{}, fmt.Errorf("field %s: %w", field, err) + } + return tlkCompiledValue{Value: value}, nil + } + if refID, ok := typed["id"]; ok { + if refKey, ok := refID.(string); ok && strings.Contains(refKey, ":") { + id, ok := r.keyToID[refKey] + if !ok { + return tlkCompiledValue{}, fmt.Errorf("unknown key reference: %s (row=%s field=%s)", refKey, rowCacheIdentity(r.dataset.Name, row), field) + } + return tlkCompiledValue{Value: id}, nil + } + } + if refKey, ok := typed["ref"]; ok { + fieldValue, hasField := typed["field"] + refKeyString, okKey := refKey.(string) + fieldString, okField := fieldValue.(string) + if hasField && okKey && okField { + targetRow, ok := r.rowByKey[refKeyString] + if !ok { + return tlkCompiledValue{}, fmt.Errorf("unknown row reference: %s", refKeyString) + } + return r.resolveField(targetRow, fieldString, stack) + } + } + if tableValue, ok := typed["table"]; ok { + tableKey, ok := tableValue.(string) + if ok { + outputName, ok := r.tableRegistry.stemByKey[tableKey] + if !ok { + return tlkCompiledValue{}, fmt.Errorf("unknown table reference: %s", tableKey) + } + return tlkCompiledValue{Value: outputName}, nil + } + } + + resolved := map[string]any{} + for _, key := range sortedKeys(typed) { + item, err := r.resolveValue(row, key, typed[key], stack) + if err != nil { + return tlkCompiledValue{}, err + } + resolved[key] = item.Value + } + return tlkCompiledValue{Value: resolved}, nil + case []any: + resolved := make([]any, 0, len(typed)) + for _, item := range typed { + value, err := r.resolveValue(row, field, item, stack) + if err != nil { + return tlkCompiledValue{}, err + } + resolved = append(resolved, value.Value) + } + return tlkCompiledValue{Value: resolved}, nil + default: + return tlkCompiledValue{Value: typed}, nil + } +} + +func (r *valueResolver) valueEncodingForField(field string) (project.TopDataValueEncodingConfig, bool) { + if len(r.dataset.ValueEncodings) == 0 { + return project.TopDataValueEncodingConfig{}, false + } + encoding, ok := r.dataset.ValueEncodings[strings.ToLower(strings.TrimSpace(field))] + return encoding, ok +} + +func (r *valueResolver) valueDefaultForField(field string) (any, bool) { + if len(r.dataset.ValueDefaults) == 0 { + return nil, false + } + value, ok := r.dataset.ValueDefaults[strings.ToLower(strings.TrimSpace(field))] + return value, ok +} + +func (r *valueResolver) encodeConfiguredValueList(row map[string]any, field string, obj map[string]any, encoding project.TopDataValueEncodingConfig) (string, error) { + mode := strings.TrimSpace(encoding.Mode) + if mode != "packed_hex_list" && mode != "external_hex_list" { + return "", fmt.Errorf("unsupported value encoding mode %q", encoding.Mode) + } + values, err := expandConfiguredValueListValues(obj, encoding) + if err != nil { + return "", err + } + if mode == "external_hex_list" { + return r.sidecars.addExternalHexList(r.dataset, row, field, values, encoding) + } + return formatConfiguredPackedHexList(values, encoding), nil +} + +func encodeConfiguredValueList(obj map[string]any, encoding project.TopDataValueEncodingConfig) (string, error) { + if strings.TrimSpace(encoding.Mode) != "packed_hex_list" { + return "", fmt.Errorf("unsupported value encoding mode %q", encoding.Mode) + } + values, err := expandConfiguredValueListValues(obj, encoding) + if err != nil { + return "", err + } + return formatConfiguredPackedHexList(values, encoding), nil +} + +func expandConfiguredValueListValues(obj map[string]any, encoding project.TopDataValueEncodingConfig) ([]int, error) { + _, hasList := obj["list"] + _, hasAllExcept := obj["all_except"] + if hasList && hasAllExcept { + return nil, fmt.Errorf("list encoding object cannot contain both list and all_except") + } + for key := range obj { + if key != "list" && key != "all_except" { + return nil, fmt.Errorf("list encoding object contains unsupported key %q", key) + } + } + if hasAllExcept { + values, err := expandConfiguredValueAllExcept(obj["all_except"], encoding) + if err != nil { + return nil, err + } + return values, nil + } + rawList, ok := obj["list"].([]any) + if !ok { + return nil, fmt.Errorf("list must be an array") + } + values := make([]int, 0, len(rawList)) + for _, raw := range rawList { + expanded, err := expandConfiguredValueListItem(raw, encoding) + if err != nil { + return nil, err + } + values = append(values, expanded...) + } + return values, nil +} + +func encodeConfiguredAlignmentHexList(raw any, encoding project.TopDataValueEncodingConfig) (string, error) { + values, err := expandConfiguredAlignmentList(raw) + if err != nil { + return "", err + } + return formatConfiguredPackedHexList(values, encoding), nil +} + +func encodeConfiguredAlignmentSetMask(raw any, encoding project.TopDataValueEncodingConfig) (string, error) { + values, err := expandConfiguredAlignmentList(raw) + if err != nil { + return "", err + } + mask := 0 + for _, value := range values { + if value == 0 { + continue + } + bit, ok := configuredAlignmentSetBit(value) + if !ok { + return "", fmt.Errorf("alignment value 0x%02X is not a supported exact alignment", value) + } + mask |= bit + } + if _, err := parseConfiguredListValue(mask, encoding); err != nil { + return "", err + } + return "0x" + fmt.Sprintf("%0*X", encoding.HexWidth, mask), nil +} + +func defaultExplicitAlignmentSetMaskEncoding() project.TopDataValueEncodingConfig { + return project.TopDataValueEncodingConfig{ + Mode: "alignment_set_mask", + Min: 0, + Max: 511, + HexWidth: 3, + } +} + +func defaultExplicitIDHexListEncoding() project.TopDataValueEncodingConfig { + return project.TopDataValueEncodingConfig{ + Mode: "id_hex_list", + Min: 0, + Max: 65535, + HexWidth: 4, + } +} + +func (r *valueResolver) encodeConfiguredIDHexList(obj map[string]any, encoding project.TopDataValueEncodingConfig) (string, error) { + mode := strings.TrimSpace(encoding.Mode) + if mode != "id_hex_list" { + return "", fmt.Errorf("unsupported value encoding mode %q", encoding.Mode) + } + for key := range obj { + if key != "ids" { + return "", fmt.Errorf("id list object contains unsupported key %q", key) + } + } + rawList, ok := obj["ids"].([]any) + if !ok { + return "", fmt.Errorf("ids must be an array") + } + values := make([]int, 0, len(rawList)) + for _, raw := range rawList { + value, err := r.parseConfiguredIDListValue(raw, encoding) + if err != nil { + return "", err + } + values = append(values, value) + } + return formatConfiguredPackedHexList(values, encoding), nil +} + +func (r *valueResolver) parseConfiguredIDListValue(raw any, encoding project.TopDataValueEncodingConfig) (int, error) { + switch typed := raw.(type) { + case map[string]any: + for key := range typed { + if key != "id" { + return 0, fmt.Errorf("id list item contains unsupported key %q", key) + } + } + ref, ok := typed["id"] + if !ok { + return 0, fmt.Errorf("id list item must contain id") + } + return r.parseConfiguredIDListValue(ref, encoding) + case string: + trimmed := strings.TrimSpace(typed) + if trimmed == "" { + return 0, fmt.Errorf("id list value must not be empty") + } + if strings.Contains(trimmed, ":") { + id, ok := r.keyToID[trimmed] + if !ok { + return 0, fmt.Errorf("unknown key reference: %s", trimmed) + } + return parseConfiguredListValue(id, encoding) + } + return parseConfiguredListValue(trimmed, encoding) + default: + return parseConfiguredListValue(raw, encoding) + } +} + +func expandConfiguredAlignmentList(raw any) ([]int, error) { + if isNullLike(raw) { + return []int{0}, nil + } + switch typed := raw.(type) { + case map[string]any: + _, hasAlignments := typed["alignments"] + _, hasList := typed["list"] + if hasAlignments && hasList { + return nil, fmt.Errorf("alignment list object cannot contain both alignments and list") + } + for key := range typed { + if key != "alignments" && key != "list" { + return nil, fmt.Errorf("alignment list object contains unsupported key %q", key) + } + } + if hasAlignments { + return expandConfiguredAlignmentList(typed["alignments"]) + } + return expandConfiguredAlignmentList(typed["list"]) + case []any: + values := make([]int, 0, len(typed)) + for _, item := range typed { + value, err := parseConfiguredAlignmentValue(item) + if err != nil { + return nil, err + } + values = append(values, value) + } + if len(values) == 0 { + return []int{0}, nil + } + return values, nil + case string: + trimmed := strings.TrimSpace(typed) + if trimmed == "" { + return []int{0}, nil + } + if strings.Contains(trimmed, ",") { + parts := strings.Split(trimmed, ",") + values := make([]int, 0, len(parts)) + for _, part := range parts { + value, err := parseConfiguredAlignmentValue(part) + if err != nil { + return nil, err + } + values = append(values, value) + } + return values, nil + } + value, err := parseConfiguredAlignmentValue(trimmed) + if err != nil { + return nil, err + } + return []int{value}, nil + default: + value, err := parseConfiguredAlignmentValue(raw) + if err != nil { + return nil, err + } + return []int{value}, nil + } +} + +func parseConfiguredAlignmentValue(raw any) (int, error) { + switch typed := raw.(type) { + case int: + return validateConfiguredAlignmentNumeric(typed) + case float64: + if typed != float64(int(typed)) { + return 0, fmt.Errorf("alignment value %v must be an integer", typed) + } + return validateConfiguredAlignmentNumeric(int(typed)) + case string: + trimmed := strings.TrimSpace(typed) + if trimmed == "" || trimmed == "0" || strings.EqualFold(trimmed, "0x00") { + return 0, nil + } + if strings.HasPrefix(strings.ToLower(trimmed), "0x") { + parsed, err := strconv.ParseInt(trimmed[2:], 16, 0) + if err != nil { + return 0, fmt.Errorf("alignment value %q must be a two-letter code or hex value", typed) + } + return validateConfiguredAlignmentNumeric(int(parsed)) + } + code := strings.ToUpper(trimmed) + if code == "TN" { + return 0x01, nil + } + if len(code) != 2 { + return 0, fmt.Errorf("alignment code %q must use two letters", typed) + } + first, ok := configuredAlignmentAxisValue(code[0], true) + if !ok { + return 0, fmt.Errorf("alignment code %q has invalid law-chaos letter %q", typed, string(code[0])) + } + second, ok := configuredAlignmentAxisValue(code[1], false) + if !ok { + return 0, fmt.Errorf("alignment code %q has invalid good-evil letter %q", typed, string(code[1])) + } + return first | second, nil + default: + return 0, fmt.Errorf("alignment value %v must be a two-letter code or integer", raw) + } +} + +func validateConfiguredAlignmentNumeric(value int) (int, error) { + if value < 0 || value > 31 { + return 0, fmt.Errorf("alignment value %d outside range 0-31", value) + } + return value, nil +} + +func configuredAlignmentAxisValue(letter byte, lawChaos bool) (int, bool) { + switch letter { + case 'N': + return 0x01, true + case 'L': + return 0x02, lawChaos + case 'C': + return 0x04, lawChaos + case 'G': + return 0x08, !lawChaos + case 'E': + return 0x10, !lawChaos + default: + return 0, false + } +} + +func configuredAlignmentSetBit(value int) (int, bool) { + switch value { + case 0x0A: // lawful good + return 1 << 0, true + case 0x03: // lawful neutral + return 1 << 1, true + case 0x12: // lawful evil + return 1 << 2, true + case 0x09: // neutral good + return 1 << 3, true + case 0x01: // true neutral + return 1 << 4, true + case 0x11: // neutral evil + return 1 << 5, true + case 0x0C: // chaotic good + return 1 << 6, true + case 0x05: // chaotic neutral + return 1 << 7, true + case 0x14: // chaotic evil + return 1 << 8, true + default: + return 0, false + } +} + +func formatConfiguredPackedHexList(values []int, encoding project.TopDataValueEncodingConfig) string { + if len(values) == 0 { + return "" + } + var builder strings.Builder + builder.WriteString("0x") + for _, value := range values { + builder.WriteString(fmt.Sprintf("%0*X", encoding.HexWidth, value)) + } + return builder.String() +} + +func expandConfiguredValueAllExcept(raw any, encoding project.TopDataValueEncodingConfig) ([]int, error) { + obj, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("all_except must be an object") + } + for key := range obj { + if key != "min" && key != "max" && key != "values" { + return nil, fmt.Errorf("all_except contains unsupported key %q", key) + } + } + min := encoding.Min + if rawMin, ok := obj["min"]; ok { + parsed, err := parseConfiguredListValue(rawMin, encoding) + if err != nil { + return nil, fmt.Errorf("all_except min: %w", err) + } + min = parsed + } + max := encoding.Max + if rawMax, ok := obj["max"]; ok { + parsed, err := parseConfiguredListValue(rawMax, encoding) + if err != nil { + return nil, fmt.Errorf("all_except max: %w", err) + } + max = parsed + } + if max < min { + return nil, fmt.Errorf("all_except max %d is less than min %d", max, min) + } + rawValues, ok := obj["values"].([]any) + if !ok { + return nil, fmt.Errorf("all_except values must be an array") + } + excluded := map[int]struct{}{} + for _, rawValue := range rawValues { + expanded, err := expandConfiguredValueListItem(rawValue, encoding) + if err != nil { + return nil, err + } + for _, value := range expanded { + if value < min || value > max { + return nil, fmt.Errorf("value %d outside all_except range %d-%d", value, min, max) + } + excluded[value] = struct{}{} + } + } + values := make([]int, 0, max-min+1-len(excluded)) + for value := min; value <= max; value++ { + if _, skip := excluded[value]; skip { + continue + } + values = append(values, value) + } + return values, nil +} + +func expandConfiguredValueListItem(raw any, encoding project.TopDataValueEncodingConfig) ([]int, error) { + if obj, ok := raw.(map[string]any); ok { + rawRange, ok := obj["range"] + if !ok { + return nil, fmt.Errorf("list object item must contain range") + } + if len(obj) != 1 { + return nil, fmt.Errorf("range item contains unsupported keys") + } + rangeList, ok := rawRange.([]any) + if !ok || len(rangeList) != 2 { + return nil, fmt.Errorf("range must be an array with two values") + } + start, err := parseConfiguredListValue(rangeList[0], encoding) + if err != nil { + return nil, err + } + end, err := parseConfiguredListValue(rangeList[1], encoding) + if err != nil { + return nil, err + } + if end < start { + return nil, fmt.Errorf("range %d-%d ends before it starts", start, end) + } + values := make([]int, 0, end-start+1) + for value := start; value <= end; value++ { + values = append(values, value) + } + return values, nil + } + value, err := parseConfiguredListValue(raw, encoding) + if err != nil { + return nil, err + } + return []int{value}, nil +} + +func parseConfiguredListValue(raw any, encoding project.TopDataValueEncodingConfig) (int, error) { + var value int + switch typed := raw.(type) { + case int: + value = typed + case float64: + if typed != float64(int(typed)) { + return 0, fmt.Errorf("list value %v must be an integer", typed) + } + value = int(typed) + case string: + trimmed := strings.TrimSpace(typed) + parsed, err := strconv.Atoi(trimmed) + if err != nil { + return 0, fmt.Errorf("list value %q must be numeric", typed) + } + value = parsed + default: + return 0, fmt.Errorf("list value %v must be numeric", raw) + } + if value < encoding.Min || value > encoding.Max { + return 0, fmt.Errorf("value %d outside range %d-%d", value, encoding.Min, encoding.Max) + } + return value, nil +} + +func (r *valueResolver) resolveLiteralFeatObject(row map[string]any, field string, value any) (string, bool) { + if r.dataset.Name != "feat" { + return "", false + } + typed, ok := value.(map[string]any) + if !ok { + return "", false + } + if refID, ok := typed["id"].(string); ok && len(typed) == 1 && strings.Contains(refID, ":") { + if !shouldPreserveLiteralFeatIDObject(row, field, refID) { + return "", false + } + if _, exists := r.keyToID[refID]; !exists { + return "", false + } + return "{'id': '" + refID + "'}", true + } + if refKey, ok := typed["key"].(string); ok && len(typed) == 1 && strings.Contains(refKey, ":") { + if !shouldPreserveLiteralFeatKeyObject(field) { + return "", false + } + if _, exists := r.keyToID[refKey]; !exists && !isLegacyLiteralFeatKeyException(field, refKey) { + return "", false + } + return "{'key': '" + refKey + "'}", true + } + return "", false +} + +func shouldPreserveLiteralFeatKeyObject(field string) bool { + switch field { + case "SUCCESSOR", "PREREQFEAT1", "PREREQFEAT2": + return true + default: + return false + } +} + +func shouldPreserveLiteralFeatIDObject(row map[string]any, field, refID string) bool { + switch field { + case "MinLevelClass": + return shouldPreserveFeatMinLevelClass(row) + case "SUCCESSOR": + switch refID { + case "feat:weaponproficiencysimple": + return true + default: + return false + } + default: + return false + } +} + +func shouldPreserveFeatMinLevelClass(row map[string]any) bool { + rowKey, _ := row["key"].(string) + if prefix, suffix, ok := strings.Cut(strings.TrimSpace(rowKey), ":"); ok && prefix == "masterfeats" { + switch normalizeKeyIdentity(suffix) { + case "weaponfocus", "weaponspecialization", "greaterweaponfocus", "greaterweaponspecialization", "improvedcritical", "overwhelmingcritical": + return true + } + } + switch { + case strings.HasPrefix(rowKey, "feat:weaponfocus_"): + return true + case strings.HasPrefix(rowKey, "feat:weaponspecialization_"): + return true + case strings.HasPrefix(rowKey, "feat:greaterweaponfocus_"): + return true + case strings.HasPrefix(rowKey, "feat:greaterweaponspecialization_"): + return true + case strings.HasPrefix(rowKey, "feat:improvedcritical_"): + return true + case strings.HasPrefix(rowKey, "feat:overwhelmingcritical_"): + return true + case strings.HasPrefix(rowKey, "feat:epicweaponfocus_"): + return true + case strings.HasPrefix(rowKey, "feat:epicweaponspecialization_"): + return true + default: + return false + } +} + +func isLegacyLiteralFeatKeyException(field, refKey string) bool { + if field != "SUCCESSOR" { + return false + } + switch refKey { + case "feat:improveddodge", "feat:improveduncannydodge": + return true + default: + return false + } +} + +func (r *valueResolver) resolveTLKPayload(row map[string]any, field string, payload parsedTLKPayload, stack []string) (tlkCompiledValue, error) { + if payload.Ref != "" { + targetRow, ok := r.rowByKey[payload.Ref] + if !ok { + return tlkCompiledValue{}, fmt.Errorf("unknown TLK ref row %s", payload.Ref) + } + resolved, err := r.resolveField(targetRow, payload.RefField, stack) + if err != nil { + return tlkCompiledValue{}, err + } + if resolved.TLKKey == "" { + switch resolved.Value.(type) { + case string, int, float64: + return resolved, nil + default: + return tlkCompiledValue{}, fmt.Errorf("TLK ref %s.%s does not resolve to TLK-backed text", payload.Ref, payload.RefField) + } + } + return resolved, nil + } + if payload.Text != "" { + key := payload.Key + if key == "" { + key = deriveAutoTLKKey(rowIdentity(r.dataset.Name, row), field) + } + return r.compiler.registerInline(key, tlkEntryData{ + Text: payload.Text, + SoundResRef: payload.SoundResRef, + SoundLength: payload.SoundLength, + }) + } + return r.compiler.resolveLegacyKey(payload.Key) +} + +func rowColumns(row map[string]any, ordered []string) []string { + columns := make([]string, 0, len(row)) + seen := map[string]struct{}{} + for _, key := range ordered { + if _, ok := row[key]; ok { + columns = append(columns, key) + seen[key] = struct{}{} + } + } + for key := range row { + if key == "id" || key == "key" || key == "__columns" || key == "inherit" || isMetadataField(key) { + continue + } + if _, ok := seen[key]; ok { + continue + } + columns = append(columns, key) + } + return columns +} + +func columnMatchesSpec(spec datasetTextSpec, field string) bool { + return roleForField(spec, field) == textRoleName || roleForField(spec, field) == textRoleDescription +} + +func rowIdentity(datasetName string, row map[string]any) string { + if key, ok := row["key"].(string); ok && key != "" { + return key + } + return fmt.Sprintf("%s:id:%d", datasetName, row["id"].(int)) +} + +func rowCacheIdentity(datasetName string, row map[string]any) string { + return rowIdentity(datasetName, row) +} + +func lookupField(row map[string]any, field string) (any, bool) { + if value, ok := row[field]; ok { + return value, true + } + for key, value := range row { + if strings.EqualFold(key, field) { + return value, true + } + } + return nil, false +} + +func parseColumns(baseData map[string]any, datasetName string) ([]string, error) { + rawColumns, ok := baseData["columns"].([]any) + if !ok { + return nil, fmt.Errorf("dataset %s: columns must be an array", datasetName) + } + columns := make([]string, 0, len(rawColumns)) + for _, raw := range rawColumns { + column, ok := raw.(string) + if !ok { + return nil, fmt.Errorf("dataset %s: columns must contain strings", datasetName) + } + if isDeprecatedAuthoringOnlyField(datasetName, column) || isMetadataField(column) { + continue + } + columns = append(columns, column) + } + return columns, nil +} + +func extendColumns(columns []string, obj map[string]any, datasetName, sourceLabel string) ([]string, error) { + rawColumns, ok := obj["columns"] + if !ok { + return columns, nil + } + columnList, ok := rawColumns.([]any) + if !ok { + return nil, fmt.Errorf("dataset %s: columns in %s must be an array", datasetName, sourceLabel) + } + extended := append([]string(nil), columns...) + for _, raw := range columnList { + column, ok := raw.(string) + if !ok { + return nil, fmt.Errorf("dataset %s: columns in %s must contain strings", datasetName, sourceLabel) + } + if isDeprecatedAuthoringOnlyField(datasetName, column) || isMetadataField(column) { + continue + } + if _, exists := canonicalColumn(extended, column); exists { + continue + } + extended = append(extended, column) + } + return extended, nil +} + +func ensureRowsExposeColumns(rows []map[string]any, columns []string) { + for _, row := range rows { + for _, column := range columns { + if _, ok := row[column]; !ok { + row[column] = nullValue + } + } + } +} + +func isNativeAllocationNullRow(row map[string]any, columns []string) bool { + if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" && strings.TrimSpace(key) != nullValue { + return false + } + for _, column := range columns { + if !isNullLikeValue(row[column]) { + return false + } + } + return true +} + +func canonicalizeBaseRow(dataset nativeDataset, columns []string, raw map[string]any, index int) (map[string]any, error) { + rowID := index + if value, ok := raw["id"]; ok { + parsed, err := asInt(value) + if err != nil { + return nil, fmt.Errorf("row id %v is not numeric", value) + } + rowID = parsed + } + + row := map[string]any{"id": rowID} + for _, column := range columns { + row[column] = nullValue + } + for key, value := range raw { + switch key { + case "id", "_tlk": + case "key": + if value == nil { + continue + } + text, ok := value.(string) + if !ok { + return nil, fmt.Errorf("row key must be a string") + } + row["key"] = text + case "inherit": + row["inherit"] = value + default: + if isMetadataField(key) { + meta, err := parseRowMetadata(value) + if err != nil { + return nil, err + } + if meta != nil { + row["meta"] = meta + } + continue + } + if isDeprecatedAuthoringOnlyField(dataset.Name, key) { + if err := setLegacyWikiMetadata(row, key, value); err != nil { + return nil, err + } + continue + } + columnName, ok := canonicalColumn(columns, key) + if !ok { + if isDeprecatedAuthoringOnlyField(dataset.Name, key) { + continue + } + return nil, fmt.Errorf("unknown column %q", key) + } + row[columnName] = value + } + } + + return expandRowAuthoringSugar(dataset, columns, raw, row) +} + +func canonicalizeEntry(dataset nativeDataset, columns []string, rowID int, key string, raw map[string]any) (map[string]any, error) { + row := map[string]any{ + "id": rowID, + "key": key, + } + for _, column := range columns { + row[column] = nullValue + } + for field, value := range raw { + if field == "_tlk" { + continue + } + if field == "inherit" { + row["inherit"] = value + continue + } + if isMetadataField(field) { + meta, err := parseRowMetadata(value) + if err != nil { + return nil, err + } + if meta != nil { + row["meta"] = meta + } + continue + } + if isDeprecatedAuthoringOnlyField(dataset.Name, field) { + if err := setLegacyWikiMetadata(row, field, value); err != nil { + return nil, err + } + continue + } + columnName, ok := canonicalColumn(columns, field) + if !ok { + if isDeprecatedAuthoringOnlyField(dataset.Name, field) { + continue + } + return nil, fmt.Errorf("entry %q references unknown column %q", key, field) + } + row[columnName] = value + } + return expandRowAuthoringSugar(dataset, columns, raw, row) +} + +func isAuthoringOnlyField(name string) bool { + return strings.HasPrefix(strings.ToUpper(strings.TrimSpace(name)), "WIKI") +} + +func preserveLegacyWikiColumn(datasetName, field string) bool { + return filepath.ToSlash(strings.TrimSpace(datasetName)) == "classes/core" && + strings.EqualFold(strings.TrimSpace(field), "WIKIGENERATE") +} + +func isDeprecatedAuthoringOnlyField(datasetName, field string) bool { + return isAuthoringOnlyField(field) && !preserveLegacyWikiColumn(datasetName, field) +} + +func isMetadataField(name string) bool { + return strings.EqualFold(strings.TrimSpace(name), "meta") +} + +func expandRowAuthoringSugar(dataset nativeDataset, columns []string, source map[string]any, row map[string]any) (map[string]any, error) { + expanded, err := expandRowTLKSugar(dataset, columns, source, row) + if err != nil { + return nil, err + } + return expandRowInheritanceSugar(dataset, columns, source, expanded) +} + +func expandRowTLKSugar(dataset nativeDataset, columns []string, source map[string]any, row map[string]any) (map[string]any, error) { + rawTLK, ok := source["_tlk"] + if !ok { + return row, nil + } + tlkMap, ok := rawTLK.(map[string]any) + if !ok { + return nil, fmt.Errorf("dataset %s: _tlk must be an object", dataset.Name) + } + + out := map[string]any{} + for key, value := range row { + out[key] = value + } + for roleName, rawValue := range tlkMap { + var role textRole + switch strings.ToLower(roleName) { + case "name": + role = textRoleName + case "description": + role = textRoleDescription + default: + return nil, fmt.Errorf("dataset %s: unsupported _tlk role %q", dataset.Name, roleName) + } + column := columnForRole(columns, dataset.Spec, role) + if column == "" { + return nil, fmt.Errorf("dataset %s: _tlk.%s has no matching text column", dataset.Name, roleName) + } + if _, hasField := source[column]; hasField { + return nil, fmt.Errorf("dataset %s: cannot use _tlk.%s and field %s together", dataset.Name, roleName, column) + } + + switch typed := rawValue.(type) { + case string: + out[column] = map[string]any{"tlk": map[string]any{"text": typed}} + case map[string]any: + if _, ok := typed["tlk"]; ok { + out[column] = typed + } else { + out[column] = map[string]any{"tlk": typed} + } + default: + return nil, fmt.Errorf("dataset %s: _tlk.%s must be a string or object", dataset.Name, roleName) + } + } + return out, nil +} + +func expandRowInheritanceSugar(dataset nativeDataset, columns []string, source map[string]any, row map[string]any) (map[string]any, error) { + spec, ok, err := parseRowInheritanceSpec(source) + if err != nil { + return nil, fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + if !ok { + return row, nil + } + + parentKey, err := resolveInheritanceParentKey(row, spec.From) + if err != nil { + return nil, fmt.Errorf("dataset %s: %w", dataset.Name, err) + } + + out := map[string]any{} + for key, value := range row { + out[key] = value + } + delete(out, "inherit") + for _, field := range spec.Fields { + columnName, ok := canonicalColumn(columns, field) + if !ok { + return nil, fmt.Errorf("inherit field %q is not a known column", field) + } + if rowFieldWasAuthored(source, columns, columnName, dataset.Spec) { + continue + } + if current, ok := lookupField(out, columnName); ok && current != nullValue { + continue + } + out[columnName] = map[string]any{ + "ref": parentKey, + "field": columnName, + } + } + return out, nil +} + +type rowInheritanceSpec struct { + From string + Fields []string +} + +type genericInheritanceSpec struct { + Ref string + Field string +} + +func parseRowInheritanceSpec(source map[string]any) (rowInheritanceSpec, bool, error) { + raw, ok := source["inherit"] + if !ok { + return rowInheritanceSpec{}, false, nil + } + obj, ok := raw.(map[string]any) + if !ok { + return rowInheritanceSpec{}, false, fmt.Errorf("inherit must be an object") + } + if _, hasRef := obj["ref"]; hasRef { + if _, hasFrom := obj["from"]; hasFrom { + return rowInheritanceSpec{}, false, fmt.Errorf("inherit cannot mix row sugar (from/fields) with object inheritance (ref)") + } + return rowInheritanceSpec{}, false, nil + } + rawFrom, ok := obj["from"] + if !ok { + return rowInheritanceSpec{}, false, fmt.Errorf("inherit.from is required") + } + from, ok := rawFrom.(string) + if !ok || strings.TrimSpace(from) == "" { + return rowInheritanceSpec{}, false, fmt.Errorf("inherit.from must be a non-empty string") + } + rawFields, ok := obj["fields"] + if !ok { + return rowInheritanceSpec{}, false, fmt.Errorf("inherit.fields is required") + } + fieldList, ok := rawFields.([]any) + if !ok || len(fieldList) == 0 { + return rowInheritanceSpec{}, false, fmt.Errorf("inherit.fields must be a non-empty array") + } + fields := make([]string, 0, len(fieldList)) + seen := map[string]struct{}{} + for _, rawField := range fieldList { + field, ok := rawField.(string) + if !ok || strings.TrimSpace(field) == "" { + return rowInheritanceSpec{}, false, fmt.Errorf("inherit.fields must contain non-empty strings") + } + normalized := strings.ToLower(field) + if _, exists := seen[normalized]; exists { + continue + } + seen[normalized] = struct{}{} + fields = append(fields, field) + } + return rowInheritanceSpec{From: from, Fields: fields}, true, nil +} + +func parseGenericInheritanceSpec(source map[string]any) (genericInheritanceSpec, bool, error) { + raw, ok := source["inherit"] + if !ok { + return genericInheritanceSpec{}, false, nil + } + obj, ok := raw.(map[string]any) + if !ok { + return genericInheritanceSpec{}, false, fmt.Errorf("inherit must be an object") + } + if _, hasFrom := obj["from"]; hasFrom { + if _, hasRef := obj["ref"]; hasRef { + return genericInheritanceSpec{}, false, fmt.Errorf("inherit cannot mix row sugar (from/fields) with object inheritance (ref)") + } + return genericInheritanceSpec{}, false, nil + } + rawRef, ok := obj["ref"] + if !ok { + return genericInheritanceSpec{}, false, fmt.Errorf("inherit.ref is required") + } + ref, ok := rawRef.(string) + if !ok || strings.TrimSpace(ref) == "" { + return genericInheritanceSpec{}, false, fmt.Errorf("inherit.ref must be a non-empty string") + } + spec := genericInheritanceSpec{Ref: strings.TrimSpace(ref)} + if rawField, ok := obj["field"]; ok { + field, ok := rawField.(string) + if !ok || strings.TrimSpace(field) == "" { + return genericInheritanceSpec{}, false, fmt.Errorf("inherit.field must be a non-empty string when present") + } + spec.Field = strings.TrimSpace(field) + } + return spec, true, nil +} + +func resolveInheritanceParentKey(row map[string]any, from string) (string, error) { + value, ok := lookupField(row, from) + if !ok { + return "", fmt.Errorf("inherit.from field %q was not found on row", from) + } + switch typed := value.(type) { + case string: + if strings.Contains(typed, ":") { + return typed, nil + } + case map[string]any: + if rawID, ok := typed["id"]; ok { + if key, ok := rawID.(string); ok && strings.Contains(key, ":") { + return key, nil + } + } + if rawRef, ok := typed["ref"]; ok { + if key, ok := rawRef.(string); ok && strings.Contains(key, ":") { + if _, hasField := typed["field"]; !hasField { + return key, nil + } + } + } + } + return "", fmt.Errorf("inherit.from field %q must resolve to a keyed parent row reference", from) +} + +func (r *valueResolver) resolveInheritedObject(ownerRow map[string]any, identity string, obj map[string]any, stack []string) (map[string]any, error) { + spec, ok, err := parseGenericInheritanceSpec(obj) + if err != nil { + return nil, err + } + + out := map[string]any{} + for key, value := range obj { + if key == "inherit" { + continue + } + out[key] = cloneAuthoringValue(value) + } + if !ok { + return out, nil + } + + marker := "inherit:" + identity + for _, existing := range stack { + if existing == marker { + return nil, fmt.Errorf("cyclic inheritance detected at %s", identity) + } + } + + parent, err := r.resolveGenericInheritanceTarget(ownerRow, spec, append(stack, marker)) + if err != nil { + return nil, err + } + return mergeInheritedMaps(parent, out), nil +} + +func (r *valueResolver) resolveGenericInheritanceTarget(ownerRow map[string]any, spec genericInheritanceSpec, stack []string) (map[string]any, error) { + targetRow, ok := r.rowByKey[spec.Ref] + if !ok { + return nil, fmt.Errorf("inherit target %q was not found", spec.Ref) + } + if spec.Field == "" { + return r.resolveInheritedObject(targetRow, spec.Ref, targetRow, stack) + } + value, ok := lookupField(targetRow, spec.Field) + if !ok { + return nil, fmt.Errorf("inherit target field %q was not found on %s", spec.Field, spec.Ref) + } + targetObj, ok := value.(map[string]any) + if !ok { + return nil, fmt.Errorf("inherit target %s.%s must resolve to an object", spec.Ref, spec.Field) + } + return r.resolveInheritedObject(targetRow, spec.Ref+"."+spec.Field, targetObj, stack) +} + +func mergeInheritedMaps(parent, local map[string]any) map[string]any { + out := map[string]any{} + for key, value := range parent { + out[key] = cloneAuthoringValue(value) + } + for key, value := range local { + if key == "inherit" { + continue + } + parentValue, hasParent := out[key] + if !hasParent { + out[key] = cloneAuthoringValue(value) + continue + } + out[key] = mergeInheritedValues(parentValue, value) + } + return out +} + +func mergeInheritedValues(parent, local any) any { + if isNullLike(local) { + return cloneAuthoringValue(parent) + } + localMap, localIsMap := local.(map[string]any) + parentMap, parentIsMap := parent.(map[string]any) + if localIsMap && parentIsMap && !isAtomicAuthoringMap(localMap) && !isAtomicAuthoringMap(parentMap) { + return mergeInheritedMaps(parentMap, localMap) + } + return cloneAuthoringValue(local) +} + +func isAtomicAuthoringMap(value map[string]any) bool { + for _, key := range []string{"tlk", "ref", "id", "table"} { + if _, ok := value[key]; ok { + return true + } + } + return false +} + +func cloneAuthoringValue(value any) any { + switch typed := value.(type) { + case map[string]any: + out := map[string]any{} + for key, item := range typed { + out[key] = cloneAuthoringValue(item) + } + return out + case []any: + out := make([]any, 0, len(typed)) + for _, item := range typed { + out = append(out, cloneAuthoringValue(item)) + } + return out + default: + return typed + } +} + +func rowFieldWasAuthored(source map[string]any, columns []string, field string, spec datasetTextSpec) bool { + if _, ok := source[field]; ok { + return true + } + for key := range source { + if canonical, ok := canonicalColumn(columns, key); ok && canonical == field { + return true + } + } + rawTLK, ok := source["_tlk"].(map[string]any) + if !ok { + return false + } + for roleName := range rawTLK { + var role textRole + switch strings.ToLower(roleName) { + case "name": + role = textRoleName + case "description": + role = textRoleDescription + default: + continue + } + column := columnForRole(columns, spec, role) + if column == field { + return true + } + } + return false +} + +func resolveOverrideTarget(datasetName string, override map[string]any, rowByID map[int]map[string]any, rowByKey map[string]map[string]any) (map[string]any, error) { + if rawID, ok := override["id"]; ok { + rowID, err := asInt(rawID) + if err != nil { + return nil, fmt.Errorf("dataset %s: override id %v is not numeric", datasetName, rawID) + } + row, ok := rowByID[rowID] + if !ok { + return nil, fmt.Errorf("dataset %s: override target id %d was not found", datasetName, rowID) + } + return row, nil + } + if rawKey, ok := override["key"]; ok { + key, ok := rawKey.(string) + if !ok || key == "" { + return nil, fmt.Errorf("dataset %s: override key must be a string", datasetName) + } + if key != "" { + row, ok := rowByKey[key] + if ok { + return row, nil + } + return nil, fmt.Errorf("dataset %s: override target key %q was not found", datasetName, key) + } + } + return nil, fmt.Errorf("dataset %s: override must specify id or key", datasetName) +} + +func overrideRequestsNullRow(override map[string]any) bool { + value, ok := override["null"] + if !ok { + return false + } + switch typed := value.(type) { + case bool: + return typed + case string: + trimmed := strings.TrimSpace(strings.ToLower(typed)) + return trimmed == "true" || trimmed == "1" || trimmed == "yes" + case int: + return typed != 0 + case float64: + return typed != 0 + default: + return false + } +} + +func nullifyOverrideRow(row map[string]any, columns []string, rowByKey map[string]map[string]any) { + if oldKey, ok := row["key"].(string); ok && oldKey != "" { + if mapped, exists := rowByKey[oldKey]; exists { + mappedID, mappedHasID := mapped["id"].(int) + rowID, rowHasID := row["id"].(int) + if mappedHasID && rowHasID && mappedID == rowID { + delete(rowByKey, oldKey) + } + } + } + for field := range row { + if field != "id" { + delete(row, field) + } + } + for _, column := range columns { + row[column] = nullValue + } +} + +func retireRowIdentity(row map[string]any, lockData map[string]int, retiredKeys map[string]struct{}) { + if row == nil { + return + } + oldKey, ok := row["key"].(string) + if !ok || oldKey == "" { + return + } + rowID, _ := row["id"].(int) + if lockedID, ok := lockData[oldKey]; ok && lockedID == rowID { + delete(lockData, oldKey) + } + if retiredKeys != nil { + retiredKeys[oldKey] = struct{}{} + } +} + +func updateOverrideRowKey(datasetName string, row map[string]any, expanded map[string]any, rowByKey map[string]map[string]any, lockData map[string]int) (bool, string, error) { + oldKey, _ := row["key"].(string) + newKeyValue, hasKey := expanded["key"] + if !hasKey { + return false, "", nil + } + rowID, _ := row["id"].(int) + changed := false + retiredKey := "" + if oldKey != "" { + if mapped, ok := rowByKey[oldKey]; ok { + mappedID, mappedHasID := mapped["id"].(int) + rowHasID := rowID != 0 || row["id"] != nil + if mappedHasID && rowHasID && mappedID == rowID { + delete(rowByKey, oldKey) + } + } + } + if newKeyValue == nil { + delete(row, "key") + if oldKey != "" { + retiredKey = oldKey + if lockedID, ok := lockData[oldKey]; ok && lockedID == rowID { + delete(lockData, oldKey) + } + changed = true + } + return changed, retiredKey, nil + } + newKey, ok := newKeyValue.(string) + if !ok || newKey == "" || strings.TrimSpace(newKey) == nullValue { + delete(row, "key") + if oldKey != "" { + retiredKey = oldKey + if lockedID, ok := lockData[oldKey]; ok && lockedID == rowID { + delete(lockData, oldKey) + } + changed = true + } + return changed, retiredKey, nil + } + if oldKey == newKey { + row["key"] = newKey + rowByKey[newKey] = row + if _, ok := lockData[newKey]; !ok { + lockData[newKey] = rowID + changed = true + } + return changed, "", nil + } + conflictingID := -1 + if conflicting, ok := rowByKey[newKey]; ok && conflicting != nil { + var conflictingHasID bool + conflictingID, conflictingHasID = conflicting["id"].(int) + rowHasID := rowID != 0 || row["id"] != nil + if !conflictingHasID || !rowHasID || conflictingID != rowID { + delete(conflicting, "key") + changed = true + } + } + row["key"] = newKey + rowByKey[newKey] = row + if oldKey != "" && oldKey != newKey { + retiredKey = oldKey + if lockedID, ok := lockData[oldKey]; ok && lockedID == rowID { + delete(lockData, oldKey) + } + changed = true + } + if existingID, ok := lockData[newKey]; ok { + if existingID != rowID { + if conflictingID >= 0 && existingID == conflictingID { + lockData[newKey] = rowID + changed = true + } else { + return false, "", fmt.Errorf("dataset %s: key %q is locked to id %d and cannot be rewritten to id %d", datasetName, newKey, existingID, rowID) + } + } + } else { + lockData[newKey] = rowID + changed = true + } + return changed, retiredKey, nil +} + +func isTopDataNullLike(value any) bool { + if value == nil { + return true + } + text, ok := value.(string) + if !ok { + return false + } + trimmed := strings.TrimSpace(text) + return trimmed == "" || trimmed == nullValue +} + +type parsedGlobalDefaultField struct { + Field string + Column string + Value parsedGlobalDefaultValue +} + +type parsedGlobalDefaultValue struct { + Literal any + Format string + IsFormat bool +} + +func globalDefaultMatchesRow(rawMatch any, source nativeRowSource) (bool, error) { + if rawMatch == nil { + return true, nil + } + if text, ok := rawMatch.(string); ok { + if strings.EqualFold(strings.TrimSpace(text), "all") { + return true, nil + } + return false, fmt.Errorf("unsupported defaults match value %q", text) + } + obj, ok := rawMatch.(map[string]any) + if !ok { + return false, fmt.Errorf("defaults match must be \"all\" or an object") + } + rawSource, ok := obj["source"] + if !ok { + return false, fmt.Errorf("defaults match object must contain source") + } + sourceText, ok := rawSource.(string) + if !ok || strings.TrimSpace(sourceText) == "" { + return false, fmt.Errorf("defaults match source must be a non-empty string") + } + for key := range obj { + if key != "source" { + return false, fmt.Errorf("defaults match field %q is not supported", key) + } + } + switch strings.ToLower(strings.TrimSpace(sourceText)) { + case string(nativeRowSourceBase): + return source == nativeRowSourceBase, nil + case string(nativeRowSourceEntries): + return source == nativeRowSourceEntries, nil + case string(nativeRowSourceOverride): + return source == nativeRowSourceOverride, nil + default: + return false, fmt.Errorf("defaults match source %q is not supported", sourceText) + } +} + +func parseGlobalDefaultValue(value any) (parsedGlobalDefaultValue, error) { + obj, ok := value.(map[string]any) + if !ok { + return parsedGlobalDefaultValue{Literal: value}, nil + } + rawFormat, hasFormat := obj["format"] + if !hasFormat { + return parsedGlobalDefaultValue{Literal: value}, nil + } + if len(obj) != 1 { + return parsedGlobalDefaultValue{}, fmt.Errorf("default format object must contain only format") + } + format, ok := rawFormat.(string) + if !ok { + return parsedGlobalDefaultValue{}, fmt.Errorf("default format must be a string") + } + if err := validateTopDataRowFormat(format); err != nil { + return parsedGlobalDefaultValue{}, err + } + return parsedGlobalDefaultValue{Format: format, IsFormat: true}, nil +} + +func evaluateGlobalDefaultValue(value parsedGlobalDefaultValue, row map[string]any) (any, error) { + if !value.IsFormat { + return value.Literal, nil + } + return formatTopDataRowValue(value.Format, row) +} + +func validateTopDataRowFormat(format string) error { + return walkTopDataRowFormat(format, nil, nil) +} + +func formatTopDataRowValue(format string, row map[string]any) (string, error) { + var builder strings.Builder + err := walkTopDataRowFormat(format, func(text string) { + builder.WriteString(text) + }, func(token string) error { + switch token { + case "id": + rowID, ok := row["id"].(int) + if !ok { + return fmt.Errorf("default format token {id} requires numeric row id") + } + builder.WriteString(strconv.Itoa(rowID)) + case "key": + key, ok := row["key"].(string) + if !ok { + return fmt.Errorf("default format token {key} requires row key") + } + builder.WriteString(key) + } + return nil + }) + if err != nil { + return "", err + } + return builder.String(), nil +} + +func walkTopDataRowFormat(format string, literal func(string), tokenValue func(string) error) error { + for index := 0; index < len(format); { + switch format[index] { + case '{': + end := strings.IndexByte(format[index+1:], '}') + if end < 0 { + return fmt.Errorf("unterminated default format token in %q", format) + } + token := format[index+1 : index+1+end] + switch token { + case "id", "key": + default: + return fmt.Errorf("default format token {%s} is not supported", token) + } + if tokenValue != nil { + if err := tokenValue(token); err != nil { + return err + } + } + index += end + 2 + case '}': + return fmt.Errorf("unexpected default format token terminator in %q", format) + default: + start := index + for index < len(format) && format[index] != '{' && format[index] != '}' { + index++ + } + if literal != nil { + literal(format[start:index]) + } + continue + } + } + return nil +} + +func write2DA(data map[string]any, path string, denseRows bool, denseRowMax int) error { + columns, ok := data["columns"].([]string) + if !ok { + return fmt.Errorf("2da columns must be []string") + } + rows, ok := data["rows"].([]map[string]any) + if !ok { + return fmt.Errorf("2da rows must be []map[string]any") + } + + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create 2da output parent: %w", err) + } + + var builder strings.Builder + builder.WriteString("2DA V2.0\n\n") + builder.WriteString(strings.Join(columns, "\t")) + builder.WriteByte('\n') + nextID := 0 + for _, row := range rows { + rowID := row["id"].(int) + if denseRows { + for nextID < rowID { + builder.WriteString(strconv.Itoa(nextID)) + for range columns { + builder.WriteByte('\t') + builder.WriteString(nullValue) + } + builder.WriteByte('\n') + nextID++ + } + } + builder.WriteString(strconv.Itoa(rowID)) + for _, column := range columns { + builder.WriteByte('\t') + builder.WriteString(format2DAValue(row[column])) + } + builder.WriteByte('\n') + if denseRows { + nextID = rowID + 1 + } + } + if denseRows { + for nextID <= denseRowMax { + builder.WriteString(strconv.Itoa(nextID)) + for range columns { + builder.WriteByte('\t') + builder.WriteString(nullValue) + } + builder.WriteByte('\n') + nextID++ + } + } + return os.WriteFile(path, []byte(builder.String()), 0o644) +} + +func format2DAValue(value any) string { + switch typed := value.(type) { + case nil: + return nullValue + case string: + if typed == nullValue { + return typed + } + if strings.ContainsAny(typed, " \t\r\n") { + return strconv.Quote(typed) + } + return typed + case int: + return strconv.Itoa(typed) + case float64: + if typed == float64(int64(typed)) { + return strconv.FormatInt(int64(typed), 10) + } + return strconv.FormatFloat(typed, 'f', -1, 64) + case bool: + if typed { + return "1" + } + return "0" + default: + raw, err := json.Marshal(typed) + if err != nil { + return fmt.Sprint(typed) + } + text := string(raw) + if strings.ContainsAny(text, " \t\r\n") { + return strconv.Quote(text) + } + return text + } +} + +func outputStem(outputName string) string { + return strings.TrimSuffix(outputName, filepath.Ext(outputName)) +} + +func nativeDatasetDefaultOutputName(path string, kind nativeDatasetKind) string { + name := filepath.Base(path) + if kind == nativeDatasetPlain { + name = strings.TrimSuffix(name, filepath.Ext(name)) + } + return name + ".2da" +} + +func loadJSONObject(path string) (map[string]any, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + var payload any + if err := json.Unmarshal(raw, &payload); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + obj, ok := payload.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s must contain a JSON object", path) + } + return obj, nil +} + +func loadLockfile(path string) (map[string]int, error) { + lockData := map[string]int{} + if _, err := os.Stat(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + return lockData, nil + } + return nil, err + } + + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + for key, value := range obj { + rowID, err := asInt(value) + if err != nil { + return nil, fmt.Errorf("lock entry %q is not numeric", key) + } + lockData[key] = rowID + } + return lockData, nil +} + +func saveLockfile(path string, lockData map[string]int) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create lockfile parent: %w", err) + } + keys, err := orderedLockfileKeys(path, lockData) + if err != nil { + return fmt.Errorf("read lockfile order: %w", err) + } + formatting, err := resolveJSONFileFormatting(path) + if err != nil { + return fmt.Errorf("resolve lockfile formatting: %w", err) + } + raw, err := marshalOrderedLockfile(keys, lockData, formatting) + if err != nil { + return fmt.Errorf("marshal lockfile: %w", err) + } + current, err := os.ReadFile(path) + if err == nil && bytes.Equal(current, raw) { + return nil + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("read lockfile %s: %w", path, err) + } + if err := os.WriteFile(path, raw, 0o644); err != nil { + return fmt.Errorf("write lockfile %s: %w", path, err) + } + return nil +} + +func orderedLockfileKeys(path string, lockData map[string]int) ([]string, error) { + existingKeys, err := readLockfileKeyOrder(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + keys := make([]string, 0, len(lockData)) + seen := make(map[string]struct{}, len(lockData)) + for _, key := range existingKeys { + if _, ok := lockData[key]; !ok { + continue + } + keys = append(keys, key) + seen[key] = struct{}{} + } + + extraKeys := make([]string, 0, len(lockData)-len(keys)) + for key := range lockData { + if _, ok := seen[key]; ok { + continue + } + extraKeys = append(extraKeys, key) + } + slices.Sort(extraKeys) + keys = append(keys, extraKeys...) + return keys, nil +} + +func readLockfileKeyOrder(path string) ([]string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(bytes.NewReader(raw)) + token, err := decoder.Token() + if err != nil { + return nil, err + } + delim, ok := token.(json.Delim) + if !ok || delim != '{' { + return nil, fmt.Errorf("%s must contain a JSON object", path) + } + + keys := []string{} + for decoder.More() { + token, err := decoder.Token() + if err != nil { + return nil, err + } + key, ok := token.(string) + if !ok { + return nil, fmt.Errorf("%s contains a non-string object key", path) + } + var value json.RawMessage + if err := decoder.Decode(&value); err != nil { + return nil, err + } + keys = append(keys, key) + } + + token, err = decoder.Token() + if err != nil { + return nil, err + } + delim, ok = token.(json.Delim) + if !ok || delim != '}' { + return nil, fmt.Errorf("%s must contain a JSON object", path) + } + return keys, nil +} + +func marshalOrderedLockfile(keys []string, lockData map[string]int, formatting jsonFileFormatting) ([]byte, error) { + lineEnding := formatting.LineEnding + if lineEnding == "" { + lineEnding = "\n" + } + if len(keys) == 0 { + if formatting.FinalNewline { + return []byte("{}" + lineEnding), nil + } + return []byte("{}"), nil + } + + var buffer bytes.Buffer + buffer.WriteString("{") + buffer.WriteString(lineEnding) + for index, key := range keys { + keyRaw, err := json.Marshal(key) + if err != nil { + return nil, err + } + valueRaw, err := json.Marshal(lockData[key]) + if err != nil { + return nil, err + } + buffer.WriteString(formatting.Indent) + buffer.Write(keyRaw) + buffer.WriteString(": ") + buffer.Write(valueRaw) + if index < len(keys)-1 { + buffer.WriteByte(',') + } + buffer.WriteString(lineEnding) + } + buffer.WriteString("}") + if formatting.FinalNewline { + buffer.WriteString(lineEnding) + } + return buffer.Bytes(), nil +} + +func loadLegacyTLK(root string) (*legacyTLKData, error) { + info, err := os.Stat(root) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("legacy tlk root must be a directory: %s", root) + } + + result := &legacyTLKData{ + Language: "en", + Entries: map[string]tlkEntryData{}, + Lock: map[string]int{}, + } + + lockPath := filepath.Join(root, "lock.json") + lockData, err := loadLockfile(lockPath) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + for key, id := range lockData { + result.Lock[key] = id + } + + basePath := filepath.Join(root, "base.json") + if _, err := os.Stat(basePath); err == nil { + base, err := loadJSONObject(basePath) + if err != nil { + return nil, err + } + if language, ok := base["language"].(string); ok && language != "" { + result.Language = language + } + if entries, ok := base["entries"].(map[string]any); ok { + normalized, err := normalizeLegacyTLKEntries(entries) + if err != nil { + return nil, err + } + for key, entry := range normalized { + result.Entries[key] = entry + } + } + } + + modulesDir := filepath.Join(root, "modules") + modulePaths, err := collectModulePaths(modulesDir) + if err != nil { + if os.IsNotExist(err) { + return result, nil + } + return nil, err + } + for _, path := range modulePaths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + entries, ok := obj["entries"].(map[string]any) + if !ok { + continue + } + normalized, err := normalizeLegacyTLKEntries(entries) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + for key, entry := range normalized { + result.Entries[key] = entry + } + } + return result, nil +} + +func normalizeLegacyTLKEntries(entries map[string]any) (map[string]tlkEntryData, error) { + normalized := map[string]tlkEntryData{} + for _, key := range sortedKeys(entries) { + value := entries[key] + switch typed := value.(type) { + case string: + normalized[key] = tlkEntryData{Text: typed} + case map[string]any: + if shouldExpandLegacyKey(key, typed) { + for _, subKey := range sortedKeys(typed) { + subObj, ok := typed[subKey].(map[string]any) + if !ok { + continue + } + entry, err := parseLegacyTLKLeaf(subObj) + if err != nil { + return nil, err + } + normalized[key+"."+subKey] = entry + } + continue + } + entry, err := parseLegacyTLKLeaf(typed) + if err != nil { + return nil, err + } + normalized[key] = entry + default: + return nil, fmt.Errorf("legacy TLK entry %q must be an object or string", key) + } + } + return normalized, nil +} + +func shouldExpandLegacyKey(key string, obj map[string]any) bool { + if strings.Contains(key[strings.LastIndex(key, ":")+1:], ".") { + return false + } + for _, value := range obj { + if _, ok := value.(map[string]any); ok { + return true + } + } + return false +} + +func parseLegacyTLKLeaf(obj map[string]any) (tlkEntryData, error) { + entry := tlkEntryData{} + if rawText, ok := obj["text"]; ok { + text, ok := rawText.(string) + if !ok { + return entry, fmt.Errorf("legacy TLK text must be a string") + } + entry.Text = text + } + if rawSound, ok := obj["sound_resref"]; ok { + sound, ok := rawSound.(string) + if !ok { + return entry, fmt.Errorf("legacy TLK sound_resref must be a string") + } + entry.SoundResRef = sound + } + if rawLength, ok := obj["sound_length"]; ok { + switch typed := rawLength.(type) { + case float64: + entry.SoundLength = float32(typed) + case int: + entry.SoundLength = float32(typed) + default: + return entry, fmt.Errorf("legacy TLK sound_length must be numeric") + } + } + return entry, nil +} + +func collectModulePaths(modulesDir string) ([]string, error) { + if _, err := os.Stat(modulesDir); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, err + } + var paths []string + err := filepath.WalkDir(modulesDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.HasSuffix(strings.ToLower(d.Name()), ".json") { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return nil, err + } + slices.Sort(paths) + return paths, nil +} + +func canonicalColumn(columns []string, name string) (string, bool) { + for _, column := range columns { + if strings.EqualFold(column, name) { + return column, true + } + } + return "", false +} + +func optionalBool(value any, fallback bool) bool { + switch typed := value.(type) { + case bool: + return typed + case nil: + return fallback + default: + return fallback + } +} + +func asInt(value any) (int, error) { + switch typed := value.(type) { + case int: + return typed, nil + case int64: + return int(typed), nil + case float64: + return int(typed), nil + case json.Number: + n, err := typed.Int64() + return int(n), err + case string: + return strconv.Atoi(typed) + default: + return 0, fmt.Errorf("not an int") + } +} + +func nextAvailableID(used map[int]struct{}) int { + return nextAvailableIDAtLeast(used, 0) +} + +func nextAvailableIDAtLeast(used map[int]struct{}, minimum int) int { + next := minimum + for { + if _, ok := used[next]; !ok { + return next + } + next++ + } +} + +func hasJSONFiles(root string) (bool, error) { + found := false + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.HasSuffix(strings.ToLower(d.Name()), ".json") { + found = true + return fs.SkipAll + } + return nil + }) + if err != nil && !errors.Is(err, fs.SkipAll) { + return false, err + } + return found, nil +} + +func collectReferenceLockIDs(root string) (map[string]int, error) { + result := map[string]int{} + if _, err := os.Stat(root); err != nil { + if os.IsNotExist(err) { + return result, nil + } + return nil, err + } + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || d.Name() != "lock.json" { + return nil + } + lockData, err := loadLockfile(path) + if err != nil { + return err + } + for key, rowID := range lockData { + if _, exists := result[key]; !exists { + result[key] = rowID + } + } + return nil + }) + if err != nil { + return nil, err + } + return result, nil +} + +func collectSupplementalLockIDs(sourceDir, referenceBuilderDir string) (map[string]int, error) { + result := map[string]int{} + snapshotIDs, err := collectReferenceLockIDs(filepath.Join(sourceDir, "migration_snapshot", "data")) + if err != nil { + return nil, err + } + for key, rowID := range snapshotIDs { + result[key] = rowID + } + if strings.TrimSpace(referenceBuilderDir) == "" { + return result, nil + } + referenceIDs, err := collectReferenceLockIDs(filepath.Join(referenceBuilderDir, "data")) + if err != nil { + return nil, err + } + for key, rowID := range referenceIDs { + if _, ok := result[key]; ok { + continue + } + result[key] = rowID + } + return result, nil +} diff --git a/internal/topdata/parts_discovery.go b/internal/topdata/parts_discovery.go new file mode 100644 index 0000000..87ad0d5 --- /dev/null +++ b/internal/topdata/parts_discovery.go @@ -0,0 +1,692 @@ +package topdata + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +// supportedPartCategories defines the part types that should be auto-discovered +// from the sow-assets repository. +var supportedPartCategories = []string{ + "belt", + "bicep", + "chest", + "foot", + "forearm", + "leg", + "neck", + "pelvis", + "robe", + "shin", + "shoulder", +} + +var partDatasetToAssetCategory = map[string]string{ + "belt": "belt", + "bicep": "bicep", + "chest": "chest", + "foot": "foot", + "forearm": "forearm", + "legs": "leg", + "neck": "neck", + "pelvis": "pelvis", + "robe": "robe", + "shin": "shin", + "shoulder": "shoulder", +} + +// trailingNumberRegex matches trailing digits at the end of a string +var trailingNumberRegex = regexp.MustCompile(`(\d+)$`) + +// isPartsDataset checks if a dataset name corresponds to a parts dataset +func isPartsDataset(datasetName string) bool { + // Check if the dataset is under parts/ directory + if strings.HasPrefix(datasetName, "parts/") { + return true + } + // Check if it's exactly a parts file (e.g., "parts/belt") + parts := strings.Split(datasetName, "/") + if len(parts) == 2 && parts[0] == "parts" { + return true + } + return false +} + +func isAutogenEligiblePartsDataset(dataset nativeCollectedDataset) bool { + if !isPartsDataset(dataset.Dataset.Name) { + return false + } + for _, row := range dataset.Rows { + key, ok := row["key"].(string) + if ok && strings.TrimSpace(key) != "" { + return false + } + } + return true +} + +// extractPartCategory extracts the part category from a dataset name +// e.g., "parts/belt" -> "belt", "parts/bicep" -> "bicep" +func extractPartCategory(datasetName string) string { + parts := strings.Split(datasetName, "/") + if len(parts) >= 2 { + return parts[len(parts)-1] + } + return "" +} + +func partAssetCategoryForDataset(datasetName string) string { + category := extractPartCategory(datasetName) + if mapped, ok := partDatasetToAssetCategory[category]; ok { + return mapped + } + return category +} + +// isSupportedPartCategory checks if the given category is in the supported list +func isSupportedPartCategory(category string) bool { + for _, c := range supportedPartCategories { + if c == category { + return true + } + } + return false +} + +// extractTrailingNumericSuffix extracts the trailing numeric suffix from a filename stem +// e.g., "pfa0_belt018" -> 18, "pfa0_belt171" -> 171 +// Returns 0 and false if no trailing numeric suffix is found +func extractTrailingNumericSuffix(filenameStem string) (int, bool) { + matches := trailingNumberRegex.FindStringSubmatch(filenameStem) + if len(matches) < 2 { + return 0, false + } + num, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, false + } + return num, true +} + +// DiscoveredPartModel represents a discovered model file with its row ID +type DiscoveredPartModel struct { + RowID int + Filename string +} + +// DiscoverPartModels scans the assets/part directory for .mdl files +// and returns a map of rowID -> default row data for the given category. +// The assetsDir should be the root assets directory (containing the part/ subdirectory). +func DiscoverPartModels(assetsDir string, category string) (map[int]*DiscoveredPartModel, error) { + result := make(map[int]*DiscoveredPartModel) + + if !isSupportedPartCategory(category) { + return result, nil + } + + partRoot, err := resolvePartsRoot(assetsDir) + if err != nil { + return nil, err + } + if partRoot == "" { + return result, nil + } + + // Construct the path to the part category directory + partDir := filepath.Join(partRoot, category) + + // Check if the directory exists + info, err := os.Stat(partDir) + if err != nil { + if os.IsNotExist(err) { + return result, nil + } + return nil, fmt.Errorf("failed to access part directory %s: %w", partDir, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("part directory %s is not a directory", partDir) + } + + // Walk the directory and find .mdl files + err = filepath.Walk(partDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + // Only process .mdl files + if !strings.HasSuffix(strings.ToLower(info.Name()), ".mdl") { + return nil + } + + // Extract the filename stem (without extension) + filenameStem := strings.TrimSuffix(info.Name(), filepath.Ext(info.Name())) + + // Extract trailing numeric suffix + rowID, hasSuffix := extractTrailingNumericSuffix(filenameStem) + if !hasSuffix { + return nil + } + + // Store the discovered model + if existing, ok := result[rowID]; ok { + // If we already have this row ID, keep the first one found (alphabetically) + if path < existing.Filename { + result[rowID] = &DiscoveredPartModel{ + RowID: rowID, + Filename: path, + } + } + } else { + result[rowID] = &DiscoveredPartModel{ + RowID: rowID, + Filename: path, + } + } + + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to scan part directory %s: %w", partDir, err) + } + + return result, nil +} + +// CreateDefaultPartRow creates a default row for a discovered part model. +// Base defaults are COSTMODIFIER=0 and ACBONUS=0.00. Any robe HIDE* columns +// present in the dataset columns default to 0 as well. +func CreateDefaultPartRow(rowID int) map[string]any { + return CreateDefaultPartRowForColumns(rowID, nil) +} + +func CreateDefaultPartRowForColumns(rowID int, columns []string) map[string]any { + return createDefaultPartRowForDataset(rowID, "", columns, project.PartsRowsConfig{}) +} + +func createDefaultPartRowForDataset(rowID int, datasetName string, columns []string, cfg project.PartsRowsConfig) map[string]any { + row := map[string]any{ + "id": rowID, + "COSTMODIFIER": "0", + "ACBONUS": "0.00", + } + for field, value := range cfg.RowDefaults { + if strings.TrimSpace(field) == "" { + continue + } + row[field] = value + } + if value, ok := configuredPartsRowsACBonusValue(rowID, datasetName, cfg); ok { + row["ACBONUS"] = value + } + for _, column := range columns { + if strings.HasPrefix(column, "HIDE") { + row[column] = "0" + } + } + return row +} + +func isUnsetPartValue(value any) bool { + switch typed := value.(type) { + case nil: + return true + case string: + return typed == "" || typed == "****" + } + return false +} + +func applyDiscoveredPartDefaults(row map[string]any, columns []string) { + applyDiscoveredPartDefaultsWithConfig(row, "", columns, project.PartsRowsConfig{}) +} + +func applyDiscoveredPartDefaultsWithConfig(row map[string]any, datasetName string, columns []string, cfg project.PartsRowsConfig) { + if isUnsetPartValue(row["COSTMODIFIER"]) { + row["COSTMODIFIER"] = "0" + } + for field, value := range cfg.RowDefaults { + if strings.TrimSpace(field) == "" { + continue + } + if isUnsetPartValue(row[field]) { + row[field] = value + } + } + if isUnsetPartValue(row["ACBONUS"]) { + row["ACBONUS"] = "0.00" + } + if value, ok := configuredPartsRowsACBonusValue(rowIDFromPartRow(row), datasetName, cfg); ok { + row["ACBONUS"] = value + } + for _, column := range columns { + if !strings.HasPrefix(column, "HIDE") { + continue + } + if isUnsetPartValue(row[column]) { + row[column] = "0" + } + } +} + +func rowIDFromPartRow(row map[string]any) int { + rowID, _ := row["id"].(int) + return rowID +} + +func resolvePartsRoot(root string) (string, error) { + if strings.TrimSpace(root) == "" { + return "", nil + } + candidates := []string{ + filepath.Join(root, "assets", "parts"), + filepath.Join(root, "assets", "part"), + filepath.Join(root, "parts"), + filepath.Join(root, "part"), + } + for _, candidate := range candidates { + info, err := os.Stat(candidate) + if err == nil && info.IsDir() { + return candidate, nil + } + if err != nil && !os.IsNotExist(err) { + return "", fmt.Errorf("failed to access parts root %s: %w", candidate, err) + } + } + return "", nil +} + +// scanAutogeneratedParts scans the assets/part directory for all supported part categories +// and returns a map of category -> set of row IDs that should be auto-generated. +// The assetsDir parameter should be the path to the assets directory that contains the part/ subdirectory. +func scanAutogeneratedParts(assetsDir string) (map[string]map[int]struct{}, error) { + if assetsDir == "" { + return nil, nil + } + + result := make(map[string]map[int]struct{}) + + for _, category := range supportedPartCategories { + discovered, err := DiscoverPartModels(assetsDir, category) + if err != nil { + return nil, err + } + // Always include the category, even if no IDs were discovered + ids := make(map[int]struct{}, len(discovered)) + for rowID := range discovered { + ids[rowID] = struct{}{} + } + result[category] = ids + } + + return result, nil +} + +func resolveAutogeneratedPartsInventory(p *project.Project, progress func(string)) (map[string]map[int]struct{}, error) { + overrideDir, err := resolvePartsAssetsOverrideDir(p) + if err != nil { + return nil, err + } + if overrideDir != "" { + if progress != nil { + progress(fmt.Sprintf("Scanning part models from local topdata.assets override %s...", overrideDir)) + } + return scanAutogeneratedParts(overrideDir) + } + return resolveReleasedPartsManifest(p, progress) +} + +func hasCollectedPartsDatasets(collected []nativeCollectedDataset) bool { + for _, dataset := range collected { + if isPartsDataset(dataset.Dataset.Name) { + return true + } + } + return false +} + +func loadPartOverrides(path string) ([]map[string]any, error) { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + rawOverrides, ok := obj["overrides"] + if !ok { + return nil, nil + } + overrideList, ok := rawOverrides.([]any) + if !ok { + return nil, fmt.Errorf("%s: overrides must be an array", path) + } + overrides := make([]map[string]any, 0, len(overrideList)) + for index, rawOverride := range overrideList { + override, ok := rawOverride.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: override %d must be an object", path, index) + } + overrides = append(overrides, override) + } + return overrides, nil +} + +func applyPartOverrides(sourceDir string, collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) { + return applyPartOverridesWithConfig(sourceDir, collected, project.PartsRowsConfig{}) +} + +func applyPartOverridesWithConfig(sourceDir string, collected []nativeCollectedDataset, cfg project.PartsRowsConfig) ([]nativeCollectedDataset, error) { + result := make([]nativeCollectedDataset, len(collected)) + copy(result, collected) + for i, dataset := range result { + if !strings.HasPrefix(dataset.Dataset.Name, "parts/") { + continue + } + category := extractPartCategory(dataset.Dataset.Name) + if category == "" { + continue + } + overrides, err := loadPartOverridesForCategory(sourceDir, category) + if err != nil { + return nil, err + } + if len(overrides) == 0 { + continue + } + + rows := make([]map[string]any, len(dataset.Rows)) + rowByID := make(map[int]map[string]any, len(dataset.Rows)) + for index, row := range dataset.Rows { + cloned := cloneRowMap(row) + rows[index] = cloned + if rowID, ok := cloned["id"].(int); ok { + rowByID[rowID] = cloned + } + } + + for index, override := range overrides { + rawID, ok := override["id"] + if !ok { + return nil, fmt.Errorf("parts/%s override %d is missing id", category, index) + } + rowID, err := asInt(rawID) + if err != nil { + return nil, fmt.Errorf("parts/%s override %d id is not numeric", category, index) + } + row, ok := rowByID[rowID] + if !ok { + row = createDefaultPartRowForDataset(rowID, dataset.Dataset.Name, dataset.Columns, cfg) + rows = append(rows, row) + rowByID[rowID] = row + } + for field, value := range override { + if field == "id" { + continue + } + row[field] = normalizePartOverrideValue(value) + } + } + + slices.SortFunc(rows, func(a, b map[string]any) int { + idA := a["id"].(int) + idB := b["id"].(int) + return idA - idB + }) + result[i].Rows = rows + } + return result, nil +} + +func normalizePartsRowsACBonus(collected []nativeCollectedDataset, cfg project.PartsRowsConfig) ([]nativeCollectedDataset, error) { + if !partsRowsACBonusConfigured(cfg) && len(cfg.RowDefaults) == 0 { + return collected, nil + } + result := make([]nativeCollectedDataset, len(collected)) + copy(result, collected) + for i, dataset := range result { + if !strings.HasPrefix(dataset.Dataset.Name, "parts/") { + continue + } + rows := make([]map[string]any, len(dataset.Rows)) + for index, row := range dataset.Rows { + cloned := cloneRowMap(row) + for field, value := range cfg.RowDefaults { + if strings.TrimSpace(field) == "" { + continue + } + if isUnsetPartValue(cloned[field]) { + cloned[field] = value + } + } + rowID, ok := cloned["id"].(int) + if isUnsetPartValue(cloned["ACBONUS"]) { + cloned["COSTMODIFIER"] = nullValue + } else if ok { + if value, configured := configuredPartsRowsACBonusValue(rowID, dataset.Dataset.Name, cfg); configured { + cloned["ACBONUS"] = value + } + } + rows[index] = cloned + } + result[i].Rows = rows + } + return result, nil +} + +func partsRowsACBonusConfigured(cfg project.PartsRowsConfig) bool { + if strings.TrimSpace(cfg.ACBonus.Default.Strategy) != "" { + return true + } + if len(cfg.ACBonus.Datasets) > 0 { + return true + } + for _, dataset := range cfg.Datasets { + if strings.TrimSpace(dataset.ACBonus.Strategy) != "" { + return true + } + } + return false +} + +func configuredPartsRowsACBonusValue(rowID int, datasetName string, cfg project.PartsRowsConfig) (string, bool) { + policy, ok := partsRowsACBonusPolicyForDataset(datasetName, cfg) + if !ok { + return "", false + } + switch strings.TrimSpace(policy.Strategy) { + case "ascending_row_id_sort_key": + format := strings.TrimSpace(policy.Format) + if format == "" { + format = "%.2f" + } + return fmt.Sprintf(format, float64(rowID)/float64(policy.Divisor)), true + case "descending_row_id_sort_key": + format := strings.TrimSpace(policy.Format) + if format == "" { + format = "%.2f" + } + return fmt.Sprintf(format, float64(policy.MaxRowID-rowID)/float64(policy.Divisor)), true + case "fixed": + return policy.Value, true + default: + return "", false + } +} + +func partsRowsACBonusPolicyForDataset(datasetName string, cfg project.PartsRowsConfig) (project.PartsRowsACBonusPolicy, bool) { + category := extractPartCategory(datasetName) + if policy, ok := cfg.ACBonus.Datasets[category]; ok && strings.TrimSpace(policy.Strategy) != "" { + return policy, true + } + if dataset, ok := cfg.Datasets[category]; ok && strings.TrimSpace(dataset.ACBonus.Strategy) != "" { + return dataset.ACBonus, true + } + if strings.TrimSpace(cfg.ACBonus.Default.Strategy) != "" { + return cfg.ACBonus.Default, true + } + return project.PartsRowsACBonusPolicy{}, false +} + +func loadPartOverridesForCategory(sourceDir, category string) ([]map[string]any, error) { + overrides := []map[string]any{} + + legacyPath := filepath.Join(sourceDir, "data", "parts", "overrides", category+".json") + legacyOverrides, err := loadPartOverrides(legacyPath) + if err != nil { + return nil, err + } + overrides = append(overrides, legacyOverrides...) + + moduleDir := filepath.Join(sourceDir, "data", "parts", "modules") + modulePaths, err := partModuleOverridePaths(moduleDir, category) + if err != nil { + return nil, err + } + for _, path := range modulePaths { + moduleOverrides, err := loadPartOverrides(path) + if err != nil { + return nil, err + } + overrides = append(overrides, moduleOverrides...) + } + return overrides, nil +} + +func partModuleOverridePaths(moduleDir, category string) ([]string, error) { + info, err := os.Stat(moduleDir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("parts module override path %s is not a directory", moduleDir) + } + + var paths []string + err = filepath.WalkDir(moduleDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.ToLower(filepath.Ext(path)) != ".json" { + return nil + } + if partModuleOverrideMatchesCategory(path, category) { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return nil, err + } + slices.Sort(paths) + return paths, nil +} + +func partModuleOverrideMatchesCategory(path, category string) bool { + stem := strings.TrimSuffix(strings.ToLower(filepath.Base(path)), strings.ToLower(filepath.Ext(path))) + category = strings.ToLower(strings.TrimSpace(category)) + return stem == category || + strings.HasPrefix(stem, category+"_") || + strings.HasSuffix(stem, "_"+category) || + strings.Contains(stem, "_"+category+"_") +} + +func normalizePartOverrideValue(value any) any { + switch typed := value.(type) { + case float64: + if typed == float64(int(typed)) { + return strconv.Itoa(int(typed)) + } + case int: + return strconv.Itoa(typed) + case json.Number: + return typed.String() + } + return value +} + +// augmentWithAutogeneratedParts applies repository-backed discovery defaults. +// Missing discovered IDs are added with default values. Existing discovered rows +// retain authored values unless they still use placeholder values, in which case +// discovery activates them with engine-visible defaults. +func augmentWithAutogeneratedParts(collected []nativeCollectedDataset, autogenerated map[string]map[int]struct{}) []nativeCollectedDataset { + return augmentWithAutogeneratedPartsWithConfig(collected, autogenerated, project.PartsRowsConfig{}) +} + +func augmentWithAutogeneratedPartsWithConfig(collected []nativeCollectedDataset, autogenerated map[string]map[int]struct{}, cfg project.PartsRowsConfig) []nativeCollectedDataset { + if len(autogenerated) == 0 { + return collected + } + + result := make([]nativeCollectedDataset, len(collected)) + copy(result, collected) + + for i, dataset := range collected { + if !isAutogenEligiblePartsDataset(dataset) { + continue + } + category := partAssetCategoryForDataset(dataset.Dataset.Name) + if !isSupportedPartCategory(category) { + continue + } + + ids, ok := autogenerated[category] + if !ok || len(ids) == 0 { + continue + } + + // Build a map of existing row IDs. + existingRows := make(map[int]map[string]any, len(dataset.Rows)) + for _, row := range dataset.Rows { + if id, ok := row["id"].(int); ok { + existingRows[id] = row + } + } + + // Add rows for discovered IDs that don't already exist. + newRows := make([]map[string]any, 0, len(dataset.Rows)) + newRows = append(newRows, dataset.Rows...) + + for rowID := range ids { + if row, exists := existingRows[rowID]; exists { + applyDiscoveredPartDefaultsWithConfig(row, dataset.Dataset.Name, dataset.Columns, cfg) + continue + } + if _, exists := existingRows[rowID]; !exists { + newRow := createDefaultPartRowForDataset(rowID, dataset.Dataset.Name, dataset.Columns, cfg) + newRows = append(newRows, newRow) + } + } + + // Sort rows by ID + slices.SortFunc(newRows, func(a, b map[string]any) int { + idA := a["id"].(int) + idB := b["id"].(int) + return idA - idB + }) + + result[i].Rows = newRows + } + + return result +} diff --git a/internal/topdata/parts_manifest.go b/internal/topdata/parts_manifest.go new file mode 100644 index 0000000..5c97731 --- /dev/null +++ b/internal/topdata/parts_manifest.go @@ -0,0 +1,287 @@ +package topdata + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +const ( + defaultSowAssetsRepoOwner = "ShadowsOverWestgate" + defaultSowAssetsRepoName = "sow-assets" + defaultGiteaBaseURL = "https://gitea.westgate.pw" + partsManifestReleaseTag = "parts-manifest-current" + partsManifestAssetName = "sow-parts-manifest.json" + partsManifestCacheFileName = "sow-parts-manifest.json" + partsManifestCacheMaxAge = time.Hour + partsManifestRequestTimeout = 30 * time.Second +) + +type partsManifest struct { + Repo string `json:"repo"` + Ref string `json:"ref"` + GeneratedAt string `json:"generated_at"` + Categories map[string][]int `json:"categories"` +} + +type giteaRepoSpec struct { + BaseURL string + Owner string + Repo string +} + +var partsManifestHTTPClient = &http.Client{Timeout: partsManifestRequestTimeout} + +func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (map[string]map[int]struct{}, error) { + if progress == nil { + progress = func(string) {} + } + cachePath := p.AutogenCachePath(partsManifestCacheFileName) + if strings.TrimSpace(os.Getenv(p.PartsManifestRefreshEnv())) == "" { + manifest, fresh, err := readFreshPartsManifestCache(cachePath, partsManifestCacheMaxAge) + if err != nil { + progress(fmt.Sprintf("Ignoring cached parts manifest: %v", err)) + } else if fresh { + progress(fmt.Sprintf("Using cached released parts manifest from %s...", cachePath)) + return inventoryFromPartsManifest(manifest), nil + } + } + spec, err := deriveSowAssetsRepoSpec(p) + if err != nil { + return nil, err + } + manifestURL, err := resolvePartsManifestAssetURL(spec) + if err != nil { + return nil, err + } + progress(fmt.Sprintf("Fetching released parts manifest from %s...", manifestURL)) + manifest, err := fetchPartsManifest(manifestURL) + if err != nil { + return nil, err + } + if err := writePartsManifestCache(cachePath, manifest); err != nil { + return nil, err + } + return inventoryFromPartsManifest(manifest), nil +} + +func deriveSowAssetsRepoSpec(p *project.Project) (giteaRepoSpec, error) { + serverOverride := strings.TrimSpace(os.Getenv(p.AssetsServerURLEnv())) + repoOverride := strings.TrimSpace(os.Getenv(p.AssetsRepoEnv())) + remote, err := gitOutput(p.Root, "remote", "get-url", "origin") + spec := giteaRepoSpec{BaseURL: defaultGiteaBaseURL, Owner: defaultSowAssetsRepoOwner, Repo: defaultSowAssetsRepoName} + if err == nil && strings.TrimSpace(remote) != "" { + if derived, ok := parseRepoSpec(strings.TrimSpace(remote)); ok { + derived.Repo = defaultSowAssetsRepoName + spec = derived + } + } + + if serverOverride != "" { + spec.BaseURL = strings.TrimRight(serverOverride, "/") + } + if repoOverride != "" { + owner, repo, ok := parseOwnerRepo(repoOverride) + if !ok { + return giteaRepoSpec{}, fmt.Errorf("invalid SOW_ASSETS_REPO %q; expected owner/repo", repoOverride) + } + spec.Owner = owner + spec.Repo = repo + } + return spec, nil +} + +func parseOwnerRepo(raw string) (string, string, bool) { + parts := strings.Split(strings.Trim(strings.TrimSuffix(raw, ".git"), "/"), "/") + if len(parts) < 2 { + return "", "", false + } + owner := strings.TrimSpace(parts[len(parts)-2]) + repo := strings.TrimSpace(parts[len(parts)-1]) + if owner == "" || repo == "" { + return "", "", false + } + return owner, repo, true +} + +func parseRepoSpec(raw string) (giteaRepoSpec, bool) { + if strings.HasPrefix(raw, "ssh://") || strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") { + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return giteaRepoSpec{}, false + } + parts := strings.Split(strings.Trim(strings.TrimSuffix(u.Path, ".git"), "/"), "/") + if len(parts) < 2 { + return giteaRepoSpec{}, false + } + base := (&url.URL{Scheme: "https", Host: normalizeGiteaHTTPHost(u.Hostname())}).String() + if u.Scheme == "http" || u.Scheme == "https" { + base = (&url.URL{Scheme: u.Scheme, Host: u.Host}).String() + } + return giteaRepoSpec{BaseURL: strings.TrimRight(base, "/"), Owner: parts[len(parts)-2], Repo: parts[len(parts)-1]}, true + } + if strings.HasPrefix(raw, "git@") { + withoutUser := strings.TrimPrefix(raw, "git@") + hostAndPath := strings.SplitN(withoutUser, ":", 2) + if len(hostAndPath) != 2 { + return giteaRepoSpec{}, false + } + parts := strings.Split(strings.Trim(strings.TrimSuffix(hostAndPath[1], ".git"), "/"), "/") + if len(parts) < 2 { + return giteaRepoSpec{}, false + } + base := (&url.URL{Scheme: "https", Host: normalizeGiteaHTTPHost(hostAndPath[0])}).String() + return giteaRepoSpec{BaseURL: strings.TrimRight(base, "/"), Owner: parts[len(parts)-2], Repo: parts[len(parts)-1]}, true + } + return giteaRepoSpec{}, false +} + +func normalizeGiteaHTTPHost(host string) string { + host = strings.TrimSpace(host) + if strings.HasPrefix(host, "git-ssh.") { + return "gitea." + strings.TrimPrefix(host, "git-ssh.") + } + return host +} + +func resolvePartsManifestAssetURL(spec giteaRepoSpec) (string, error) { + apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), url.PathEscape(spec.Owner), url.PathEscape(spec.Repo), url.PathEscape(partsManifestReleaseTag)) + var release struct { + Assets []struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + DownloadURL string `json:"download_url"` + } `json:"assets"` + } + if err := fetchJSON(apiURL, &release); err != nil { + return "", fmt.Errorf("fetch parts manifest release metadata: %w", err) + } + for _, asset := range release.Assets { + if asset.Name != partsManifestAssetName { + continue + } + if asset.DownloadURL != "" { + return asset.DownloadURL, nil + } + if asset.BrowserDownloadURL != "" { + return asset.BrowserDownloadURL, nil + } + } + return "", fmt.Errorf("parts manifest asset %s not found in release %s/%s:%s", partsManifestAssetName, spec.Owner, spec.Repo, partsManifestReleaseTag) +} + +func fetchPartsManifest(manifestURL string) (*partsManifest, error) { + var manifest partsManifest + if err := fetchJSON(manifestURL, &manifest); err != nil { + return nil, fmt.Errorf("download parts manifest: %w", err) + } + if len(manifest.Categories) == 0 { + return nil, fmt.Errorf("download parts manifest: categories is empty") + } + return &manifest, nil +} + +func readFreshPartsManifestCache(path string, maxAge time.Duration) (*partsManifest, bool, error) { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil, false, nil + } + return nil, false, err + } + if maxAge > 0 && time.Since(info.ModTime()) > maxAge { + return nil, false, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, false, err + } + var manifest partsManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, false, err + } + if len(manifest.Categories) == 0 { + return nil, false, fmt.Errorf("categories is empty") + } + return &manifest, true, nil +} + +func fetchJSON(target string, out any) error { + req, err := http.NewRequest(http.MethodGet, target, nil) + if err != nil { + return err + } + if token := strings.TrimSpace(os.Getenv("SOW_ASSETS_TOKEN")); token != "" { + req.Header.Set("Authorization", "token "+token) + } else if token := strings.TrimSpace(os.Getenv("GITEA_TOKEN")); token != "" { + req.Header.Set("Authorization", "token "+token) + } + resp, err := partsManifestHTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + return fmt.Errorf("HTTP %d from %s; set SOW_ASSETS_TOKEN or GITEA_TOKEN if the repo is private", resp.StatusCode, target) + } + return fmt.Errorf("HTTP %d from %s: %s", resp.StatusCode, target, strings.TrimSpace(string(body))) + } + decoder := json.NewDecoder(resp.Body) + if err := decoder.Decode(out); err != nil { + return err + } + return nil +} + +func writePartsManifestCache(path string, manifest *partsManifest) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create parts manifest cache dir: %w", err) + } + payload, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return fmt.Errorf("marshal parts manifest cache: %w", err) + } + payload = append(payload, '\n') + current, err := os.ReadFile(path) + if err == nil && bytes.Equal(current, payload) { + now := time.Now() + if err := os.Chtimes(path, now, now); err != nil { + return fmt.Errorf("refresh parts manifest cache timestamp: %w", err) + } + return nil + } + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("read parts manifest cache: %w", err) + } + tmpPath := path + ".tmp" + if err := os.WriteFile(tmpPath, payload, 0o644); err != nil { + return fmt.Errorf("write parts manifest cache: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("replace parts manifest cache: %w", err) + } + return nil +} + +func inventoryFromPartsManifest(manifest *partsManifest) map[string]map[int]struct{} { + result := make(map[string]map[int]struct{}, len(supportedPartCategories)) + for _, category := range supportedPartCategories { + ids := make(map[int]struct{}) + for _, rowID := range manifest.Categories[category] { + ids[rowID] = struct{}{} + } + result[category] = ids + } + return result +} diff --git a/internal/topdata/parts_manifest_test.go b/internal/topdata/parts_manifest_test.go new file mode 100644 index 0000000..aa5424c --- /dev/null +++ b/internal/topdata/parts_manifest_test.go @@ -0,0 +1,26 @@ +package topdata + +import "testing" + +func TestParseRepoSpecNormalizesWestgateSSHHostToPublicGiteaHost(t *testing.T) { + spec, ok := parseRepoSpec("ssh://git@git-ssh.westgate.pw:2222/ShadowsOverWestgate/sow-module.git") + if !ok { + t.Fatal("expected SSH remote to parse") + } + if spec.BaseURL != "https://gitea.westgate.pw" { + t.Fatalf("expected public Gitea base URL, got %q", spec.BaseURL) + } + if spec.Owner != "ShadowsOverWestgate" || spec.Repo != "sow-module" { + t.Fatalf("unexpected repo spec: %#v", spec) + } +} + +func TestParseRepoSpecKeepsHTTPRemoteHost(t *testing.T) { + spec, ok := parseRepoSpec("https://example.invalid/owner/repo.git") + if !ok { + t.Fatal("expected HTTPS remote to parse") + } + if spec.BaseURL != "https://example.invalid" { + t.Fatalf("expected HTTPS base URL to be preserved, got %q", spec.BaseURL) + } +} diff --git a/internal/topdata/placeables_migrate.go b/internal/topdata/placeables_migrate.go new file mode 100644 index 0000000..2535307 --- /dev/null +++ b/internal/topdata/placeables_migrate.go @@ -0,0 +1,6 @@ +package topdata + +func importLegacyPlaceables(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "placeables", "placeables.2da", nil) +} diff --git a/internal/topdata/portraits_migrate.go b/internal/topdata/portraits_migrate.go new file mode 100644 index 0000000..d3f48dc --- /dev/null +++ b/internal/topdata/portraits_migrate.go @@ -0,0 +1,8 @@ +package topdata + +func importLegacyPortraits(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + if legacyTLK == nil { + return 0, nil + } + return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "portraits", "portraits.2da", nil) +} diff --git a/internal/topdata/progfx_migrate.go b/internal/topdata/progfx_migrate.go new file mode 100644 index 0000000..17ba8ff --- /dev/null +++ b/internal/topdata/progfx_migrate.go @@ -0,0 +1,6 @@ +package topdata + +func importLegacyProgfx(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "progfx", "progfx.2da", nil) +} diff --git a/internal/topdata/racialtypes_migrate.go b/internal/topdata/racialtypes_migrate.go new file mode 100644 index 0000000..a012906 --- /dev/null +++ b/internal/topdata/racialtypes_migrate.go @@ -0,0 +1,282 @@ +package topdata + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" +) + +type legacyRacialtypesFeatTable struct { + Output string + Rows []map[string]any +} + +func importLegacyRacialtypes(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + if strings.TrimSpace(referenceBuilderDir) == "" { + return 0, nil + } + + legacyCoreDir := filepath.Join(referenceBuilderDir, "data", "racialtypes", "core") + legacyFeatsDir := filepath.Join(referenceBuilderDir, "data", "racialtypes", "feats") + if _, err := os.Stat(legacyCoreDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + if _, err := os.Stat(legacyFeatsDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetRoot := filepath.Join(dataDir, "racialtypes", "registry") + targetBasePath := filepath.Join(targetRoot, "base.json") + targetLockPath := filepath.Join(targetRoot, "lock.json") + targetRacesDir := filepath.Join(targetRoot, "races") + if fileExists(targetBasePath) && fileExists(targetLockPath) { + entries, err := os.ReadDir(targetRacesDir) + if err == nil { + raceFiles := 0 + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(strings.ToLower(entry.Name()), ".json") { + raceFiles++ + } + } + if raceFiles == 16 { + return 0, nil + } + } + } + + baseColumns, baseRows, lockData, raceRows, err := collectLegacyRacialtypesRegistry(legacyCoreDir, legacyFeatsDir, legacyTLK) + if err != nil { + return 0, err + } + + if err := removePathIfExists(filepath.Join(dataDir, "racialtypes", "core")); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(dataDir, "racialtypes", "feats")); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(targetRoot, "base_rows.json")); err != nil { + return 0, err + } + if err := removePathIfExists(targetRacesDir); err != nil { + return 0, err + } + if err := os.MkdirAll(targetRacesDir, 0o755); err != nil { + return 0, err + } + + writes := []struct { + path string + obj any + }{ + { + path: targetBasePath, + obj: map[string]any{ + "columns": stringSliceToAny(baseColumns), + "rows": rowsToAny(baseRows), + }, + }, + { + path: targetLockPath, + obj: anyMapInt(lockData), + }, + } + + raceKeys := make([]string, 0, len(raceRows)) + for key := range raceRows { + raceKeys = append(raceKeys, key) + } + slices.Sort(raceKeys) + for _, key := range raceKeys { + fileName := strings.TrimPrefix(key, "racialtypes:") + ".json" + writes = append(writes, struct { + path string + obj any + }{ + path: filepath.Join(targetRacesDir, fileName), + obj: raceRows[key], + }) + } + + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + return len(writes), nil +} + +func collectLegacyRacialtypesRegistry(coreDir, featsDir string, legacyTLK *legacyTLKData) ([]string, []map[string]any, map[string]int, map[string]map[string]any, error) { + coreCollected, err := collectBaseDataset(nativeDataset{ + Name: "racialtypes", + BasePath: filepath.Join(coreDir, "base.json"), + LockPath: filepath.Join(coreDir, "lock.json"), + ModulesDir: filepath.Join(coreDir, "modules"), + Spec: specForDataset("racialtypes"), + }) + if err != nil { + return nil, nil, nil, nil, err + } + + featTables, err := loadLegacyRacialtypesFeatTables(featsDir) + if err != nil { + return nil, nil, nil, nil, err + } + + baseRows := make([]map[string]any, 0) + raceRows := map[string]map[string]any{} + lockData := map[string]int{} + for key, id := range coreCollected.LockData { + if strings.HasPrefix(key, "racialtypes:") { + lockData[key] = id + } + } + + for _, collectedRow := range coreCollected.Rows { + row, ok := deepCopyValue(collectedRow).(map[string]any) + if !ok { + continue + } + if legacyTLK != nil { + if _, _, err := inlineLegacyTLKValue(row, legacyTLK); err != nil { + return nil, nil, nil, nil, err + } + } + key, _ := row["key"].(string) + if !strings.HasPrefix(key, "racialtypes:") { + assignRacialtypesBaseRowKey(row) + if key, _ := row["key"].(string); key != "" { + rowID, err := asInt(row["id"]) + if err != nil { + return nil, nil, nil, nil, err + } + lockData[key] = rowID + } + baseRows = append(baseRows, row) + continue + } + + delete(row, "id") + delete(row, "key") + + raceFile := map[string]any{ + "key": key, + "core": row, + } + if tableKey := extractRacialtypesFeatTableKey(row["FeatsTable"]); tableKey != "" { + table, ok := featTables[tableKey] + if !ok { + return nil, nil, nil, nil, fmt.Errorf("%s: unknown feat table %s", key, tableKey) + } + delete(row, "FeatsTable") + raceFile["feat_output"] = table.Output + raceFile["feats"] = rowsToAny(table.Rows) + } + raceRows[key] = raceFile + } + + slices.SortFunc(baseRows, func(a, b map[string]any) int { + left, _ := asInt(a["id"]) + right, _ := asInt(b["id"]) + return left - right + }) + return coreCollected.Columns, baseRows, lockData, raceRows, nil +} + +func loadLegacyRacialtypesFeatTables(featsDir string) (map[string]legacyRacialtypesFeatTable, error) { + entries, err := os.ReadDir(featsDir) + if err != nil { + return nil, err + } + + tables := map[string]legacyRacialtypesFeatTable{} + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + continue + } + obj, err := loadJSONObject(filepath.Join(featsDir, entry.Name())) + if err != nil { + return nil, err + } + tableKey, _ := obj["key"].(string) + outputName, _ := obj["output"].(string) + rawRows, ok := obj["rows"].([]any) + if tableKey == "" || outputName == "" || !ok { + return nil, fmt.Errorf("%s: racialtypes feat table must define key, output, and rows", entry.Name()) + } + rows := make([]map[string]any, 0, len(rawRows)) + for index, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: row %d must be an object", entry.Name(), index) + } + rows = append(rows, row) + } + tables[tableKey] = legacyRacialtypesFeatTable{ + Output: outputName, + Rows: rows, + } + } + return tables, nil +} + +func extractRacialtypesFeatTableKey(value any) string { + obj, ok := value.(map[string]any) + if !ok { + return "" + } + tableKey, _ := obj["table"].(string) + return tableKey +} + +func assignRacialtypesBaseRowKey(row map[string]any) { + name, _ := row["Name"].(string) + if strings.TrimSpace(name) == "" || name == nullValue { + delete(row, "key") + return + } + label, _ := row["Label"].(string) + keySuffix := normalizeRacialtypesKeySuffix(label) + if keySuffix == "" { + delete(row, "key") + return + } + row["key"] = "racialtypes:" + keySuffix +} + +func normalizeRacialtypesKeySuffix(label string) string { + normalized := strings.ToLower(strings.TrimSpace(label)) + if normalized == "" { + return "" + } + var b strings.Builder + lastUnderscore := false + for _, ch := range normalized { + isAlphaNum := (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') + if isAlphaNum { + b.WriteRune(ch) + lastUnderscore = false + continue + } + if lastUnderscore { + continue + } + b.WriteByte('_') + lastUnderscore = true + } + return strings.Trim(b.String(), "_") +} diff --git a/internal/topdata/racialtypes_registry.go b/internal/topdata/racialtypes_registry.go new file mode 100644 index 0000000..1584d2a --- /dev/null +++ b/internal/topdata/racialtypes_registry.go @@ -0,0 +1,270 @@ +package topdata + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" +) + +const ( + racialtypesRegistryDirName = "racialtypes/registry" + racialtypesRegistryLock = "lock.json" +) + +type racialtypesRegistry struct { + RootPath string + LockPath string + LockData map[string]int + BaseColumns []string + BaseRows []map[string]any + Races []map[string]any +} + +func loadRacialtypesRegistry(dataDir string) (*racialtypesRegistry, error) { + root := filepath.Join(dataDir, filepath.FromSlash(racialtypesRegistryDirName)) + info, err := os.Stat(root) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("racialtypes registry root must be a directory: %s", root) + } + + lockPath := filepath.Join(root, racialtypesRegistryLock) + lockData, err := loadLockfile(lockPath) + if err != nil { + return nil, err + } + + basePath := filepath.Join(root, "base.json") + baseObj, err := loadJSONObject(basePath) + if err != nil { + return nil, err + } + baseColumns, err := parseColumns(baseObj, "racialtypes/registry/core") + if err != nil { + return nil, err + } + rawBaseRows, ok := baseObj["rows"].([]any) + if !ok { + return nil, fmt.Errorf("%s: rows must be an array", basePath) + } + baseRows := make([]map[string]any, 0, len(rawBaseRows)) + for index, raw := range rawBaseRows { + row, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: row %d must be an object", basePath, index) + } + baseRows = append(baseRows, row) + } + + raceRows, err := loadRacialtypesRegistryRaceRows(filepath.Join(root, "races")) + if err != nil { + return nil, err + } + + return &racialtypesRegistry{ + RootPath: root, + LockPath: lockPath, + LockData: lockData, + BaseColumns: baseColumns, + BaseRows: baseRows, + Races: raceRows, + }, nil +} + +func loadRacialtypesRegistryRaceRows(dir string) ([]map[string]any, error) { + info, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("racialtypes registry races path must be a directory: %s", dir) + } + + paths := make([]string, 0) + err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.HasSuffix(strings.ToLower(d.Name()), ".json") { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return nil, err + } + slices.Sort(paths) + + rows := make([]map[string]any, 0, len(paths)) + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + rows = append(rows, obj) + } + return rows, nil +} + +func collectRacialtypesRegistryDatasets(dataDir string) ([]nativeCollectedDataset, error) { + registry, err := loadRacialtypesRegistry(dataDir) + if err != nil { + return nil, err + } + if registry == nil { + return nil, nil + } + + lockModified, err := syncRacialtypesBaseLockData(registry.BaseRows, registry.LockData) + if err != nil { + return nil, err + } + raceIDs, raceLockModified, err := assignRegistryIDs(registry.Races, registry.LockData, "racialtypes:") + if err != nil { + return nil, err + } + lockModified = lockModified || raceLockModified + if lockModified { + if err := saveLockfile(registry.LockPath, registry.LockData); err != nil { + return nil, err + } + } + + coreDataset := nativeDataset{ + Kind: nativeDatasetBase, + Name: "racialtypes/registry/core", + OutputName: "racialtypes.2da", + Spec: specForDataset("racialtypes"), + } + featDataset := nativeDataset{ + Kind: nativeDatasetPlain, + Name: "racialtypes/registry/feats", + Spec: specForDataset("racialtypes"), + } + + coreRows := make([]map[string]any, 0, len(registry.BaseRows)+len(registry.Races)) + rowIDSeen := map[int]string{} + for index, raw := range registry.BaseRows { + row, err := canonicalizeBaseRow(coreDataset, registry.BaseColumns, raw, index) + if err != nil { + return nil, fmt.Errorf("racialtypes registry base row %d: %w", index, err) + } + rowID := row["id"].(int) + if existing, ok := rowIDSeen[rowID]; ok { + return nil, fmt.Errorf("racialtypes registry base row id %d conflicts with %s", rowID, existing) + } + rowIDSeen[rowID] = "base" + coreRows = append(coreRows, row) + } + + datasets := make([]nativeCollectedDataset, 0, 16) + for _, race := range registry.Races { + key := stringField(race, "key") + if key == "" { + return nil, fmt.Errorf("racialtypes registry row is missing key") + } + rowID, ok := raceIDs[key] + if !ok { + return nil, fmt.Errorf("racialtypes registry row %s is missing lock id", key) + } + if existing, ok := rowIDSeen[rowID]; ok { + return nil, fmt.Errorf("racialtypes registry row id %d for %s conflicts with %s", rowID, key, existing) + } + rowIDSeen[rowID] = key + + coreRaw, ok := race["core"].(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: core must be an object", key) + } + row, err := canonicalizeEntry(coreDataset, registry.BaseColumns, rowID, key, coreRaw) + if err != nil { + return nil, fmt.Errorf("%s: %w", key, err) + } + + featOutput, _ := race["feat_output"].(string) + if featOutput != "" { + row["FeatsTable"] = outputStem(featOutput) + } + coreRows = append(coreRows, row) + + if featOutput == "" { + continue + } + rawFeats, ok := race["feats"].([]any) + if !ok { + return nil, fmt.Errorf("%s: feats must be an array when feat_output is set", key) + } + featRows := make([]map[string]any, 0, len(rawFeats)) + for index, rawFeat := range rawFeats { + featRow, ok := rawFeat.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: feat row %d must be an object", key, index) + } + row, err := canonicalizeBaseRow(featDataset, []string{"FeatLabel", "FeatIndex"}, featRow, index) + if err != nil { + return nil, fmt.Errorf("%s: feat row %d: %w", key, index, err) + } + featRows = append(featRows, row) + } + datasets = append(datasets, newGeneratedDataset( + "racialtypes/registry/"+strings.TrimPrefix(key, "racialtypes:"), + featOutput, + []string{"FeatLabel", "FeatIndex"}, + featRows, + )) + } + + slices.SortFunc(coreRows, func(a, b map[string]any) int { + return a["id"].(int) - b["id"].(int) + }) + slices.SortFunc(datasets, func(a, b nativeCollectedDataset) int { + return strings.Compare(a.Dataset.OutputName, b.Dataset.OutputName) + }) + + coreCollected := newGeneratedDataset("racialtypes/registry/core", "racialtypes.2da", registry.BaseColumns, coreRows) + out := make([]nativeCollectedDataset, 0, len(datasets)+1) + out = append(out, coreCollected) + out = append(out, datasets...) + return out, nil +} + +func syncRacialtypesBaseLockData(rows []map[string]any, lockData map[string]int) (bool, error) { + modified := false + for _, raw := range rows { + key := stringField(raw, "key") + if key == "" { + continue + } + rowID, ok := raw["id"] + if !ok { + return false, fmt.Errorf("racialtypes registry base row %s is missing id", key) + } + id, err := asInt(rowID) + if err != nil { + return false, fmt.Errorf("racialtypes registry base row %s has invalid id: %w", key, err) + } + if existing, ok := lockData[key]; ok { + if existing != id { + return false, fmt.Errorf("racialtypes registry base row %s id %d does not match lock id %d", key, id, existing) + } + continue + } + lockData[key] = id + modified = true + } + return modified, nil +} diff --git a/internal/topdata/repadjust_migrate.go b/internal/topdata/repadjust_migrate.go new file mode 100644 index 0000000..0a3bcc1 --- /dev/null +++ b/internal/topdata/repadjust_migrate.go @@ -0,0 +1,63 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacyRepadjust(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "repadjust") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "repadjust") + targetBasePath := filepath.Join(targetDir, "base.json") + targetLockPath := filepath.Join(targetDir, "lock.json") + if fileExists(targetBasePath) && fileExists(targetLockPath) { + obj, err := loadJSONObject(targetBasePath) + if err == nil && countLegacyTLKRefsInValue(obj) == 0 { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + baseObj["output"] = "repadjust.2da" + + lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) + if err != nil { + return 0, err + } + + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return 0, err + } + + writes := []struct { + path string + obj map[string]any + }{ + {path: targetBasePath, obj: baseObj}, + {path: targetLockPath, obj: lockObj}, + } + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + return len(writes), nil +} diff --git a/internal/topdata/ruleset_migrate.go b/internal/topdata/ruleset_migrate.go new file mode 100644 index 0000000..ffb5d0c --- /dev/null +++ b/internal/topdata/ruleset_migrate.go @@ -0,0 +1,171 @@ +package topdata + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" +) + +func importLegacyRuleset(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "ruleset") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "ruleset") + targetPath := filepath.Join(targetDir, "ruleset.json") + if _, err := os.Stat(targetPath); err == nil { + obj, err := loadJSONObject(targetPath) + if err == nil && countLegacyTLKRefsInValue(obj) == 0 { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + rows, err := mergeRulesetRows(baseObj, filepath.Join(legacyDir, "modules")) + if err != nil { + return 0, err + } + + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(targetDir, "base.json")); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(targetDir, "lock.json")); err != nil { + return 0, err + } + if err := removePathIfExists(filepath.Join(targetDir, "modules")); err != nil { + return 0, err + } + + out := map[string]any{ + "output": "ruleset.2da", + "columns": baseObj["columns"], + "rows": rows, + } + raw, err := json.MarshalIndent(out, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(targetPath, raw, 0o644); err != nil { + return 0, err + } + return 1, nil +} + +func mergeRulesetRows(baseObj map[string]any, modulesDir string) ([]any, error) { + rawRows, ok := baseObj["rows"].([]any) + if !ok { + return nil, nil + } + + rows := make([]map[string]any, 0, len(rawRows)) + byLabel := map[string]map[string]any{} + for _, raw := range rawRows { + row, ok := deepCopyValue(raw).(map[string]any) + if !ok { + continue + } + delete(row, "key") + if rawID, ok := row["id"]; ok { + id, err := asInt(rawID) + if err != nil { + return nil, err + } + row["id"] = id + } + if label, ok := row["Label"].(string); ok && label != "" && label != "****" { + if _, exists := byLabel[label]; exists { + return nil, fmt.Errorf("duplicate ruleset label %q", label) + } + byLabel[label] = row + } + rows = append(rows, row) + } + + modulePaths, err := collectModulePaths(modulesDir) + if err != nil { + return nil, err + } + for _, path := range modulePaths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + overrideList, ok := obj["overrides"].([]any) + if !ok { + continue + } + for _, raw := range overrideList { + override, ok := raw.(map[string]any) + if !ok { + continue + } + matchObj, ok := override["match"].(map[string]any) + if !ok { + return nil, fmt.Errorf("%s: ruleset override is missing match object", path) + } + rawLabel, ok := matchObj["Label"] + if !ok { + return nil, fmt.Errorf("%s: ruleset override match is missing Label", path) + } + labels, err := normalizeRulesetMatchLabels(rawLabel) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + for _, label := range labels { + row, ok := byLabel[label] + if !ok { + return nil, fmt.Errorf("%s: ruleset override target label %q was not found", path, label) + } + for key, value := range override { + if key == "id" || key == "key" || key == "_tlk" || isMetadataField(key) || key == "match" { + continue + } + row[key] = deepCopyValue(value) + } + } + } + } + + sort.Slice(rows, func(i, j int) bool { + return rows[i]["id"].(int) < rows[j]["id"].(int) + }) + out := make([]any, 0, len(rows)) + for _, row := range rows { + out = append(out, row) + } + return out, nil +} + +func normalizeRulesetMatchLabels(raw any) ([]string, error) { + switch typed := raw.(type) { + case string: + return []string{typed}, nil + case []any: + out := make([]string, 0, len(typed)) + for _, item := range typed { + label, ok := item.(string) + if !ok { + return nil, fmt.Errorf("ruleset match.Label entries must be strings") + } + out = append(out, label) + } + return out, nil + default: + return nil, fmt.Errorf("ruleset match.Label must be a string or string array") + } +} diff --git a/internal/topdata/skills_migrate.go b/internal/topdata/skills_migrate.go new file mode 100644 index 0000000..01f168a --- /dev/null +++ b/internal/topdata/skills_migrate.go @@ -0,0 +1,140 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacySkills(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "skills") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "skills") + targetBasePath := filepath.Join(targetDir, "base.json") + targetLockPath := filepath.Join(targetDir, "lock.json") + targetModulesDir := filepath.Join(targetDir, "modules") + moduleNames := []string{ + "add_athletics.json", + "add_disguise.json", + "add_linguistics.json", + "add_perception.json", + "add_scriptcraft.json", + "add_sensemotive.json", + "add_stealth.json", + "add_survival.json", + "add_userope.json", + filepath.Join("craft", "add_craftalchemy.json"), + filepath.Join("craft", "add_craftcooking.json"), + filepath.Join("craft", "add_craftjewelry.json"), + filepath.Join("craft", "add_craftleatherworking.json"), + filepath.Join("craft", "add_craftstonework.json"), + filepath.Join("craft", "add_crafttextiles.json"), + filepath.Join("craft", "ovr_craftarmorsmithing.json"), + filepath.Join("craft", "ovr_crafttinkering.json"), + filepath.Join("craft", "ovr_craftweaponsmithing.json"), + filepath.Join("craft", "ovr_craftwoodworking.json"), + filepath.Join("knowledge", "add_knowledgearcana.json"), + filepath.Join("knowledge", "add_knowledgearchitecture.json"), + filepath.Join("knowledge", "add_knowledgedungeoneering.json"), + filepath.Join("knowledge", "add_knowledgegeography.json"), + filepath.Join("knowledge", "add_knowledgehistory.json"), + filepath.Join("knowledge", "add_knowledgelocal.json"), + filepath.Join("knowledge", "add_knowledgenature.json"), + filepath.Join("knowledge", "add_knowledgenobility.json"), + filepath.Join("knowledge", "add_knowledgeplanar.json"), + filepath.Join("knowledge", "add_knowledgereligion.json"), + "ovr_acrobatics.json", + "ovr_animalhandling.json", + "ovr_appraise.json", + "ovr_concentration.json", + "ovr_disabledevice.json", + "ovr_heal.json", + "ovr_hiddenskills.json", + "ovr_influence.json", + "ovr_openlock.json", + "ovr_parry.json", + "ovr_searchtowis.json", + "ovr_sleightofhand.json", + "ovr_taunttointimidate.json", + "rmv_removedskills.json", + } + targetModulePaths := make([]string, 0, len(moduleNames)) + for _, name := range moduleNames { + targetModulePaths = append(targetModulePaths, filepath.Join(targetModulesDir, name)) + } + if fileExists(targetBasePath) && fileExists(targetLockPath) { + allModulesPresent := true + for _, path := range targetModulePaths { + if !fileExists(path) { + allModulesPresent = false + break + } + } + if allModulesPresent { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + baseObj["output"] = "skills.2da" + canonicalizeWikiMetadataDocument(baseObj) + + lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) + if err != nil { + return 0, err + } + + moduleObjs := make([]map[string]any, 0, len(moduleNames)) + for _, name := range moduleNames { + obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", name)) + if err != nil { + return 0, err + } + canonicalizeWikiMetadataDocument(obj) + moduleObjs = append(moduleObjs, obj) + } + + if err := os.MkdirAll(targetModulesDir, 0o755); err != nil { + return 0, err + } + + writes := []struct { + path string + obj map[string]any + }{ + {path: targetBasePath, obj: baseObj}, + {path: targetLockPath, obj: lockObj}, + } + for i, path := range targetModulePaths { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return 0, err + } + writes = append(writes, struct { + path string + obj map[string]any + }{path: path, obj: moduleObjs[i]}) + } + + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + return len(writes), nil +} diff --git a/internal/topdata/skyboxes_migrate.go b/internal/topdata/skyboxes_migrate.go new file mode 100644 index 0000000..6b09eb3 --- /dev/null +++ b/internal/topdata/skyboxes_migrate.go @@ -0,0 +1,6 @@ +package topdata + +func importLegacySkyboxes(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "skyboxes", "skyboxes.2da", nil) +} diff --git a/internal/topdata/spells_migrate.go b/internal/topdata/spells_migrate.go new file mode 100644 index 0000000..1dca1f2 --- /dev/null +++ b/internal/topdata/spells_migrate.go @@ -0,0 +1,103 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacySpells(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "spells") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "spells") + targetBasePath := filepath.Join(targetDir, "base.json") + targetLockPath := filepath.Join(targetDir, "lock.json") + targetModulesDir := filepath.Join(targetDir, "modules") + moduleNames := []string{ + "specialattacks.json", + "ovr_fixduplicates.json", + filepath.Join("bardicmusic", "fascinate.json"), + filepath.Join("bardicmusic", "inspirecompetence.json"), + filepath.Join("bardicmusic", "inspirecourage.json"), + filepath.Join("bardicmusic", "inspiregreatness.json"), + filepath.Join("bardicmusic", "inspireheroics.json"), + filepath.Join("bardicmusic", "songoffreedom.json"), + } + targetModulePaths := make([]string, 0, len(moduleNames)) + for _, name := range moduleNames { + targetModulePaths = append(targetModulePaths, filepath.Join(targetModulesDir, name)) + } + if fileExists(targetBasePath) && fileExists(targetLockPath) { + allModulesPresent := true + for _, path := range targetModulePaths { + if !fileExists(path) { + allModulesPresent = false + break + } + } + if allModulesPresent { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + baseObj["output"] = "spells.2da" + + lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) + if err != nil { + return 0, err + } + + moduleObjs := make([]map[string]any, 0, len(moduleNames)) + for _, name := range moduleNames { + obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", name)) + if err != nil { + return 0, err + } + moduleObjs = append(moduleObjs, obj) + } + + if err := os.MkdirAll(targetModulesDir, 0o755); err != nil { + return 0, err + } + + writes := []struct { + path string + obj map[string]any + }{ + {path: targetBasePath, obj: baseObj}, + {path: targetLockPath, obj: lockObj}, + } + for i, path := range targetModulePaths { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return 0, err + } + writes = append(writes, struct { + path string + obj map[string]any + }{path: path, obj: moduleObjs[i]}) + } + + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + return len(writes), nil +} diff --git a/internal/topdata/tailmodel_migrate.go b/internal/topdata/tailmodel_migrate.go new file mode 100644 index 0000000..4c44ee9 --- /dev/null +++ b/internal/topdata/tailmodel_migrate.go @@ -0,0 +1,86 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacyTailmodel(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "tailmodel") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "tailmodel") + targetBasePath := filepath.Join(targetDir, "base.json") + targetLockPath := filepath.Join(targetDir, "lock.json") + targetModulesDir := filepath.Join(targetDir, "modules") + targetModulePaths := []string{ + filepath.Join(targetModulesDir, "add.json"), + filepath.Join(targetModulesDir, "ovr_tailprefixes.json"), + } + if fileExists(targetBasePath) && fileExists(targetLockPath) { + allModulesPresent := true + for _, path := range targetModulePaths { + if !fileExists(path) { + allModulesPresent = false + break + } + } + if allModulesPresent { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + baseObj["output"] = "tailmodel.2da" + + lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) + if err != nil { + return 0, err + } + + moduleNames := []string{"add.json", "ovr_tailprefixes.json"} + moduleObjs := make([]map[string]any, 0, len(moduleNames)) + for _, name := range moduleNames { + obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", name)) + if err != nil { + return 0, err + } + moduleObjs = append(moduleObjs, obj) + } + + if err := os.MkdirAll(targetModulesDir, 0o755); err != nil { + return 0, err + } + + writes := []struct { + path string + obj map[string]any + }{ + {path: targetBasePath, obj: baseObj}, + {path: targetLockPath, obj: lockObj}, + {path: targetModulePaths[0], obj: moduleObjs[0]}, + {path: targetModulePaths[1], obj: moduleObjs[1]}, + } + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + return len(writes), nil +} diff --git a/internal/topdata/tlk_native.go b/internal/topdata/tlk_native.go new file mode 100644 index 0000000..01da36f --- /dev/null +++ b/internal/topdata/tlk_native.go @@ -0,0 +1,729 @@ +package topdata + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + + "golang.org/x/text/encoding/charmap" +) + +const ( + defaultTLKName = "sow_tlk.tlk" + tlkStateFile = ".tlk_state.json" + customTLKBase = 0x01000000 +) + +type textRole string + +const ( + textRoleName textRole = "name" + textRoleDescription textRole = "description" +) + +type datasetTextSpec struct { + NameFields []string + DescriptionFields []string +} + +var datasetTextSpecs = map[string]datasetTextSpec{ + "baseitems": { + NameFields: []string{"Name"}, + DescriptionFields: []string{"Description"}, + }, + "classes": { + NameFields: []string{"Short", "Name", "Plural", "Lower", "CLASS"}, + DescriptionFields: []string{"Description"}, + }, + "feat": { + NameFields: []string{"FEAT"}, + DescriptionFields: []string{"DESCRIPTION"}, + }, + "masterfeats": { + NameFields: []string{"STRREF"}, + DescriptionFields: []string{"DESCRIPTION"}, + }, + "itemprops": { + NameFields: []string{"Name", "StringRef"}, + DescriptionFields: []string{"Description", "GameStrRef"}, + }, + "racialtypes": { + NameFields: []string{"Name", "RACIALTYPE"}, + DescriptionFields: []string{"Description"}, + }, + "skills": { + NameFields: []string{"Name", "SKILL"}, + DescriptionFields: []string{"Description"}, + }, + "spells": { + NameFields: []string{"Name", "SPELL"}, + DescriptionFields: []string{"SpellDesc", "DESCRIPTION"}, + }, +} + +type tlkEntryData struct { + Text string + SoundResRef string + SoundLength float32 +} + +type tlkStateDocument struct { + Version int `json:"version"` + Language string `json:"language"` + Entries map[string]tlkStateMapping `json:"entries"` +} + +type tlkStateMapping struct { + ID int `json:"id"` + Retired bool `json:"retired,omitempty"` +} + +type tlkCompiledValue struct { + Value any + TLKKey string +} + +type tlkCompiler struct { + language string + statePath string + state tlkStateDocument + legacy map[string]tlkEntryData + active map[string]tlkEntryData + activeKeys map[string]struct{} + reservedByID map[int]string + nextID int +} + +func newTLKCompiler(sourceDir string, legacy *legacyTLKData) (*tlkCompiler, error) { + statePath := filepath.Join(sourceDir, tlkStateFile) + state, err := loadTLKState(statePath) + if err != nil { + return nil, err + } + baseDialog, err := loadBaseDialogData(filepath.Join(sourceDir, "base_dialog.json")) + if err != nil { + return nil, err + } + legacy = mergeLegacyTLK(baseDialog, legacy) + language := state.Language + if language == "" { + language = "en" + } + if legacy != nil && legacy.Language != "" { + if state.Language == "" { + language = legacy.Language + } + for key, id := range legacy.Lock { + if _, ok := state.Entries[key]; !ok { + state.Entries[key] = tlkStateMapping{ID: id} + } + } + } + + reserved := map[int]string{} + nextID := 0 + for key, entry := range state.Entries { + if owner, ok := reserved[entry.ID]; ok && owner != key { + return nil, fmt.Errorf("tlk state id collision between %q and %q", owner, key) + } + reserved[entry.ID] = key + if entry.ID >= nextID { + nextID = entry.ID + 1 + } + } + + compiler := &tlkCompiler{ + language: language, + statePath: statePath, + state: state, + legacy: map[string]tlkEntryData{}, + active: map[string]tlkEntryData{}, + activeKeys: map[string]struct{}{}, + reservedByID: reserved, + nextID: nextID, + } + if legacy != nil { + for key, entry := range legacy.Entries { + compiler.legacy[key] = entry + } + } + return compiler, nil +} + +func loadBaseDialogData(path string) (*legacyTLKData, error) { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("stat %s: %w", path, err) + } + obj, err := loadJSONObject(path) + if err != nil { + return nil, err + } + result := &legacyTLKData{ + Language: "en", + Entries: map[string]tlkEntryData{}, + Lock: map[string]int{}, + } + if language, ok := obj["language"].(string); ok && strings.TrimSpace(language) != "" { + result.Language = language + } + rawEntries, ok := obj["entries"] + if !ok { + return result, nil + } + entries, err := normalizeBaseDialogEntries(rawEntries) + if err != nil { + return nil, err + } + normalized, err := normalizeLegacyTLKEntries(entries) + if err != nil { + return nil, err + } + for key, entry := range normalized { + result.Entries[key] = entry + } + return result, nil +} + +func normalizeBaseDialogEntries(raw any) (map[string]any, error) { + switch typed := raw.(type) { + case map[string]any: + return typed, nil + case []any: + entries := make(map[string]any, len(typed)) + for index, item := range typed { + obj, ok := item.(map[string]any) + if !ok { + return nil, fmt.Errorf("base_dialog.json entry %d must be an object", index) + } + rawID, ok := obj["id"] + if !ok { + return nil, fmt.Errorf("base_dialog.json entry %d is missing id", index) + } + key, err := legacyDialogIDKey(rawID) + if err != nil { + return nil, fmt.Errorf("base_dialog.json entry %d: %w", index, err) + } + entry := map[string]any{} + if text, ok := obj["text"]; ok { + entry["text"] = text + } + if sound, ok := obj["sound_resref"]; ok { + entry["sound_resref"] = sound + } + if length, ok := obj["sound_length"]; ok { + entry["sound_length"] = length + } + entries[key] = entry + } + return entries, nil + default: + return nil, fmt.Errorf("base_dialog.json entries must be an object or array") + } +} + +func legacyDialogIDKey(raw any) (string, error) { + switch typed := raw.(type) { + case string: + if strings.TrimSpace(typed) == "" { + return "", fmt.Errorf("id must not be empty") + } + return typed, nil + case float64: + return strconv.Itoa(int(typed)), nil + case int: + return strconv.Itoa(typed), nil + default: + return "", fmt.Errorf("id must be a string or number") + } +} + +func loadTLKState(path string) (tlkStateDocument, error) { + doc := tlkStateDocument{ + Version: 1, + Entries: map[string]tlkStateMapping{}, + } + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return doc, nil + } + return doc, fmt.Errorf("read %s: %w", path, err) + } + if err := json.Unmarshal(raw, &doc); err != nil { + return doc, fmt.Errorf("parse %s: %w", path, err) + } + if doc.Version == 0 { + doc.Version = 1 + } + if doc.Entries == nil { + doc.Entries = map[string]tlkStateMapping{} + } + return doc, nil +} + +func saveTLKState(path string, doc tlkStateDocument) error { + doc.Version = 1 + if doc.Language == "" { + doc.Language = "en" + } + if doc.Entries == nil { + doc.Entries = map[string]tlkStateMapping{} + } + raw, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return fmt.Errorf("marshal tlk state: %w", err) + } + raw = append(raw, '\n') + current, err := os.ReadFile(path) + if err == nil && bytes.Equal(current, raw) { + return nil + } + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("read %s: %w", path, err) + } + if err := os.WriteFile(path, raw, 0o644); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + return nil +} + +func (c *tlkCompiler) resolveLegacyKey(key string) (tlkCompiledValue, error) { + if _, ok := c.legacy[key]; !ok { + if active, ok := c.active[key]; ok { + return tlkCompiledValue{ + Value: c.customStrrefForKey(key), + TLKKey: key, + }, c.markActive(key, active) + } + if len(c.legacy) == 0 { + return tlkCompiledValue{}, fmt.Errorf("unknown TLK reference: %s (inline TLK text is required for native builds)", key) + } + return tlkCompiledValue{}, fmt.Errorf("unknown TLK reference: %s", key) + } + if err := c.markActive(key, c.legacy[key]); err != nil { + return tlkCompiledValue{}, err + } + return tlkCompiledValue{ + Value: c.customStrrefForKey(key), + TLKKey: key, + }, nil +} + +func (c *tlkCompiler) registerInline(key string, entry tlkEntryData) (tlkCompiledValue, error) { + if err := c.markActive(key, entry); err != nil { + return tlkCompiledValue{}, err + } + return tlkCompiledValue{ + Value: c.customStrrefForKey(key), + TLKKey: key, + }, nil +} + +func (c *tlkCompiler) customStrrefForKey(key string) int { + return customTLKBase + c.state.Entries[key].ID +} + +func (c *tlkCompiler) markActive(key string, entry tlkEntryData) error { + mapping, ok := c.state.Entries[key] + if !ok { + mapping = tlkStateMapping{ID: c.allocateID(), Retired: false} + c.state.Entries[key] = mapping + } + existing, ok := c.active[key] + if ok && existing != entry { + return fmt.Errorf("conflicting TLK content for key %q", key) + } + c.active[key] = entry + c.activeKeys[key] = struct{}{} + mapping.Retired = false + c.state.Entries[key] = mapping + return nil +} + +func (c *tlkCompiler) allocateID() int { + id := c.nextID + c.nextID++ + return id +} + +func (c *tlkCompiler) finish(outputDir, tlkName string) (int, error) { + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return 0, fmt.Errorf("create tlk output dir: %w", err) + } + + for key, mapping := range c.state.Entries { + if _, ok := c.activeKeys[key]; ok { + mapping.Retired = false + } else { + mapping.Retired = true + } + c.state.Entries[key] = mapping + } + pruneRetiredGeneratedFeatTLKEntries(filepath.Dir(c.statePath), c.state.Entries) + + if err := saveTLKState(c.statePath, c.state); err != nil { + return 0, err + } + + maxID := -1 + for key := range c.activeKeys { + id := c.state.Entries[key].ID + if id > maxID { + maxID = id + } + } + entries := make([]tlkEntryData, maxID+1) + for key := range c.activeKeys { + id := c.state.Entries[key].ID + entries[id] = c.active[key] + } + + if strings.TrimSpace(tlkName) == "" { + tlkName = defaultTLKName + } + if err := writeTLKBinary(filepath.Join(outputDir, tlkName), entries, c.language); err != nil { + return 0, err + } + if maxID < 0 { + return 1, nil + } + return 1, nil +} + +func pruneRetiredGeneratedFeatTLKEntries(sourceDir string, entries map[string]tlkStateMapping) { + prefixes := collectGeneratedFeatTLKPrefixes(sourceDir) + for key, mapping := range entries { + if !mapping.Retired || !isRetiredGeneratedFeatTLKKey(key, prefixes) { + continue + } + delete(entries, key) + } +} + +func isRetiredGeneratedFeatTLKKey(key string, prefixes []string) bool { + if !strings.HasPrefix(key, "feat:") { + return false + } + for _, prefix := range prefixes { + if strings.HasPrefix(key, prefix) { + return true + } + } + return false +} + +func collectGeneratedFeatTLKPrefixes(sourceDir string) []string { + featGeneratedDir := filepath.Join(sourceDir, "data", "feat", "generated") + paths, err := collectModulePaths(featGeneratedDir) + if err != nil { + return nil + } + prefixes := make([]string, 0, len(paths)) + seen := map[string]struct{}{} + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil || !isFamilyExpansionObject(obj) { + continue + } + spec, err := parseFamilyExpansionSpec(path, obj) + if err != nil || spec.FamilyKey == "" { + continue + } + prefix := "feat:" + spec.FamilyKey + "_" + if _, ok := seen[prefix]; ok { + continue + } + seen[prefix] = struct{}{} + prefixes = append(prefixes, prefix) + } + slices.Sort(prefixes) + return prefixes +} + +func writeTLKBinary(path string, entries []tlkEntryData, language string) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create tlk output parent: %w", err) + } + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("create tlk output: %w", err) + } + defer file.Close() + + langID, err := languageID(language) + if err != nil { + return err + } + + if _, err := file.Write([]byte("TLK V3.0")); err != nil { + return err + } + if err := binary.Write(file, binary.LittleEndian, uint32(langID)); err != nil { + return err + } + if err := binary.Write(file, binary.LittleEndian, uint32(len(entries))); err != nil { + return err + } + if err := binary.Write(file, binary.LittleEndian, uint32(20+len(entries)*40)); err != nil { + return err + } + + stringData := make([][]byte, len(entries)) + offset := 0 + for index, entry := range entries { + flags := uint32(0) + if entry.Text != "" { + flags |= 0x1 + } + if entry.SoundResRef != "" { + flags |= 0x2 + } + if entry.SoundLength != 0 { + flags |= 0x4 + } + encodedText, err := charmap.Windows1252.NewEncoder().Bytes([]byte(entry.Text)) + if err != nil { + return fmt.Errorf("encode tlk text at row %d: %w", index, err) + } + stringData[index] = encodedText + + soundBytes := make([]byte, 16) + if len(entry.SoundResRef) > 16 { + return fmt.Errorf("sound resref at row %d exceeds 16 characters", index) + } + copy(soundBytes, []byte(entry.SoundResRef)) + + if err := binary.Write(file, binary.LittleEndian, flags); err != nil { + return err + } + if _, err := file.Write(soundBytes); err != nil { + return err + } + if err := binary.Write(file, binary.LittleEndian, uint32(0)); err != nil { + return err + } + if err := binary.Write(file, binary.LittleEndian, uint32(0)); err != nil { + return err + } + if err := binary.Write(file, binary.LittleEndian, uint32(offset)); err != nil { + return err + } + if err := binary.Write(file, binary.LittleEndian, uint32(len(encodedText))); err != nil { + return err + } + if err := binary.Write(file, binary.LittleEndian, entry.SoundLength); err != nil { + return err + } + offset += len(encodedText) + } + + for _, encoded := range stringData { + if _, err := file.Write(encoded); err != nil { + return err + } + } + return nil +} + +func languageID(code string) (int, error) { + switch strings.ToLower(strings.TrimSpace(code)) { + case "", "en": + return 0, nil + case "fr": + return 1, nil + case "de": + return 2, nil + case "it": + return 3, nil + case "es": + return 4, nil + case "pl": + return 5, nil + default: + return 0, fmt.Errorf("unknown TLK language code %q", code) + } +} + +func specForDataset(name string) datasetTextSpec { + best := datasetTextSpecs[name] + bestLength := 0 + for prefix, spec := range datasetTextSpecs { + if name != prefix && !strings.HasPrefix(name, prefix+"/") { + continue + } + if len(prefix) <= bestLength { + continue + } + best = spec + bestLength = len(prefix) + } + return best +} + +func roleForField(spec datasetTextSpec, field string) textRole { + for _, candidate := range spec.NameFields { + if strings.EqualFold(candidate, field) { + return textRoleName + } + } + for _, candidate := range spec.DescriptionFields { + if strings.EqualFold(candidate, field) { + return textRoleDescription + } + } + return textRole(strings.ToLower(field)) +} + +func columnForRole(columns []string, spec datasetTextSpec, role textRole) string { + var candidates []string + switch role { + case textRoleName: + candidates = spec.NameFields + case textRoleDescription: + candidates = spec.DescriptionFields + default: + return "" + } + for _, candidate := range candidates { + if column, ok := canonicalColumn(columns, candidate); ok { + return column + } + } + return "" +} + +type parsedTLKPayload struct { + Key string + Text string + Ref string + RefField string + SoundResRef string + SoundLength float32 +} + +func parseTLKPayload(value any, allowBare bool) (parsedTLKPayload, bool, error) { + switch typed := value.(type) { + case map[string]any: + if rawTLK, ok := typed["tlk"]; ok { + if key, ok := rawTLK.(string); ok { + return parsedTLKPayload{Key: key}, true, nil + } + return parseTLKPayload(rawTLK, true) + } + if allowBare || hasNonRefTLKPayloadKeys(typed) { + if _, ok := typed["text"]; ok { + return buildParsedTLKPayload(typed) + } + if _, ok := typed["ref"]; ok { + return buildParsedTLKPayload(typed) + } + if _, ok := typed["key"]; ok { + return buildParsedTLKPayload(typed) + } + } + return parsedTLKPayload{}, false, nil + case string: + return parsedTLKPayload{}, false, nil + default: + return parsedTLKPayload{}, false, nil + } +} + +func hasNonRefTLKPayloadKeys(obj map[string]any) bool { + for _, key := range []string{"text", "key", "sound_resref", "sound_length"} { + if _, ok := obj[key]; ok { + return true + } + } + return false +} + +func buildParsedTLKPayload(obj map[string]any) (parsedTLKPayload, bool, error) { + payload := parsedTLKPayload{} + if rawKey, ok := obj["key"]; ok { + key, ok := rawKey.(string) + if !ok { + return payload, false, fmt.Errorf("tlk key must be a string") + } + payload.Key = key + } + if rawText, ok := obj["text"]; ok { + text, ok := rawText.(string) + if !ok { + return payload, false, fmt.Errorf("tlk text must be a string") + } + payload.Text = text + } + if rawRef, ok := obj["ref"]; ok { + ref, ok := rawRef.(string) + if !ok { + return payload, false, fmt.Errorf("tlk ref must be a string") + } + payload.Ref = ref + } + if rawField, ok := obj["field"]; ok { + field, ok := rawField.(string) + if !ok { + return payload, false, fmt.Errorf("tlk field must be a string") + } + payload.RefField = field + } + if rawSound, ok := obj["sound_resref"]; ok { + sound, ok := rawSound.(string) + if !ok { + return payload, false, fmt.Errorf("tlk sound_resref must be a string") + } + payload.SoundResRef = sound + } + if rawLength, ok := obj["sound_length"]; ok { + switch typed := rawLength.(type) { + case float64: + payload.SoundLength = float32(typed) + case int: + payload.SoundLength = float32(typed) + default: + return payload, false, fmt.Errorf("tlk sound_length must be numeric") + } + } + if payload.Ref != "" && payload.RefField == "" { + return payload, false, fmt.Errorf("tlk ref requires field") + } + if payload.Ref != "" && payload.Text != "" { + return payload, false, fmt.Errorf("tlk payload cannot contain both text and ref") + } + if payload.Ref == "" && payload.Text == "" && payload.Key == "" { + return payload, false, fmt.Errorf("tlk payload must contain text, ref, or key") + } + return payload, true, nil +} + +func deriveAutoTLKKey(rowIdentity string, field string) string { + fieldText := strings.ToLower(strings.TrimSpace(field)) + if fieldText == "" { + fieldText = "text" + } + return rowIdentity + "." + fieldText +} + +func sortedKeys[M ~map[string]V, V any](input M) []string { + keys := make([]string, 0, len(input)) + for key := range input { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +func almostEqualFloat32(a, b float32) bool { + return math.Abs(float64(a-b)) < 0.0001 +} diff --git a/internal/topdata/top_package.go b/internal/topdata/top_package.go new file mode 100644 index 0000000..313b829 --- /dev/null +++ b/internal/topdata/top_package.go @@ -0,0 +1,713 @@ +package topdata + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +const ( + PackageHAKResRef = "sow_top" + PackageHAKFileName = PackageHAKResRef + ".hak" + PackageTLKFileName = defaultTLKName +) + +func compiled2DAOutputDir(p *project.Project) string { + return p.TopDataCompiled2DADir() +} + +func topDataBuildUsesRepoCache(p *project.Project) bool { + return p.TopDataUsesRepoCache() +} + +func compiledTLKOutputPath(p *project.Project) string { + return p.TopDataCompiledTLKPath() +} + +func compiledTLKOutputDir(p *project.Project) string { + return p.TopDataCompiledTLKDir() +} + +func wikiOutputRootDir(p *project.Project) string { + return p.TopDataWikiRootDir() +} + +func wikiOutputPagesDir(p *project.Project) string { + return p.TopDataWikiPagesDir() +} + +func wikiOutputStatePath(p *project.Project) string { + return p.TopDataWikiStatePath() +} + +func legacyCompiled2DAOutputDir(p *project.Project) string { + return filepath.Join(p.TopDataSourceDir(), "assets", "2da") +} + +func legacyWikiOutputRootDir(p *project.Project) string { + return filepath.Join(p.TopDataSourceDir(), legacyWikiRootDirName) +} + +func BuildPackage(p *project.Project, progress func(string)) (PackageResult, error) { + if progress == nil { + progress = func(string) {} + } + nativeResult, err := packagedBuildResult(p) + if err != nil { + return PackageResult{}, err + } + return packageBuiltTopData(p, nativeResult, progress) +} + +func BuildAndPackage(p *project.Project, progress func(string)) (PackageResult, error) { + return BuildAndPackageWithOptions(p, BuildAndPackageOptions{}, progress) +} + +type BuildAndPackageOptions struct { + Force bool + BuildWiki bool +} + +func BuildAndPackageWithOptions(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) { + if progress == nil { + progress = func(string) {} + } + if !opts.Force { + incrementalResult, ok, err := currentTopPackageResult(p) + if err != nil { + return PackageResult{}, err + } + if ok { + progress("Topdata outputs are current; skipping native build and top package refresh.") + return incrementalResult, nil + } + + nativeResult, ok, err := currentCompiledBuildResult(p) + if err != nil { + return PackageResult{}, err + } + if ok { + progress(fmt.Sprintf("Compiled topdata outputs are current; refreshing %s and %s only.", p.TopDataPackageHAKName(), p.TopDataPackageTLKName())) + return packageBuiltTopData(p, nativeResult, progress) + } + } + + return buildAndPackageFull(p, opts, progress) +} + +func buildAndPackageFull(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) { + nativeResult, err := BuildNativeWithOptions(p, NativeBuildOptions{BuildWiki: opts.BuildWiki, Force: opts.Force}, progress) + if err != nil { + return PackageResult{}, err + } + return packageBuiltTopData(p, nativeResult, progress) +} + +func BuildPackageFromBuild(p *project.Project, nativeResult BuildResult, progress func(string)) (PackageResult, error) { + if progress == nil { + progress = func(string) {} + } + return packageBuiltTopData(p, nativeResult, progress) +} + +func packageBuiltTopData(p *project.Project, nativeResult BuildResult, progress func(string)) (PackageResult, error) { + if progress == nil { + progress = func(string) {} + } + outputHAK := p.TopDataPackageHAKPath() + outputTLK := p.TopDataPackageTLKPath() + + progress(fmt.Sprintf("Packaging compiled topdata resources into %s...", filepath.Base(outputHAK))) + resources, assetFiles, err := collectTopPackageResources(p, nativeResult.Output2DADir) + if err != nil { + return PackageResult{}, err + } + if err := writeHAK(outputHAK, resources); err != nil { + return PackageResult{}, err + } + + progress(fmt.Sprintf("Preparing compiled TLK release output %s...", filepath.Base(outputTLK))) + sourceTLK := compiledTLKOutputPath(p) + if nativeResult.OutputTLKDir != "" { + sourceTLK, err = resolveCompiledTLKPath(nativeResult.OutputTLKDir) + if err != nil { + return PackageResult{}, err + } + } + if err := copyFile(outputTLK, sourceTLK); err != nil { + return PackageResult{}, err + } + + return PackageResult{ + Mode: nativeResult.Mode, + Output2DADir: nativeResult.Output2DADir, + OutputTLKDir: nativeResult.OutputTLKDir, + OutputHAKPath: outputHAK, + OutputTLKPath: outputTLK, + Files2DA: nativeResult.Files2DA, + FilesTLK: nativeResult.FilesTLK, + HAKResources: len(resources), + AssetFiles: assetFiles, + WikiOutputDir: nativeResult.WikiOutputDir, + WikiPages: nativeResult.WikiPages, + WikiStatus: nativeResult.WikiStatus, + }, nil +} + +func collectTopPackageResources(p *project.Project, compiled2DADir string) ([]erf.Resource, int, error) { + resourceByKey := map[string]erf.Resource{} + resourceSourceByKey := map[string]string{} + + entries, err := os.ReadDir(compiled2DADir) + if err != nil { + return nil, 0, fmt.Errorf("read compiled 2da output dir: %w", err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".2da") { + continue + } + path := filepath.Join(compiled2DADir, entry.Name()) + resource, err := topPackageResourceFromPath(path) + if err != nil { + return nil, 0, err + } + key := topPackageResourceKey(resource) + resourceByKey[key] = resource + resourceSourceByKey[key] = path + } + + assetFiles := 0 + assetsDir := filepath.Join(p.TopDataSourceDir(), "assets") + skipDirs := map[string]struct{}{ + filepath.Clean(compiled2DADir): {}, + filepath.Clean(legacyCompiled2DAOutputDir(p)): {}, + } + info, err := os.Stat(assetsDir) + if err == nil && info.IsDir() { + err = filepath.WalkDir(assetsDir, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + if _, ok := skipDirs[filepath.Clean(path)]; ok { + return filepath.SkipDir + } + } + if d.IsDir() { + return nil + } + if strings.HasPrefix(filepath.Base(path), ".") { + return nil + } + resource, err := topPackageResourceFromPath(path) + if err != nil { + return err + } + key := topPackageResourceKey(resource) + if existing, ok := resourceSourceByKey[key]; ok { + return fmt.Errorf("topdata asset %s collides with %s for top package resource %s", path, existing, key) + } + resourceByKey[key] = resource + resourceSourceByKey[key] = path + assetFiles++ + return nil + }) + if err != nil { + return nil, 0, err + } + } else if err != nil && !os.IsNotExist(err) { + return nil, 0, fmt.Errorf("stat topdata assets dir: %w", err) + } + + keys := make([]string, 0, len(resourceByKey)) + for key := range resourceByKey { + keys = append(keys, key) + } + slices.Sort(keys) + + resources := make([]erf.Resource, 0, len(keys)) + for _, key := range keys { + resources = append(resources, resourceByKey[key]) + } + return resources, assetFiles, nil +} + +func shouldSkipTopDataSourceDir(path string, skipDirs map[string]struct{}) bool { + _, ok := skipDirs[filepath.Clean(path)] + return ok +} + +func topDataSourceSkipDirs(p *project.Project) map[string]struct{} { + return map[string]struct{}{ + filepath.Clean(filepath.Join(p.TopDataSourceDir(), "templates")): {}, + filepath.Clean(legacyWikiOutputRootDir(p)): {}, + filepath.Clean(legacyCompiled2DAOutputDir(p)): {}, + filepath.Clean(wikiOutputRootDir(p)): {}, + } +} + +func newestTopDataSource(p *project.Project) (time.Time, string, error) { + newest := time.Time{} + newestPath := "" + skipDirs := topDataSourceSkipDirs(p) + sourceDir := p.TopDataSourceDir() + err := filepath.WalkDir(sourceDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() && shouldSkipTopDataSourceDir(path, skipDirs) { + return filepath.SkipDir + } + if path == sourceDir { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if newest.IsZero() || info.ModTime().After(newest) { + newest = info.ModTime() + newestPath = path + } + return nil + }) + if err != nil { + return time.Time{}, "", fmt.Errorf("scan topdata sources: %w", err) + } + return newest, newestPath, nil +} + +func newestTopDataInput(p *project.Project, now time.Time) (time.Time, string, error) { + newest, newestPath, err := newestTopDataSource(p) + if err != nil { + return time.Time{}, "", err + } + autogenTime, autogenPath, err := newestAutogenInput(p, now) + if err != nil { + return time.Time{}, "", err + } + if autogenTime.After(newest) { + return autogenTime, autogenPath, nil + } + return newest, newestPath, nil +} + +func newestAutogenInput(p *project.Project, now time.Time) (time.Time, string, error) { + newest := time.Time{} + newestPath := "" + for _, consumer := range p.Config.Autogen.Consumers { + overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer) + if err != nil { + return time.Time{}, "", err + } + if overrideRoot != "" { + candidateRoot := filepath.Join(overrideRoot, consumer.Root) + candidateTime, candidatePath, err := newestMatchingAutogenOverrideInput(candidateRoot, consumer.Include) + if err != nil { + return time.Time{}, "", err + } + if candidateTime.After(newest) { + newest = candidateTime + newestPath = candidatePath + } + continue + } + + cachePath := p.AutogenCachePath(consumer.Manifest.CacheName) + candidateTime, candidatePath, err := newestReleasedAutogenManifestInput(p, consumer, now, cachePath) + if err != nil { + return time.Time{}, "", err + } + if candidateTime.After(newest) { + newest = candidateTime + newestPath = candidatePath + } + } + return newest, newestPath, nil +} + +func newestMatchingAutogenOverrideInput(scanRoot string, include []string) (time.Time, string, error) { + info, err := os.Stat(scanRoot) + if err != nil { + if os.IsNotExist(err) { + return time.Time{}, "", nil + } + return time.Time{}, "", fmt.Errorf("stat autogen override root %s: %w", scanRoot, err) + } + if !info.IsDir() { + return time.Time{}, "", fmt.Errorf("autogen override root %s is not a directory", scanRoot) + } + + newest := time.Time{} + newestPath := "" + err = filepath.WalkDir(scanRoot, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(scanRoot, path) + if err != nil { + return err + } + if !matchesAutogenInclude(filepath.ToSlash(rel), include) { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if info.ModTime().After(newest) { + newest = info.ModTime() + newestPath = path + } + return nil + }) + if err != nil { + return time.Time{}, "", fmt.Errorf("scan autogen override %s: %w", scanRoot, err) + } + return newest, newestPath, nil +} + +func topPackageResourceFromPath(path string) (erf.Resource, error) { + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") + resourceType, ok := erf.HAKResourceTypeForExtension(extension) + if !ok { + return erf.Resource{}, fmt.Errorf("unsupported top package HAK resource extension %q", filepath.Ext(path)) + } + info, err := os.Stat(path) + if err != nil { + return erf.Resource{}, fmt.Errorf("stat %s: %w", path, err) + } + return erf.Resource{ + Name: strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))), + Type: resourceType, + SourcePath: path, + Size: info.Size(), + }, nil +} + +func topPackageResourceKey(resource erf.Resource) string { + extension, ok := erf.ExtensionForResourceType(resource.Type) + if !ok { + return strings.ToLower(resource.Name) + } + return strings.ToLower(resource.Name) + "." + strings.TrimPrefix(strings.ToLower(extension), ".") +} + +func writeHAK(path string, resources []erf.Resource) error { + output, err := os.Create(path) + if err != nil { + return fmt.Errorf("create top package hak: %w", err) + } + defer output.Close() + if err := erf.Write(output, erf.New("HAK ", resources)); err != nil { + return fmt.Errorf("write top package hak: %w", err) + } + return nil +} + +func resolveCompiledTLKPath(outputDir string) (string, error) { + entries, err := os.ReadDir(outputDir) + if err != nil { + return "", fmt.Errorf("read compiled tlk output dir: %w", err) + } + var tlkFiles []string + for _, entry := range entries { + if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".tlk") { + continue + } + tlkFiles = append(tlkFiles, filepath.Join(outputDir, entry.Name())) + } + if len(tlkFiles) != 1 { + return "", fmt.Errorf("expected exactly 1 compiled tlk output, found %d", len(tlkFiles)) + } + return tlkFiles[0], nil +} + +func copyFile(dst, src string) error { + if samePath(dst, src) { + return nil + } + input, err := os.Open(src) + if err != nil { + return fmt.Errorf("open %s: %w", src, err) + } + defer input.Close() + + output, err := os.Create(dst) + if err != nil { + return fmt.Errorf("create %s: %w", dst, err) + } + defer output.Close() + + if _, err := io.Copy(output, input); err != nil { + return fmt.Errorf("copy %s to %s: %w", src, dst, err) + } + return nil +} + +func samePath(left, right string) bool { + leftAbs, leftErr := filepath.Abs(left) + rightAbs, rightErr := filepath.Abs(right) + if leftErr != nil || rightErr != nil { + return left == right + } + return leftAbs == rightAbs +} + +func packagedBuildResult(p *project.Project) (BuildResult, error) { + if !p.HasTopData() { + return BuildResult{}, errors.New("topdata is not configured for this project") + } + + output2DA := compiled2DAOutputDir(p) + outputTLK := compiledTLKOutputDir(p) + if _, err := os.Stat(output2DA); err != nil { + return BuildResult{}, fmt.Errorf("topdata build output missing: run build-topdata first") + } + if _, err := os.Stat(compiledTLKOutputPath(p)); err != nil { + return BuildResult{}, fmt.Errorf("topdata tlk output missing: run build-topdata first") + } + + files2DA, oldest2DA, err := countCompiledFiles(output2DA, ".2da") + if err != nil { + return BuildResult{}, err + } + filesTLK := 1 + tlkInfo, err := os.Stat(compiledTLKOutputPath(p)) + if err != nil { + return BuildResult{}, fmt.Errorf("stat compiled tlk output %s: %w", compiledTLKOutputPath(p), err) + } + oldestTLK := tlkInfo.ModTime() + if files2DA == 0 { + return BuildResult{}, fmt.Errorf("topdata build output missing: run build-topdata first") + } + if err := validateCompiled2DAOutputCatalog(p, output2DA); err != nil { + return BuildResult{}, err + } + + newestSource, newestPath, err := newestTopDataInput(p, time.Now()) + if err != nil { + return BuildResult{}, err + } + oldestBuild := oldest2DA + if oldestBuild.IsZero() || (!oldestTLK.IsZero() && oldestTLK.Before(oldestBuild)) { + oldestBuild = oldestTLK + } + if !newestSource.IsZero() && !oldestBuild.IsZero() && newestSource.After(oldestBuild) { + return BuildResult{}, fmt.Errorf("topdata build output is older than source file %s; run build-topdata first", newestPath) + } + + return BuildResult{ + Mode: "native", + Output2DADir: output2DA, + OutputTLKDir: outputTLK, + Files2DA: files2DA, + FilesTLK: filesTLK, + }, nil +} + +func validateCompiled2DAOutputCatalog(p *project.Project, output2DA string) error { + expected, err := expectedCompiled2DAOutputs(p) + if err != nil { + return err + } + actual, err := compiled2DAOutputNames(output2DA) + if err != nil { + return err + } + + missing := make([]string, 0) + for outputName := range expected { + if _, ok := actual[outputName]; !ok { + missing = append(missing, outputName) + } + } + stale := make([]string, 0) + for outputName := range actual { + if _, ok := expected[outputName]; !ok { + stale = append(stale, outputName) + } + } + slices.Sort(missing) + slices.Sort(stale) + if len(missing) > 0 { + return fmt.Errorf("topdata build output is missing compiled 2da %s; run build-topdata first", strings.Join(missing, ", ")) + } + if len(stale) > 0 { + return fmt.Errorf("topdata build output contains stale compiled 2da %s; run build-topdata first", strings.Join(stale, ", ")) + } + return nil +} + +func expectedCompiled2DAOutputs(p *project.Project) (map[string]struct{}, error) { + dataDir := filepath.Join(p.TopDataSourceDir(), "data") + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return nil, err + } + registryDatasets, err := collectGeneratedRegistryDatasets(dataDir) + if err != nil { + return nil, err + } + outputs := make(map[string]struct{}, len(datasets)+len(registryDatasets)) + for _, dataset := range datasets { + outputs[dataset.OutputName] = struct{}{} + } + for _, dataset := range registryDatasets { + outputs[dataset.Dataset.OutputName] = struct{}{} + } + return outputs, nil +} + +func compiled2DAOutputNames(output2DA string) (map[string]struct{}, error) { + entries, err := os.ReadDir(output2DA) + if err != nil { + return nil, fmt.Errorf("read compiled 2da output dir %s: %w", output2DA, err) + } + outputs := make(map[string]struct{}) + for _, entry := range entries { + if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".2da") { + continue + } + outputs[entry.Name()] = struct{}{} + } + return outputs, nil +} + +func currentCompiledBuildResult(p *project.Project) (BuildResult, bool, error) { + nativeResult, err := packagedBuildResult(p) + if err == nil { + return nativeResult, true, nil + } + if strings.Contains(err.Error(), "run build-topdata first") { + return BuildResult{}, false, nil + } + return BuildResult{}, false, err +} + +func currentTopPackageResult(p *project.Project) (PackageResult, bool, error) { + nativeResult, ok, err := currentCompiledBuildResult(p) + if err != nil || !ok { + return PackageResult{}, ok, err + } + + outputHAK := p.TopDataPackageHAKPath() + outputTLK := p.TopDataPackageTLKPath() + hakInfo, err := os.Stat(outputHAK) + if err != nil { + if os.IsNotExist(err) { + return PackageResult{}, false, nil + } + return PackageResult{}, false, fmt.Errorf("stat top package hak %s: %w", outputHAK, err) + } + tlkInfo, err := os.Stat(outputTLK) + if err != nil { + if os.IsNotExist(err) { + return PackageResult{}, false, nil + } + return PackageResult{}, false, fmt.Errorf("stat top package tlk %s: %w", outputTLK, err) + } + + newestSource, _, err := newestTopDataInput(p, time.Now()) + if err != nil { + return PackageResult{}, false, err + } + newestCompiled, err := newestFileModTime(nativeResult.Output2DADir, ".2da") + if err != nil { + return PackageResult{}, false, err + } + compiledTLKInfo, err := os.Stat(compiledTLKOutputPath(p)) + if err != nil { + return PackageResult{}, false, fmt.Errorf("stat compiled tlk output %s: %w", compiledTLKOutputPath(p), err) + } + if compiledTLKInfo.ModTime().After(newestCompiled) { + newestCompiled = compiledTLKInfo.ModTime() + } + newestInput := newestSource + if newestCompiled.After(newestInput) { + newestInput = newestCompiled + } + + oldestPackage := hakInfo.ModTime() + if tlkInfo.ModTime().Before(oldestPackage) { + oldestPackage = tlkInfo.ModTime() + } + if !newestInput.IsZero() && newestInput.After(oldestPackage) { + return PackageResult{}, false, nil + } + + resources, assetFiles, err := collectTopPackageResources(p, nativeResult.Output2DADir) + if err != nil { + return PackageResult{}, false, err + } + return PackageResult{ + Mode: "incremental-noop", + Output2DADir: nativeResult.Output2DADir, + OutputTLKDir: nativeResult.OutputTLKDir, + OutputHAKPath: outputHAK, + OutputTLKPath: outputTLK, + Files2DA: nativeResult.Files2DA, + FilesTLK: nativeResult.FilesTLK, + HAKResources: len(resources), + AssetFiles: assetFiles, + WikiOutputDir: nativeResult.WikiOutputDir, + WikiPages: nativeResult.WikiPages, + WikiStatus: nativeResult.WikiStatus, + }, true, nil +} + +func newestFileModTime(dir, extension string) (time.Time, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return time.Time{}, fmt.Errorf("read output dir %s: %w", dir, err) + } + newest := time.Time{} + for _, entry := range entries { + if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), extension) { + continue + } + info, err := entry.Info() + if err != nil { + return time.Time{}, fmt.Errorf("stat output %s: %w", filepath.Join(dir, entry.Name()), err) + } + if newest.IsZero() || info.ModTime().After(newest) { + newest = info.ModTime() + } + } + return newest, nil +} + +func countCompiledFiles(dir, extension string) (int, time.Time, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return 0, time.Time{}, fmt.Errorf("read compiled output dir %s: %w", dir, err) + } + count := 0 + oldest := time.Time{} + for _, entry := range entries { + if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), extension) { + continue + } + info, err := entry.Info() + if err != nil { + return 0, time.Time{}, fmt.Errorf("stat compiled output %s: %w", filepath.Join(dir, entry.Name()), err) + } + count++ + if oldest.IsZero() || info.ModTime().Before(oldest) { + oldest = info.ModTime() + } + } + return count, oldest, nil +} diff --git a/internal/topdata/topdata.go b/internal/topdata/topdata.go new file mode 100644 index 0000000..4059c59 --- /dev/null +++ b/internal/topdata/topdata.go @@ -0,0 +1,3009 @@ +package topdata + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +type Severity string + +const ( + SeverityError Severity = "error" + SeverityWarning Severity = "warning" +) + +type Diagnostic struct { + Severity Severity + Path string + Message string +} + +type ValidationReport struct { + Files int + DataFiles int + TLKFiles int + Diagnostics []Diagnostic +} + +func (r ValidationReport) SummaryLines(maxPaths int) []string { + type key struct { + severity Severity + message string + } + groups := map[key][]string{} + order := make([]key, 0) + for _, diagnostic := range r.Diagnostics { + k := key{severity: diagnostic.Severity, message: diagnostic.Message} + if _, ok := groups[k]; !ok { + order = append(order, k) + } + groups[k] = append(groups[k], diagnostic.Path) + } + slices.SortFunc(order, func(a, b key) int { + if a.severity != b.severity { + if a.severity == SeverityError { + return -1 + } + return 1 + } + return strings.Compare(a.message, b.message) + }) + lines := make([]string, 0, len(order)) + for _, k := range order { + paths := append([]string(nil), groups[k]...) + slices.Sort(paths) + paths = slices.Compact(paths) + lines = append(lines, formatDiagnosticSummaryLine(string(k.severity), k.message, paths, maxPaths)) + } + return lines +} + +func (r ValidationReport) HasErrors() bool { + for _, diagnostic := range r.Diagnostics { + if diagnostic.Severity == SeverityError { + return true + } + } + return false +} + +func (r ValidationReport) ErrorCount() int { + count := 0 + for _, diagnostic := range r.Diagnostics { + if diagnostic.Severity == SeverityError { + count++ + } + } + return count +} + +func (r ValidationReport) WarningCount() int { + count := 0 + for _, diagnostic := range r.Diagnostics { + if diagnostic.Severity == SeverityWarning { + count++ + } + } + return count +} + +func formatDiagnosticSummaryLine(prefix, message string, paths []string, maxPaths int) string { + if len(paths) == 0 { + return fmt.Sprintf("%s: %s", prefix, message) + } + if maxPaths <= 0 { + maxPaths = 3 + } + display := paths + if len(display) > maxPaths { + display = display[:maxPaths] + } + location := strings.Join(display, ", ") + if len(paths) > len(display) { + location += fmt.Sprintf(", and %d more", len(paths)-len(display)) + } + return fmt.Sprintf("%s: %s [%s]", prefix, message, location) +} + +type BuildResult struct { + Mode string + Output2DADir string + OutputTLKDir string + Files2DA int + FilesTLK int + WikiOutputDir string + WikiPages int + WikiStatus string +} + +type PackageResult struct { + Mode string + Output2DADir string + OutputTLKDir string + OutputHAKPath string + OutputTLKPath string + Files2DA int + FilesTLK int + HAKResources int + AssetFiles int + WikiOutputDir string + WikiPages int + WikiStatus string +} + +type CompareResult struct { + Mode string + Compared2DA int + ComparedTLK int + NativePass int + NotYetCanonical int + NativeMismatch int +} + +func ValidateProject(p *project.Project) ValidationReport { + report := ValidationReport{} + if !p.HasTopData() { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: project.ConfigFile, + Message: "topdata is not configured for this project", + }) + return report + } + + sourceDir := p.TopDataSourceDir() + info, err := os.Stat(sourceDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: sourceDir, + Message: err.Error(), + }) + return report + } + if !info.IsDir() { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: sourceDir, + Message: "topdata source must be a directory", + }) + return report + } + + dataDir := filepath.Join(sourceDir, "data") + // Canonical topdata no longer requires a dedicated source tlk directory. + // When present, it is only validated for disallowed legacy module input. + tlkDir := filepath.Join(sourceDir, "tlk") + baseDialog := filepath.Join(sourceDir, "base_dialog.json") + statePath := filepath.Join(sourceDir, tlkStateFile) + + for _, requiredDir := range []string{dataDir} { + info, err := os.Stat(requiredDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: requiredDir, + Message: err.Error(), + }) + continue + } + if !info.IsDir() { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: requiredDir, + Message: "must be a directory", + }) + } + } + + if info, err := os.Stat(tlkDir); err == nil && info.IsDir() { + modulesDir := filepath.Join(tlkDir, "modules") + legacyModuleFiles := 0 + if moduleInfo, moduleErr := os.Stat(modulesDir); moduleErr == nil && moduleInfo.IsDir() { + _ = walkJSON(modulesDir, func(path string) { + legacyModuleFiles++ + report.Files++ + report.TLKFiles++ + _ = validateJSONFile(path, "tlk", &report) + }) + } else if moduleErr != nil && !errors.Is(moduleErr, os.ErrNotExist) { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: modulesDir, + Message: moduleErr.Error(), + }) + } + if legacyModuleFiles > 0 { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: modulesDir, + Message: "topdata/tlk/modules is no longer allowed for canonical authored input; inline TLK-in-data is required", + }) + } + if lockInfo, lockErr := os.Stat(filepath.Join(tlkDir, "lock.json")); lockErr == nil && !lockInfo.IsDir() { + report.Files++ + report.TLKFiles++ + _ = validateJSONFile(filepath.Join(tlkDir, "lock.json"), "tlk", &report) + } else if lockErr != nil && !errors.Is(lockErr, os.ErrNotExist) { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: filepath.Join(tlkDir, "lock.json"), + Message: lockErr.Error(), + }) + } + } + + if _, err := os.Stat(baseDialog); err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: baseDialog, + Message: "base_dialog.json is missing; dialog migration is not canonical yet", + }) + } else { + report.Files++ + if err := validateJSONFile(baseDialog, "base_dialog", &report); err == nil { + // count only once for base dialog. + } + } + + _ = walkJSON(dataDir, func(path string) { + report.Files++ + report.DataFiles++ + _ = validateJSONFile(path, "data", &report) + }) + validateTopPackageAssets(sourceDir, dataDir, &report) + validateNativeOutputCatalog(dataDir, &report) + validateNativeEntryKeyUniqueness(dataDir, &report) + validateNativeLockAllocation(dataDir, p.EffectiveConfig().TopData.RowGeneration, &report) + validateGeneratedFeatFamilies(dataDir, &report) + validateClassSpellbooks(dataDir, &report) + validateItempropsRegistryGraph(dataDir, &report) + if _, err := os.Stat(statePath); err == nil { + report.Files++ + _ = validateJSONFile(statePath, "state", &report) + } + if !report.HasErrors() { + validateNativeAuthoringWarnings(dataDir, p.EffectiveConfig().TopData.RowGeneration, &report) + validateNativeBuildability(p, &report) + } + + return report +} + +func validateJSONFile(path, domain string, report *ValidationReport) error { + raw, err := os.ReadFile(path) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return err + } + + var payload any + validateDuplicateJSONKeys(path, raw, report) + if err := json.Unmarshal(raw, &payload); err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("invalid JSON: %v", err), + }) + return err + } + + obj, ok := payload.(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "topdata files must contain a JSON object at the root", + }) + return errors.New("topdata file root must be object") + } + + switch domain { + case "data": + validateDataObject(path, obj, report) + case "tlk": + validateTLKObject(path, obj, report) + case "state": + validateStateObject(path, obj, report) + case "base_dialog": + validateBaseDialogObject(path, obj, report) + } + return nil +} + +func validateDuplicateJSONKeys(path string, raw []byte, report *ValidationReport) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil { + return + } +} + +func scanJSONValueForDuplicateKeys(decoder *json.Decoder, path string, report *ValidationReport) error { + token, err := decoder.Token() + if err != nil { + return err + } + switch delim := token.(type) { + case json.Delim: + switch delim { + case '{': + seen := map[string]struct{}{} + for decoder.More() { + rawKey, err := decoder.Token() + if err != nil { + return err + } + key, ok := rawKey.(string) + if !ok { + return fmt.Errorf("expected object key") + } + if _, ok := seen[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("duplicate JSON object key %q", key), + }) + } + seen[key] = struct{}{} + if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil { + return err + } + } + _, err := decoder.Token() + return err + case '[': + for decoder.More() { + if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil { + return err + } + } + _, err := decoder.Token() + return err + default: + return nil + } + default: + return nil + } +} + +func validateDataObject(path string, obj map[string]any, report *ValidationReport) { + base := filepath.Base(path) + slashed := filepath.ToSlash(path) + inModules := strings.Contains(slashed, "/modules/") + dataPath := parseTopdataDataPath(path) + + if invariant, ok := datasetInvariantForPath(dataPath); ok { + if dataPath.IsRootBase { + validateDatasetInvariantBaseFile(path, obj, invariant, report) + return + } + if dataPath.IsRootLock { + validateDatasetInvariantLockFile(path, obj, invariant, report) + return + } + } + if strings.Contains(slashed, "/data/feat/generated/") { + validateFeatGeneratedFile(path, obj, report) + return + } + if strings.Contains(slashed, "/data/parts/overrides/") { + validateOverridesFile(path, obj, report) + return + } + if isClassSpellbookPath(path) { + validateClassSpellbookShape(path, obj, report) + return + } + + if strings.Contains(slashed, "/racialtypes/registry/races/") { + validateRacialtypesRegistryRaceFile(path, obj, report) + return + } + + if base == "global.json" { + validateGlobalJSONFile(path, obj, report) + return + } + if base == "lock.json" { + validateLockObject(path, obj, report) + return + } + if base == "base.json" { + validateRowsFile(path, obj, report, true) + return + } + if inModules { + if _, hasEntries := obj["entries"]; hasEntries { + validateEntriesFile(path, obj, report) + return + } + if _, hasOverrides := obj["overrides"]; hasOverrides { + validateOverridesFile(path, obj, report) + return + } + if _, hasRows := obj["rows"]; hasRows { + validateRowsFile(path, obj, report, false) + return + } + if _, hasColumns := obj["columns"]; hasColumns { + validateColumnsFile(path, obj, report) + return + } + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: path, + Message: "module file does not use a recognized canonical shape yet; expected one of columns, entries, overrides, or rows", + }) + return + } + + if _, hasRows := obj["rows"]; hasRows { + validateRowsFile(path, obj, report, false) + return + } + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: path, + Message: "unrecognized data file shape; parseable for compatibility, but not canonical yet", + }) +} + +type topdataDataPath struct { + Segments []string + IsRootBase bool + IsRootLock bool +} + +func parseTopdataDataPath(path string) topdataDataPath { + slashed := filepath.ToSlash(path) + marker := "/data/" + index := strings.LastIndex(slashed, marker) + if index == -1 { + return topdataDataPath{} + } + rel := strings.Trim(slashed[index+len(marker):], "/") + if rel == "" { + return topdataDataPath{} + } + segments := strings.Split(rel, "/") + return topdataDataPath{ + Segments: segments, + IsRootBase: len(segments) == 2 && segments[1] == "base.json", + IsRootLock: len(segments) == 2 && segments[1] == "lock.json", + } +} + +type datasetValidationInvariant struct { + Name string + KeyPrefix string + RequiredColumns []string +} + +var datasetValidationInvariants = map[string]datasetValidationInvariant{ + "feat": { + Name: "feat", + KeyPrefix: "feat:", + }, + "masterfeats": { + Name: "masterfeats", + KeyPrefix: "masterfeats:", + RequiredColumns: []string{"LABEL", "STRREF", "DESCRIPTION"}, + }, +} + +func datasetInvariantForPath(dataPath topdataDataPath) (datasetValidationInvariant, bool) { + if len(dataPath.Segments) == 0 { + return datasetValidationInvariant{}, false + } + invariant, ok := datasetValidationInvariants[dataPath.Segments[0]] + return invariant, ok +} + +func validateDatasetInvariantBaseFile(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) { + validateRowsFile(path, obj, report, true) + validateDerivedBaseOutput(path, obj, invariant.Name, report) + validateRequiredColumns(path, obj, invariant, report) + validateRowsKeyPrefix(path, obj, invariant, report) +} + +func validateRequiredColumns(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) { + if len(invariant.RequiredColumns) == 0 { + return + } + columns := extractValidationColumns(obj) + for _, required := range invariant.RequiredColumns { + if _, ok := canonicalColumn(columns, required); ok { + continue + } + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s base.json must include %s in columns", invariant.Name, required), + }) + } +} + +func validateRowsKeyPrefix(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) { + if invariant.KeyPrefix == "" { + return + } + rows, ok := obj["rows"].([]any) + if !ok { + return + } + for index, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + key, ok := row["key"].(string) + if !ok || strings.TrimSpace(key) == "" { + continue + } + if !strings.HasPrefix(key, invariant.KeyPrefix) { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s row %d key %q must start with %s", invariant.Name, index, key, invariant.KeyPrefix), + }) + } + } +} + +func validateDatasetInvariantLockFile(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) { + validateLockObject(path, obj, report) + if invariant.KeyPrefix == "" { + return + } + for key := range obj { + if !strings.HasPrefix(key, invariant.KeyPrefix) { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s lock key %q must start with %s", invariant.Name, key, invariant.KeyPrefix), + }) + } + } +} + +func validateDerivedBaseOutput(path string, obj map[string]any, datasetLabel string, report *ValidationReport) { + if output, ok := obj["output"]; !ok { + return + } else if outputText, ok := output.(string); !ok || strings.TrimSpace(outputText) != nativeDatasetDefaultOutputName(filepath.Dir(path), nativeDatasetBase) { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s output must be %s", datasetLabel, nativeDatasetDefaultOutputName(filepath.Dir(path), nativeDatasetBase)), + }) + } +} + +func validateFeatGeneratedFile(path string, obj map[string]any, report *ValidationReport) { + family, ok := obj["family"].(string) + if !ok || strings.TrimSpace(family) == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "feat generated file must contain a non-empty family string", + }) + return + } + if isFamilyExpansionObject(obj) { + validateFeatFamilyExpansionFile(path, obj, report) + return + } + switch family { + default: + _, hasEntries := obj["entries"] + _, hasOverrides := obj["overrides"] + if !hasEntries && !hasOverrides { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "feat generated file must contain entries and/or overrides", + }) + return + } + if hasEntries { + validateEntriesFile(path, obj, report) + } + if hasOverrides { + validateOverridesFile(path, obj, report) + } + } +} + +func validateFeatFamilyExpansionFile(path string, obj map[string]any, report *ValidationReport) { + spec, err := parseFamilyExpansionSpec(path, obj) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return + } + if !strings.HasPrefix(spec.Template, "masterfeats:") { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "compact feat generated template must start with masterfeats:", + }) + } + if spec.NamePrefix == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "family expansion file must contain a non-empty name_prefix", + }) + } + if spec.LabelPrefix == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "family expansion file must contain a non-empty label_prefix", + }) + } + if spec.ConstantPrefix == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "family expansion file must contain a non-empty constant_prefix", + }) + } + if raw, ok := obj["overrides"]; ok { + if _, ok := raw.(map[string]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "family expansion overrides must be an object keyed by source dataset key", + }) + } + } +} + +func validateRacialtypesRegistryRaceFile(path string, obj map[string]any, report *ValidationReport) { + key, ok := obj["key"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "racialtypes registry race file must contain a key field", + }) + } else if _, ok := key.(string); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "racialtypes registry race key must be a string", + }) + } + + core, ok := obj["core"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "racialtypes registry race file must contain a core object", + }) + } else if _, ok := core.(map[string]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "racialtypes registry race core must be a JSON object", + }) + } + + if rawOutput, ok := obj["feat_output"]; ok { + if _, ok := rawOutput.(string); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "racialtypes registry race feat_output must be a string", + }) + } + if rawFeats, ok := obj["feats"]; ok { + if _, ok := rawFeats.([]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "racialtypes registry race feats must be an array", + }) + } + } else { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "racialtypes registry race feat_output requires a feats array", + }) + } + return + } + + if _, ok := obj["feats"]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "racialtypes registry race feats requires feat_output", + }) + } +} + +func validateTLKObject(path string, obj map[string]any, report *ValidationReport) { + if filepath.Base(path) == "lock.json" { + validateLockObject(path, obj, report) + return + } + validateLegacyDialogEntries(path, obj, report, "TLK") +} + +func validateBaseDialogObject(path string, obj map[string]any, report *ValidationReport) { + validateLegacyDialogEntries(path, obj, report, "base_dialog") +} + +func validateLegacyDialogEntries(path string, obj map[string]any, report *ValidationReport, label string) { + entries, ok := obj["entries"] + if ok { + switch entries.(type) { + case map[string]any, []any: + // accepted compatibility shapes + default: + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s entries must be a JSON object or array", label), + }) + } + } + if language, ok := obj["language"]; ok { + switch language.(type) { + case string, float64: + // accepted compatibility shapes + default: + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s language must be a string or numeric language id when present", label), + }) + } + } + if label == "TLK" && !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "TLK file must contain an entries object", + }) + } +} + +func validateLockObject(path string, obj map[string]any, report *ValidationReport) { + ids := map[int]string{} + checkDuplicateIDs := !strings.Contains(filepath.ToSlash(path), "/data/itemprops/registry/lock.json") + for key, value := range obj { + var rowID int + switch value.(type) { + case float64: + // JSON numbers land here. + if parsed, err := asInt(value); err == nil { + rowID = parsed + } + default: + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("lock entry %q must be numeric", key), + }) + } + if checkDuplicateIDs { + if rowID <= 0 { + continue + } + if other, ok := ids[rowID]; ok && other != key { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("lock id collision between %q and %q at id %d", other, key, rowID), + }) + } + ids[rowID] = key + } + } +} + +func validateRowsFile(path string, obj map[string]any, report *ValidationReport, requireColumns bool) { + rows, ok := obj["rows"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "rows file must contain a rows array", + }) + return + } + if _, ok := rows.([]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "rows must be a JSON array", + }) + } + if rowsList, ok := rows.([]any); ok { + validateRowCollection(path, obj, rowsList, report) + } + if _, ok := obj["columns"]; ok { + validateColumnsFile(path, obj, report) + } else if requireColumns { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "base dataset file must contain columns", + }) + } +} + +func validateColumnsFile(path string, obj map[string]any, report *ValidationReport) { + columns, ok := obj["columns"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "columns file must contain a columns array", + }) + return + } + list, ok := columns.([]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "columns must be a JSON array when present", + }) + return + } + for _, raw := range list { + column, ok := raw.(string) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "columns must contain only strings", + }) + continue + } + if isAuthoringOnlyField(column) && !preserveLegacyWikiColumnForPath(path, column) { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("deprecated wiki column %q must be moved into meta.wiki", column), + }) + } + } +} + +func validateEntriesFile(path string, obj map[string]any, report *ValidationReport) { + entries, ok := obj["entries"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "entries file must contain an entries object", + }) + return + } + if _, ok := entries.(map[string]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "entries must be a JSON object", + }) + } + if entryMap, ok := entries.(map[string]any); ok { + for key, raw := range entryMap { + entry, ok := raw.(map[string]any) + if !ok { + continue + } + for field := range entry { + if isAuthoringOnlyField(field) && !preserveLegacyWikiColumnForPath(path, field) { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("entry %q uses deprecated wiki field %q; move it into meta.wiki", key, field), + }) + } + } + validateRowMetadata(path, fmt.Sprintf("entry %q", key), entry, report) + validateInlineTextUsage(path, key, entry, report) + } + } + if _, ok := obj["columns"]; ok { + validateColumnsFile(path, obj, report) + } +} + +func validateOverridesFile(path string, obj map[string]any, report *ValidationReport) { + overrides, ok := obj["overrides"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "override file must contain an overrides array", + }) + return + } + if _, ok := overrides.([]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "overrides must be a JSON array", + }) + } + if overrideList, ok := overrides.([]any); ok { + container := map[string]any{"rows": overrideList} + validateRowCollection(path, container, overrideList, report) + } + if _, ok := obj["columns"]; ok { + validateColumnsFile(path, obj, report) + } +} + +func validateGlobalJSONFile(path string, obj map[string]any, report *ValidationReport) { + if _, ok := obj["columns"]; ok { + validateColumnsFile(path, obj, report) + } + if _, ok := obj["entries"]; ok { + validateEntriesFile(path, obj, report) + } + if _, ok := obj["overrides"]; ok { + validateOverridesFile(path, obj, report) + } + if _, ok := obj["defaults"]; ok { + validateGlobalDefaults(path, obj, report) + } + if rawPosition, ok := obj["position"]; ok { + position, ok := rawPosition.(string) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "global position must be a string", + }) + } else { + switch strings.ToLower(strings.TrimSpace(position)) { + case "", "append", "prepend": + default: + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "global position must be append or prepend", + }) + } + } + } + rawInjections, hasInjections := obj["injections"] + if !hasInjections { + return + } + injections, ok := rawInjections.([]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "global injections must be a JSON array", + }) + return + } + rows := make([]any, 0, len(injections)) + for index, rawInjection := range injections { + injection, ok := rawInjection.(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global injection %d must be an object", index), + }) + continue + } + row, ok := injection["row"].(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global injection %d must contain a row object", index), + }) + } else { + rows = append(rows, row) + } + validateGlobalConditionList(path, fmt.Sprintf("global injection %d require_present", index), injection["require_present"], report) + validateGlobalConditionList(path, fmt.Sprintf("global injection %d unless_present", index), injection["unless_present"], report) + } + validateRowCollection(path, obj, rows, report) +} + +func topdataColumnsForGlobalValidation(path string, obj map[string]any) ([]string, bool) { + basePath := filepath.Join(filepath.Dir(path), "base.json") + baseData, err := loadJSONObject(basePath) + if err != nil { + return nil, false + } + datasetName := filepath.ToSlash(filepath.Base(filepath.Dir(path))) + columns, err := parseColumns(baseData, datasetName) + if err != nil { + return nil, true + } + modulePaths, err := collectModulePaths(filepath.Join(filepath.Dir(path), "modules")) + if err == nil { + for _, modulePath := range modulePaths { + moduleData, err := loadJSONObject(modulePath) + if err != nil { + continue + } + columns, err = extendColumns(columns, moduleData, datasetName, modulePath) + if err != nil { + return columns, true + } + } + } + columns, err = extendColumns(columns, obj, datasetName, path) + if err != nil { + return columns, true + } + return columns, true +} + +func validateGlobalDefaults(path string, obj map[string]any, report *ValidationReport) { + rawDefaults, ok := obj["defaults"] + if !ok { + return + } + defaults, ok := rawDefaults.([]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "global defaults must be a JSON array", + }) + return + } + columns, hasBaseDataset := topdataColumnsForGlobalValidation(path, obj) + if !hasBaseDataset { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "global defaults require a sibling base.json dataset", + }) + return + } + for index, rawDefault := range defaults { + defaultRule, ok := rawDefault.(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d must be an object", index), + }) + continue + } + validateGlobalDefaultMatch(path, index, defaultRule["match"], report) + rawValues, ok := defaultRule["values"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d values is required", index), + }) + continue + } + values, ok := rawValues.(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d values must be an object", index), + }) + continue + } + for field, value := range values { + if _, ok := canonicalColumn(columns, field); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d values.%s is not a known column", index, field), + }) + } + validateGlobalDefaultValue(path, index, field, value, report) + } + } +} + +func validateGlobalDefaultMatch(path string, index int, rawMatch any, report *ValidationReport) { + if rawMatch == nil { + return + } + if text, ok := rawMatch.(string); ok { + if strings.EqualFold(strings.TrimSpace(text), "all") { + return + } + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d match must be all or an object", index), + }) + return + } + match, ok := rawMatch.(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d match must be all or an object", index), + }) + return + } + for key := range match { + if key != "source" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d match.%s is not supported", index, key), + }) + } + } + source, ok := match["source"].(string) + if !ok || strings.TrimSpace(source) == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d match.source is required", index), + }) + return + } + switch strings.ToLower(strings.TrimSpace(source)) { + case string(nativeRowSourceBase), string(nativeRowSourceEntries), string(nativeRowSourceOverride): + default: + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d match.source is not supported", index), + }) + } +} + +func validateGlobalDefaultValue(path string, index int, field string, value any, report *ValidationReport) { + obj, ok := value.(map[string]any) + if !ok { + return + } + rawFormat, hasFormat := obj["format"] + if !hasFormat { + return + } + if len(obj) != 1 { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d values.%s format object must contain only format", index, field), + }) + return + } + format, ok := rawFormat.(string) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d values.%s format must be a string", index, field), + }) + return + } + if err := validateTopDataRowFormat(format); err != nil { + message := strings.TrimPrefix(err.Error(), "default ") + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("global default %d values.%s %s", index, field, message), + }) + } +} + +func validateGlobalConditionList(path, label string, raw any, report *ValidationReport) { + if raw == nil { + return + } + conditions, ok := raw.([]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s must be a JSON array", label), + }) + return + } + for index, rawCondition := range conditions { + condition, ok := rawCondition.(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s[%d] must be an object", label, index), + }) + continue + } + if field, ok := condition["field"].(string); !ok || strings.TrimSpace(field) == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s[%d].field is required", label, index), + }) + } + if id, ok := condition["id"].(string); !ok || strings.TrimSpace(id) == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s[%d].id is required", label, index), + }) + } + } +} + +func validateStateObject(path string, obj map[string]any, report *ValidationReport) { + entries, ok := obj["entries"].(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "TLK state file must contain an entries object", + }) + return + } + ids := map[int]string{} + for key, raw := range entries { + entry, ok := raw.(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("TLK state entry %q must be an object", key), + }) + continue + } + idValue, ok := entry["id"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("TLK state entry %q is missing id", key), + }) + continue + } + id, err := asInt(idValue) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("TLK state entry %q id must be numeric", key), + }) + continue + } + if other, ok := ids[id]; ok && other != key { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("TLK state id collision between %q and %q", other, key), + }) + } + ids[id] = key + } +} + +func validateNativeOutputCatalog(dataDir string, report *ValidationReport) { + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataDir, + Message: err.Error(), + }) + return + } + registryDatasets, err := collectGeneratedRegistryDatasets(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataDir, + Message: err.Error(), + }) + return + } + outputOwners := map[string]string{} + for _, dataset := range datasets { + recordNativeOutputOwner(dataset.OutputName, dataset.Name, outputOwners, report) + } + for _, dataset := range registryDatasets { + recordNativeOutputOwner(dataset.Dataset.OutputName, dataset.Dataset.Name, outputOwners, report) + } +} + +func validateNativeEntryKeyUniqueness(dataDir string, report *ValidationReport) { + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataDir, + Message: fmt.Sprintf("discover native datasets for entry key uniqueness validation: %v", err), + }) + return + } + for _, dataset := range datasets { + if dataset.Kind != nativeDatasetBase { + continue + } + entryKeyPath := map[string]string{} + for _, dir := range []string{dataset.ModulesDir, dataset.GeneratedDir} { + paths, err := collectModulePaths(dir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dir, + Message: fmt.Sprintf("collect module paths for entry key uniqueness validation: %v", err), + }) + continue + } + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("load module for entry key uniqueness validation: %v", err), + }) + continue + } + entries, ok := obj["entries"].(map[string]any) + if !ok { + continue + } + for _, key := range sortedKeys(entries) { + if key == "" { + continue + } + if previousPath, exists := entryKeyPath[key]; exists { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("duplicate entries key %q declared by %s and %s", key, previousPath, path), + }) + continue + } + entryKeyPath[key] = path + } + } + } + } +} + +func validateNativeLockAllocation(dataDir string, rowGeneration []project.TopDataRowGenerationConfig, report *ValidationReport) { + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataDir, + Message: fmt.Sprintf("discover native datasets for lock allocation validation: %v", err), + }) + return + } + datasets = applyTopDataRowGeneration(datasets, rowGeneration) + for _, dataset := range datasets { + if dataset.Kind != nativeDatasetBase { + continue + } + baseObj, err := loadJSONObject(dataset.BasePath) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataset.BasePath, + Message: fmt.Sprintf("load base rows for lock allocation validation: %v", err), + }) + continue + } + rawRows, ok := baseObj["rows"].([]any) + if !ok || len(rawRows) == 0 { + continue + } + baseBoundaryID := len(rawRows) - 1 + baseKeys := map[string]struct{}{} + nullBaseIDs := map[int]struct{}{} + columns, _ := parseColumns(baseObj, dataset.Name) + for index, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + rowID := index + if rawID, ok := row["id"]; ok { + if parsed, err := asInt(rawID); err == nil { + rowID = parsed + } + } + baseBoundaryID = rowID + if key, ok := row["key"].(string); ok && key != "" { + baseKeys[key] = struct{}{} + } + if len(columns) > 0 { + canonical, err := canonicalizeBaseRow(dataset, columns, row, index) + if err == nil && isNativeAllocationNullRow(canonical, columns) { + nullBaseIDs[rowID] = struct{}{} + } + } + } + explicitModuleIDs := map[string]int{} + for _, dir := range []string{dataset.ModulesDir, dataset.GeneratedDir} { + paths, err := collectModulePaths(dir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dir, + Message: fmt.Sprintf("collect module paths for lock allocation validation: %v", err), + }) + continue + } + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("load module for lock allocation validation: %v", err), + }) + continue + } + ids, err := collectExplicitModuleIDs(dataset.Name, path, obj) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + continue + } + for key, rowID := range ids { + explicitModuleIDs[key] = rowID + } + } + } + lockData, err := loadLockfile(dataset.LockPath) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataset.LockPath, + Message: fmt.Sprintf("load lockfile for allocation validation: %v", err), + }) + continue + } + if dataset.Name == "feat" { + generatedModules, err := collectGeneratedModules(dataset, lockData) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataset.GeneratedDir, + Message: fmt.Sprintf("collect generated feat modules for lock allocation validation: %v", err), + }) + continue + } + for _, module := range generatedModules { + ids, err := collectExplicitModuleIDs(dataset.Name, module.Name, module.Data) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: module.Name, + Message: err.Error(), + }) + continue + } + for key, rowID := range ids { + explicitModuleIDs[key] = rowID + } + } + } + invalid := 0 + for key, rowID := range lockData { + if _, ok := baseKeys[key]; ok { + continue + } + if explicitID, ok := explicitModuleIDs[key]; ok && explicitID == rowID { + continue + } + if dataset.RowGenerationMinRow > 0 && rowID < dataset.RowGenerationMinRow { + invalid++ + continue + } + if rowID > baseBoundaryID { + continue + } + if dataset.RowGeneration == "first_null_row" { + if _, ok := nullBaseIDs[rowID]; ok { + if rowID >= dataset.RowGenerationMinRow { + continue + } + } + } + invalid++ + } + if invalid > 0 { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: dataset.LockPath, + Message: fmt.Sprintf("%d lock entrie(s) do not match configured generated-row allocation and will be regenerated", invalid), + }) + } + } +} + +func validateNativeAuthoringWarnings(dataDir string, rowGeneration []project.TopDataRowGenerationConfig, report *ValidationReport) { + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return + } + datasets = applyTopDataRowGeneration(datasets, rowGeneration) + for _, dataset := range datasets { + if dataset.Kind != nativeDatasetBase { + continue + } + validateDatasetOverrideTargetWarnings(dataset, report) + validateDatasetNormalizedLockWarnings(dataset, report) + validateDatasetGhostRowWarnings(dataset, report) + } +} + +func validateDatasetOverrideTargetWarnings(dataset nativeDataset, report *ValidationReport) { + if dataset.BasePath == "" { + return + } + if _, err := os.Stat(dataset.BasePath); err != nil { + return + } + baseObj, err := loadJSONObject(dataset.BasePath) + if err != nil { + return + } + rawRows, ok := baseObj["rows"].([]any) + if !ok { + return + } + columns, _ := parseColumns(baseObj, dataset.Name) + rowIDToKey := map[int]string{} + keyToID := map[string]int{} + usedIDs := map[int]struct{}{} + for index, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + rowID := index + if rawID, ok := row["id"]; ok { + if parsed, err := asInt(rawID); err == nil { + rowID = parsed + } + } + if dataset.RowGeneration == "first_null_row" && len(columns) > 0 && rowID >= dataset.RowGenerationMinRow { + canonical, err := canonicalizeBaseRow(dataset, columns, row, index) + if err == nil && isNativeAllocationNullRow(canonical, columns) { + continue + } + } + usedIDs[rowID] = struct{}{} + if key, ok := row["key"].(string); ok && key != "" { + rowIDToKey[rowID] = key + keyToID[key] = rowID + } + } + nextID := nextAvailableIDAtLeast(usedIDs, dataset.RowGenerationMinRow) + allocateNextID := func() int { + rowID := nextID + usedIDs[rowID] = struct{}{} + nextID = nextAvailableIDAtLeast(usedIDs, dataset.RowGenerationMinRow) + return rowID + } + modulePaths, err := collectModulePaths(dataset.ModulesDir) + if err != nil { + return + } + for _, path := range modulePaths { + obj, err := loadJSONObject(path) + if err != nil { + continue + } + if rawEntries, ok := obj["entries"].(map[string]any); ok { + for key := range rawEntries { + if key == "" { + continue + } + if _, exists := keyToID[key]; exists { + continue + } + rowID := allocateNextID() + keyToID[key] = rowID + rowIDToKey[rowID] = key + } + } + rawOverrides, ok := obj["overrides"].([]any) + if !ok { + continue + } + for index, raw := range rawOverrides { + override, ok := raw.(map[string]any) + if !ok { + continue + } + keyOnlyTarget := false + keyText, hasKeyText := override["key"].(string) + if _, hasID := override["id"]; !hasID && hasKeyText && keyText != "" { + keyOnlyTarget = true + if _, exists := keyToID[keyText]; !exists { + canonical := resolveCanonicalDatasetKey(keyText, keyToID) + if canonical != keyText { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: path, + Message: fmt.Sprintf("override %d targets key %q, but the live canonical key at that point is %q; this key-only override will create or retarget a new row", index, keyText, canonical), + }) + } else { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: path, + Message: fmt.Sprintf("override %d targets key %q, but no live row owns that key at that point; this key-only override will create or retarget a new row", index, keyText), + }) + } + } + } + rowID := -1 + if rawID, ok := override["id"]; ok { + parsed, err := asInt(rawID) + if err == nil { + rowID = parsed + } + } else if hasKeyText && keyText != "" { + if mappedID, exists := keyToID[keyText]; exists { + rowID = mappedID + } + } + if rowID < 0 { + if keyOnlyTarget { + rowID = allocateNextID() + keyToID[keyText] = rowID + rowIDToKey[rowID] = keyText + } + continue + } + currentKey := rowIDToKey[rowID] + if overrideRequestsNullRow(override) { + if currentKey != "" { + delete(keyToID, currentKey) + delete(rowIDToKey, rowID) + } + continue + } + if rawKey, present := override["key"]; present { + switch typed := rawKey.(type) { + case nil: + if currentKey != "" { + delete(keyToID, currentKey) + delete(rowIDToKey, rowID) + } + case string: + if typed == "" || strings.TrimSpace(typed) == nullValue { + if currentKey != "" { + delete(keyToID, currentKey) + delete(rowIDToKey, rowID) + } + continue + } + if currentKey != "" && currentKey != typed { + delete(keyToID, currentKey) + } + if previousID, exists := keyToID[typed]; exists && previousID != rowID { + delete(rowIDToKey, previousID) + } + keyToID[typed] = rowID + rowIDToKey[rowID] = typed + } + } + } + } +} + +func validateDatasetNormalizedLockWarnings(dataset nativeDataset, report *ValidationReport) { + lockData, err := loadLockfile(dataset.LockPath) + if err != nil { + return + } + groups := map[string][]string{} + for key := range lockData { + prefix, suffix, ok := strings.Cut(key, ":") + if !ok || prefix == "" || suffix == "" { + continue + } + groups[prefix+":"+normalizeKeyIdentity(suffix)] = append(groups[prefix+":"+normalizeKeyIdentity(suffix)], key) + } + for _, keys := range groups { + if len(keys) < 2 { + continue + } + slices.Sort(keys) + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: dataset.LockPath, + Message: fmt.Sprintf("lockfile keeps multiple normalized-equivalent keys alive: %s", strings.Join(keys, ", ")), + }) + } +} + +func validateDatasetGhostRowWarnings(dataset nativeDataset, report *ValidationReport) { + if dataset.BasePath == "" { + return + } + if _, err := os.Stat(dataset.BasePath); err != nil { + return + } + baseObj, err := loadJSONObject(dataset.BasePath) + if err != nil { + return + } + rawRows, ok := baseObj["rows"].([]any) + if !ok { + return + } + baseKeyedIDs := map[int]struct{}{} + for index, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + if key, ok := row["key"].(string); !ok || key == "" { + continue + } + rowID := index + if rawID, ok := row["id"]; ok { + if parsed, err := asInt(rawID); err == nil { + rowID = parsed + } + } + baseKeyedIDs[rowID] = struct{}{} + } + collected, err := collectNativeDataset(dataset) + if err != nil { + return + } + for _, row := range collected.Rows { + if key, ok := row["key"].(string); ok && key != "" { + continue + } + rowID, _ := row["id"].(int) + if _, hadBaseKey := baseKeyedIDs[rowID]; !hadBaseKey { + continue + } + nonNullFields := make([]string, 0, 4) + for field, value := range row { + if field == "id" || field == "key" || field == "meta" { + continue + } + if isEffectivelyNullValue(value) { + continue + } + nonNullFields = append(nonNullFields, field) + } + if len(nonNullFields) == 0 || len(nonNullFields) > 3 { + continue + } + slices.Sort(nonNullFields) + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: dataset.BasePath, + Message: fmt.Sprintf("row id %d has no canonical key but still carries fields %s; this usually indicates a key-only override hit a retired row or a row was partially de-identified", rowID, strings.Join(nonNullFields, ", ")), + }) + } +} + +func isEffectivelyNullValue(value any) bool { + switch typed := value.(type) { + case nil: + return true + case string: + return strings.TrimSpace(typed) == "" || strings.TrimSpace(typed) == nullValue + default: + return false + } +} + +type requiredFeatFamily struct { + FamilyKey string + Dataset string + Column string + Predicate string +} + +func validateGeneratedFeatFamilies(dataDir string, report *ValidationReport) { + featDataset, ok, err := discoverNamedNativeDataset(dataDir, "feat") + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataDir, + Message: fmt.Sprintf("discover feat dataset for generated family validation: %v", err), + }) + return + } + if !ok { + return + } + lockData, err := loadLockfile(featDataset.LockPath) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: featDataset.LockPath, + Message: fmt.Sprintf("load feat lockfile for generated family validation: %v", err), + }) + return + } + ctx, err := newFeatGeneratedContext(featDataset, lockData) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: featDataset.GeneratedDir, + Message: fmt.Sprintf("prepare generated family validation: %v", err), + }) + return + } + specs, specPaths, err := collectGeneratedFamilySpecs(featDataset.GeneratedDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: featDataset.GeneratedDir, + Message: fmt.Sprintf("load generated family specs: %v", err), + }) + return + } + for familyKey, spec := range specs { + ctx.familySpecs[familyKey] = spec + } + + required := []requiredFeatFamily{ + {FamilyKey: "skillfocus", Dataset: "skills", Predicate: "accessible"}, + {FamilyKey: "greaterskillfocus", Dataset: "skills", Predicate: "accessible"}, + {FamilyKey: "weaponfocus", Dataset: "baseitems", Column: "WeaponFocusFeat"}, + {FamilyKey: "weaponspecialization", Dataset: "baseitems", Column: "WeaponSpecializationFeat"}, + {FamilyKey: "improvedcritical", Dataset: "baseitems", Column: "WeaponImprovedCriticalFeat"}, + {FamilyKey: "overwhelmingcritical", Dataset: "baseitems", Column: "EpicWeaponOverwhelmingCriticalFeat"}, + {FamilyKey: "greaterweaponfocus", Dataset: "baseitems", Column: "EpicWeaponFocusFeat"}, + {FamilyKey: "greaterweaponspecialization", Dataset: "baseitems", Column: "EpicWeaponSpecializationFeat"}, + {FamilyKey: "weaponofchoice", Dataset: "baseitems", Column: "WeaponOfChoiceFeat"}, + } + for _, requirement := range required { + spec, path, ok := generatedFamilySpecByNormalizedKey(specs, specPaths, requirement.FamilyKey) + if !ok { + continue + } + if spec.ChildSource.Dataset != requirement.Dataset || + spec.ChildSource.Column != requirement.Column || + spec.ChildSource.Predicate != requirement.Predicate { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("generated feat family %q must use child source dataset=%q column=%q predicate=%q", requirement.FamilyKey, requirement.Dataset, requirement.Column, requirement.Predicate), + }) + continue + } + validateGeneratedFeatFamilyCompleteness(path, spec, ctx, report) + } + if hasGeneratedFamilySpecs(specs, coreWeaponGeneratedFamilyKeys) { + validateBaseitemsWeaponFeatColumnCompleteness(featDataset.GeneratedDir, ctx, report) + } + validateRequiredProductionGeneratedFamilies(featDataset.GeneratedDir, specs, report) +} + +var coreWeaponGeneratedFamilyKeys = []string{ + "weaponfocus", + "weaponspecialization", + "improvedcritical", + "overwhelmingcritical", + "greaterweaponfocus", + "greaterweaponspecialization", +} + +func hasGeneratedFamilySpecs(specs map[string]familyExpansionSpec, familyKeys []string) bool { + for _, familyKey := range familyKeys { + if _, _, ok := generatedFamilySpecByNormalizedKey(specs, nil, familyKey); !ok { + return false + } + } + return true +} + +func validateBaseitemsWeaponFeatColumnCompleteness(path string, ctx *featGeneratedContext, report *ValidationReport) { + rows, err := ctx.datasetRows("baseitems") + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("validate baseitems weapon feat coverage: %v", err), + }) + return + } + coreColumns := []string{ + "WeaponFocusFeat", + "WeaponSpecializationFeat", + "WeaponImprovedCriticalFeat", + "EpicWeaponFocusFeat", + "EpicWeaponSpecializationFeat", + "EpicWeaponOverwhelmingCriticalFeat", + } + for _, sourceKey := range sortedStringMapKeys(rows) { + row := rows[sourceKey] + hasAny := false + missing := make([]string, 0) + for _, column := range coreColumns { + value, ok := lookupField(row, column) + if !ok || isNullishValue(value) { + missing = append(missing, column) + continue + } + hasAny = true + } + if !hasAny || len(missing) == 0 { + continue + } + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("baseitems row %q declares partial core weapon feat coverage; missing %s", sourceKey, strings.Join(missing, ", ")), + }) + } +} + +func generatedFamilySpecByNormalizedKey(specs map[string]familyExpansionSpec, specPaths map[string]string, familyKey string) (familyExpansionSpec, string, bool) { + normalized := normalizeKeyIdentity(familyKey) + for candidateKey, spec := range specs { + if normalizeKeyIdentity(candidateKey) == normalized { + path := "" + if specPaths != nil { + path = specPaths[candidateKey] + } + return spec, path, true + } + } + return familyExpansionSpec{}, "", false +} + +func validateRequiredProductionGeneratedFamilies(path string, specs map[string]familyExpansionSpec, report *ValidationReport) { + if !hasGeneratedFamilySpecs(specs, coreWeaponGeneratedFamilyKeys) { + return + } + if _, _, ok := generatedFamilySpecByNormalizedKey(specs, nil, "weaponofchoice"); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "missing generated feat family \"weaponofchoice\"", + }) + } +} + +func discoverNamedNativeDataset(dataDir, name string) (nativeDataset, bool, error) { + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return nativeDataset{}, false, err + } + for _, dataset := range datasets { + if dataset.Name == name { + return dataset, true, nil + } + } + return nativeDataset{}, false, nil +} + +func collectGeneratedFamilySpecs(generatedDir string) (map[string]familyExpansionSpec, map[string]string, error) { + paths, err := collectModulePaths(generatedDir) + if err != nil { + return nil, nil, err + } + specs := map[string]familyExpansionSpec{} + pathsByFamily := map[string]string{} + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + return nil, nil, err + } + if !isFamilyExpansionObject(obj) { + continue + } + spec, err := parseFamilyExpansionSpec(path, obj) + if err != nil { + return nil, nil, err + } + if previous, ok := pathsByFamily[spec.FamilyKey]; ok { + return nil, nil, fmt.Errorf("generated family %q is declared by both %s and %s", spec.FamilyKey, previous, path) + } + specs[spec.FamilyKey] = spec + pathsByFamily[spec.FamilyKey] = path + } + return specs, pathsByFamily, nil +} + +func validateGeneratedFeatFamilyCompleteness(path string, spec familyExpansionSpec, ctx *featGeneratedContext, report *ValidationReport) { + sourceRows, err := ctx.datasetRows(spec.ChildSource.Dataset) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return + } + moduleData, err := buildFamilyExpansionGeneratedModule(path, map[string]any{ + "family": spec.Family, + "family_key": spec.FamilyKey, + "template": spec.Template, + "name_prefix": spec.NamePrefix, + "label_prefix": spec.LabelPrefix, + "constant_prefix": spec.ConstantPrefix, + "template_fields": validationStringSliceToAny(spec.TemplateFields), + "default_fields": spec.DefaultFields, + "apply_after_modules": spec.ApplyAfterModules, + "child_ref_field": spec.ChildRefField, + "identity_source": spec.IdentitySource, + "allow_existing_only": spec.AllowExistingOnly, + "auto_prereq_fields": stringMapToAny(spec.AutoPrereqFields), + "legacy_family_keys": validationStringSliceToAny(spec.LegacyFamilyKeys), + "child_source": map[string]any{ + "dataset": spec.ChildSource.Dataset, + "column": spec.ChildSource.Column, + "predicate": spec.ChildSource.Predicate, + }, + "overrides": map[string]any{}, + }, ctx) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return + } + expected := map[string]struct{}{} + familyAllowlist := spec.AllowExistingOnly && ctx.familyHasExistingRows(spec.FamilyKey) + for sourceKey, row := range sourceRows { + include, err := familyExpansionSourceEligible(spec, row) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return + } + if include { + if familyAllowlist { + slug := strings.TrimPrefix(sourceKey, spec.ChildSource.Dataset+":") + featKey, _, _, err := ctx.resolveGeneratedFeatIdentity(spec, slug, row) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return + } + if !ctx.featKeyExists(featKey) { + continue + } + } + expected[sourceKey] = struct{}{} + } + } + actual := map[string]string{} + overrides, _ := moduleData["overrides"].([]any) + for index, raw := range overrides { + override, ok := raw.(map[string]any) + if !ok { + continue + } + source, ok := familySourceFromMeta(override["meta"]) + if !ok || source == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("generated family %q row %d is missing family source metadata", spec.FamilyKey, index), + }) + continue + } + if previousKey, ok := actual[source]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("generated family %q emits source %q more than once (%s and row %d)", spec.FamilyKey, source, previousKey, index), + }) + } + key, _ := override["key"].(string) + actual[source] = key + } + for source := range expected { + if _, ok := actual[source]; !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("generated family %q is missing source %q", spec.FamilyKey, source), + }) + } + } + for source := range actual { + if _, ok := expected[source]; !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("generated family %q includes unexpected source %q", spec.FamilyKey, source), + }) + } + } +} + +func familySourceFromMeta(raw any) (string, bool) { + meta, ok := raw.(map[string]any) + if !ok { + return "", false + } + family, ok := meta["family"].(map[string]any) + if !ok { + return "", false + } + source, ok := family["source"].(string) + return source, ok +} + +func validationStringSliceToAny(items []string) []any { + if items == nil { + return nil + } + out := make([]any, 0, len(items)) + for _, item := range items { + out = append(out, item) + } + return out +} + +func stringMapToAny(items map[string]string) map[string]any { + if items == nil { + return nil + } + out := make(map[string]any, len(items)) + for key, value := range items { + out[key] = value + } + return out +} + +func validateTopPackageAssets(sourceDir, dataDir string, report *ValidationReport) { + assetsDir := filepath.Join(sourceDir, "assets") + generated2DADir := filepath.Join(assetsDir, "2da") + info, err := os.Stat(assetsDir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return + } + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: assetsDir, + Message: err.Error(), + }) + return + } + if !info.IsDir() { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: assetsDir, + Message: "must be a directory", + }) + return + } + + projectOutputs, err := discoverNativeOutputCatalog(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: assetsDir, + Message: err.Error(), + }) + return + } + + seen := map[string]string{} + err = filepath.WalkDir(assetsDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() && path == generated2DADir { + return filepath.SkipDir + } + if d.IsDir() { + return nil + } + report.Files++ + rel, err := filepath.Rel(assetsDir, path) + if err != nil { + return err + } + if strings.HasPrefix(filepath.Base(path), ".") { + return nil + } + base := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) + ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") + if _, ok := erf.HAKResourceTypeForExtension(ext); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("unsupported topdata asset HAK resource extension %q", filepath.Ext(path)), + }) + return nil + } + key := base + "." + ext + if previous, ok := seen[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("duplicate topdata asset resource %q also defined by %s", key, previous), + }) + return nil + } + seen[key] = rel + if owner, ok := projectOutputs[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("topdata asset resource %q collides with compiled topdata output from %s", key, owner), + }) + } + return nil + }) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: assetsDir, + Message: err.Error(), + }) + } +} + +func recordNativeOutputOwner(outputName, owner string, outputOwners map[string]string, report *ValidationReport) { + if existing, ok := outputOwners[outputName]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: owner, + Message: fmt.Sprintf("duplicate 2da output %q also produced by %s", outputName, existing), + }) + return + } + outputOwners[outputName] = owner +} + +func validateNativeBuildability(p *project.Project, report *ValidationReport) { + dataDir := filepath.Join(p.TopDataSourceDir(), "data") + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataDir, + Message: err.Error(), + }) + return + } + registryDatasets, err := collectGeneratedRegistryDatasets(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataDir, + Message: err.Error(), + }) + return + } + if len(datasets) == 0 && len(registryDatasets) == 0 { + return + } + tempBuild, err := os.MkdirTemp("", "sow-topdata-validate-*") + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: p.TopDataSourceDir(), + Message: fmt.Sprintf("create validation temp dir: %v", err), + }) + return + } + defer os.RemoveAll(tempBuild) + + clone := *p + clone.Config.TopData.Build = tempBuild + lockSnapshot, err := snapshotLockfiles(dataDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: dataDir, + Message: fmt.Sprintf("snapshot lockfiles for validation build: %v", err), + }) + return + } + defer restoreLockfiles(lockSnapshot) + if _, err := buildNativeUnchecked(&clone, NativeBuildOptions{}, nil, nil); err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: p.TopDataSourceDir(), + Message: fmt.Sprintf("native topdata buildability check failed: %v", err), + }) + } +} + +type lockfileSnapshot struct { + Root string + Files map[string][]byte +} + +func snapshotLockfiles(dataDir string) (lockfileSnapshot, error) { + snapshot := lockfileSnapshot{ + Root: dataDir, + Files: map[string][]byte{}, + } + err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() || filepath.Base(path) != "lock.json" { + return nil + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + snapshot.Files[path] = raw + return nil + }) + return snapshot, err +} + +func restoreLockfiles(snapshot lockfileSnapshot) { + seen := map[string]struct{}{} + for path, raw := range snapshot.Files { + seen[path] = struct{}{} + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + continue + } + _ = os.WriteFile(path, raw, 0o644) + } + _ = filepath.WalkDir(snapshot.Root, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil || d.IsDir() || filepath.Base(path) != "lock.json" { + return nil + } + if _, ok := seen[path]; !ok { + _ = os.Remove(path) + } + return nil + }) +} + +func validateRowCollection(path string, obj map[string]any, rows []any, report *ValidationReport) { + columns := extractValidationColumns(obj) + declaredKeys := map[string]int{} + checkDuplicateKeys := filepath.Base(path) != "base.json" + rowHasIdentity := func(row map[string]any, fallbackID int) (string, bool) { + if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" { + return key, true + } + if _, ok := row["id"]; ok { + return fmt.Sprintf("id:%d", fallbackID), true + } + return "", false + } + + for index, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + if checkDuplicateKeys { + if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" { + if previous, ok := declaredKeys[key]; ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("row %d repeats key %q already declared by row %d", index, key, previous), + }) + } + declaredKeys[key] = index + } + } + for key := range row { + if isAuthoringOnlyField(key) && !preserveLegacyWikiColumnForPath(path, key) { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("row %d uses deprecated wiki field %q; move it into meta.wiki", index, key), + }) + } + } + if _, hasTLK := row["_tlk"]; hasTLK { + if _, ok := rowHasIdentity(row, index); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("row %d uses _tlk but has no resolvable identity", index), + }) + } + } + validateRowMetadata(path, fmt.Sprintf("row %d", index), row, report) + validateInheritanceUsage(path, fmt.Sprintf("row %d", index), row, columns, report) + validateGenericInheritanceObject(path, fmt.Sprintf("row %d", index), row, report) + validateInlineTextUsage(path, fmt.Sprintf("row %d", index), row, report) + } + _ = obj +} + +func validateRowMetadata(path, label string, row map[string]any, report *ValidationReport) { + rawMeta, ok := lookupField(row, "meta") + if !ok { + return + } + if _, err := parseRowMetadata(rawMeta); err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s meta: %v", label, err), + }) + } +} + +func extractValidationColumns(obj map[string]any) []string { + rawColumns, ok := obj["columns"].([]any) + if !ok { + return nil + } + columns := make([]string, 0, len(rawColumns)) + for _, raw := range rawColumns { + column, ok := raw.(string) + if !ok { + continue + } + columns = append(columns, column) + } + return columns +} + +func preserveLegacyWikiColumnForPath(path, field string) bool { + return strings.HasSuffix(filepath.ToSlash(path), "/data/classes/core/base.json") && + strings.EqualFold(strings.TrimSpace(field), "WIKIGENERATE") +} + +func validateInheritanceUsage(path, label string, row map[string]any, columns []string, report *ValidationReport) { + spec, ok, err := parseRowInheritanceSpec(row) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s inherit: %v", label, err), + }) + return + } + if !ok { + return + } + if _, ok := row[spec.From]; !ok { + found := false + for key := range row { + if strings.EqualFold(key, spec.From) { + found = true + break + } + } + if !found { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s inherit.from field %q was not found on row", label, spec.From), + }) + } + } + for _, field := range spec.Fields { + if len(columns) == 0 { + continue + } + if _, ok := canonicalColumn(columns, field); ok { + continue + } + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s inherit field %q is not a known column", label, field), + }) + } +} + +func validateGenericInheritanceObject(path, label string, value any, report *ValidationReport) { + switch typed := value.(type) { + case map[string]any: + spec, ok, err := parseGenericInheritanceSpec(typed) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s inherit: %v", label, err), + }) + } else if ok { + if !strings.Contains(spec.Ref, ":") { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s inherit.ref %q must use dataset:key identity", label, spec.Ref), + }) + } + } + for key, child := range typed { + if key == "inherit" { + continue + } + validateGenericInheritanceObject(path, label+"."+key, child, report) + } + case []any: + for index, child := range typed { + validateGenericInheritanceObject(path, fmt.Sprintf("%s[%d]", label, index), child, report) + } + } +} + +func validateInlineTextUsage(path, label string, obj map[string]any, report *ValidationReport) { + for key, value := range obj { + if key == "_tlk" { + typed, ok := value.(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s _tlk must be an object", label), + }) + continue + } + for role, roleValue := range typed { + if strings.ToLower(role) != "name" && strings.ToLower(role) != "description" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s _tlk role %q is not supported", label, role), + }) + continue + } + if _, _, err := parseTLKPayload(roleValue, true); err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s _tlk.%s: %v", label, role, err), + }) + } + } + continue + } + + allowBare := true + if typed, ok := value.(map[string]any); ok { + _, hasWrappedTLK := typed["tlk"] + if !hasWrappedTLK { + if _, hasRef := typed["ref"]; hasRef { + if _, hasText := typed["text"]; !hasText { + if _, hasKey := typed["key"]; !hasKey { + continue + } + } + } + allowBare = false + } + } + payload, ok, err := parseTLKPayload(value, allowBare) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s %s: %v", label, key, err), + }) + continue + } + if !ok { + continue + } + if payload.Key == "" && payload.Text == "" && payload.Ref == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("%s %s TLK payload is empty", label, key), + }) + } + } +} + +func Compare(p *project.Project, progress func(string)) (CompareResult, error) { + if progress == nil { + progress = func(string) {} + } + output2DA := compiled2DAOutputDir(p) + outputTLK := compiledTLKOutputDir(p) + if _, err := os.Stat(output2DA); err != nil { + return CompareResult{}, fmt.Errorf("topdata build output missing: run build-topdata first") + } + compared2DA, comparedTLK, nativePass, nativeMismatch, err := compareNativeSelfCheck(p, output2DA, outputTLK, progress) + if err != nil { + return CompareResult{}, err + } + + return CompareResult{ + Mode: "native", + Compared2DA: compared2DA, + ComparedTLK: comparedTLK, + NativePass: nativePass, + NotYetCanonical: 0, + NativeMismatch: nativeMismatch, + }, nil +} + +func BuildReference(p *project.Project, progress func(string)) (BuildResult, error) { + return BuildNative(p, progress) +} + +func CompareReference(p *project.Project, progress func(string)) (CompareResult, error) { + return Compare(p, progress) +} + +func compareDirs(actualRoot, expectedRoot string) (int, error) { + expectedFiles := make([]string, 0) + err := filepath.WalkDir(expectedRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(expectedRoot, path) + if err != nil { + return err + } + expectedFiles = append(expectedFiles, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return 0, err + } + slices.Sort(expectedFiles) + + actualFiles := make([]string, 0) + err = filepath.WalkDir(actualRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(actualRoot, path) + if err != nil { + return err + } + actualFiles = append(actualFiles, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return 0, err + } + slices.Sort(actualFiles) + if !slices.Equal(actualFiles, expectedFiles) { + return 0, fmt.Errorf("topdata output file list differs from reference") + } + + for _, rel := range expectedFiles { + actualBytes, err := os.ReadFile(filepath.Join(actualRoot, rel)) + if err != nil { + return 0, err + } + expectedBytes, err := os.ReadFile(filepath.Join(expectedRoot, rel)) + if err != nil { + return 0, err + } + if !bytes.Equal(actualBytes, expectedBytes) { + return 0, fmt.Errorf("topdata output differs from reference: %s", rel) + } + } + return len(expectedFiles), nil +} + +func compareNativeSelfCheck(p *project.Project, actual2DA, actualTLK string, progress func(string)) (int, int, int, int, error) { + if progress == nil { + progress = func(string) {} + } + projectOutputs, err := discoverNativeOutputCatalog(filepath.Join(p.TopDataSourceDir(), "data")) + if err != nil { + return 0, 0, 0, 0, err + } + if len(projectOutputs) == 0 { + return 0, 0, 0, 0, nil + } + + tempBuild, err := os.MkdirTemp("", "sow-topdata-native-compare-*") + if err != nil { + return 0, 0, 0, 0, fmt.Errorf("create native compare temp dir: %w", err) + } + defer os.RemoveAll(tempBuild) + + nativeProject := *p + nativeProject.Config.TopData.Build = tempBuild + nativeResult, err := buildNativeUnchecked(&nativeProject, NativeBuildOptions{}, nil, nil) + if err != nil { + return 0, 0, 0, len(projectOutputs), nil + } + + pass := 0 + mismatch := 0 + for outputName := range projectOutputs { + actualBytes, err := os.ReadFile(filepath.Join(actual2DA, outputName)) + if err != nil { + mismatch++ + continue + } + expectedBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, outputName)) + if err != nil { + mismatch++ + continue + } + if bytes.Equal(normalize2DAForCompare(outputName, actualBytes), normalize2DAForCompare(outputName, expectedBytes)) { + pass++ + continue + } + mismatch++ + } + actual2DAFiles, err := listRelativeFiles(actual2DA) + if err != nil { + return 0, 0, 0, 0, err + } + expected2DAFiles, err := listRelativeFiles(nativeResult.Output2DADir) + if err != nil { + return 0, 0, 0, 0, err + } + if !slices.Equal(actual2DAFiles, expected2DAFiles) { + mismatch += symmetricDiffCount(actual2DAFiles, expected2DAFiles) + } + comparedTLK, tlkMismatch, err := compareExactDirFilesByExtension(actualTLK, nativeResult.OutputTLKDir, ".tlk") + if err != nil { + return 0, 0, 0, 0, err + } + mismatch += tlkMismatch + return len(projectOutputs), comparedTLK, pass, mismatch, nil +} + +func compareNativeCoverage(p *project.Project, actual2DA string, progress func(string)) (int, int, int, error) { + actualTLK := compiledTLKOutputDir(p) + compared2DA, _, pass, mismatch, err := compareNativeSelfCheck(p, actual2DA, actualTLK, progress) + if err != nil { + return 0, 0, 0, err + } + if compared2DA == 0 { + return 0, 0, mismatch, nil + } + return pass, 0, mismatch, nil +} + +func normalize2DAForCompare(outputName string, raw []byte) []byte { + text := strings.ReplaceAll(string(raw), "\r\n", "\n") + lines := strings.Split(text, "\n") + if len(lines) < 3 || !strings.HasPrefix(lines[0], "2DA V2.0") { + return raw + } + headerIndex := 2 + columns := strings.Split(lines[headerIndex], "\t") + keep := make([]int, 0, len(columns)) + filtered := make([]string, 0, len(columns)) + for index, column := range columns { + if isAuthoringOnlyField(column) { + continue + } + keep = append(keep, index) + filtered = append(filtered, column) + } + var builder strings.Builder + builder.WriteString(lines[0]) + builder.WriteByte('\n') + builder.WriteByte('\n') + builder.WriteString(strings.Join(filtered, "\t")) + builder.WriteByte('\n') + normalizedRows := make([][]string, 0, len(lines)) + for _, line := range lines[headerIndex+1:] { + if line == "" { + continue + } + fields := strings.Split(line, "\t") + if len(fields) == 0 { + continue + } + row := make([]string, 0, len(keep)) + for _, index := range keep { + if index+1 < len(fields) { + row = append(row, fields[index+1]) + } else { + row = append(row, nullValue) + } + } + normalizedRows = append(normalizedRows, row) + } + if shouldIgnore2DAPlacement(outputName) { + slices.SortFunc(normalizedRows, func(a, b []string) int { + return strings.Compare(strings.Join(a, "\x00"), strings.Join(b, "\x00")) + }) + } + for index, row := range normalizedRows { + builder.WriteString(strconv.Itoa(index)) + for _, value := range row { + builder.WriteByte('\t') + builder.WriteString(value) + } + builder.WriteByte('\n') + } + return []byte(builder.String()) +} + +func shouldIgnore2DAPlacement(outputName string) bool { + return strings.HasPrefix(outputName, "cls_feat_") && strings.HasSuffix(outputName, ".2da") +} + +func walkJSON(root string, fn func(path string)) error { + return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.HasSuffix(strings.ToLower(d.Name()), ".json") { + fn(path) + } + return nil + }) +} + +func countFiles(root string) (int, error) { + count := 0 + err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + count++ + } + return nil + }) + return count, err +} + +type progressWriter struct { + progress func(string) +} + +func (w progressWriter) Write(p []byte) (int, error) { + text := strings.TrimSpace(string(p)) + if text != "" { + w.progress(text) + } + return len(p), nil +} + +func ioDiscardProgress(progress func(string)) *progressWriter { + return &progressWriter{progress: progress} +} + +func listRelativeFiles(root string) ([]string, error) { + return listRelativeFilesByExtension(root, "") +} + +func listRelativeFilesByExtension(root, extension string) ([]string, error) { + files := make([]string, 0) + if _, err := os.Stat(root); err != nil { + if os.IsNotExist(err) { + return files, nil + } + return nil, err + } + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if extension != "" && !strings.EqualFold(filepath.Ext(path), extension) { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + files = append(files, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return nil, err + } + slices.Sort(files) + return files, nil +} + +func symmetricDiffCount(left, right []string) int { + leftSet := map[string]struct{}{} + rightSet := map[string]struct{}{} + for _, item := range left { + leftSet[item] = struct{}{} + } + for _, item := range right { + rightSet[item] = struct{}{} + } + count := 0 + for item := range leftSet { + if _, ok := rightSet[item]; !ok { + count++ + } + } + for item := range rightSet { + if _, ok := leftSet[item]; !ok { + count++ + } + } + return count +} + +func compareExactDirFiles(actualRoot, expectedRoot string) (int, int, error) { + return compareExactDirFilesByExtension(actualRoot, expectedRoot, "") +} + +func compareExactDirFilesByExtension(actualRoot, expectedRoot, extension string) (int, int, error) { + actualFiles, err := listRelativeFilesByExtension(actualRoot, extension) + if err != nil { + return 0, 0, err + } + expectedFiles, err := listRelativeFilesByExtension(expectedRoot, extension) + if err != nil { + return 0, 0, err + } + mismatch := 0 + if !slices.Equal(actualFiles, expectedFiles) { + mismatch += symmetricDiffCount(actualFiles, expectedFiles) + } + shared := make([]string, 0, len(expectedFiles)) + for _, rel := range expectedFiles { + if slices.Contains(actualFiles, rel) { + shared = append(shared, rel) + } + } + for _, rel := range shared { + actualBytes, err := os.ReadFile(filepath.Join(actualRoot, rel)) + if err != nil { + mismatch++ + continue + } + expectedBytes, err := os.ReadFile(filepath.Join(expectedRoot, rel)) + if err != nil { + mismatch++ + continue + } + if !bytes.Equal(actualBytes, expectedBytes) { + mismatch++ + } + } + return len(expectedFiles), mismatch, nil +} diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go new file mode 100644 index 0000000..6f038e7 --- /dev/null +++ b/internal/topdata/topdata_test.go @@ -0,0 +1,15115 @@ +package topdata + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func TestValidateProjectAcceptsInlineTLKSchema(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [ + { + "id": 0, + "key": "feat:test", + "LABEL": "FEAT_TEST", + "_tlk": { + "name": "Test Feat", + "description": "Test Description" + } + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + report := ValidateProject(testProject(root)) + if report.HasErrors() { + t.Fatalf("expected no topdata validation errors, got %#v", report.Diagnostics) + } +} + +func TestValidateAndBuildDerivesFeatOutputWhenOmitted(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [ + { + "id": 0, + "key": "feat:test", + "LABEL": "FEAT_TEST", + "FEAT": {"tlk": {"key": "feat:test.name", "text": "Test Feat"}}, + "DESCRIPTION": {"tlk": {"key": "feat:test.description", "text": "Test Description"}} + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0}`+"\n") + + proj := testProject(root) + report := ValidateProject(proj) + if report.HasErrors() { + t.Fatalf("expected omitted feat output to validate cleanly, got:\n%s", diagnosticsText(report.Diagnostics)) + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("expected omitted feat output to build: %v", err) + } + if _, err := os.Stat(filepath.Join(result.Output2DADir, "feat.2da")); err != nil { + t.Fatalf("expected derived feat.2da output: %v", err) + } +} + +func TestValidateProjectRejectsInvalidFeatInvariantContract(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [ + { + "id": 0, + "key": "masterfeats:test", + "LABEL": "FEAT_TEST", + "FEAT": "100", + "DESCRIPTION": "200" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"masterfeats:test":0}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatal("expected validation errors for invalid feat invariants") + } + text := diagnosticsText(report.Diagnostics) + for _, want := range []string{ + "feat output must be feat.2da", + `feat row 0 key "masterfeats:test" must start with feat:`, + `feat lock key "masterfeats:test" must start with feat:`, + } { + if !strings.Contains(text, want) { + t.Fatalf("expected validation message %q, got:\n%s", want, text) + } + } +} + +func TestValidateProjectRejectsDuplicateJSONKeys(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "output": "other.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatal("expected duplicate JSON key validation error") + } + if !diagnosticsContain(report.Diagnostics, "duplicate JSON object key") { + t.Fatalf("expected duplicate JSON object key diagnostic, got %#v", report.Diagnostics) + } +} + +func TestValidateProjectAcceptsModuleColumnExpansionFile(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "00_newcolumns.json"), `{ + "columns": ["ECL", "AvailableHeadsMale", "AvailableSkinColors"] +}`+"\n") + + report := ValidateProject(testProject(root)) + text := diagnosticsText(report.Diagnostics) + if strings.Contains(text, "module file does not use a recognized canonical shape") { + t.Fatalf("expected module column expansion file to validate as canonical, got:\n%s", text) + } + if report.HasErrors() { + t.Fatalf("expected module column expansion file to avoid validation errors, got:\n%s", text) + } +} + +func TestValidateProjectRejectsInvalidModuleColumnExpansionFile(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "00_badcolumns.json"), `{ + "columns": "ECL" +}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatalf("expected invalid module columns file to fail validation, got:\n%s", diagnosticsText(report.Diagnostics)) + } + if !diagnosticsContain(report.Diagnostics, "columns must be a JSON array when present") { + t.Fatalf("expected columns array diagnostic, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + +func TestValidateProjectRejectsInvalidGlobalJSONDefaults(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "base.json"), `{ + "output": "spells.2da", + "columns": ["Label", "ImpactScript"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "global.json"), `{ + "defaults": [ + {"match": {"source": "entries"}, "values": {"ImpactScript": {"format": "ss_{missing}"}}}, + {"match": {"source": "unknown"}, "values": {"ImpactScript": "x"}}, + {"match": {"source": "entries", "extra": "bad"}, "values": {"ImpactScript": "x"}}, + {"match": {"source": "entries"}, "values": {"NotAColumn": "x"}}, + {"match": {"source": "entries"}} + ] +}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatal("expected invalid global defaults to fail validation") + } + text := diagnosticsText(report.Diagnostics) + for _, want := range []string{ + "default 0 values.ImpactScript format token {missing} is not supported", + "default 1 match.source is not supported", + "default 2 match.extra is not supported", + "default 3 values.NotAColumn is not a known column", + "default 4 values is required", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected diagnostic %q, got:\n%s", want, text) + } + } +} + +func TestValidateProjectRejectsGlobalJSONDefaultsWithoutBaseDataset(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "global.json"), `{ + "defaults": [ + {"match": "all", "values": {"ClassSkill": 1}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "fighter.json"), `{ + "columns": ["SkillLabel", "ClassSkill"], + "rows": [ + {"id": 0, "SkillLabel": "Concentration", "ClassSkill": 1} + ] +}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatal("expected global defaults without a base dataset to fail validation") + } + if !diagnosticsContain(report.Diagnostics, "global defaults require a sibling base.json dataset") { + t.Fatalf("expected unsupported defaults diagnostic, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + +func TestBuildNativeEncodesConfiguredPackedHexLists(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "StartLanguages", "BonusLanguages", "AvailableHeadsMale", "AvailableHeadsFemale", "AvailableSkinColors"], + "rows": [ + { + "id": 6, + "key": "racialtypes:human", + "Label": "Human", + "StartLanguages": "regional", + "BonusLanguages": "all" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "10_human.json"), `{ + "overrides": [ + { + "key": "racialtypes:human", + "AvailableHeadsMale": {"list": [1, 10, 999]}, + "AvailableHeadsFemale": {"all_except": {"min": "001", "max": "003", "values": []}}, + "AvailableSkinColors": { + "list": [ + {"range": [0, 12]}, + 24, + 74, + {"range": [116, 119]}, + {"range": [128, 131]}, + 156, + 157, + 168, + 170, + 174 + ] + } + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "racialtypes/core", Column: "AvailableHeadsMale", Mode: "packed_hex_list", Min: 0, Max: 999, HexWidth: 3}, + {Dataset: "racialtypes/core", Column: "AvailableHeadsFemale", Mode: "packed_hex_list", Min: 0, Max: 999, HexWidth: 3}, + {Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 175, HexWidth: 2}, + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "racialtypes.2da")) + if err != nil { + t.Fatalf("read racialtypes.2da: %v", err) + } + got := string(raw) + for _, want := range []string{ + "regional", + "all", + "0x00100A3E7", + "0x001002003", + "0x000102030405060708090A0B0C184A74757677808182839C9DA8AAAE", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected compiled racialtypes.2da to contain %q, got:\n%s", want, got) + } + } +} + +func TestBuildNativeEncodesConfiguredAlignmentHexLists(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "PreferredAlignments"], + "rows": [ + {"id": 6, "key": "racialtypes:human", "Label": "Human"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "10_human.json"), `{ + "overrides": [ + { + "key": "racialtypes:human", + "PreferredAlignments": {"alignments": ["le", "CG", "tn"]} + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "racialtypes/core", Column: "PreferredAlignments", Mode: "alignment_hex_list", Min: 0, Max: 31, HexWidth: 2}, + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "racialtypes.2da")) + if err != nil { + t.Fatalf("read racialtypes.2da: %v", err) + } + got := string(raw) + if !strings.Contains(got, "0x120C01") { + t.Fatalf("expected compiled racialtypes.2da to contain encoded alignments, got:\n%s", got) + } +} + +func TestBuildNativeEncodesConfiguredAlignmentSetMasks(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "core", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "base.json"), `{ + "output": "classes.2da", + "columns": ["Label", "PreferredAlignments"], + "rows": [ + {"id": 0, "key": "classes:barbarian", "Label": "Barbarian"}, + {"id": 1, "key": "classes:paladin", "Label": "Paladin"}, + {"id": 2, "key": "classes:fighter", "Label": "Fighter"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "lock.json"), `{"classes:barbarian":0,"classes:paladin":1,"classes:fighter":2}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "modules", "10_alignments.json"), `{ + "overrides": [ + { + "key": "classes:barbarian", + "PreferredAlignments": {"alignments": ["ng", "cg", "tn", "cn", "ne", "ce"]} + }, + { + "key": "classes:paladin", + "PreferredAlignments": {"alignments": ["lg"]} + }, + { + "key": "classes:fighter", + "PreferredAlignments": "0x00" + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "*", Column: "PreferredAlignments", Mode: "alignment_set_mask", Min: 0, Max: 511, HexWidth: 3}, + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "classes.2da")) + if err != nil { + t.Fatalf("read classes.2da: %v", err) + } + got := string(raw) + for _, want := range []string{ + "0\tBarbarian\t0x1F8\n", + "1\tPaladin\t0x001\n", + "2\tFighter\t0x000\n", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected compiled classes.2da to contain %q, got:\n%s", want, got) + } + } +} + +func TestBuildNativeEncodesConfiguredIDHexLists(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 1, "key": "feat:weapon_proficiency_martial", "LABEL": "WeaponProfMartial"}, + {"id": 7, "key": "feat:weapon_proficiency_elf", "LABEL": "WeaponProfElf"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:weapon_proficiency_martial": 1, + "feat:weapon_proficiency_elf": 7 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["Label", "ProficiencyFeats"], + "rows": [ + {"id": 42, "key": "baseitems:longsword", "Label": "Longsword"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:longsword":42}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_longsword.json"), `{ + "overrides": [ + { + "key": "baseitems:longsword", + "ProficiencyFeats": { + "ids": [ + "feat:weapon_proficiency_martial", + {"id": "feat:weapon_proficiency_elf"} + ] + } + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "baseitems", Column: "ProficiencyFeats", Mode: "id_hex_list", Min: 0, Max: 65535, HexWidth: 4}, + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "baseitems.2da")) + if err != nil { + t.Fatalf("read baseitems.2da: %v", err) + } + got := string(raw) + if !strings.Contains(got, "0x00010007") { + t.Fatalf("expected compiled baseitems.2da to contain encoded proficiency feat IDs, got:\n%s", got) + } +} + +func TestBuildNativeEncodesConfiguredExternalHexListsAndPackedIDs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 1, "key": "feat:first", "LABEL": "First"}, + {"id": 2, "key": "feat:second", "LABEL": "Second"}, + {"id": 3, "key": "feat:third", "LABEL": "Third"}, + {"id": 7, "key": "feat:seventh", "LABEL": "Seventh"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:first": 1, + "feat:second": 2, + "feat:third": 3, + "feat:seventh": 7 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "AvailableSkinColors"], + "rows": [ + {"id": 6, "key": "racialtypes:human", "Label": "Human"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "10_human.json"), `{ + "overrides": [ + { + "key": "racialtypes:human", + "AvailableSkinColors": {"list": [1, 2, 3, 5, {"range": [7, 9]}]} + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["Label", "ProficiencyFeats"], + "rows": [ + {"id": 42, "key": "baseitems:testclub", "Label": "Test Club"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:testclub":42}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_testclub.json"), `{ + "overrides": [ + { + "key": "baseitems:testclub", + "ProficiencyFeats": {"ids": ["feat:first", "feat:second", "feat:third", "feat:seventh"]} + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "external_hex_list", Min: 0, Max: 255, HexWidth: 2}, + {Dataset: "*", Column: "ProficiencyFeats", Mode: "id_hex_list", Min: 0, Max: 65535, HexWidth: 4}, + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + racialtypesRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "racialtypes.2da")) + if err != nil { + t.Fatalf("read racialtypes.2da: %v", err) + } + if got := string(racialtypesRaw); !strings.Contains(got, "6\tHuman\thuman\n") { + t.Fatalf("expected compiled racialtypes.2da to contain a sidecar reference, got:\n%s", got) + } + sidecarOutputName := sidecarOutputName("racialtypes.2da", "AvailableSkinColors") + if stem := outputStem(sidecarOutputName); len(stem) > 16 { + t.Fatalf("expected generated sidecar stem to fit NWN resource limit, got %q", stem) + } + sidecarRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, sidecarOutputName)) + if err != nil { + t.Fatalf("read %s: %v", sidecarOutputName, err) + } + for _, want := range []string{ + "Label\tValue\n", + "0\thuman\t0x01\n", + "1\thuman\t0x02\n", + "2\thuman\t0x03\n", + "3\thuman\t0x05\n", + "4\thuman\t0x07\n", + "5\thuman\t0x08\n", + "6\thuman\t0x09\n", + } { + if got := string(sidecarRaw); !strings.Contains(got, want) { + t.Fatalf("expected sidecar 2da to contain %q, got:\n%s", want, got) + } + } + baseitemsRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "baseitems.2da")) + if err != nil { + t.Fatalf("read baseitems.2da: %v", err) + } + if got := string(baseitemsRaw); !strings.Contains(got, "0x0001000200030007") { + t.Fatalf("expected compiled baseitems.2da to contain packed IDs, got:\n%s", got) + } +} + +func TestBuildNativeAppliesWildcardConfiguredValueEncodings(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "core", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 2, "key": "feat:weapon_proficiency_simple", "LABEL": "WeaponProfSimple"}, + {"id": 9, "key": "feat:weapon_proficiency_martial", "LABEL": "WeaponProfMartial"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:weapon_proficiency_simple": 2, + "feat:weapon_proficiency_martial": 9 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "base.json"), `{ + "output": "classes.2da", + "columns": ["Label", "PreferredAlignments"], + "rows": [ + {"id": 0, "key": "classes:barbarian", "Label": "Barbarian"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "lock.json"), `{"classes:barbarian":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "modules", "10_barbarian.json"), `{ + "overrides": [ + { + "key": "classes:barbarian", + "PreferredAlignments": {"alignments": ["ng", "cg", "tn", "cn", "ne", "ce"]} + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["Label", "ProficiencyFeats"], + "rows": [ + {"id": 42, "key": "baseitems:testclub", "Label": "Test Club"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:testclub":42}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_testclub.json"), `{ + "overrides": [ + { + "key": "baseitems:testclub", + "ProficiencyFeats": {"ids": ["feat:weapon_proficiency_simple", "feat:weapon_proficiency_martial"]} + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "classes/core", Column: "PreferredAlignments", Mode: "alignment_hex_list", Min: 0, Max: 31, HexWidth: 3}, + {Dataset: "*", Column: "PreferredAlignments", Mode: "alignment_hex_list", Min: 0, Max: 31, HexWidth: 2}, + {Dataset: "*", Column: "ProficiencyFeats", Mode: "id_hex_list", Min: 0, Max: 65535, HexWidth: 4}, + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + classesRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "classes.2da")) + if err != nil { + t.Fatalf("read classes.2da: %v", err) + } + if got := string(classesRaw); !strings.Contains(got, "0x00900C001005011014") { + t.Fatalf("expected wildcard alignment encoding in classes.2da, got:\n%s", got) + } + baseitemsRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "baseitems.2da")) + if err != nil { + t.Fatalf("read baseitems.2da: %v", err) + } + if got := string(baseitemsRaw); !strings.Contains(got, "0x00020009") { + t.Fatalf("expected wildcard id encoding in baseitems.2da, got:\n%s", got) + } +} + +func TestBuildNativeEncodesExplicitAlignmentAndIDListsInAnyDataset(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "futuretable", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 2, "key": "feat:first_future", "LABEL": "FirstFuture"}, + {"id": 17, "key": "feat:second_future", "LABEL": "SecondFuture"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:first_future": 2, + "feat:second_future": 17 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "futuretable", "base.json"), `{ + "output": "futuretable.2da", + "columns": ["Label", "AnyAlignmentMask", "AnyIDList"], + "rows": [ + {"id": 0, "key": "futuretable:first", "Label": "First"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "futuretable", "lock.json"), `{"futuretable:first":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "futuretable", "modules", "10_first.json"), `{ + "overrides": [ + { + "key": "futuretable:first", + "AnyAlignmentMask": {"alignments": ["lg", "tn", "ce"]}, + "AnyIDList": {"ids": ["feat:first_future", {"id": "feat:second_future"}]} + } + ] +}`+"\n") + + proj := testProject(root) + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "futuretable.2da")) + if err != nil { + t.Fatalf("read futuretable.2da: %v", err) + } + got := string(raw) + if !strings.Contains(got, "0\tFirst\t0x111\t0x00020011\n") { + t.Fatalf("expected generic explicit list encodings in futuretable.2da, got:\n%s", got) + } +} + +func TestBuildNativeAppliesConfiguredValueDefaults(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "ECL", "DefaultScale"], + "rows": [ + {"id": 6, "key": "racialtypes:human", "Label": "Human"}, + {"id": 7, "key": "racialtypes:elf", "Label": "Elf", "ECL": null, "DefaultScale": null}, + {"id": 8, "key": "racialtypes:drow", "Label": "Drow", "ECL": 2, "DefaultScale": "0.95"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6,"racialtypes:elf":7,"racialtypes:drow":8}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueDefaults = []project.TopDataValueDefaultConfig{ + {Dataset: "racialtypes/core", Column: "ECL", Value: 0}, + {Dataset: "racialtypes/core", Column: "DefaultScale", Value: 1}, + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "racialtypes.2da")) + if err != nil { + t.Fatalf("read racialtypes.2da: %v", err) + } + got := string(raw) + for _, want := range []string{ + "6\tHuman\t0\t1\n", + "7\tElf\t0\t1\n", + "8\tDrow\t2\t0.95\n", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected compiled racialtypes.2da to contain %q, got:\n%s", want, got) + } + } +} + +func TestBuildNativeGlobalJSONDefaultsReplaceConfiguredValueDefaults(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "ECL", "DefaultScale"], + "rows": [ + {"id": 6, "key": "racialtypes:human", "Label": "Human"}, + {"id": 7, "key": "racialtypes:elf", "Label": "Elf", "ECL": null, "DefaultScale": null}, + {"id": 8, "key": "racialtypes:drow", "Label": "Drow", "ECL": 2, "DefaultScale": "0.95"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6,"racialtypes:elf":7,"racialtypes:drow":8}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "global.json"), `{ + "defaults": [ + { + "match": "all", + "values": { + "ECL": 0, + "DefaultScale": 1 + } + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "racialtypes.2da")) + if err != nil { + t.Fatalf("read racialtypes.2da: %v", err) + } + got := string(raw) + for _, want := range []string{ + "6\tHuman\t0\t1\n", + "7\tElf\t0\t1\n", + "8\tDrow\t2\t0.95\n", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected racialtypes.2da to contain %q, got:\n%s", want, got) + } + } +} + +func TestValidateProjectRejectsInvalidConfiguredPackedHexList(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "AvailableSkinColors"], + "rows": [ + {"id": 6, "key": "racialtypes:human", "Label": "Human"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "10_human.json"), `{ + "overrides": [ + { + "key": "racialtypes:human", + "AvailableSkinColors": {"list": [{"range": [12, 0]}]} + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 175, HexWidth: 2}, + } + + report := ValidateProject(proj) + if !report.HasErrors() { + t.Fatalf("expected invalid packed hex list to fail validation, got:\n%s", diagnosticsText(report.Diagnostics)) + } + if !diagnosticsContain(report.Diagnostics, "range 12-0 ends before it starts") { + t.Fatalf("expected reversed range diagnostic, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + +func TestValidateProjectRejectsInvalidConfiguredAlignmentHexList(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "PreferredAlignments"], + "rows": [ + {"id": 6, "key": "racialtypes:human", "Label": "Human"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "10_human.json"), `{ + "overrides": [ + { + "key": "racialtypes:human", + "PreferredAlignments": {"alignments": ["lawful evil"]} + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "racialtypes/core", Column: "PreferredAlignments", Mode: "alignment_hex_list", Min: 0, Max: 31, HexWidth: 2}, + } + + report := ValidateProject(proj) + if !report.HasErrors() { + t.Fatalf("expected invalid alignment list to fail validation, got:\n%s", diagnosticsText(report.Diagnostics)) + } + if !diagnosticsContain(report.Diagnostics, `alignment code "lawful evil" must use two letters`) { + t.Fatalf("expected invalid alignment diagnostic, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + +func TestValidateProjectRejectsUnknownConfiguredIDHexListReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 1, "key": "feat:weapon_proficiency_martial", "LABEL": "WeaponProfMartial"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:weapon_proficiency_martial":1}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["Label", "ProficiencyFeats"], + "rows": [ + {"id": 42, "key": "baseitems:longsword", "Label": "Longsword"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:longsword":42}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_longsword.json"), `{ + "overrides": [ + { + "key": "baseitems:longsword", + "ProficiencyFeats": {"ids": ["feat:missing_proficiency"]} + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "baseitems", Column: "ProficiencyFeats", Mode: "id_hex_list", Min: 0, Max: 65535, HexWidth: 4}, + } + + report := ValidateProject(proj) + if !report.HasErrors() { + t.Fatalf("expected unknown ID list reference to fail validation, got:\n%s", diagnosticsText(report.Diagnostics)) + } + if !diagnosticsContain(report.Diagnostics, "unknown key reference: feat:missing_proficiency") { + t.Fatalf("expected unknown key reference diagnostic, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + +func TestEncodeConfiguredPackedHexListSupportsAllExcept(t *testing.T) { + encoding := project.TopDataValueEncodingConfig{ + Dataset: "racialtypes/core", + Column: "AvailableHeadsMale", + Mode: "packed_hex_list", + Min: 0, + Max: 999, + HexWidth: 3, + } + + got, err := encodeConfiguredValueList(map[string]any{ + "all_except": map[string]any{ + "min": "001", + "max": "010", + "values": []any{"002", float64(5), map[string]any{"range": []any{"007", "008"}}}, + }, + }, encoding) + if err != nil { + t.Fatalf("encodeConfiguredValueList failed: %v", err) + } + want := "0x00100300400600900A" + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} + +func TestEncodeConfiguredPackedHexListAllExceptUsesConfiguredBounds(t *testing.T) { + encoding := project.TopDataValueEncodingConfig{ + Dataset: "racialtypes/core", + Column: "AvailableSkinColors", + Mode: "packed_hex_list", + Min: 0, + Max: 5, + HexWidth: 2, + } + + got, err := encodeConfiguredValueList(map[string]any{ + "all_except": map[string]any{ + "values": []any{float64(1), "04"}, + }, + }, encoding) + if err != nil { + t.Fatalf("encodeConfiguredValueList failed: %v", err) + } + want := "0x00020305" + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} + +func TestEncodeConfiguredPackedHexListAllExceptRejectsInvalidInput(t *testing.T) { + encoding := project.TopDataValueEncodingConfig{ + Dataset: "racialtypes/core", + Column: "AvailableHeadsMale", + Mode: "packed_hex_list", + Min: 0, + Max: 999, + HexWidth: 3, + } + tests := []struct { + name string + obj map[string]any + want string + }{ + { + name: "mixed list and all_except", + obj: map[string]any{ + "list": []any{float64(1)}, + "all_except": map[string]any{"values": []any{float64(2)}}, + }, + want: "cannot contain both list and all_except", + }, + { + name: "unknown outer key", + obj: map[string]any{ + "all_except": map[string]any{"values": []any{float64(2)}}, + "notes": "unused", + }, + want: `contains unsupported key "notes"`, + }, + { + name: "unknown all_except key", + obj: map[string]any{ + "all_except": map[string]any{ + "values": []any{float64(2)}, + "notes": "unused", + }, + }, + want: `all_except contains unsupported key "notes"`, + }, + { + name: "excluded value outside effective range", + obj: map[string]any{ + "all_except": map[string]any{ + "max": "010", + "values": []any{"011"}, + }, + }, + want: "value 11 outside all_except range 0-10", + }, + { + name: "reversed effective range", + obj: map[string]any{ + "all_except": map[string]any{ + "min": "010", + "max": "009", + "values": []any{}, + }, + }, + want: "all_except max 9 is less than min 10", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := encodeConfiguredValueList(test.obj, encoding) + if err == nil { + t.Fatalf("expected error containing %q", test.want) + } + if !strings.Contains(err.Error(), test.want) { + t.Fatalf("expected error containing %q, got %q", test.want, err.Error()) + } + }) + } +} + +func TestValidateProjectRejectsDuplicateEntryKeysAcrossModules(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["Label", "MaxRange"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_add_sign.json"), `{ + "entries": { + "baseitems:holdable_sign": {"Label": "Holdable Sign", "MaxRange": "100"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "20_add_sign_again.json"), `{ + "entries": { + "baseitems:holdable_sign": {"Label": "Duplicate Sign", "MaxRange": "200"} + } +}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatal("expected duplicate entry key validation error") + } + text := diagnosticsText(report.Diagnostics) + if !strings.Contains(text, `duplicate entries key "baseitems:holdable_sign"`) || + !strings.Contains(text, "10_add_sign.json") || + !strings.Contains(text, "20_add_sign_again.json") { + t.Fatalf("expected duplicate entry key diagnostic with both module paths, got:\n%s", text) + } +} + +func TestResolvedTableRegistryRegistersConsistentTables(t *testing.T) { + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Kind: nativeDatasetPlain, + Name: "classes/savthr/fortitude_high", + OutputName: "cls_savthr_fort.2da", + }, + Columns: []string{"Level", "FortSave"}, + Rows: []map[string]any{ + {"id": 1, "Level": "1", "FortSave": "2"}, + }, + LockData: map[string]int{}, + TableKey: "classes/saves:fortitude", + }, + { + Dataset: nativeDataset{ + Kind: nativeDatasetPlain, + Name: "classes/bfeat/fighter", + OutputName: "cls_bfeat_fight.2da", + }, + Columns: []string{"Bonus"}, + Rows: []map[string]any{ + {"id": 1, "Bonus": "1"}, + }, + LockData: map[string]int{"classes:dummy": 99}, + TableKey: "classes/bonusfeats:fighter", + }, + } + + registry, err := newResolvedTableRegistry(collected) + if err != nil { + t.Fatalf("newResolvedTableRegistry failed: %v", err) + } + + saveTable, ok := registry.resolveTableByKey("classes/saves:fortitude") + if !ok { + t.Fatal("expected save table to resolve") + } + if saveTable.Key != "classes/saves:fortitude" || + saveTable.DatasetName != "classes/savthr/fortitude_high" || + saveTable.OutputName != "cls_savthr_fort.2da" || + saveTable.OutputStem != "cls_savthr_fort" || + saveTable.Kind != nativeDatasetPlain { + t.Fatalf("unexpected save table metadata: %#v", saveTable) + } + if len(saveTable.Columns) != 2 || len(saveTable.Rows) != 1 { + t.Fatalf("unexpected save table shape: %#v", saveTable) + } + + collected[0].Rows[0]["FortSave"] = "99" + if got := saveTable.Rows[0]["FortSave"]; got != "2" { + t.Fatalf("expected registry table rows to be cloned, got %v", got) + } + + bonusTable, ok := registry.resolveTableByKey("classes/bonusfeats:fighter") + if !ok { + t.Fatal("expected bonus feat table to resolve") + } + if bonusTable.OutputStem != "cls_bfeat_fight" || bonusTable.LockData["classes:dummy"] != 99 { + t.Fatalf("unexpected bonus feat table: %#v", bonusTable) + } +} + +func TestSaveLockfilePreservesExistingOrderAndAppendsNewKeys(t *testing.T) { + root := t.TempDir() + lockPath := filepath.Join(root, "lock.json") + writeFile(t, lockPath, "{\n \"b\": 2,\n \"a\": 1\n}\n") + + if err := saveLockfile(lockPath, map[string]int{ + "b": 2, + "a": 1, + "c": 3, + }); err != nil { + t.Fatalf("saveLockfile failed: %v", err) + } + + got, err := os.ReadFile(lockPath) + if err != nil { + t.Fatalf("read lockfile: %v", err) + } + want := "{\n \"b\": 2,\n \"a\": 1,\n \"c\": 3\n}\n" + if string(got) != want { + t.Fatalf("unexpected lockfile contents:\n%s", string(got)) + } +} + +func TestSaveLockfileUsesEditorConfigJSONFormatting(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, ".editorconfig"), `root = true + +[topdata/data/*.json] +indent_style = space +indent_size = 4 +end_of_line = lf +insert_final_newline = true + +[topdata/data/**/*.json] +indent_style = space +indent_size = 4 +end_of_line = lf +insert_final_newline = true +`) + lockPath := filepath.Join(root, "topdata", "data", "feat", "lock.json") + mkdirAll(t, filepath.Dir(lockPath)) + writeFile(t, lockPath, "{\n \"b\": 2,\n \"a\": 1\n}\n") + + if err := saveLockfile(lockPath, map[string]int{ + "b": 2, + "a": 1, + "c": 3, + }); err != nil { + t.Fatalf("saveLockfile failed: %v", err) + } + + got, err := os.ReadFile(lockPath) + if err != nil { + t.Fatalf("read lockfile: %v", err) + } + want := "{\n \"b\": 2,\n \"a\": 1,\n \"c\": 3\n}\n" + if string(got) != want { + t.Fatalf("unexpected lockfile contents:\n%s", string(got)) + } +} + +func TestSaveNativeLockfilesNormalizesEditorConfigFormattingWhenDataIsUnchanged(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, ".editorconfig"), `root = true + +[topdata/data/**/*.json] +indent_style = space +indent_size = 4 +end_of_line = lf +insert_final_newline = true +`) + lockPath := filepath.Join(root, "topdata", "data", "feat", "lock.json") + mkdirAll(t, filepath.Dir(lockPath)) + writeFile(t, lockPath, "{\n \"feat:a\": 1,\n \"feat:b\": 2\n}\n") + + added, pruned, err := saveNativeLockfiles([]nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Name: "feat", + LockPath: lockPath, + }, + LockData: map[string]int{ + "feat:a": 1, + "feat:b": 2, + }, + }, + }) + if err != nil { + t.Fatalf("saveNativeLockfiles failed: %v", err) + } + if added != 0 || pruned != 0 { + t.Fatalf("expected formatting-only normalization not to count as lock data change, got added=%d pruned=%d", added, pruned) + } + + got, err := os.ReadFile(lockPath) + if err != nil { + t.Fatalf("read lockfile: %v", err) + } + want := "{\n \"feat:a\": 1,\n \"feat:b\": 2\n}\n" + if string(got) != want { + t.Fatalf("unexpected lockfile contents:\n%s", string(got)) + } +} + +func TestBuildNativeAllocatesConfiguredDatasetsIntoFirstNullBaseRows(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ + "output": "portraits.2da", + "columns": ["BaseResRef", "Sex"], + "rows": [ + {"id": 0, "key": "portraits:none", "BaseResRef": "****", "Sex": 4}, + {"id": 1, "BaseResRef": "****", "Sex": "****"}, + {"id": 2, "BaseResRef": null, "Sex": null}, + {"id": 3, "key": "portraits:existing", "BaseResRef": "existing_", "Sex": 1}, + {"id": 4, "BaseResRef": "****", "Sex": "****"}, + {"id": 5, "BaseResRef": "****", "Sex": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), `{ + "portraits:none": 0, + "portraits:existing": 3, + "portraits:first": 6, + "portraits:second": 7 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "modules", "custom.json"), `{ + "entries": { + "portraits:first": {"BaseResRef": "first_", "Sex": 4}, + "portraits:second": {"BaseResRef": "second_", "Sex": 4} + } +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.RowGeneration = []project.TopDataRowGenerationConfig{ + {Dataset: "portraits", Mode: "first_null_row"}, + } + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "portraits", "lock.json")) + if err != nil { + t.Fatalf("read lockfile: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"portraits:first": 1`) || !strings.Contains(lockText, `"portraits:second": 2`) { + t.Fatalf("expected new portrait locks to use first null base rows, got:\n%s", lockText) + } + outputRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da")) + if err != nil { + t.Fatalf("read portraits.2da: %v", err) + } + output := string(outputRaw) + for _, want := range []string{ + "1\tfirst_\t4", + "2\tsecond_\t4", + "4\t****\t****", + "5\t****\t****", + } { + if !strings.Contains(output, want) { + t.Fatalf("expected generated portraits row %q, got:\n%s", want, output) + } + } + if strings.Contains(output, "4\tfirst_") || strings.Contains(output, "5\tsecond_") { + t.Fatalf("expected portraits not to allocate after the base boundary, got:\n%s", output) + } +} + +func TestBuildNativeAllocatesFirstNullRowsAtOrAboveConfiguredMinimumRow(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ + "output": "portraits.2da", + "columns": ["BaseResRef", "Sex"], + "rows": [ + {"id": 0, "key": "portraits:none", "BaseResRef": "po_none_", "Sex": 0}, + {"id": 1, "BaseResRef": "****", "Sex": "****"}, + {"id": 15000, "BaseResRef": "****", "Sex": "****"}, + {"id": 15001, "BaseResRef": "****", "Sex": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), `{ + "portraits:none": 0, + "portraits:first": 1, + "portraits:second": 15002 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "modules", "custom.json"), `{ + "entries": { + "portraits:first": {"BaseResRef": "first_", "Sex": 4}, + "portraits:second": {"BaseResRef": "second_", "Sex": 4}, + "portraits:third": {"BaseResRef": "third_", "Sex": 4} + } +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.RowGeneration = []project.TopDataRowGenerationConfig{ + {Dataset: "portraits", Mode: "first_null_row", MinimumRow: 15000}, + } + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "portraits", "lock.json")) + if err != nil { + t.Fatalf("read lockfile: %v", err) + } + lockText := string(lockRaw) + for _, want := range []string{ + `"portraits:first": 15000`, + `"portraits:second": 15001`, + `"portraits:third": 15002`, + } { + if !strings.Contains(lockText, want) { + t.Fatalf("expected portrait locks to respect minimum row, missing %s in:\n%s", want, lockText) + } + } + outputRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da")) + if err != nil { + t.Fatalf("read portraits.2da: %v", err) + } + output := string(outputRaw) + for _, want := range []string{ + "1\t****\t****", + "15000\tfirst_\t4", + "15001\tsecond_\t4", + "15002\tthird_\t4", + } { + if !strings.Contains(output, want) { + t.Fatalf("expected generated portraits row %q, got:\n%s", want, output) + } + } +} + +func TestMergeExpansionDataAllocatesConfiguredDatasetsIntoFirstNullRows(t *testing.T) { + root := t.TempDir() + lockPath := filepath.Join(root, "portraits-lock.json") + writeFile(t, lockPath, `{ + "portraits:existing": 0, + "portraits:expanded": 5 +}`+"\n") + + collected, err := mergeExpansionData([]nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Kind: nativeDatasetBase, + Name: "appearance", + OutputName: "appearance.2da", + RowGeneration: "after_base", + }, + Columns: []string{"LABEL", "PORTRAIT"}, + Rows: []map[string]any{{"id": 0, "key": "appearance:source", "PORTRAIT": map[string]any{"value": "po_expanded_", "data": map[string]any{"portraits": map[string]any{"key": "portraits:expanded", "BaseResRef": "expanded_", "Sex": 4}}}}}, + LockData: map[string]int{"appearance:source": 0}, + }, + { + Dataset: nativeDataset{ + Kind: nativeDatasetBase, + Name: "portraits", + LockPath: lockPath, + OutputName: "portraits.2da", + RowGeneration: "first_null_row", + }, + Columns: []string{"BaseResRef", "Sex"}, + Rows: []map[string]any{{"id": 0, "key": "portraits:existing", "BaseResRef": "existing_", "Sex": 4}}, + LockData: map[string]int{"portraits:existing": 0}, + }, + }) + if err != nil { + t.Fatalf("mergeExpansionData failed: %v", err) + } + + portraits := collected[1] + if got := portraits.LockData["portraits:expanded"]; got != 1 { + t.Fatalf("expected expansion lock to use first null row 1, got %d in %#v", got, portraits.LockData) + } + found := false + for _, row := range portraits.Rows { + if row["key"] == "portraits:expanded" && row["id"] == 1 { + found = true + } + } + if !found { + t.Fatalf("expected expanded row at id 1, got %#v", portraits.Rows) + } +} + +func TestMergeExpansionDataAllocatesFirstNullRowsAtOrAboveConfiguredMinimumRow(t *testing.T) { + root := t.TempDir() + lockPath := filepath.Join(root, "portraits-lock.json") + writeFile(t, lockPath, `{ + "portraits:existing": 0 +}`+"\n") + + collected, err := mergeExpansionData([]nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Kind: nativeDatasetBase, + Name: "appearance", + OutputName: "appearance.2da", + RowGeneration: "after_base", + }, + Columns: []string{"LABEL", "PORTRAIT"}, + Rows: []map[string]any{{"id": 0, "key": "appearance:source", "PORTRAIT": map[string]any{"value": "po_expanded_", "data": map[string]any{"portraits": map[string]any{"key": "portraits:expanded", "BaseResRef": "expanded_", "Sex": 4}}}}}, + LockData: map[string]int{"appearance:source": 0}, + }, + { + Dataset: nativeDataset{ + Kind: nativeDatasetBase, + Name: "portraits", + LockPath: lockPath, + OutputName: "portraits.2da", + RowGeneration: "first_null_row", + RowGenerationMinRow: 15000, + }, + Columns: []string{"BaseResRef", "Sex"}, + Rows: []map[string]any{{"id": 0, "key": "portraits:existing", "BaseResRef": "existing_", "Sex": 4}, {"id": 1, "BaseResRef": "****", "Sex": "****"}}, + LockData: map[string]int{"portraits:existing": 0}, + }, + }) + if err != nil { + t.Fatalf("mergeExpansionData failed: %v", err) + } + + portraits := collected[1] + if got := portraits.LockData["portraits:expanded"]; got != 15000 { + t.Fatalf("expected expansion lock to respect minimum row 15000, got %d in %#v", got, portraits.LockData) + } +} + +func TestResolvedTableRegistryRejectsDuplicateKeys(t *testing.T) { + _, err := newResolvedTableRegistry([]nativeCollectedDataset{ + { + Dataset: nativeDataset{Name: "classes/skills/barbarian", OutputName: "cls_skill_barb.2da"}, + Columns: []string{"SkillLabel"}, + Rows: []map[string]any{{"id": 0, "SkillLabel": "Athletics"}}, + TableKey: "classes/skills:barbarian", + }, + { + Dataset: nativeDataset{Name: "classes/skills/barbarian_alt", OutputName: "cls_skill_barb_alt.2da"}, + Columns: []string{"SkillLabel"}, + Rows: []map[string]any{{"id": 0, "SkillLabel": "Ride"}}, + TableKey: "classes/skills:barbarian", + }, + }) + if err == nil || !strings.Contains(err.Error(), `duplicate table key "classes/skills:barbarian"`) { + t.Fatalf("expected duplicate table key error, got %v", err) + } +} + +func TestResolveNativeDatasetPreservesScalarTableReferenceBehavior(t *testing.T) { + tableRegistry, err := newResolvedTableRegistry([]nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Kind: nativeDatasetPlain, + Name: "classes/skills/barbarian", + OutputName: "cls_skill_barb.2da", + }, + Columns: []string{"SkillLabel", "SkillIndex", "ClassSkill"}, + Rows: []map[string]any{ + {"id": 0, "SkillLabel": "Athletics", "SkillIndex": 47, "ClassSkill": "1"}, + }, + TableKey: "classes/skills:barbarian", + }, + }) + if err != nil { + t.Fatalf("newResolvedTableRegistry failed: %v", err) + } + + dataset := nativeCollectedDataset{ + Dataset: nativeDataset{ + Kind: nativeDatasetBase, + Name: "classes", + OutputName: "classes.2da", + Columns: []string{"Label", "SkillsTable"}, + }, + Columns: []string{"Label", "SkillsTable"}, + Rows: []map[string]any{ + { + "id": 0, + "key": "classes:barbarian", + "Label": "Barbarian", + "SkillsTable": map[string]any{"table": "classes/skills:barbarian"}, + }, + }, + } + + resolved, err := resolveNativeDataset( + dataset, + map[string]int{"classes:barbarian": 0}, + map[string]map[string]any{"classes:barbarian": dataset.Rows[0]}, + tableRegistry, + nil, + project.TopDataClassFeatInjectionConfig{}, + nil, + ) + if err != nil { + t.Fatalf("resolveNativeDataset failed: %v", err) + } + rows := resolved["rows"].([]map[string]any) + if got := rows[0]["SkillsTable"]; got != "cls_skill_barb" { + t.Fatalf("expected scalar table reference to resolve to output stem, got %v", got) + } +} + +func TestNormalizeProjectImportsLegacyClasses(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + + referenceRoot := filepath.Join(root, "reference") + mkdirAll(t, filepath.Join(referenceRoot, "data", "classes", "core", "modules")) + mkdirAll(t, filepath.Join(referenceRoot, "data", "classes", "feats")) + mkdirAll(t, filepath.Join(referenceRoot, "data", "classes", "skills")) + mkdirAll(t, filepath.Join(referenceRoot, "data", "classes", "savthr")) + mkdirAll(t, filepath.Join(referenceRoot, "data", "classes", "bfeat")) + mkdirAll(t, filepath.Join(referenceRoot, "tlk", "modules", "classes")) + + writeFile(t, filepath.Join(referenceRoot, "data", "classes", "core", "base.json"), `{ + "output": "classes.2da", + "columns": ["Label", "Description", "FeatsTable", "SavingThrowTable", "SkillsTable", "BonusFeatsTable", "PreReqTable"], + "rows": [ + { + "id": 4, + "key": "classes:fighter", + "Label": "Fighter", + "Description": "240", + "FeatsTable": "cls_feat_fight", + "SavingThrowTable": "cls_savthr_fort", + "SkillsTable": "cls_skill_fight", + "BonusFeatsTable": "cls_bfeat_fight", + "PreReqTable": "****" + }, + { + "id": 11, + "key": "classes:aberration", + "Label": "Aberration", + "Description": "241", + "FeatsTable": "CLS_FEAT_ABER", + "SavingThrowTable": "CLS_SAVTHR_WIZ", + "SkillsTable": "CLS_SKILL_FIGHT", + "BonusFeatsTable": "CLS_BFEAT_BARB", + "PreReqTable": "****" + } + ] +}`+"\n") + writeFile(t, filepath.Join(referenceRoot, "data", "classes", "core", "lock.json"), `{ + "classes:fighter": 4, + "classes:aberration": 11 +}`+"\n") + writeFile(t, filepath.Join(referenceRoot, "data", "classes", "core", "modules", "fighter.json"), `{ + "overrides": [ + { + "key": "classes:fighter", + "Description": { "tlk": "classes:fighter.description" } + } + ] +}`+"\n") + writeFile(t, filepath.Join(referenceRoot, "data", "classes", "feats", "fighter.json"), `{ + "key": "classes/feats:fighter", + "output": "cls_feat_fight.2da", + "columns": ["FeatIndex"], + "rows": [ + {"FeatIndex": {"id": "feat:literate"}} + ] +}`+"\n") + writeFile(t, filepath.Join(referenceRoot, "data", "classes", "skills", "fighter.json"), `{ + "key": "classes/skills:fighter", + "output": "cls_skill_fight.2da", + "columns": ["SkillIndex"], + "rows": [ + {"SkillIndex": {"id": "skills:spot"}} + ] +}`+"\n") + writeFile(t, filepath.Join(referenceRoot, "data", "classes", "savthr", "fortitude_high.json"), `{ + "key": "classes/saves:fortitude", + "output": "cls_savthr_fort.2da", + "columns": ["Level", "FortSave"], + "rows": [ + {"Level": "1", "FortSave": "2"} + ] +}`+"\n") + writeFile(t, filepath.Join(referenceRoot, "data", "classes", "bfeat", "fighter.json"), `{ + "key": "classes/bonusfeats:fighter", + "output": "cls_bfeat_fight.2da", + "columns": ["Bonus"], + "rows": [ + {"id": 1, "Bonus": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(referenceRoot, "tlk", "lock.json"), `{ + "classes:fighter.description": 1234 +}`+"\n") + writeFile(t, filepath.Join(referenceRoot, "tlk", "modules", "classes", "fighter.json"), `{ + "entries": { + "classes:fighter.description": { + "text": "Martial expert." + } + } +}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "reference" + result, err := NormalizeProject(p) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles == 0 { + t.Fatal("expected classes import to write canonical files") + } + + coreObj := map[string]any{} + coreRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "classes", "core", "base.json")) + if err != nil { + t.Fatalf("read classes core: %v", err) + } + if err := json.Unmarshal(coreRaw, &coreObj); err != nil { + t.Fatalf("parse classes core: %v", err) + } + rows := coreObj["rows"].([]any) + row := rows[0].(map[string]any) + + description := row["Description"].(map[string]any)["tlk"].(map[string]any) + if description["key"] != "classes:fighter.description" || description["text"] != "Martial expert." { + t.Fatalf("unexpected inline TLK payload: %#v", description) + } + if table := row["FeatsTable"].(map[string]any)["table"]; table != "classes/feats:fighter" { + t.Fatalf("expected feats table ref rewrite, got %#v", row["FeatsTable"]) + } + if table := row["SavingThrowTable"].(map[string]any)["table"]; table != "classes/saves:fortitude" { + t.Fatalf("expected save table ref rewrite, got %#v", row["SavingThrowTable"]) + } + if table := row["SkillsTable"].(map[string]any)["table"]; table != "classes/skills:fighter" { + t.Fatalf("expected skills table ref rewrite, got %#v", row["SkillsTable"]) + } + if table := row["BonusFeatsTable"].(map[string]any)["table"]; table != "classes/bonusfeats:fighter" { + t.Fatalf("expected bonus feats table ref rewrite, got %#v", row["BonusFeatsTable"]) + } + monsterRow := rows[1].(map[string]any) + if got := monsterRow["SkillsTable"]; got != "CLS_SKILL_FIGHT" { + t.Fatalf("expected legacy uppercase monster table stem to remain literal, got %#v", got) + } + + state := tlkStateDocument{} + stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", ".tlk_state.json")) + if err != nil { + t.Fatalf("read tlk state: %v", err) + } + if err := json.Unmarshal(stateRaw, &state); err != nil { + t.Fatalf("parse tlk state: %v", err) + } + if got := state.Entries["classes:fighter.description"].ID; got != 1234 { + t.Fatalf("expected preserved TLK state ID 1234, got %d", got) + } +} + +func TestBuildNativeSupportsClassesInlineTLKFields(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "core")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "base.json"), `{ + "output": "classes.2da", + "columns": ["Label", "Short", "Name", "Plural", "Lower", "Description"], + "rows": [ + { + "id": 48, + "key": "classes:swashbuckler", + "Label": "Swashbuckler", + "Short": { "tlk": { "key": "classes:swashbuckler.short", "text": "Swa" } }, + "Name": { "tlk": { "key": "classes:swashbuckler.name", "text": "Swashbuckler" } }, + "Plural": { "tlk": { "key": "classes:swashbuckler.plural", "text": "Swashbucklers" } }, + "Lower": { "tlk": { "key": "classes:swashbuckler.lower", "text": "swashbuckler" } }, + "Description": { "tlk": { "key": "classes:swashbuckler.description", "text": "Agile duelist." } } + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "lock.json"), `{ + "classes:swashbuckler": 48 +}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + result, err := BuildNative(p, nil) + if err != nil { + report := ValidateProject(p) + t.Fatalf("BuildNative failed: %v\n%s", err, diagnosticsText(report.Diagnostics)) + } + + classesBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "classes.2da")) + if err != nil { + t.Fatalf("read classes.2da: %v", err) + } + classesText := string(classesBytes) + if !strings.Contains(classesText, "Label\tShort\tName\tPlural\tLower\tDescription\n") { + t.Fatalf("unexpected classes.2da header:\n%s", classesText) + } + if !strings.Contains(classesText, "48\tSwashbuckler\t16777216\t16777217\t16777218\t16777219\t16777220\n") { + t.Fatalf("expected compiled TLK strrefs in classes.2da, got:\n%s", classesText) + } +} + +func TestBuildNativePreservesClassesLegacyWikiGenerateColumn(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "core")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "base.json"), `{ + "output": "classes.2da", + "columns": ["Label", "WIKIGENERATE", "SkipSpellSelection"], + "rows": [ + { + "id": 4, + "key": "classes:fighter", + "Label": "Fighter", + "WIKIGENERATE": "****", + "SkipSpellSelection": "0" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "lock.json"), `{ + "classes:fighter": 4 +}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + report := ValidateProject(p) + if report.HasErrors() { + t.Fatalf("expected classes/core legacy WIKIGENERATE compatibility, got %#v", report.Diagnostics) + } + + result, err := BuildNative(p, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + classesBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "classes.2da")) + if err != nil { + t.Fatalf("read classes.2da: %v", err) + } + classesText := string(classesBytes) + if !strings.Contains(classesText, "Label\tWIKIGENERATE\tSkipSpellSelection\n") { + t.Fatalf("expected classes.2da to keep WIKIGENERATE column, got:\n%s", classesText) + } + if !strings.Contains(classesText, "4\tFighter\t****\t0\n") { + t.Fatalf("expected classes.2da row to keep WIKIGENERATE value, got:\n%s", classesText) + } +} + +func TestBuildNativeExpandsClassesFeatMasterfeatRows(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "feats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "feats", "acolyte.json"), `{ + "key": "classes/feats:acolyte", + "output": "cls_feat_acolyte.2da", + "columns": ["FeatLabel", "FeatIndex", "List", "GrantedOnLevel", "OnMenu"], + "rows": [ + {"FeatIndex": {"id": "feat:literate"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "0"}, + {"FeatIndex": {"id": "masterfeats:spellfocus"}, "List": "1", "GrantedOnLevel": "-1", "OnMenu": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION", "MASTERFEAT"], + "rows": [ + {"id": 35, "key": "feat:spellfocus_abjuration", "LABEL": "SpellFocusAbj", "FEAT": "425", "DESCRIPTION": "426", "MASTERFEAT": "3"}, + {"id": 36, "key": "feat:spellfocus_conjuration", "LABEL": "SpellFocusCon", "FEAT": "166", "DESCRIPTION": "427", "MASTERFEAT": "3"}, + {"id": 1135, "key": "feat:literate", "LABEL": "FEAT_LITERATE", "FEAT": "9000", "DESCRIPTION": "9001", "MASTERFEAT": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:spellfocus_abjuration": 35, + "feat:spellfocus_conjuration": 36, + "feat:literate": 1135 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON"], + "rows": [ + {"id": 3, "key": "masterfeats:spellfocus", "LABEL": "SpellFocus", "STRREF": "6490", "DESCRIPTION": "436", "ICON": "ife_skfoc"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{ + "masterfeats:spellfocus": 3 +}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + result, err := BuildNative(p, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "cls_feat_acolyte.2da")) + if err != nil { + t.Fatalf("read cls_feat_acolyte.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "0\tFEAT_LITERATE\t1135\t3\t1\t0\n") { + t.Fatalf("expected direct feat row label fill, got:\n%s", text) + } + if !strings.Contains(text, "1\tSpellFocusAbj\t35\t1\t-1\t0\n") || !strings.Contains(text, "2\tSpellFocusCon\t36\t1\t-1\t0\n") { + t.Fatalf("expected masterfeat expansion rows, got:\n%s", text) + } +} + +func TestBuildNativeClassFeatMasterfeatExpansionRespectsAllClassesCanUse(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "feats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "base.json"), `{ + "output": "classes.2da", + "columns": ["Label", "Name", "FeatsTable"], + "rows": [ + {"id": 4, "key": "classes:fighter", "Label": "Fighter", "Name": "111", "FeatsTable": "cls_feat_fighter"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "lock.json"), `{"classes:fighter":4}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "feats", "fighter.json"), `{ + "key": "classes/feats:fighter", + "output": "cls_feat_fighter.2da", + "columns": ["FeatLabel", "FeatIndex", "List", "GrantedOnLevel", "OnMenu"], + "rows": [ + {"FeatIndex": {"id": "masterfeats:weaponfocus"}, "List": "1", "GrantedOnLevel": "-1", "OnMenu": "0"}, + {"FeatIndex": {"id": "masterfeats:weaponspecialization"}, "List": "1", "GrantedOnLevel": "4", "OnMenu": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION", "MASTERFEAT", "ALLCLASSESCANUSE", "MinLevelClass"], + "rows": [ + {"id": 100, "key": "feat:weaponfocus_longsword", "LABEL": "WeaponFocusLongsword", "FEAT": "1000", "DESCRIPTION": "1001", "MASTERFEAT": "1", "ALLCLASSESCANUSE": "1", "MinLevelClass": "****"}, + {"id": 101, "key": "feat:weaponfocus_creature", "LABEL": "WeaponFocusCreature", "FEAT": "1002", "DESCRIPTION": "1003", "MASTERFEAT": "1", "ALLCLASSESCANUSE": "0", "MinLevelClass": "****"}, + {"id": 200, "key": "feat:weaponspecialization_longsword", "LABEL": "WeaponSpecLongsword", "FEAT": "2000", "DESCRIPTION": "2001", "MASTERFEAT": "2", "ALLCLASSESCANUSE": "0", "MinLevelClass": {"id": "classes:fighter"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:weaponfocus_longsword": 100, + "feat:weaponfocus_creature": 101, + "feat:weaponspecialization_longsword": 200 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON"], + "rows": [ + {"id": 1, "key": "masterfeats:weaponfocus", "LABEL": "WeaponFocus", "STRREF": "6490", "DESCRIPTION": "436", "ICON": "ife_wepfoc"}, + {"id": 2, "key": "masterfeats:weaponspecialization", "LABEL": "WeaponSpecialization", "STRREF": "6491", "DESCRIPTION": "437", "ICON": "ife_wepspec"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{ + "masterfeats:weaponfocus": 1, + "masterfeats:weaponspecialization": 2 +}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + result, err := BuildNative(p, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "cls_feat_fighter.2da")) + if err != nil { + t.Fatalf("read cls_feat_fighter.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "WeaponFocusLongsword") { + t.Fatalf("expected globally selectable weapon focus row, got:\n%s", text) + } + if strings.Contains(text, "WeaponFocusCreature") { + t.Fatalf("expected ALLCLASSESCANUSE=0 creature row without class restriction to be filtered, got:\n%s", text) + } + if !strings.Contains(text, "WeaponSpecLongsword") { + t.Fatalf("expected class-restricted ALLCLASSESCANUSE=0 row to remain available to fighter, got:\n%s", text) + } +} + +func TestBuildNativeExpandsClassesFeatMasterfeatRowsWithFilter(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "feats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "skills")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "feats", "fighter.json"), `{ + "key": "classes/feats:fighter", + "output": "cls_feat_fighter.2da", + "columns": ["FeatLabel", "FeatIndex", "List", "GrantedOnLevel", "OnMenu"], + "rows": [ + {"FeatIndex": {"id": "feat:toughness", "filter": "successors"}, "List": "1", "GrantedOnLevel": "-1", "OnMenu": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "fighter.json"), `{ + "key": "classes/skills:fighter", + "output": "cls_skill_fighter.2da", + "columns": ["SkillLabel", "SkillIndex", "ClassSkill"], + "rows": [ + {"key": "skills:athletics", "SkillLabel": "Athletics", "SkillIndex": {"id": "skills:athletics"}, "ClassSkill": "1"}, + {"key": "skills:persuade", "SkillLabel": "Persuade", "SkillIndex": {"id": "skills:persuade"}, "ClassSkill": "1"}, + {"key": "skills:alchemy", "SkillLabel": "Alchemy", "SkillIndex": {"id": "skills:alchemy"}, "ClassSkill": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION", "MASTERFEAT", "REQSKILL", "SUCCESSOR"], + "rows": [ + {"id": 1, "key": "feat:toughness", "LABEL": "Toughness", "FEAT": "100", "DESCRIPTION": "101", "MASTERFEAT": "****", "REQSKILL": "****", "SUCCESSOR": "feat:toughness_1"}, + {"id": 2, "key": "feat:toughness_1", "LABEL": "Toughness1", "FEAT": "102", "DESCRIPTION": "103", "MASTERFEAT": "****", "REQSKILL": "****", "SUCCESSOR": "feat:toughness_2"}, + {"id": 3, "key": "feat:toughness_2", "LABEL": "Toughness2", "FEAT": "104", "DESCRIPTION": "105", "MASTERFEAT": "****", "REQSKILL": "****", "SUCCESSOR": "****"}, + {"id": 10, "key": "feat:skillfocus_athletics", "LABEL": "SkillFocusAthletics", "FEAT": "200", "DESCRIPTION": "201", "MASTERFEAT": "4", "REQSKILL": "skills:athletics", "SUCCESSOR": "****"}, + {"id": 11, "key": "feat:skillfocus_persuade", "LABEL": "SkillFocusPersuade", "FEAT": "202", "DESCRIPTION": "203", "MASTERFEAT": "4", "REQSKILL": "skills:persuade", "SUCCESSOR": "****"}, + {"id": 12, "key": "feat:skillfocus_alchemy", "LABEL": "SkillFocusAlchemy", "FEAT": "203", "DESCRIPTION": "204", "MASTERFEAT": "4", "REQSKILL": "skills:alchemy", "SUCCESSOR": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:toughness": 1, + "feat:toughness_1": 2, + "feat:toughness_2": 3, + "feat:skillfocus_athletics": 10, + "feat:skillfocus_persuade": 11, + "feat:skillfocus_alchemy": 12 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON"], + "rows": [ + {"id": 4, "key": "masterfeats:skillfocus", "LABEL": "SkillFocus", "STRREF": "6493", "DESCRIPTION": "426", "ICON": "ife_skfoc"}, + {"id": 5, "key": "masterfeats:greaterskillfocus", "LABEL": "GreaterSkillFocus", "STRREF": "6494", "DESCRIPTION": "427", "ICON": "ife_grtskfoc"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{ + "masterfeats:skillfocus": 4, + "masterfeats:greaterskillfocus": 5 +}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + result, err := BuildNative(p, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "cls_feat_fighter.2da")) + if err != nil { + t.Fatalf("read cls_feat_fighter.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "2\tToughness\t1\t1\t-1\t0\n") || !strings.Contains(text, "3\tToughness1\t2\t1\t-1\t0\n") || !strings.Contains(text, "4\tToughness2\t3\t1\t-1\t0\n") { + t.Fatalf("expected successor expansion, got:\n%s", text) + } +} + +func TestBuildNativeExpandsClassesFeatClassskillsFilter(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "feats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "skills")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "feats", "fighter.json"), `{ + "key": "classes/feats:fighter", + "output": "cls_feat_fighter.2da", + "columns": ["FeatLabel", "FeatIndex", "List", "GrantedOnLevel", "OnMenu"], + "rows": [ + {"FeatIndex": {"id": "masterfeats:skillfocus", "filter": "classskills"}, "List": "1", "GrantedOnLevel": "-1", "OnMenu": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "fighter.json"), `{ + "key": "classes/skills:fighter", + "output": "cls_skill_fighter.2da", + "columns": ["SkillLabel", "SkillIndex", "ClassSkill"], + "rows": [ + {"key": "skills:athletics", "SkillLabel": "Athletics", "SkillIndex": {"id": "skills:athletics"}, "ClassSkill": "1"}, + {"key": "skills:persuade", "SkillLabel": "Persuade", "SkillIndex": {"id": "skills:persuade"}, "ClassSkill": "1"}, + {"key": "skills:alchemy", "SkillLabel": "Alchemy", "SkillIndex": {"id": "skills:alchemy"}, "ClassSkill": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION", "MASTERFEAT", "REQSKILL", "SUCCESSOR"], + "rows": [ + {"id": 10, "key": "feat:skillfocus_athletics", "LABEL": "SkillFocusAthletics", "FEAT": "200", "DESCRIPTION": "201", "MASTERFEAT": "4", "REQSKILL": "skills:athletics", "SUCCESSOR": "****"}, + {"id": 11, "key": "feat:skillfocus_persuade", "LABEL": "SkillFocusPersuade", "FEAT": "202", "DESCRIPTION": "203", "MASTERFEAT": "4", "REQSKILL": "skills:persuade", "SUCCESSOR": "****"}, + {"id": 12, "key": "feat:skillfocus_alchemy", "LABEL": "SkillFocusAlchemy", "FEAT": "203", "DESCRIPTION": "204", "MASTERFEAT": "4", "REQSKILL": "skills:alchemy", "SUCCESSOR": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:skillfocus_athletics": 10, + "feat:skillfocus_persuade": 11, + "feat:skillfocus_alchemy": 12 +}`+"\n") + /* CLASSKILLS_FILTER_TEST_INSERT_MARKER */ + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON"], + "rows": [ + {"id": 4, "key": "masterfeats:skillfocus", "LABEL": "SkillFocus", "STRREF": "6493", "DESCRIPTION": "426", "ICON": "ife_skfoc"}, + {"id": 5, "key": "masterfeats:greaterskillfocus", "LABEL": "GreaterSkillFocus", "STRREF": "6494", "DESCRIPTION": "427", "ICON": "ife_grtskfoc"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{ + "masterfeats:skillfocus": 4, + "masterfeats:greaterskillfocus": 5 +}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + result, err := BuildNative(p, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "cls_feat_fighter.2da")) + if err != nil { + t.Fatalf("read cls_feat_fighter.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "SkillFocusAthletics") || !strings.Contains(text, "SkillFocusPersuade") { + t.Fatalf("expected classskill filter expansion (athletics + persuade), got:\n%s", text) + } + if strings.Contains(text, "SkillFocusAlchemy") { + t.Fatalf("expected alchemy to be filtered out (not a class skill), got:\n%s", text) + } +} + +func TestBuildNativeAppliesConfiguredClassFeatInjections(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "feats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "skills")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "feats", "fighter.json"), `{ + "key": "classes/feats:fighter", + "output": "cls_feat_fighter.2da", + "columns": ["FeatLabel", "FeatIndex", "List", "GrantedOnLevel", "OnMenu"], + "rows": [ + {"FeatIndex": {"id": "feat:illiterate"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "fighter.json"), `{ + "key": "classes/skills:fighter", + "output": "cls_skill_fighter.2da", + "columns": ["SkillLabel", "SkillIndex", "ClassSkill"], + "rows": [ + {"key": "skills:athletics", "SkillLabel": "Athletics", "SkillIndex": {"id": "skills:athletics"}, "ClassSkill": "1"}, + {"key": "skills:persuade", "SkillLabel": "Persuade", "SkillIndex": {"id": "skills:persuade"}, "ClassSkill": "1"}, + {"key": "skills:alchemy", "SkillLabel": "Alchemy", "SkillIndex": {"id": "skills:alchemy"}, "ClassSkill": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION", "MASTERFEAT", "REQSKILL", "SUCCESSOR"], + "rows": [ + {"id": 1, "key": "feat:illiterate", "LABEL": "Illiterate", "FEAT": "100", "DESCRIPTION": "101", "MASTERFEAT": "****", "REQSKILL": "****", "SUCCESSOR": "****"}, + {"id": 2, "key": "feat:literate", "LABEL": "Literate", "FEAT": "102", "DESCRIPTION": "103", "MASTERFEAT": "****", "REQSKILL": "****", "SUCCESSOR": "****"}, + {"id": 3, "key": "feat:customglobal", "LABEL": "CustomGlobal", "FEAT": "104", "DESCRIPTION": "105", "MASTERFEAT": "****", "REQSKILL": "****", "SUCCESSOR": "****"}, + {"id": 10, "key": "feat:skillfocus_athletics", "LABEL": "SkillFocusAthletics", "FEAT": "200", "DESCRIPTION": "201", "MASTERFEAT": "4", "REQSKILL": "skills:athletics", "SUCCESSOR": "****"}, + {"id": 11, "key": "feat:skillfocus_persuade", "LABEL": "SkillFocusPersuade", "FEAT": "202", "DESCRIPTION": "203", "MASTERFEAT": "4", "REQSKILL": "skills:persuade", "SUCCESSOR": "****"}, + {"id": 12, "key": "feat:skillfocus_alchemy", "LABEL": "SkillFocusAlchemy", "FEAT": "204", "DESCRIPTION": "205", "MASTERFEAT": "4", "REQSKILL": "skills:alchemy", "SUCCESSOR": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:illiterate": 1, + "feat:literate": 2, + "feat:customglobal": 3, + "feat:skillfocus_athletics": 10, + "feat:skillfocus_persuade": 11, + "feat:skillfocus_alchemy": 12 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON"], + "rows": [ + {"id": 4, "key": "masterfeats:skillfocus", "LABEL": "SkillFocus", "STRREF": "6493", "DESCRIPTION": "426", "ICON": "ife_skfoc"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{"masterfeats:skillfocus": 4}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label"], + "rows": [ + {"id": 1, "key": "skills:athletics", "Label": "Athletics"}, + {"id": 2, "key": "skills:alchemy", "Label": "Alchemy"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":1,"skills:alchemy":2}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + p.Config.TopData.ClassFeatInjections = project.TopDataClassFeatInjectionConfig{ + GlobalFeats: []project.TopDataClassFeatGlobalRule{ + {Feat: "feat:literate", List: "3", GrantedOnLevel: "1", OnMenu: "0", UnlessPresent: []string{"feat:ill_iterate"}}, + {Feat: "feat:custom_global", List: "3", GrantedOnLevel: "2", OnMenu: "1"}, + }, + ClassSkillMasterfeats: []project.TopDataClassFeatMasterfeatRule{ + {Masterfeat: "masterfeats:skillfocus", List: "1", GrantedOnLevel: "2", OnMenu: "0"}, + }, + } + result, err := BuildNative(p, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "cls_feat_fighter.2da")) + if err != nil { + t.Fatalf("read cls_feat_fighter.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "CustomGlobal\t3\t3\t2\t1\n") { + t.Fatalf("expected configured global feat injection, got:\n%s", text) + } + if strings.Contains(text, "Literate\t2\t3\t1\t0\n") { + t.Fatalf("expected literate injection to be skipped when illiterate is present, got:\n%s", text) + } + if !strings.Contains(text, "SkillFocusAthletics\t10\t1\t2\t0\n") || !strings.Contains(text, "SkillFocusPersuade\t11\t1\t2\t0\n") { + t.Fatalf("expected configured class-skill masterfeat expansion, got:\n%s", text) + } + if strings.Contains(text, "SkillFocusAlchemy") { + t.Fatalf("expected non-class skill focus to be filtered out, got:\n%s", text) + } +} + +func TestBuildNativeAppliesClassFeatGlobalJSONInjections(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "feats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "skills")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "feats", "global.json"), `{ + "position": "prepend", + "injections": [ + { + "row": {"FeatIndex": {"id": "feat:literate"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "0"}, + "unless_present": [{"field": "FeatIndex", "id": "feat:illiterate"}] + }, + { + "row": {"FeatIndex": {"id": "feat:customglobal"}, "List": "3", "GrantedOnLevel": "2", "OnMenu": "1"} + }, + { + "row": {"FeatIndex": {"id": "masterfeats:skillfocus", "filter": "classskills"}, "List": "1", "GrantedOnLevel": "2", "OnMenu": "0"} + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "feats", "fighter.json"), `{ + "key": "classes/feats:fighter", + "output": "cls_feat_fighter.2da", + "columns": ["FeatLabel", "FeatIndex", "List", "GrantedOnLevel", "OnMenu"], + "rows": [ + {"FeatIndex": {"id": "feat:illiterate"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "fighter.json"), `{ + "key": "classes/skills:fighter", + "output": "cls_skill_fighter.2da", + "columns": ["SkillLabel", "SkillIndex", "ClassSkill"], + "rows": [ + {"SkillLabel": "Athletics", "SkillIndex": {"id": "skills:athletics"}, "ClassSkill": "1"}, + {"SkillLabel": "Alchemy", "SkillIndex": {"id": "skills:alchemy"}, "ClassSkill": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION", "MASTERFEAT", "REQSKILL", "SUCCESSOR"], + "rows": [ + {"id": 1, "key": "feat:illiterate", "LABEL": "Illiterate", "FEAT": "100", "DESCRIPTION": "101", "MASTERFEAT": "****", "REQSKILL": "****", "SUCCESSOR": "****"}, + {"id": 2, "key": "feat:literate", "LABEL": "Literate", "FEAT": "102", "DESCRIPTION": "103", "MASTERFEAT": "****", "REQSKILL": "****", "SUCCESSOR": "****"}, + {"id": 3, "key": "feat:customglobal", "LABEL": "CustomGlobal", "FEAT": "104", "DESCRIPTION": "105", "MASTERFEAT": "****", "REQSKILL": "****", "SUCCESSOR": "****"}, + {"id": 10, "key": "feat:skillfocus_athletics", "LABEL": "SkillFocusAthletics", "FEAT": "200", "DESCRIPTION": "201", "MASTERFEAT": "4", "REQSKILL": "skills:athletics", "SUCCESSOR": "****"}, + {"id": 11, "key": "feat:skillfocus_alchemy", "LABEL": "SkillFocusAlchemy", "FEAT": "204", "DESCRIPTION": "205", "MASTERFEAT": "4", "REQSKILL": "skills:alchemy", "SUCCESSOR": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:illiterate": 1, + "feat:literate": 2, + "feat:customglobal": 3, + "feat:skillfocus_athletics": 10, + "feat:skillfocus_alchemy": 11 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON"], + "rows": [ + {"id": 4, "key": "masterfeats:skillfocus", "LABEL": "SkillFocus", "STRREF": "6493", "DESCRIPTION": "426", "ICON": "ife_skfoc"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{"masterfeats:skillfocus": 4}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label"], + "rows": [ + {"id": 1, "key": "skills:athletics", "Label": "Athletics"}, + {"id": 2, "key": "skills:alchemy", "Label": "Alchemy"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":1,"skills:alchemy":2}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + result, err := BuildNativeWithOptions(p, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "cls_feat_fighter.2da")) + if err != nil { + t.Fatalf("read cls_feat_fighter.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "CustomGlobal\t3\t3\t2\t1\n") { + t.Fatalf("expected global.json feat injection, got:\n%s", text) + } + if strings.Contains(text, "Literate\t2\t3\t1\t0\n") { + t.Fatalf("expected literate global injection to be skipped when illiterate is present, got:\n%s", text) + } + if !strings.Contains(text, "SkillFocusAthletics\t10\t1\t2\t0\n") { + t.Fatalf("expected global.json class-skill masterfeat expansion, got:\n%s", text) + } + if strings.Contains(text, "SkillFocusAlchemy") { + t.Fatalf("expected non-class skill focus to be filtered out, got:\n%s", text) + } +} + +func TestResolveCanonicalDatasetKeyMatchesMasterfeatAliases(t *testing.T) { + keyToID := map[string]int{ + "masterfeats:skill_focus": 4, + "masterfeats:greater_skill_focus": 15, + "masterfeats:weapon_focus": 1, + "masterfeats:greater_weapon_specialization": 11, + } + + cases := map[string]string{ + "masterfeats:skillfocus": "masterfeats:skill_focus", + "masterfeats:greaterskillfocus": "masterfeats:greater_skill_focus", + "masterfeats:weaponfocus": "masterfeats:weapon_focus", + "masterfeats:greaterweaponspecialization": "masterfeats:greater_weapon_specialization", + "masterfeats:skill_focus": "masterfeats:skill_focus", + } + + for input, want := range cases { + if got := resolveCanonicalDatasetKey(input, keyToID); got != want { + t.Fatalf("resolveCanonicalDatasetKey(%q) = %q, want %q", input, got, want) + } + } +} + +func TestValidateProjectRejectsLegacyAuthoredTLKModules(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [ + { + "id": 0, + "key": "feat:test", + "LABEL": "FEAT_TEST", + "FEAT": {"tlk": {"key": "feat:test.name", "text": "Test Feat"}}, + "DESCRIPTION": {"tlk": {"key": "feat:test.description", "text": "Test Description"}} + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "modules", "feat", "test.json"), `{"entries":{"feat:test.name":{"text":"Legacy name"}}}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatal("expected validation error for legacy authored TLK modules") + } + if !strings.Contains(diagnosticsText(report.Diagnostics), "topdata/tlk/modules is no longer allowed for canonical authored input") { + t.Fatalf("expected inline-TLK enforcement error, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + +func TestNormalizeProjectRemovesLegacyAuthoredTLKModules(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [ + { + "id": 0, + "key": "feat:test", + "LABEL": "FEAT_TEST", + "FEAT": {"tlk": "feat:test.name"}, + "DESCRIPTION": {"tlk": "feat:test.description"} + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "lock.json"), `{"feat:test.name":9,"feat:test.description":10}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "modules", "feat", "test.json"), `{ + "entries": { + "feat:test.name": {"text": "Legacy Name"}, + "feat:test.description": {"text": "Legacy Description"} + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.RemainingLegacyRefs != 0 { + t.Fatalf("expected no remaining legacy refs, got %#v", result) + } + if _, err := os.Stat(filepath.Join(root, "topdata", "tlk", "modules")); !os.IsNotExist(err) { + t.Fatalf("expected legacy tlk/modules to be removed, got err=%v", err) + } + if _, err := os.Stat(filepath.Join(root, "topdata", "tlk", "lock.json")); !os.IsNotExist(err) { + t.Fatalf("expected legacy tlk/lock.json to be removed, got err=%v", err) + } + report := ValidateProject(testProject(root)) + if report.HasErrors() { + t.Fatalf("expected normalized project to validate cleanly, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + +func TestValidateProjectAcceptsCanonicalMasterfeatsContract(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON"], + "rows": [ + { + "id": 0, + "key": "masterfeats:weaponfocus", + "LABEL": "WeaponFocus", + "STRREF": "6490", + "DESCRIPTION": "436", + "ICON": "ife_wepfoc" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{ + "masterfeats:weaponfocus": 0 +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + report := ValidateProject(testProject(root)) + if report.HasErrors() { + t.Fatalf("expected no validation errors, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + +func TestValidateAndBuildDerivesMasterfeatsOutputWhenOmitted(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON"], + "rows": [ + { + "id": 0, + "key": "masterfeats:weaponfocus", + "LABEL": "WeaponFocus", + "STRREF": "6490", + "DESCRIPTION": "436", + "ICON": "ife_wepfoc" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{ + "masterfeats:weaponfocus": 0 +}`+"\n") + + proj := testProject(root) + report := ValidateProject(proj) + if report.HasErrors() { + t.Fatalf("expected omitted masterfeats output to validate cleanly, got:\n%s", diagnosticsText(report.Diagnostics)) + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("expected omitted masterfeats output to build: %v", err) + } + if _, err := os.Stat(filepath.Join(result.Output2DADir, "masterfeats.2da")); err != nil { + t.Fatalf("expected derived masterfeats.2da output: %v", err) + } +} + +func TestValidateProjectRejectsInvalidMasterfeatsContract(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "STRREF"], + "rows": [ + { + "id": 0, + "key": "feat:weaponfocus", + "LABEL": "WeaponFocus", + "STRREF": "6490" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{ + "feat:weaponfocus": 0 +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatal("expected validation errors for invalid masterfeats contract") + } + text := diagnosticsText(report.Diagnostics) + for _, want := range []string{ + "masterfeats output must be masterfeats.2da", + "masterfeats base.json must include DESCRIPTION in columns", + `masterfeats row 0 key "feat:weaponfocus" must start with masterfeats:`, + `masterfeats lock key "feat:weaponfocus" must start with masterfeats:`, + } { + if !strings.Contains(text, want) { + t.Fatalf("expected validation message %q, got:\n%s", want, text) + } + } +} + +func TestBuildReferenceAndCompareReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["LABEL"], + "rows": [{"id": 0, "LABEL": "TEST"}] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "lock.json"), "{}\n") + + p := testProject(root) + + result, err := BuildReference(p, nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + if result.Files2DA != 1 || result.FilesTLK != 1 { + t.Fatalf("unexpected build result: %#v", result) + } + + compare, err := CompareReference(p, nil) + if err != nil { + t.Fatalf("CompareReference failed: %v", err) + } + if compare.Mode != "native" || compare.NativeMismatch != 0 || compare.NativePass != 1 { + t.Fatalf("unexpected compare result: %#v", compare) + } +} + +func TestCompareReferenceHandlesMixedModeOutputListMismatch(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "parts")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER"], + "rows": [{"id": 0, "COSTMODIFIER": "0"}] +}`+"\n") + + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + writeFile(t, filepath.Join(nativeResult.Output2DADir, "unexpected.2da"), "2DA V2.0\n\nLABEL\n0\tExtra\n") + + compare, err := CompareReference(testProject(root), nil) + if err != nil { + t.Fatalf("CompareReference failed: %v", err) + } + if compare.NativeMismatch == 0 { + t.Fatalf("unexpected mixed-mode compare result: %#v", compare) + } +} + +func TestBuildUsesNativeModeForInlineTLKAndWritesState(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [ + {"id": 0, "key": "feat:test", "LABEL": "FeatTest", "_tlk": {"name": "Base Name", "description": "Base Description"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0,"feat:module":5}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "module.json"), `{ + "entries": { + "feat:module": { + "LABEL": "Feat Module", + "_tlk": { + "name": {"ref": "feat:test", "field": "FEAT"}, + "description": {"text": "Module Description"} + } + } + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + if report := ValidateProject(testProject(root)); report.HasErrors() { + t.Fatalf("unexpected validation errors:\n%s", diagnosticsText(report.Diagnostics)) + } + + result, err := Build(testProject(root), nil) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + if result.Mode != "native" { + t.Fatalf("expected native mode, got %#v", result) + } + if result.Files2DA != 1 || result.FilesTLK != 1 { + t.Fatalf("unexpected build result: %#v", result) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read native 2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "LABEL\tFEAT\tDESCRIPTION\n") || + !strings.Contains(text, "0\tFeatTest\t16777216\t16777217\n") || + !strings.Contains(text, "5\t\"Feat Module\"\t16777216\t16777218\n") { + t.Fatalf("unexpected 2da output:\n%s", string(got)) + } + + stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile)) + if err != nil { + t.Fatalf("read tlk state: %v", err) + } + var state tlkStateDocument + if err := json.Unmarshal(stateRaw, &state); err != nil { + t.Fatalf("parse state: %v", err) + } + if state.Language != "en" { + t.Fatalf("unexpected state language: %#v", state) + } + if state.Entries["feat:test.feat"].ID != 0 || state.Entries["feat:test.description"].ID != 1 || state.Entries["feat:module.description"].ID != 2 { + t.Fatalf("unexpected state entries: %#v", state.Entries) + } + + if _, err := os.Stat(filepath.Join(result.OutputTLKDir, defaultTLKName)); err != nil { + t.Fatalf("expected tlk output: %v", err) + } +} + +func TestBuildPreservesTLKStateAcrossTextChanges(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description"], + "rows": [ + {"id": 0, "key": "skills:athletics", "Label": "Athletics", "_tlk": {"name": "Athletics", "description": "Old text"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + p := testProject(root) + if _, err := Build(p, nil); err != nil { + t.Fatalf("first build failed: %v", err) + } + + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description"], + "rows": [ + {"id": 0, "key": "skills:athletics", "Label": "Athletics", "_tlk": {"name": "Athletics", "description": "New text"}} + ] +}`+"\n") + if _, err := Build(p, nil); err != nil { + t.Fatalf("second build failed: %v", err) + } + + stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile)) + if err != nil { + t.Fatalf("read tlk state: %v", err) + } + var state tlkStateDocument + if err := json.Unmarshal(stateRaw, &state); err != nil { + t.Fatalf("parse state: %v", err) + } + if state.Entries["skills:athletics.name"].ID != 0 || state.Entries["skills:athletics.description"].ID != 1 { + t.Fatalf("unexpected stable IDs after rebuild: %#v", state.Entries) + } +} + +func TestBuildAutoTLKKeysUseLiteralColumnNamesForIDRowsAndPlainTables(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "parts")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description"], + "rows": [ + {"id": 0, "Label": "Athletics", "Name": {"tlk": {"text": "Athletics"}}, "Description": {"tlk": {"text": "Athletics description"}}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["LABEL", "NAME"], + "rows": [ + {"key": "parts:belt:test", "LABEL": "BeltTest", "NAME": {"tlk": {"text": "Belt Name"}}} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + if _, err := Build(testProject(root), nil); err != nil { + t.Fatalf("Build failed: %v", err) + } + + stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile)) + if err != nil { + t.Fatalf("read tlk state: %v", err) + } + var state tlkStateDocument + if err := json.Unmarshal(stateRaw, &state); err != nil { + t.Fatalf("parse state: %v", err) + } + idName, ok := state.Entries["skills:id:0.name"] + if !ok { + t.Fatalf("missing id-row name auto key: %#v", state.Entries) + } + idDescription, ok := state.Entries["skills:id:0.description"] + if !ok { + t.Fatalf("missing id-row description auto key: %#v", state.Entries) + } + plainName, ok := state.Entries["parts:belt:test.name"] + if !ok { + t.Fatalf("missing plain-table auto key: %#v", state.Entries) + } + if idName.ID == idDescription.ID || idName.ID == plainName.ID || idDescription.ID == plainName.ID { + t.Fatalf("expected distinct ids for auto-generated keys, got %#v", state.Entries) + } +} + +func TestBuildAutoTLKKeyConflictsFailClearly(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT"], + "rows": [ + {"id": 0, "key": "feat:test", "LABEL": "FeatTest", "_tlk": {"name": "Auto Name"}}, + {"id": 1, "key": "feat:other", "LABEL": "FeatOther", "FEAT": {"tlk": {"key": "feat:test.feat", "text": "Different Name"}}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0,"feat:other":1}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + _, err := Build(testProject(root), nil) + if err == nil || !strings.Contains(err.Error(), `conflicting TLK content for key "feat:test.feat"`) { + t.Fatalf("expected TLK conflict, got %v", err) + } +} + +func TestBuildRejectsLegacyAuthoredTLKRefs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [ + {"id": 0, "key": "feat:test", "LABEL": "FeatTest", "FEAT": {"tlk": "feat:test.name"}, "DESCRIPTION": {"tlk": "feat:test.description"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + _, err := Build(testProject(root), nil) + if err == nil || !strings.Contains(err.Error(), "inline TLK text is required for native builds") { + t.Fatalf("expected legacy bare tlk ref failure, got %v", err) + } +} + +func TestBuildSupportsPlainTableDatasets(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "parts")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER", "ACBONUS"], + "rows": [ + {"id": 0, "COSTMODIFIER": "0", "ACBONUS": "0.00"}, + {"key": "parts:belt:test", "COSTMODIFIER": "1", "ACBONUS": "0.25"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + result, err := Build(testProject(root), nil) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + if result.Mode != "native" { + t.Fatalf("expected native mode, got %#v", result) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "parts_belt.2da")) + if err != nil { + t.Fatalf("read 2da: %v", err) + } + want := "2DA V2.0\n\nCOSTMODIFIER\tACBONUS\n0\t0\t0.00\n1\t1\t0.25\n" + if string(got) != want { + t.Fatalf("unexpected plain-table output:\n%s", string(got)) + } + + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "parts", "lock.json")) + if err != nil { + t.Fatalf("read lockfile: %v", err) + } + if !strings.Contains(string(lockRaw), `"parts:belt:test": 1`) { + t.Fatalf("expected generated plain-table lock entry, got:\n%s", string(lockRaw)) + } +} + +func TestBuildSurfacesNativeFailuresWithoutReferenceFallback(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "broken")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "broken", "base.json"), `{ + "output": "broken.2da", + "columns": ["LABEL"], + "rows": [ + {"id": "bad", "LABEL": "Broken"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "broken", "lock.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + _, err := Build(testProject(root), nil) + if err == nil || !strings.Contains(err.Error(), "row id bad is not numeric") { + t.Fatalf("expected native build error, got %v", err) + } +} + +func TestBuildAppliesNonTLKOverrides(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "itemprops", "sets", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "sets", "base.json"), `{ + "output": "itemprops.2da", + "columns": ["0_Melee", "6_Arm_Shld", "StringRef", "Label"], + "rows": [ + {"id": 81, "0_Melee": "****", "6_Arm_Shld": "****", "StringRef": "58325", "Label": "Weight_Increase"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "sets", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "sets", "modules", "ovr_eos.json"), `{ + "overrides": [ + {"id": 81, "6_Arm_Shld": "1"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + result, err := Build(testProject(root), nil) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + if result.Mode != "native" { + t.Fatalf("expected native mode, got %#v", result) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "itemprops.2da")) + if err != nil { + t.Fatalf("read 2da: %v", err) + } + if !strings.Contains(string(got), "81\t****\t1\t58325\tWeight_Increase\n") { + t.Fatalf("expected override-applied row, got:\n%s", string(got)) + } +} + +func TestBuildFallsBackWhenProjectMigrationIsIncomplete(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "parts")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER"], + "rows": [{"id": 0, "COSTMODIFIER": "0"}] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "parts")) + mkdirAll(t, filepath.Join(root, "reference", "data", "feat")) + writeFile(t, filepath.Join(root, "reference", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER"], + "rows": [{"id": 0, "COSTMODIFIER": "0"}] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL"], + "rows": [{"id": 0, "LABEL": "Feat"}] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "parts_belt.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\\n\\nCOSTMODIFIER\\n0\\t0\\n") +with open(os.path.join(out, "feat.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\\n\\nLABEL\\n0\\tFeat\\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + + result, err := Build(testProject(root), nil) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + if result.Mode != "native" { + t.Fatalf("expected native mode, got %#v", result) + } +} + +func TestNormalizeProjectInlinesLegacyTLKReferences(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [ + {"id": 0, "key": "feat:test", "LABEL": "FeatTest", "FEAT": {"tlk": "feat:test.name"}, "DESCRIPTION": {"tlk": "feat:test.description"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "lock.json"), `{"feat:test.name":9,"feat:test.description":10}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "modules", "feat", "test.json"), `{ + "entries": { + "feat:test": { + "name": {"text": "Legacy Name"}, + "description": {"text": "Legacy Description"} + } + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles == 0 || result.InlineValues != 2 { + t.Fatalf("unexpected normalize result: %#v", result) + } + + normalized, err := os.ReadFile(filepath.Join(root, "topdata", "data", "feat", "base.json")) + if err != nil { + t.Fatalf("read normalized file: %v", err) + } + text := string(normalized) + if !strings.Contains(text, `"key": "feat:test.name"`) || !strings.Contains(text, `"text": "Legacy Name"`) { + t.Fatalf("expected inlined TLK payloads, got:\n%s", text) + } + + stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile)) + if err != nil { + t.Fatalf("read state: %v", err) + } + var state tlkStateDocument + if err := json.Unmarshal(stateRaw, &state); err != nil { + t.Fatalf("parse state: %v", err) + } + if state.Entries["feat:test.name"].ID != 9 || state.Entries["feat:test.description"].ID != 10 { + t.Fatalf("expected state seeded from legacy TLK lock, got %#v", state.Entries) + } +} + +func TestNormalizeProjectHandlesPlainTablesAndReportsRemainingRefs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION"], + "rows": [ + {"key": "masterfeats:test", "LABEL": "Test", "STRREF": {"tlk": "masterfeats:test.name"}, "DESCRIPTION": {"tlk": "masterfeats:test.description"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "lock.json"), `{"masterfeats:test.name":9,"masterfeats:test.description":10}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "modules", "feat", "masterfeats.json"), `{ + "entries": { + "masterfeats:test": { + "name": {"text": "Masterfeat Name"}, + "description": {"text": "Masterfeat Description"} + } + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.ScannedFiles != 1 || result.UpdatedFiles != 1 || result.InlineValues != 2 { + t.Fatalf("unexpected normalize result: %#v", result) + } + if result.RemainingFiles != 0 || result.RemainingLegacyRefs != 0 { + t.Fatalf("expected no remaining legacy refs, got %#v", result) + } + + normalized, err := os.ReadFile(filepath.Join(root, "topdata", "data", "masterfeats", "base.json")) + if err != nil { + t.Fatalf("read normalized file: %v", err) + } + text := string(normalized) + if !strings.Contains(text, `"text": "Masterfeat Name"`) || !strings.Contains(text, `"text": "Masterfeat Description"`) { + t.Fatalf("expected normalized inline TLK payloads, got:\n%s", text) + } +} + +func TestBuildSupportsCanonicalItempropsRegistry(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "itemprops", "registry")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "properties.json"), `{ + "rows": [ + { + "key": "itemprop:basearmorclass", + "label": "BaseArmorClass", + "name": {"tlk": {"key": "itemprops:basearmorclass.name", "text": "Base Armor Class"}}, + "property_text": {"tlk": {"key": "itemprops:basearmorclass.property_text", "text": "Base armor class"}}, + "availability": ["arm_shld"], + "cost": {"value": "0", "ref": "cost_models:iprpbaseaccost"} + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "cost_models.json"), `{ + "rows": [ + { + "key": "cost_models:iprpbaseaccost", + "index_name": "iprp_baseaccost", + "label": "IPRP_BASEACCOST", + "client_load": "0", + "table": { + "ownership": "owned", + "resref": "iprp_baseaccost", + "output": "iprp_baseaccost.2da", + "columns": ["Name", "Cost"], + "rows": [ + {"id": 0, "Name": "0", "Cost": "0"} + ] + } + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "param_models.json"), `{"rows":[]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "subtype_models.json"), `{"rows":[]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "lock.json"), `{"itemprop:basearmorclass":31,"cost_models:iprpbaseaccost":31}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + if report := ValidateProject(testProject(root)); report.HasErrors() { + t.Fatalf("unexpected validation errors:\n%s", diagnosticsText(report.Diagnostics)) + } + + result, err := Build(testProject(root), nil) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + if result.Mode != "native" { + t.Fatalf("expected native mode, got %#v", result) + } + if result.Files2DA != 5 || result.FilesTLK != 1 { + t.Fatalf("unexpected build result: %#v", result) + } + + itempropsOut, err := os.ReadFile(filepath.Join(result.Output2DADir, "itemprops.2da")) + if err != nil { + t.Fatalf("read itemprops.2da: %v", err) + } + text := string(itempropsOut) + if !strings.Contains(text, "31\t****\t****\t****\t****\t****\t****\t1\t") || !strings.Contains(text, "\t16777216\tBaseArmorClass\n") { + t.Fatalf("unexpected itemprops.2da output:\n%s", text) + } + + defOut, err := os.ReadFile(filepath.Join(result.Output2DADir, "itempropdef.2da")) + if err != nil { + t.Fatalf("read itempropdef.2da: %v", err) + } + if !strings.Contains(string(defOut), "31\t16777216\tBaseArmorClass\t****\t0\t31\t****\t16777217\t****\n") { + t.Fatalf("unexpected itempropdef.2da output:\n%s", string(defOut)) + } + + stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile)) + if err != nil { + t.Fatalf("read tlk state: %v", err) + } + if !strings.Contains(string(stateRaw), `"itemprops:basearmorclass.name"`) || !strings.Contains(string(stateRaw), `"itemprops:basearmorclass.property_text"`) { + t.Fatalf("expected itemprops TLK keys in state, got:\n%s", string(stateRaw)) + } +} + +func TestNormalizeProjectImportsLegacyItempropsRegistry(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "tlk", "modules", "itemprops")) + writeFile(t, filepath.Join(root, "reference", "tlk", "lock.json"), `{ + "itemprops:basearmorclass.name": 384, + "itemprops:basearmorclass.property_text": 385 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "tlk", "modules", "itemprops", "add_itemprops.json"), `{ + "entries": { + "itemprops:basearmorclass.name": {"text": "Base Armor Class"}, + "itemprops:basearmorclass.property_text": {"text": "Base armor class"} + } +}`+"\n") + + refData := filepath.Join(root, "reference", "data", "itemprops") + mkdirAll(t, filepath.Join(refData, "defs", "modules")) + mkdirAll(t, filepath.Join(refData, "sets", "modules")) + mkdirAll(t, filepath.Join(refData, "costtables", "index", "modules")) + mkdirAll(t, filepath.Join(refData, "costtables", "baseaccost")) + mkdirAll(t, filepath.Join(refData, "paramtables", "index")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + writeFile(t, filepath.Join(refData, "defs", "base.json"), `{ + "output": "itempropdef.2da", + "columns": ["Name","Label","SubTypeResRef","Cost","CostTableResRef","Param1ResRef","GameStrRef","Description"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(refData, "defs", "lock.json"), `{"itempropdef:basearmorclass":31}`+"\n") + writeFile(t, filepath.Join(refData, "defs", "modules", "add_eos.json"), `{ + "entries": { + "itempropdef:basearmorclass": { + "Name": {"tlk": "itemprops:basearmorclass.name"}, + "Label": "BaseArmorClass", + "SubTypeResRef": "****", + "Cost": "0", + "CostTableResRef": "31", + "Param1ResRef": "****", + "GameStrRef": {"tlk": "itemprops:basearmorclass.property_text"}, + "Description": "****" + } + } +}`+"\n") + writeFile(t, filepath.Join(refData, "sets", "base.json"), `{ + "output": "itemprops.2da", + "columns": ["0_Melee","1_Ranged","2_Thrown","3_Staves","4_Rods","5_Ammo","6_Arm_Shld","7_Helm","8_Potions","9_Scrolls","10_Wands","11_Thieves","12_TrapKits","13_Hide","14_Claw","15_Misc_Uneq","16_Misc","17_No_Props","18_Containers","19_HealerKit","20_Torch","21_Glove","StringRef","Label"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(refData, "sets", "lock.json"), `{"itemprops:basearmorclass":31}`+"\n") + writeFile(t, filepath.Join(refData, "sets", "modules", "add_eos.json"), `{ + "entries": { + "itemprops:basearmorclass": { + "6_Arm_Shld": "1", + "StringRef": {"tlk": "itemprops:basearmorclass.name"}, + "Label": "BaseArmorClass" + } + } +}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "index", "base.json"), `{ + "output": "iprp_costtable.2da", + "columns": ["Name","Label","ClientLoad"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "index", "lock.json"), `{"itemprops:costtable_index:iprp_baseaccost":31}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "index", "modules", "add_eos.json"), `{ + "entries": { + "itemprops:costtable_index:iprp_baseaccost": { + "Name": "iprp_baseaccost", + "Label": "IPRP_BASEACCOST", + "ClientLoad": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "baseaccost", "base.json"), `{ + "output": "iprp_baseaccost.2da", + "columns": ["Name","Cost"], + "rows": [{"id": 0, "Name": "0", "Cost": "0"}] +}`+"\n") + writeFile(t, filepath.Join(refData, "paramtables", "index", "base.json"), `{ + "output": "iprp_paramtable.2da", + "columns": ["Name","Lable","TableResRef"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(refData, "paramtables", "index", "lock.json"), `{} `+"\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 5 { + t.Fatalf("expected registry files to be imported, got %#v", result) + } + + propertiesRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "itemprops", "registry", "properties.json")) + if err != nil { + t.Fatalf("read properties.json: %v", err) + } + text := string(propertiesRaw) + if !strings.Contains(text, `"itemprop:basearmorclass"`) || !strings.Contains(text, `"text": "Base Armor Class"`) || !strings.Contains(text, `"availability": [`) { + t.Fatalf("unexpected imported properties.json:\n%s", text) + } + if strings.Contains(text, `"tlk": "itemprops:basearmorclass.name"`) { + t.Fatalf("expected imported itemprops TLK text to be fully inlined, got:\n%s", text) + } + + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "itemprops", "registry", "lock.json")) + if err != nil { + t.Fatalf("read registry lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"itemprop:basearmorclass": 31`) || !strings.Contains(string(lockRaw), `"cost_models:iprpbaseaccost": 31`) { + t.Fatalf("unexpected registry lock:\n%s", string(lockRaw)) + } + + stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile)) + if err != nil { + t.Fatalf("read tlk state: %v", err) + } + if !strings.Contains(string(stateRaw), `"itemprops:basearmorclass.name"`) || !strings.Contains(string(stateRaw), `"id": 384`) { + t.Fatalf("expected imported itemprops ids to seed tlk state, got:\n%s", string(stateRaw)) + } +} + +func TestNormalizeProjectRefreshesLegacyItempropsRegistry(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "itemprops", "registry")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "properties.json"), `{ + "rows": [ + { + "key": "itemprop:basearmorclass", + "label": "BaseArmorClass", + "name": {"tlk": "itemprops:basearmorclass.name"}, + "property_text": {"tlk": "itemprops:basearmorclass.property_text"}, + "availability": ["arm_shld"], + "cost": {"value": "0", "ref": "cost_models:iprpbaseaccost"} + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "cost_models.json"), `{"rows":[]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "param_models.json"), `{"rows":[]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "subtype_models.json"), `{"rows":[]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "lock.json"), `{} `+"\n") + mkdirAll(t, filepath.Join(root, "reference", "tlk", "modules", "itemprops")) + writeFile(t, filepath.Join(root, "reference", "tlk", "lock.json"), `{ + "itemprops:basearmorclass.name": 384, + "itemprops:basearmorclass.property_text": 385 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "tlk", "modules", "itemprops", "add_itemprops.json"), `{ + "entries": { + "itemprops:basearmorclass.name": {"text": "Base Armor Class"}, + "itemprops:basearmorclass.property_text": {"text": "Base armor class"} + } +}`+"\n") + refData := filepath.Join(root, "reference", "data", "itemprops") + mkdirAll(t, filepath.Join(refData, "defs", "modules")) + mkdirAll(t, filepath.Join(refData, "sets", "modules")) + mkdirAll(t, filepath.Join(refData, "costtables", "index", "modules")) + mkdirAll(t, filepath.Join(refData, "costtables", "baseaccost")) + mkdirAll(t, filepath.Join(refData, "paramtables", "index")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + writeFile(t, filepath.Join(refData, "defs", "base.json"), `{"output":"itempropdef.2da","columns":["Name","Label","SubTypeResRef","Cost","CostTableResRef","Param1ResRef","GameStrRef","Description"],"rows":[]}`+"\n") + writeFile(t, filepath.Join(refData, "defs", "lock.json"), `{"itempropdef:basearmorclass":31}`+"\n") + writeFile(t, filepath.Join(refData, "defs", "modules", "add_eos.json"), `{"entries":{"itempropdef:basearmorclass":{"Name":{"tlk":"itemprops:basearmorclass.name"},"Label":"BaseArmorClass","SubTypeResRef":"****","Cost":"0","CostTableResRef":"31","Param1ResRef":"****","GameStrRef":{"tlk":"itemprops:basearmorclass.property_text"},"Description":"****"}}}`+"\n") + writeFile(t, filepath.Join(refData, "sets", "base.json"), `{"output":"itemprops.2da","columns":["0_Melee","1_Ranged","2_Thrown","3_Staves","4_Rods","5_Ammo","6_Arm_Shld","7_Helm","8_Potions","9_Scrolls","10_Wands","11_Thieves","12_TrapKits","13_Hide","14_Claw","15_Misc_Uneq","16_Misc","17_No_Props","18_Containers","19_HealerKit","20_Torch","21_Glove","StringRef","Label"],"rows":[]}`+"\n") + writeFile(t, filepath.Join(refData, "sets", "lock.json"), `{"itemprops:basearmorclass":31}`+"\n") + writeFile(t, filepath.Join(refData, "sets", "modules", "add_eos.json"), `{"entries":{"itemprops:basearmorclass":{"6_Arm_Shld":"1","StringRef":{"tlk":"itemprops:basearmorclass.name"},"Label":"BaseArmorClass"}}}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "index", "base.json"), `{"output":"iprp_costtable.2da","columns":["Name","Label","ClientLoad"],"rows":[]}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "index", "lock.json"), `{"itemprops:costtable_index:iprp_baseaccost":31}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "index", "modules", "add_eos.json"), `{"entries":{"itemprops:costtable_index:iprp_baseaccost":{"Name":"iprp_baseaccost","Label":"IPRP_BASEACCOST","ClientLoad":"0"}}}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "baseaccost", "base.json"), `{"output":"iprp_baseaccost.2da","columns":["Name","Cost"],"rows":[{"id":0,"Name":"0","Cost":"0"}]}`+"\n") + writeFile(t, filepath.Join(refData, "paramtables", "index", "base.json"), `{"output":"iprp_paramtable.2da","columns":["Name","Lable","TableResRef"],"rows":[]}`+"\n") + writeFile(t, filepath.Join(refData, "paramtables", "index", "lock.json"), `{} `+"\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 5 { + t.Fatalf("expected registry refresh, got %#v", result) + } + propertiesRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "itemprops", "registry", "properties.json")) + if err != nil { + t.Fatalf("read properties.json: %v", err) + } + if strings.Contains(string(propertiesRaw), `"tlk": "itemprops:basearmorclass.name"`) { + t.Fatalf("expected refreshed registry to inline TLK text, got:\n%s", string(propertiesRaw)) + } +} + +func TestNormalizeProjectPreservesItempropsSetProjectionDifferences(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "itemprops", "defs", "modules")) + mkdirAll(t, filepath.Join(root, "reference", "data", "itemprops", "sets", "modules")) + mkdirAll(t, filepath.Join(root, "reference", "data", "itemprops", "costtables", "index")) + mkdirAll(t, filepath.Join(root, "reference", "data", "itemprops", "paramtables", "index")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + writeFile(t, filepath.Join(root, "reference", "data", "itemprops", "defs", "base.json"), `{ + "output": "itempropdef.2da", + "columns": ["Name","Label","SubTypeResRef","Cost","CostTableResRef","Param1ResRef","GameStrRef","Description"], + "rows": [ + {"id":82,"key":"itempropdef:onhitcastspell","Name":"83400","Label":"OnHitCastSpell","SubTypeResRef":"IPRP_ONHITSPELL","Cost":"****","CostTableResRef":"26","Param1ResRef":"****","GameStrRef":"5512","Description":"1450"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "itemprops", "defs", "lock.json"), `{"itempropdef:onhitcastspell":82}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "itemprops", "sets", "base.json"), `{ + "output": "itemprops.2da", + "columns": ["0_Melee","1_Ranged","2_Thrown","3_Staves","4_Rods","5_Ammo","6_Arm_Shld","7_Helm","8_Potions","9_Scrolls","10_Wands","11_Thieves","12_TrapKits","13_Hide","14_Claw","15_Misc_Uneq","16_Misc","17_No_Props","18_Containers","19_HealerKit","20_Torch","21_Glove","StringRef","Label"], + "rows": [ + {"id":82,"0_Melee":"1","1_Ranged":"****","2_Thrown":"1","3_Staves":"1","4_Rods":"****","5_Ammo":"1","6_Arm_Shld":"1","7_Helm":"****","8_Potions":"****","9_Scrolls":"****","10_Wands":"****","11_Thieves":"****","12_TrapKits":"****","13_Hide":"1","14_Claw":"1","15_Misc_Uneq":"****","16_Misc":"****","17_No_Props":"****","18_Containers":"****","19_HealerKit":"****","20_Torch":"****","21_Glove":"1","StringRef":"723","Label":"On_Hit_Cast_Spell"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "itemprops", "sets", "lock.json"), `{"itemprops:onhitcastspell":82}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "itemprops", "costtables", "index", "base.json"), `{ + "output": "iprp_costtable.2da", + "columns": ["Name","Label","ClientLoad"], + "rows": [{"id":26,"Name":"iprp_spellcst","Label":"IPRP_SPELLCST","ClientLoad":"0"}] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "itemprops", "costtables", "index", "lock.json"), `{"cost_models:iprpspellcstr":26}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "itemprops", "paramtables", "index", "base.json"), `{"output":"iprp_paramtable.2da","columns":["Name","Lable","TableResRef"],"rows":[]}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "itemprops", "paramtables", "index", "lock.json"), `{} `+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + + propertiesRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "itemprops", "registry", "properties.json")) + if err != nil { + t.Fatalf("read properties.json: %v", err) + } + text := string(propertiesRaw) + for _, want := range []string{`"label": "OnHitCastSpell"`, `"set_label": "On_Hit_Cast_Spell"`, `"name": "83400"`, `"set_name": "723"`} { + if !strings.Contains(text, want) { + t.Fatalf("expected %s in imported registry, got:\n%s", want, text) + } + } +} + +func TestItempropsRegistryMatchesLegacyReferenceOutputs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "tlk", "modules", "itemprops")) + writeFile(t, filepath.Join(root, "reference", "tlk", "lock.json"), `{ + "itemprops:basearmorclass.name": 384, + "itemprops:basearmorclass.property_text": 385, + "itemprops:maximumdexteritybonus.name": 386, + "itemprops:maximumdexteritybonus.property_text": 387, + "itemprops:armorcheckpenalty.name": 388, + "itemprops:armorcheckpenalty.property_text": 389 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "tlk", "modules", "itemprops", "add_itemprops.json"), `{ + "entries": { + "itemprops:basearmorclass.name": {"text": "Base Armor Class"}, + "itemprops:basearmorclass.property_text": {"text": "Base Armor Class:"}, + "itemprops:maximumdexteritybonus.name": {"text": "Maximum Dexterity Bonus"}, + "itemprops:maximumdexteritybonus.property_text": {"text": "Maximum Dexterity Bonus:"}, + "itemprops:armorcheckpenalty.name": {"text": "Armor Check Penalty"}, + "itemprops:armorcheckpenalty.property_text": {"text": "Armor Check Penalty:"} + } +}`+"\n") + refData := filepath.Join(root, "reference", "data", "itemprops") + mkdirAll(t, filepath.Join(refData, "defs", "modules")) + mkdirAll(t, filepath.Join(refData, "sets", "modules")) + mkdirAll(t, filepath.Join(refData, "costtables", "index", "modules")) + mkdirAll(t, filepath.Join(refData, "costtables", "baseaccost")) + mkdirAll(t, filepath.Join(refData, "costtables", "maxdexcost")) + mkdirAll(t, filepath.Join(refData, "costtables", "accheckcost")) + mkdirAll(t, filepath.Join(refData, "paramtables", "index")) + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import json, os, sys +root = os.getcwd() +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +def write_table(src_rel): + with open(os.path.join(root, "data", src_rel), "r", encoding="utf-8") as f: + obj = json.load(f) + cols = obj["columns"] + rows = obj["rows"] + out_name = obj["output"] + with open(os.path.join(out, out_name), "w", encoding="utf-8") as f: + f.write("2DA V2.0\\n\\n") + f.write("\\t".join(cols) + "\\n") + for row in rows: + vals = [str(row.get("id", ""))] + for c in cols: + vals.append(str(row[c])) + f.write("\\t".join(vals) + "\\n") +for rel in [ + "itemprops/sets/base.json", + "itemprops/defs/base.json", + "itemprops/costtables/index/base.json", + "itemprops/paramtables/index/base.json", + "itemprops/costtables/baseaccost/base.json", + "itemprops/costtables/maxdexcost/base.json", + "itemprops/costtables/accheckcost/base.json", +]: + write_table(rel) +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`+"\n") + writeFile(t, filepath.Join(refData, "defs", "base.json"), `{ + "output": "itempropdef.2da", + "columns": ["Name","Label","SubTypeResRef","Cost","CostTableResRef","Param1ResRef","GameStrRef","Description"], + "rows": [ + {"id":31,"key":"itempropdef:basearmorclass","Name":{"tlk":"itemprops:basearmorclass.name"},"Label":"BaseArmorClass","SubTypeResRef":"****","Cost":"0","CostTableResRef":"31","Param1ResRef":"****","GameStrRef":{"tlk":"itemprops:basearmorclass.property_text"},"Description":"****"}, + {"id":32,"key":"itempropdef:maximumdexteritybonus","Name":{"tlk":"itemprops:maximumdexteritybonus.name"},"Label":"MaximumDexterityBonus","SubTypeResRef":"****","Cost":"0","CostTableResRef":"32","Param1ResRef":"****","GameStrRef":{"tlk":"itemprops:maximumdexteritybonus.property_text"},"Description":"****"}, + {"id":33,"key":"itempropdef:armorcheckpenalty","Name":{"tlk":"itemprops:armorcheckpenalty.name"},"Label":"ArmorCheckPenalty","SubTypeResRef":"****","Cost":"0","CostTableResRef":"33","Param1ResRef":"****","GameStrRef":{"tlk":"itemprops:armorcheckpenalty.property_text"},"Description":"****"} + ] +}`+"\n") + writeFile(t, filepath.Join(refData, "defs", "lock.json"), `{"itempropdef:basearmorclass":31,"itempropdef:maximumdexteritybonus":32,"itempropdef:armorcheckpenalty":33}`+"\n") + writeFile(t, filepath.Join(refData, "sets", "base.json"), `{ + "output": "itemprops.2da", + "columns": ["0_Melee","1_Ranged","2_Thrown","3_Staves","4_Rods","5_Ammo","6_Arm_Shld","7_Helm","8_Potions","9_Scrolls","10_Wands","11_Thieves","12_TrapKits","13_Hide","14_Claw","15_Misc_Uneq","16_Misc","17_No_Props","18_Containers","19_HealerKit","20_Torch","21_Glove","StringRef","Label"], + "rows": [ + {"id":31,"key":"itemprops:basearmorclass","0_Melee":"****","1_Ranged":"****","2_Thrown":"****","3_Staves":"****","4_Rods":"****","5_Ammo":"****","6_Arm_Shld":"1","7_Helm":"****","8_Potions":"****","9_Scrolls":"****","10_Wands":"****","11_Thieves":"****","12_TrapKits":"****","13_Hide":"****","14_Claw":"****","15_Misc_Uneq":"****","16_Misc":"****","17_No_Props":"****","18_Containers":"****","19_HealerKit":"****","20_Torch":"****","21_Glove":"****","StringRef":{"tlk":"itemprops:basearmorclass.name"},"Label":"BaseArmorClass"}, + {"id":32,"key":"itemprops:maximumdexteritybonus","0_Melee":"****","1_Ranged":"****","2_Thrown":"****","3_Staves":"****","4_Rods":"****","5_Ammo":"****","6_Arm_Shld":"1","7_Helm":"****","8_Potions":"****","9_Scrolls":"****","10_Wands":"****","11_Thieves":"****","12_TrapKits":"****","13_Hide":"****","14_Claw":"****","15_Misc_Uneq":"****","16_Misc":"****","17_No_Props":"****","18_Containers":"****","19_HealerKit":"****","20_Torch":"****","21_Glove":"****","StringRef":{"tlk":"itemprops:maximumdexteritybonus.name"},"Label":"MaximumDexterityBonus"}, + {"id":33,"key":"itemprops:armorcheckpenalty","0_Melee":"****","1_Ranged":"****","2_Thrown":"****","3_Staves":"****","4_Rods":"****","5_Ammo":"****","6_Arm_Shld":"1","7_Helm":"****","8_Potions":"****","9_Scrolls":"****","10_Wands":"****","11_Thieves":"****","12_TrapKits":"****","13_Hide":"****","14_Claw":"****","15_Misc_Uneq":"****","16_Misc":"****","17_No_Props":"****","18_Containers":"****","19_HealerKit":"****","20_Torch":"****","21_Glove":"****","StringRef":{"tlk":"itemprops:armorcheckpenalty.name"},"Label":"ArmorCheckPenalty"} + ] +}`+"\n") + writeFile(t, filepath.Join(refData, "sets", "lock.json"), `{"itemprops:basearmorclass":31,"itemprops:maximumdexteritybonus":32,"itemprops:armorcheckpenalty":33}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "index", "base.json"), `{ + "output": "iprp_costtable.2da", + "columns": ["Name","Label","ClientLoad"], + "rows": [ + {"id":31,"Name":"iprp_baseaccost","Label":"IPRP_BASEACCOST","ClientLoad":"0"}, + {"id":32,"Name":"iprp_maxdexcost","Label":"IPRP_MAXDEXCOST","ClientLoad":"0"}, + {"id":33,"Name":"iprp_accheckcost","Label":"IPRP_ACCHECKCOST","ClientLoad":"0"} + ] +}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "index", "lock.json"), `{"itemprops:costtable_index:iprp_baseaccost":31,"itemprops:costtable_index:iprp_maxdexcost":32,"itemprops:costtable_index:iprp_accheckcost":33}`+"\n") + writeFile(t, filepath.Join(refData, "paramtables", "index", "base.json"), `{"output":"iprp_paramtable.2da","columns":["Name","Lable","TableResRef"],"rows":[]}`+"\n") + writeFile(t, filepath.Join(refData, "paramtables", "index", "lock.json"), `{} `+"\n") + writeFile(t, filepath.Join(refData, "costtables", "baseaccost", "base.json"), `{"output":"iprp_baseaccost.2da","columns":["Name","Cost"],"rows":[{"id":0,"Name":"0","Cost":"0"}]}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "maxdexcost", "base.json"), `{"output":"iprp_maxdexcost.2da","columns":["Name","Cost"],"rows":[{"id":0,"Name":"0","Cost":"0"}]}`+"\n") + writeFile(t, filepath.Join(refData, "costtables", "accheckcost", "base.json"), `{"output":"iprp_accheckcost.2da","columns":["Name","Cost"],"rows":[{"id":0,"Name":"0","Cost":"0"}]}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + for _, outputName := range []string{"itemprops.2da", "itempropdef.2da", "iprp_costtable.2da", "iprp_paramtable.2da"} { + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, outputName)) + if err != nil { + t.Fatalf("read native %s: %v", outputName, err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, outputName)) + if err != nil { + t.Fatalf("read reference %s: %v", outputName, err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("output mismatch for %s\nnative:\n%s\nreference:\n%s", outputName, string(nativeBytes), string(referenceBytes)) + } + } + stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile)) + if err != nil { + t.Fatalf("read tlk state: %v", err) + } + for _, want := range []string{`"itemprops:basearmorclass.name"`, `"itemprops:maximumdexteritybonus.name"`, `"itemprops:armorcheckpenalty.name"`} { + if !strings.Contains(string(stateRaw), want) { + t.Fatalf("expected %s in tlk state, got:\n%s", want, string(stateRaw)) + } + } +} + +func TestValidateProjectRejectsBrokenItempropsRegistryRefs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "itemprops", "registry")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "properties.json"), `{ + "rows": [ + { + "key": "itemprop:test", + "label": "DuplicateLabel", + "name": {"tlk": {"text": "Test"}}, + "property_text": {"tlk": {"text": "Prop Text"}}, + "availability": ["not_a_group"], + "cost": {"ref": "cost_models:missing"}, + "param": {"ref": "param_models:missing"}, + "subtype": {"ref": "subtype_models:missing"} + }, + { + "key": "itemprop:test2", + "label": "DuplicateLabel", + "name": {"tlk": {"text": "Other"}}, + "property_text": {"tlk": {"text": "Other Prop"}} + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "cost_models.json"), `{"rows":[]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "param_models.json"), `{"rows":[]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "subtype_models.json"), `{"rows":[]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "itemprops", "registry", "lock.json"), `{} `+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatalf("expected validation errors, got %#v", report.Diagnostics) + } + joined := diagnosticsText(report.Diagnostics) + for _, want := range []string{ + "unknown availability group", + "unknown cost ref cost_models:missing", + "unknown param ref param_models:missing", + "unknown subtype ref subtype_models:missing", + "generated itempropdef label collision", + "generated itemprops set label collision", + } { + if !strings.Contains(joined, want) { + t.Fatalf("expected validation message %q, got:\n%s", want, joined) + } + } +} + +func TestValidateProjectRejectsInvalidInheritanceUsage(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "custom")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "example.json"), `{ + "output": "custom.2da", + "columns": ["LABEL", "PARENT", "VALUE"], + "rows": [ + { + "key": "custom:child", + "LABEL": "Child", + "inherit": { + "from": "PARENT", + "fields": ["VALUE", "MISSING"] + } + } + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatalf("expected validation errors, got %#v", report.Diagnostics) + } + joined := diagnosticsText(report.Diagnostics) + for _, want := range []string{ + `inherit.from field "PARENT" was not found on row`, + `inherit field "MISSING" is not a known column`, + } { + if !strings.Contains(joined, want) { + t.Fatalf("expected validation message %q, got:\n%s", want, joined) + } + } +} + +func TestBuildSupportsGenericInheritanceSugar(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "custom")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "example.json"), `{ + "output": "custom.2da", + "columns": ["LABEL", "PARENT", "VALUE", "ICON"], + "rows": [ + {"id": 0, "key": "custom:parent", "LABEL": "Parent", "VALUE": "7", "ICON": "parent_icon"}, + {"id": 1, "key": "custom:child", "LABEL": "Child", "PARENT": {"id": "custom:parent"}, "inherit": {"from": "PARENT", "fields": ["VALUE", "ICON"]}}, + {"id": 2, "key": "custom:override", "LABEL": "Override", "PARENT": {"id": "custom:parent"}, "VALUE": "9", "inherit": {"from": "PARENT", "fields": ["VALUE", "ICON"]}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "lock.json"), `{ + "custom:parent": 0, + "custom:child": 1, + "custom:override": 2 +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + result, err := Build(testProject(root), nil) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + if result.Mode != "native" { + t.Fatalf("expected native mode, got %#v", result) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "custom.2da")) + if err != nil { + t.Fatalf("read custom.2da: %v", err) + } + want := "2DA V2.0\n\nLABEL\tPARENT\tVALUE\tICON\n0\tParent\t****\t7\tparent_icon\n1\tChild\t0\t7\tparent_icon\n2\tOverride\t0\t9\tparent_icon\n" + if string(got) != want { + t.Fatalf("unexpected custom.2da output:\n%s", string(got)) + } +} + +func TestBuildRejectsUnresolvedInheritedParent(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "custom")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "example.json"), `{ + "output": "custom.2da", + "columns": ["LABEL", "PARENT", "VALUE"], + "rows": [ + {"id": 0, "key": "custom:child", "LABEL": "Child", "PARENT": {"field": "VALUE", "ref": "custom:parent"}, "inherit": {"from": "PARENT", "fields": ["VALUE"]}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "lock.json"), `{"custom:child":0}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + _, err := BuildNative(testProject(root), nil) + if err == nil || !strings.Contains(err.Error(), `inherit.from field "PARENT" must resolve to a keyed parent row reference`) { + t.Fatalf("expected inherit parent error, got %v", err) + } +} + +func TestValidateProjectRejectsInvalidGenericInheritanceUsage(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "custom")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "example.json"), `{ + "output": "custom.2da", + "columns": ["LABEL", "DETAILS"], + "rows": [ + { + "key": "custom:child", + "LABEL": "Child", + "DETAILS": { + "inherit": { + "ref": "not_a_key" + }, + "foo": "bar" + } + } + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatalf("expected validation errors, got %#v", report.Diagnostics) + } + joined := diagnosticsText(report.Diagnostics) + if !strings.Contains(joined, `inherit.ref "not_a_key" must use dataset:key identity`) { + t.Fatalf("expected generic inherit validation message, got:\n%s", joined) + } +} + +func TestBuildSupportsGenericNestedObjectInheritance(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "custom")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "example.json"), `{ + "output": "custom.2da", + "columns": ["LABEL", "DETAILS"], + "rows": [ + { + "id": 0, + "key": "custom:parent", + "LABEL": "Parent", + "DETAILS": { + "flags": ["base"], + "meta": { + "cost": "5", + "icon": "base_icon" + } + } + }, + { + "id": 1, + "key": "custom:child", + "LABEL": "Child", + "DETAILS": { + "inherit": { + "ref": "custom:parent", + "field": "DETAILS" + }, + "flags": ["local"], + "meta": { + "cost": "9" + } + } + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "lock.json"), `{ + "custom:parent": 0, + "custom:child": 1 +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + result, err := Build(testProject(root), nil) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "custom.2da")) + if err != nil { + t.Fatalf("read custom.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, `{"flags":["base"],"meta":{"cost":"5","icon":"base_icon"}}`) { + t.Fatalf("expected parent object in output, got:\n%s", text) + } + if !strings.Contains(text, `{"flags":["local"],"meta":{"cost":"9","icon":"base_icon"}}`) { + t.Fatalf("expected merged child object in output, got:\n%s", text) + } +} + +func TestBuildRejectsCyclicGenericInheritance(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "custom")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "example.json"), `{ + "output": "custom.2da", + "columns": ["LABEL", "VALUE"], + "rows": [ + { + "id": 0, + "key": "custom:a", + "LABEL": "A", + "inherit": { + "ref": "custom:b" + } + }, + { + "id": 1, + "key": "custom:b", + "LABEL": "B", + "inherit": { + "ref": "custom:a" + } + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "lock.json"), `{ + "custom:a": 0, + "custom:b": 1 +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + _, err := Build(testProject(root), nil) + if err == nil || !strings.Contains(err.Error(), "cyclic inheritance detected") { + t.Fatalf("expected cyclic inheritance error, got %v", err) + } +} + +func TestBuildSupportsFeatInheritanceFromMasterfeats(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON", "PreReqEpic"], + "rows": [ + { + "id": 4, + "key": "masterfeats:skillfocus", + "LABEL": "SkillFocus", + "STRREF": {"tlk": {"key": "masterfeats:skillfocus.name", "text": "Skill Focus"}}, + "DESCRIPTION": {"tlk": {"key": "masterfeats:skillfocus.description", "text": "Choose a skill."}}, + "ICON": "ife_skfoc", + "PreReqEpic": "0" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{"masterfeats:skillfocus":4}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description"], + "rows": [ + {"id": 31, "key": "skills:athletics", "Label": "Athletics", "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, "Description": {"tlk": {"key": "skills:athletics.description", "text": "Athletics description"}}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":31}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION", "ICON", "MASTERFEAT", "REQSKILL", "PreReqEpic"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:skillfocus_athletics":1158}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "skillfocus.json"), `{ + "entries": { + "feat:skillfocus_athletics": { + "inherit": { + "ref": "masterfeats:skillfocus" + }, + "LABEL": "FEAT_SKILL_FOCUS_ATHLETICS", + "FEAT": { + "tlk": { + "key": "feat:skillfocus_athletics.name", + "text": "Skill Focus: Athletics" + } + }, + "MASTERFEAT": { + "id": "masterfeats:skillfocus" + }, + "REQSKILL": { + "id": "skills:athletics" + } + } + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('reference should not run')\n") + + result, err := Build(testProject(root), nil) + if err != nil { + t.Fatalf("Build failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "1158\tFEAT_SKILL_FOCUS_ATHLETICS\t16777216\t16777217\tife_skfoc\t4\t31\t0\n") { + t.Fatalf("expected inherited feat row, got:\n%s", text) + } +} + +func TestValidateProjectRejectsInvalidFeatGeneratedFile(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "generated")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "generated", "skill_focus.json"), `{ + "family": "skill_focus", + "family_key": "skillfocus", + "template": "masterfeats:skillfocus", + "label_prefix": "FEAT_SKILL_FOCUS", + "constant_prefix": "FEAT_SKILL_FOCUS", + "child_source": {"dataset":"skills","predicate":"accessible"}, + "overrides": {} +}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatal("expected validation errors for invalid feat generated file") + } + text := diagnosticsText(report.Diagnostics) + for _, want := range []string{ + "family expansion file must contain a non-empty name_prefix", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected diagnostic %q, got:\n%s", want, text) + } + } +} + +func writeFeatGeneratedHarness(t *testing.T, root string, generatedFiles map[string]string, lock map[string]int) { + t.Helper() + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "registry", "races")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "generated")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON", "CRValue", "ReqSkillMinRanks", "MINATTACKBONUS", "SUCCESSOR", "ALLCLASSESCANUSE", "PreReqEpic", "ReqAction", "TOOLSCATEGORIES"], + "rows": [ + {"id": 4, "key": "masterfeats:skillfocus", "LABEL": "SkillFocus", "STRREF": {"tlk": {"key": "masterfeats:skillfocus.name", "text": "Skill Focus"}}, "DESCRIPTION": {"tlk": {"key": "masterfeats:skillfocus.description", "text": "Choose a skill."}}, "ICON": "ife_skfoc", "CRValue": "0.5", "ReqSkillMinRanks": "3"}, + {"id": 5, "key": "masterfeats:greaterskillfocus", "LABEL": "GreaterSkillFocus", "STRREF": {"tlk": {"key": "masterfeats:greaterskillfocus.name", "text": "Greater Skill Focus"}}, "DESCRIPTION": {"tlk": {"key": "masterfeats:greaterskillfocus.description", "text": "Choose a skill again."}}, "ICON": "ife_skfoc", "CRValue": "1", "ReqSkillMinRanks": "6"}, + {"id": 6, "key": "masterfeats:weaponfocus", "LABEL": "WeaponFocus", "STRREF": {"tlk": {"key": "masterfeats:weaponfocus.name", "text": "Weapon Focus"}}, "DESCRIPTION": {"tlk": {"key": "masterfeats:weaponfocus.description", "text": "Focus on one weapon."}}, "ICON": "ife_wepfoc", "CRValue": "1", "MINATTACKBONUS": "1", "ALLCLASSESCANUSE": "1", "PreReqEpic": "0", "ReqAction": "1", "TOOLSCATEGORIES": "1"}, + {"id": 7, "key": "masterfeats:improvedcritical", "LABEL": "ImprovedCritical", "STRREF": {"tlk": {"key": "masterfeats:improvedcritical.name", "text": "Improved Critical"}}, "DESCRIPTION": {"tlk": {"key": "masterfeats:improvedcritical.description", "text": "Improved critical threat."}}, "ICON": "ife_impcrit", "CRValue": "1", "MINATTACKBONUS": "8", "ALLCLASSESCANUSE": "1", "PreReqEpic": "0", "ReqAction": "1", "TOOLSCATEGORIES": "1"}, + {"id": 8, "key": "masterfeats:weaponproficiencycommoner", "LABEL": "WeaponProficiencyCommoner", "STRREF": {"tlk": {"key": "masterfeats:weaponproficiencycommoner.name", "text": "Commoner Weapon Proficiency"}}, "DESCRIPTION": {"tlk": {"key": "masterfeats:weaponproficiencycommoner.description", "text": "Simple commoner weapon proficiency."}}, "ICON": "ife_weppro", "CRValue": "0.2", "SUCCESSOR": "9", "ALLCLASSESCANUSE": "1", "PreReqEpic": "0", "ReqAction": "1"}, + {"id": 9, "key": "masterfeats:greaterweaponfocus", "LABEL": "GreaterWeaponFocus", "STRREF": {"tlk": {"key": "masterfeats:greaterweaponfocus.name", "text": "Greater Weapon Focus"}}, "DESCRIPTION": {"tlk": {"key": "masterfeats:greaterweaponfocus.description", "text": "Focus harder on one weapon."}}, "ICON": "ife_gwepfoc", "CRValue": "1", "MINATTACKBONUS": "8", "ALLCLASSESCANUSE": "0", "PreReqEpic": "0", "ReqAction": "1", "TOOLSCATEGORIES": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{"masterfeats:skillfocus":4,"masterfeats:greaterskillfocus":5,"masterfeats:weaponfocus":6,"masterfeats:improvedcritical":7,"masterfeats:weaponproficiencycommoner":8,"masterfeats:greaterweaponfocus":9}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"id": 1, "key": "skills:concentration", "Label": "Concentration", "Name": "270", "Description": "345", "Icon": "isk_concen", "Untrained": "1", "KeyAbility": "CON", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_CONCENTRATION", "HostileSkill": "0", "HideFromLevelUp": "0"}, + {"id": 2, "key": "skills:athletics", "Label": "Athletics", "Name": "271", "Description": "346", "Icon": "isk_athletics", "Untrained": "1", "KeyAbility": "STR", "ArmorCheckPenalty": "1", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_ATHLETICS", "HostileSkill": "0", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:concentration":1,"skills:athletics":2}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["label", "Name", "WeaponFocusFeat", "WeaponImprovedCriticalFeat", "WeaponSpecializationFeat", "EpicWeaponFocusFeat", "EpicWeaponSpecializationFeat", "EpicWeaponOverwhelmingCriticalFeat"], + "rows": [ + {"id": 1, "key": "baseitems:testclub", "label": "test_club", "Name": {"tlk": {"key": "baseitems:testclub.name", "text": "Test Club"}}, "WeaponFocusFeat": "3003", "WeaponImprovedCriticalFeat": "3004", "WeaponSpecializationFeat": "****", "EpicWeaponFocusFeat": "****", "EpicWeaponSpecializationFeat": "****", "EpicWeaponOverwhelmingCriticalFeat": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:testclub":1}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "registry", "base.json"), `{ + "columns": ["Label", "Name", "FeatsTable"], + "rows": [ + {"id": 7, "key": "racialtypes:beast", "Label": "Beast", "Name": "Beast", "FeatsTable": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "registry", "lock.json"), `{"racialtypes:beast":7,"racialtypes:testfolk":31}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "registry", "races", "testfolk.json"), `{ + "key": "racialtypes:testfolk", + "core": { + "Label": "Testfolk", + "Name": {"tlk": {"key": "racialtypes:testfolk.name", "text": "Testfolk"}} + }, + "feat_output": "race_feat_test.2da", + "feats": [ + {"FeatLabel": "SkillAffinityAthletics", "FeatIndex": {"id": "feat:skillaffinityathletics"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "base.json"), `{"output":"spells.2da","columns":["Label"],"rows":[{"id":10,"key":"spells:specialattacks","Label":"SpecialAttacks"}]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "lock.json"), `{"spells:specialattacks":10}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "compare_reference": false, + "columns": [ + "LABEL","FEAT","DESCRIPTION","ICON","MINATTACKBONUS","MINSTR","MINDEX","MININT","MINWIS","MINCON","MINCHA", + "MINSPELLLVL","PREREQFEAT1","PREREQFEAT2","GAINMULTIPLE","EFFECTSSTACK","ALLCLASSESCANUSE","CATEGORY","MAXCR", + "SPELLID","SUCCESSOR","CRValue","USESPERDAY","MASTERFEAT","TARGETSELF","OrReqFeat0","OrReqFeat1","OrReqFeat2", + "OrReqFeat3","OrReqFeat4","REQSKILL","ReqSkillMinRanks","REQSKILL2","ReqSkillMinRanks2","Constant","TOOLSCATEGORIES", + "HostileFeat","MinLevel","MinLevelClass","MaxLevel","MinFortSave","PreReqEpic","ReqAction" + ], + "rows": [] +}`+"\n") + lockJSON, err := json.MarshalIndent(lock, "", " ") + if err != nil { + t.Fatalf("marshal feat lock: %v", err) + } + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), string(lockJSON)+"\n") + for name, content := range generatedFiles { + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "generated", name), content) + } +} + +func TestSplitFamilyExpansionIdentity(t *testing.T) { + got := splitFamilyExpansionIdentity("weaponspecialization_club") + if got.Parent != "weaponspecialization" || got.Child != "club" { + t.Fatalf("unexpected family identity: %#v", got) + } + got = splitFamilyExpansionIdentity("gnome") + if got.Parent != "gnome" || got.Child != "" { + t.Fatalf("unexpected standalone family identity: %#v", got) + } +} + +func TestParseRowMetadataSupportsFamilyMetadata(t *testing.T) { + meta, err := parseRowMetadata(map[string]any{ + "family": map[string]any{ + "parent": "weaponspecialization", + "child": "club", + "source": "baseitems:club", + "template": "masterfeats:weaponspecialization", + }, + }) + if err != nil { + t.Fatalf("parseRowMetadata: %v", err) + } + family, ok := meta["family"].(map[string]any) + if !ok { + t.Fatalf("expected family metadata, got %#v", meta) + } + if family["parent"] != "weaponspecialization" || family["child"] != "club" { + t.Fatalf("unexpected family metadata: %#v", family) + } +} + +func TestPreferredGeneratedFeatKeyReusesLegacyCompactChildToken(t *testing.T) { + ctx := &featGeneratedContext{ + lockData: map[string]int{"feat:skillfocus_craftalchemy": 42}, + supplementalID: map[string]int{}, + tlkStateKeys: map[string]struct{}{}, + } + got := ctx.preferredGeneratedFeatKey("skillfocus", "craft_alchemy") + if got != "feat:skillfocus_craftalchemy" { + t.Fatalf("expected reused compact family key, got %q", got) + } +} + +func TestPreferredGeneratedFeatKeyReusesLegacyTLKStateKey(t *testing.T) { + ctx := &featGeneratedContext{ + lockData: map[string]int{}, + supplementalID: map[string]int{}, + tlkStateKeys: map[string]struct{}{"feat:greaterskillfocus_craftalchemy.name": {}}, + } + got := ctx.preferredGeneratedFeatKey("greaterskillfocus", "craft_alchemy") + if got != "feat:greaterskillfocus_craftalchemy" { + t.Fatalf("expected reused tlk-state family key, got %q", got) + } +} + +func TestResolveGeneratedFeatIdentityTranslatesLegacyFamilySourceID(t *testing.T) { + ctx := &featGeneratedContext{ + lockData: map[string]int{"feat:epicweaponfocus_club": 619}, + supplementalID: map[string]int{}, + tlkStateKeys: map[string]struct{}{}, + existingFeat: map[string]struct{}{"feat:epicweaponfocus_club": {}}, + } + got, rowID, hasID, err := ctx.resolveGeneratedFeatIdentityBySource("greaterweaponfocus", "club", "619") + if err != nil { + t.Fatalf("resolveGeneratedFeatIdentityBySource: %v", err) + } + if got != "feat:greaterweaponfocus_club" || !hasID || rowID != 619 { + t.Fatalf("expected translated greater weapon focus identity at id 619, got key=%q id=%d hasID=%v", got, rowID, hasID) + } +} + +func TestResolveGeneratedFeatIdentityMovesExplicitCanonicalRefToLegacyAliasID(t *testing.T) { + ctx := &featGeneratedContext{ + lockData: map[string]int{ + "feat:greaterweaponfocus_shortspear": 1289, + "feat:epicweaponfocus_shortspear": 627, + }, + supplementalID: map[string]int{}, + tlkStateKeys: map[string]struct{}{}, + } + got, rowID, hasID, err := ctx.resolveGeneratedFeatIdentityBySource("greaterweaponfocus", "shortspear", map[string]any{"id": "feat:greaterweaponfocus_shortspear"}) + if err != nil { + t.Fatalf("resolveGeneratedFeatIdentityBySource: %v", err) + } + if got != "feat:greaterweaponfocus_shortspear" || !hasID || rowID != 627 { + t.Fatalf("expected explicit canonical ref to move to legacy alias id 627, got key=%q id=%d hasID=%v", got, rowID, hasID) + } +} + +func TestResolveGeneratedFeatIdentityUsesLegacyAliasForNoSourceFamily(t *testing.T) { + ctx := &featGeneratedContext{ + lockData: map[string]int{ + "feat:skillfocus_animalhandling": 1307, + "feat:skillfocus_animalempathy": 34, + "feat:greaterskillfocus_appraise": 1316, + "feat:epicskillfocus_appraise": 588, + "feat:epicskillfocus_animalempathy": 587, + }, + supplementalID: map[string]int{}, + tlkStateKeys: map[string]struct{}{}, + } + spec := familyExpansionSpec{FamilyKey: "skillfocus"} + got, rowID, hasID, err := ctx.resolveGeneratedFeatIdentity(spec, "animalhandling", map[string]any{}) + if err != nil { + t.Fatalf("resolveGeneratedFeatIdentity skill focus renamed skill: %v", err) + } + if got != "feat:skillfocus_animalhandling" || !hasID || rowID != 34 { + t.Fatalf("expected skill focus renamed child to take legacy alias id 34, got key=%q id=%d hasID=%v", got, rowID, hasID) + } + + spec = familyExpansionSpec{FamilyKey: "greaterskillfocus"} + got, rowID, hasID, err = ctx.resolveGeneratedFeatIdentity(spec, "appraise", map[string]any{}) + if err != nil { + t.Fatalf("resolveGeneratedFeatIdentity: %v", err) + } + if got != "feat:greaterskillfocus_appraise" || !hasID || rowID != 588 { + t.Fatalf("expected no-source generated family to take legacy alias id 588, got key=%q id=%d hasID=%v", got, rowID, hasID) + } + got, rowID, hasID, err = ctx.resolveGeneratedFeatIdentity(spec, "animalhandling", map[string]any{}) + if err != nil { + t.Fatalf("resolveGeneratedFeatIdentity renamed skill: %v", err) + } + if got != "feat:greaterskillfocus_animalhandling" || !hasID || rowID != 587 { + t.Fatalf("expected renamed no-source generated family to take legacy alias id 587, got key=%q id=%d hasID=%v", got, rowID, hasID) + } +} + +func TestResolveGeneratedFeatIdentityUsesLegacyAliasForUnderscoreNoSourceFamily(t *testing.T) { + ctx := &featGeneratedContext{ + lockData: map[string]int{ + "feat:greater_skill_focus_appraise": 1316, + "feat:epic_skill_focus_appraise": 588, + "feat:epic_skill_focus_animal_empathy": 587, + }, + supplementalID: map[string]int{}, + tlkStateKeys: map[string]struct{}{}, + familySpecs: map[string]familyExpansionSpec{ + "greater_skill_focus": { + FamilyKey: "greater_skill_focus", + LegacyFamilyKeys: []string{"epic_skill_focus"}, + }, + }, + } + spec := familyExpansionSpec{FamilyKey: "greater_skill_focus"} + + got, rowID, hasID, err := ctx.resolveGeneratedFeatIdentity(spec, "appraise", map[string]any{}) + if err != nil { + t.Fatalf("resolveGeneratedFeatIdentity: %v", err) + } + if got != "feat:greater_skill_focus_appraise" || !hasID || rowID != 588 { + t.Fatalf("expected underscore greater skill focus to take legacy alias id 588, got key=%q id=%d hasID=%v", got, rowID, hasID) + } + + got, rowID, hasID, err = ctx.resolveGeneratedFeatIdentity(spec, "animal_handling", map[string]any{}) + if err != nil { + t.Fatalf("resolveGeneratedFeatIdentity renamed skill: %v", err) + } + if got != "feat:greater_skill_focus_animal_handling" || !hasID || rowID != 587 { + t.Fatalf("expected renamed underscore greater skill focus to take legacy alias id 587, got key=%q id=%d hasID=%v", got, rowID, hasID) + } +} + +func TestResolveGeneratedFeatIdentityTranslatesLegacyWeaponFamilySourceIDWithUnderscoreKey(t *testing.T) { + ctx := &featGeneratedContext{ + lockData: map[string]int{ + "feat:epic_weapon_focus_club": 619, + "feat:epic_weapon_specialization_club": 650, + "feat:epic_overwhelming_critical_club": 900, + }, + supplementalID: map[string]int{}, + tlkStateKeys: map[string]struct{}{}, + existingFeat: map[string]struct{}{ + "feat:epic_weapon_focus_club": {}, + "feat:epic_weapon_specialization_club": {}, + "feat:epic_overwhelming_critical_club": {}, + }, + familySpecs: map[string]familyExpansionSpec{ + "greater_weapon_focus": { + FamilyKey: "greater_weapon_focus", + LegacyFamilyKeys: []string{"epic_weapon_focus"}, + }, + "greater_weapon_specialization": { + FamilyKey: "greater_weapon_specialization", + LegacyFamilyKeys: []string{"epic_weapon_specialization"}, + }, + "overwhelming_critical": { + FamilyKey: "overwhelming_critical", + LegacyFamilyKeys: []string{"epic_overwhelming_critical"}, + }, + }, + } + + cases := []struct { + family string + want string + rowID int + }{ + {family: "greater_weapon_focus", want: "feat:greater_weapon_focus_club", rowID: 619}, + {family: "greater_weapon_specialization", want: "feat:greater_weapon_specialization_club", rowID: 650}, + {family: "overwhelming_critical", want: "feat:overwhelming_critical_club", rowID: 900}, + } + for _, tc := range cases { + got, rowID, hasID, err := ctx.resolveGeneratedFeatIdentityBySource(tc.family, "club", fmt.Sprintf("%d", tc.rowID)) + if err != nil { + t.Fatalf("resolveGeneratedFeatIdentityBySource(%s): %v", tc.family, err) + } + if got != tc.want || !hasID || rowID != tc.rowID { + t.Fatalf("expected %s at id %d, got key=%q id=%d hasID=%v", tc.want, tc.rowID, got, rowID, hasID) + } + } +} + +func TestResolveGeneratedFeatIdentityUsesConfiguredLegacyFamilyKeys(t *testing.T) { + ctx := &featGeneratedContext{ + lockData: map[string]int{ + "feat:old_weapon_training_test_club": 4000, + }, + supplementalID: map[string]int{}, + tlkStateKeys: map[string]struct{}{}, + familySpecs: map[string]familyExpansionSpec{ + "new_weapon_training": { + FamilyKey: "new_weapon_training", + LegacyFamilyKeys: []string{"old_weapon_training"}, + }, + }, + } + spec := familyExpansionSpec{FamilyKey: "new_weapon_training"} + + got, rowID, hasID, err := ctx.resolveGeneratedFeatIdentity(spec, "test_club", map[string]any{}) + if err != nil { + t.Fatalf("resolveGeneratedFeatIdentity: %v", err) + } + if got != "feat:new_weapon_training_test_club" || !hasID || rowID != 4000 { + t.Fatalf("expected configured legacy family to donate id 4000, got key=%q id=%d hasID=%v", got, rowID, hasID) + } + + if !generatedAliasLockForKey( + "feat:new_weapon_training_test_club", + "feat:old_weapon_training_test_club", + generatedFamilyAliases{"new_weapon_training": []string{"old_weapon_training"}}, + ) { + t.Fatal("expected configured legacy family lock to be movable") + } + + got, rowID, hasID, err = ctx.resolveGeneratedFeatIdentity(familyExpansionSpec{FamilyKey: "newweapontraining"}, "test_club", map[string]any{}) + if err != nil { + t.Fatalf("resolveGeneratedFeatIdentity compact family key: %v", err) + } + if got != "feat:newweapontraining_test_club" || !hasID || rowID != 4000 { + t.Fatalf("expected normalized configured legacy lookup to donate id 4000, got key=%q id=%d hasID=%v", got, rowID, hasID) + } +} + +func TestBuildFamilyExpansionMovesCanonicalLockToLegacyAliasID(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "greater_skill_focus.json": `{ + "family": "greater_skill_focus", + "family_key": "greaterskillfocus", + "template": "masterfeats:greaterskillfocus", + "name_prefix": "Greater Skill Focus", + "label_prefix": "FEAT_GREATER_SKILL_FOCUS", + "constant_prefix": "FEAT_GREATER_SKILL_FOCUS", + "legacy_family_keys": ["epicskillfocus"], + "child_ref_field": "REQSKILL", + "child_source": {"dataset":"skills","predicate":"accessible"}, + "overrides": {"skills:concentration":{"ICON":"ife_greater_concentration"}} +}` + "\n", + }, map[string]int{ + "feat:greaterskillfocus_concentration": 1317, + "feat:epicskillfocus_concentration": 589, + }) + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "589\tFEAT_GREATER_SKILL_FOCUS_CONCENTRATION") { + t.Fatalf("expected greater skill focus to move to legacy alias id 589, got:\n%s", text) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "feat", "lock.json")) + if err != nil { + t.Fatalf("read feat lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"feat:greaterskillfocus_concentration": 589`) { + t.Fatalf("expected canonical lock to move to legacy alias id, got:\n%s", lockText) + } + if strings.Contains(lockText, "epicskillfocus_concentration") { + t.Fatalf("expected stale epic skill focus lock to be pruned, got:\n%s", lockText) + } +} + +func TestBuildFamilyExpansionMovesUnderscoreCanonicalLockToUnderscoreLegacyAliasID(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "greater_skill_focus.json": `{ + "family": "greater_skill_focus", + "family_key": "greater_skill_focus", + "template": "masterfeats:greaterskillfocus", + "name_prefix": "Greater Skill Focus", + "label_prefix": "FEAT_GREATER_SKILL_FOCUS", + "constant_prefix": "FEAT_GREATER_SKILL_FOCUS", + "legacy_family_keys": ["epic_skill_focus"], + "child_ref_field": "REQSKILL", + "child_source": {"dataset":"skills","predicate":"accessible"}, + "overrides": {"skills:concentration":{"ICON":"ife_greater_concentration"}} +}` + "\n", + }, map[string]int{ + "feat:greater_skill_focus_concentration": 1317, + "feat:epic_skill_focus_concentration": 589, + }) + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "589\tFEAT_GREATER_SKILL_FOCUS_CONCENTRATION") { + t.Fatalf("expected underscore greater skill focus to move to legacy alias id 589, got:\n%s", text) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "feat", "lock.json")) + if err != nil { + t.Fatalf("read feat lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"feat:greater_skill_focus_concentration": 589`) { + t.Fatalf("expected canonical underscore key to own legacy row id 589, got:\n%s", lockText) + } + if strings.Contains(lockText, "epic_skill_focus_concentration") { + t.Fatalf("expected stale underscore epic skill focus lock to be pruned, got:\n%s", lockText) + } +} + +func TestBuildFamilyExpansionRegeneratesLegacyAliasIDWithoutFeatLock(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "greater_skill_focus.json": `{ + "family": "greater_skill_focus", + "family_key": "greater_skill_focus", + "template": "masterfeats:greaterskillfocus", + "name_prefix": "Greater Skill Focus", + "label_prefix": "FEAT_GREATER_SKILL_FOCUS", + "constant_prefix": "FEAT_GREATER_SKILL_FOCUS", + "legacy_family_keys": ["epic_skill_focus"], + "child_ref_field": "REQSKILL", + "child_source": {"dataset":"skills","predicate":"accessible"}, + "overrides": {"skills:concentration":{"ICON":"ife_greater_concentration"}} +}` + "\n", + }, map[string]int{}) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "compare_reference": false, + "columns": [ + "LABEL","FEAT","DESCRIPTION","ICON","MINATTACKBONUS","MINSTR","MINDEX","MININT","MINWIS","MINCON","MINCHA", + "MINSPELLLVL","PREREQFEAT1","PREREQFEAT2","GAINMULTIPLE","EFFECTSSTACK","ALLCLASSESCANUSE","CATEGORY","MAXCR", + "SPELLID","SUCCESSOR","CRValue","USESPERDAY","MASTERFEAT","TARGETSELF","OrReqFeat0","OrReqFeat1","OrReqFeat2", + "OrReqFeat3","OrReqFeat4","REQSKILL","ReqSkillMinRanks","REQSKILL2","ReqSkillMinRanks2","Constant","TOOLSCATEGORIES", + "HostileFeat","MinLevel","MinLevelClass","MaxLevel","MinFortSave","PreReqEpic","ReqAction" + ], + "rows": [ + { + "id": 589, + "key": "feat:epic_skill_focus_concentration", + "LABEL": "FEAT_EPIC_SKILL_FOCUS_CONCENTRATION", + "REQSKILL": 1, + "MASTERFEAT": 5, + "Constant": "FEAT_EPIC_SKILL_FOCUS_CONCENTRATION" + } + ] +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "589\tFEAT_GREATER_SKILL_FOCUS_CONCENTRATION") { + t.Fatalf("expected empty lockfile rebuild to preserve legacy row id 589, got:\n%s", text) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "feat", "lock.json")) + if err != nil { + t.Fatalf("read feat lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"feat:greater_skill_focus_concentration": 589`) { + t.Fatalf("expected regenerated lock to assign greater key to row 589, got:\n%s", lockText) + } + if strings.Contains(lockText, "epic_skill_focus_concentration") { + t.Fatalf("expected regenerated lock to prune legacy epic key, got:\n%s", lockText) + } +} + +func TestParseFamilyExpansionRejectsInvalidLegacyFamilyKeys(t *testing.T) { + base := map[string]any{ + "family": "greater_skill_focus", + "family_key": "greater_skill_focus", + "template": "masterfeats:greaterskillfocus", + "child_source": map[string]any{"dataset": "skills", "predicate": "accessible"}, + } + cases := []struct { + name string + keys []any + errMsg string + }{ + { + name: "self alias", + keys: []any{"greater_skill_focus"}, + errMsg: "legacy_family_keys must not include family_key", + }, + { + name: "normalized duplicate", + keys: []any{"epicskillfocus", "epic_skill_focus"}, + errMsg: "legacy_family_keys contains duplicate-equivalent keys", + }, + } + for _, tc := range cases { + obj := map[string]any{} + for key, value := range base { + obj[key] = value + } + obj["legacy_family_keys"] = tc.keys + _, err := parseFamilyExpansionSpec(tc.name+".json", obj) + if err == nil || !strings.Contains(err.Error(), tc.errMsg) { + t.Fatalf("%s: expected %q error, got %v", tc.name, tc.errMsg, err) + } + } +} + +func TestGeneratedWeaponFeatTitleStyleUsesSourceDisplayNameWithStableIdentity(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "weapon_focus.json": `{ + "family": "weapon_focus", + "family_key": "weapon_focus", + "template": "masterfeats:weaponfocus", + "name_prefix": "Weapon Focus", + "label_prefix": "FEAT_WEAPON_FOCUS", + "constant_prefix": "FEAT_WEAPON_FOCUS", + "identity_source": "child_source_value", + "child_source": {"dataset":"baseitems","column":"WeaponFocusFeat"}, + "title_style": {"child_case":"lower","child_parenthetical":"preserve"}, + "overrides": {} +}` + "\n", + }, map[string]int{"feat:weapon_focus_dwaxe": 3003}) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), `{ + "entries": { + "83310": {"text": "Dwarven Waraxe"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["label", "Name", "WeaponFocusFeat"], + "rows": [ + {"id": 108, "key": "baseitems:dwarvenwaraxe", "label": "dwaxe", "Name": "83310", "WeaponFocusFeat": "3003"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:dwarvenwaraxe":108}`+"\n") + + module := buildGeneratedFeatModuleForTest(t, root, "weapon_focus.json") + override := generatedFeatOverrideByKey(t, module, "feat:weapon_focus_dwaxe") + if text := generatedFeatOverrideTitle(t, override); text != "Weapon Focus (dwarven waraxe)" { + t.Fatalf("expected source display title for stable generated feat identity, got %q", text) + } + if id, ok := override["id"].(int); !ok || id != 3003 { + t.Fatalf("expected generated title style to preserve compact row id 3003, got %#v", override["id"]) + } +} + +func TestGeneratedWeaponFeatTitleStyleRewritesExistingFamilyFeatName(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "weapon_focus.json": `{ + "family": "weapon_focus", + "family_key": "weapon_focus", + "template": "masterfeats:weaponfocus", + "name_prefix": "Weapon Focus", + "label_prefix": "FEAT_WEAPON_FOCUS", + "constant_prefix": "FEAT_WEAPON_FOCUS", + "identity_source": "child_source_value", + "child_source": {"dataset":"baseitems","column":"WeaponFocusFeat"}, + "title_style": {"child_case":"lower","child_parenthetical":"preserve"}, + "overrides": {} +}` + "\n", + }, map[string]int{"feat:weapon_focus_dwaxe": 3003}) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), `{ + "entries": { + "83310": {"text": "Dwarven Waraxe"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["label", "Name", "WeaponFocusFeat"], + "rows": [ + {"id": 108, "key": "baseitems:dwarvenwaraxe", "label": "dwaxe", "Name": "83310", "WeaponFocusFeat": "3003"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:dwarvenwaraxe":108}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "compare_reference": false, + "columns": ["LABEL","FEAT","DESCRIPTION","MASTERFEAT","Constant"], + "rows": [ + { + "id": 3003, + "key": "feat:weapon_focus_dwaxe", + "LABEL": "FEAT_WEAPON_FOCUS_DWAXE", + "FEAT": 83318, + "DESCRIPTION": "****", + "MASTERFEAT": 6, + "Constant": "FEAT_WEAPON_FOCUS_DWAXE" + } + ] +}`+"\n") + + module := buildGeneratedFeatModuleForTest(t, root, "weapon_focus.json") + override := generatedFeatOverrideByKey(t, module, "feat:weapon_focus_dwaxe") + if text := generatedFeatOverrideTitle(t, override); text != "Weapon Focus (dwarven waraxe)" { + t.Fatalf("expected existing generated family row to receive normalized FEAT TLK text, got %q", text) + } +} + +func TestGeneratedSkillFeatTitleStyleFlattensChildParenthetical(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "skill_focus.json": `{ + "family": "skill_focus", + "family_key": "skill_focus", + "template": "masterfeats:skillfocus", + "name_prefix": "Skill Focus", + "label_prefix": "FEAT_SKILL_FOCUS", + "constant_prefix": "FEAT_SKILL_FOCUS", + "child_ref_field": "REQSKILL", + "child_source": {"dataset":"skills","predicate":"accessible"}, + "title_style": {"child_case":"lower","child_parenthetical":"comma"}, + "overrides": {} +}` + "\n", + }, map[string]int{}) + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "HideFromLevelUp"], + "rows": [ + {"id": 10, "key": "skills:knowledge_local", "Label": "KnowledgeLocal", "Name": {"tlk": {"text": "Knowledge (local)"}}, "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:knowledge_local":10}`+"\n") + + module := buildGeneratedFeatModuleForTest(t, root, "skill_focus.json") + override := generatedFeatOverrideByKey(t, module, "feat:skill_focus_knowledge_local") + if text := generatedFeatOverrideTitle(t, override); text != "Skill Focus (knowledge, local)" { + t.Fatalf("expected flattened generated skill title, got %q", text) + } +} + +func TestParseFamilyExpansionTitleStyleDefaultsAndValidation(t *testing.T) { + base := map[string]any{ + "family": "skill_focus", + "family_key": "skill_focus", + "template": "masterfeats:skillfocus", + "child_source": map[string]any{"dataset": "skills", "predicate": "accessible"}, + } + spec, err := parseFamilyExpansionSpec("default-title-style.json", base) + if err != nil { + t.Fatalf("parse default title style: %v", err) + } + if spec.TitleStyle.ChildCase != "preserve" || spec.TitleStyle.ChildParenthetical != "preserve" { + t.Fatalf("expected default title style to preserve legacy behavior, got %#v", spec.TitleStyle) + } + + for _, tc := range []struct { + name string + titleStyle map[string]any + want string + }{ + {name: "child case", titleStyle: map[string]any{"child_case": "headline"}, want: "title_style.child_case"}, + {name: "child parenthetical", titleStyle: map[string]any{"child_parenthetical": "nested"}, want: "title_style.child_parenthetical"}, + } { + t.Run(tc.name, func(t *testing.T) { + obj := map[string]any{} + for key, value := range base { + obj[key] = value + } + obj["title_style"] = tc.titleStyle + _, err := parseFamilyExpansionSpec("invalid-title-style.json", obj) + if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "invalid-title-style.json") { + t.Fatalf("expected %q validation error with file context, got %v", tc.want, err) + } + }) + } +} + +func buildGeneratedFeatModuleForTest(t *testing.T, root, name string) map[string]any { + t.Helper() + datasets, err := discoverNativeDatasets(filepath.Join(root, "topdata", "data")) + if err != nil { + t.Fatalf("discover datasets: %v", err) + } + var feat nativeDataset + for _, dataset := range datasets { + if dataset.Name == "feat" { + feat = dataset + break + } + } + if feat.Name == "" { + t.Fatal("expected feat dataset") + } + lock, err := loadLockfile(feat.LockPath) + if err != nil { + t.Fatalf("load feat lock: %v", err) + } + ctx, err := newFeatGeneratedContext(feat, lock) + if err != nil { + t.Fatalf("new generated feat context: %v", err) + } + path := filepath.Join(feat.GeneratedDir, name) + obj, err := loadJSONObject(path) + if err != nil { + t.Fatalf("load generated family %s: %v", name, err) + } + module, err := buildFamilyExpansionGeneratedModule(path, obj, ctx) + if err != nil { + t.Fatalf("build generated family %s: %v", name, err) + } + return module +} + +func generatedFeatOverrideByKey(t *testing.T, module map[string]any, key string) map[string]any { + t.Helper() + overrides, ok := module["overrides"].([]any) + if !ok { + t.Fatalf("expected generated overrides, got %#v", module) + } + for _, raw := range overrides { + override, ok := raw.(map[string]any) + if ok && override["key"] == key { + return override + } + } + t.Fatalf("expected generated override %q, got %#v", key, overrides) + return nil +} + +func generatedFeatOverrideTitle(t *testing.T, override map[string]any) string { + t.Helper() + feat, ok := override["FEAT"].(map[string]any) + if !ok { + t.Fatalf("expected generated FEAT TLK data, got %#v", override) + } + tlk, ok := feat["tlk"].(map[string]any) + if !ok { + t.Fatalf("expected generated FEAT tlk block, got %#v", feat) + } + text, ok := tlk["text"].(string) + if !ok { + t.Fatalf("expected generated FEAT text, got %#v", tlk) + } + return text +} + +func TestValidateBaseitemsWeaponFeatColumnCompletenessRejectsPartialCoreFamilies(t *testing.T) { + ctx := &featGeneratedContext{ + rowsByDataset: map[string]map[string]map[string]any{ + "baseitems": { + "baseitems:testblade": { + "key": "baseitems:testblade", + "WeaponFocusFeat": 1000, + "WeaponSpecializationFeat": 1001, + "WeaponImprovedCriticalFeat": 1002, + "EpicWeaponFocusFeat": 1003, + "EpicWeaponSpecializationFeat": 1004, + "EpicWeaponOverwhelmingCriticalFeat": "****", + }, + "baseitems:prop": { + "key": "baseitems:prop", + "WeaponType": 2, + }, + }, + }, + } + report := ValidationReport{} + + validateBaseitemsWeaponFeatColumnCompleteness("generated", ctx, &report) + + if !report.HasErrors() { + t.Fatalf("expected partial core weapon feat columns to be rejected") + } + if len(report.Diagnostics) != 1 { + t.Fatalf("expected one diagnostic, got %#v", report.Diagnostics) + } + if !strings.Contains(report.Diagnostics[0].Message, "baseitems:testblade") || + !strings.Contains(report.Diagnostics[0].Message, "EpicWeaponOverwhelmingCriticalFeat") { + t.Fatalf("unexpected diagnostic: %#v", report.Diagnostics[0]) + } +} + +func TestPruneRetiredGeneratedFeatTLKEntries(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "generated")) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "generated", "skill_focus.json"), `{ + "family": "anything", + "family_key": "skillfocus", + "template": "masterfeats:skillfocus", + "name_prefix": "Skill Focus", + "label_prefix": "FEAT_SKILL_FOCUS", + "constant_prefix": "FEAT_SKILL_FOCUS", + "template_fields": ["DESCRIPTION", "ICON"], + "child_source": {"dataset":"skills","predicate":"accessible"} +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "generated", "weapon_focus.json"), `{ + "family": "anything_else", + "family_key": "weaponfocus", + "template": "masterfeats:weaponfocus", + "name_prefix": "Weapon Focus", + "label_prefix": "FEAT_WEAPON_FOCUS", + "constant_prefix": "FEAT_WEAPON_FOCUS", + "template_fields": ["DESCRIPTION", "ICON"], + "identity_source": "child_source_value", + "child_source": {"dataset":"baseitems","column":"WeaponFocusFeat"} +}`+"\n") + entries := map[string]tlkStateMapping{ + "feat:skillfocus_craftalchemy.name": {ID: 10, Retired: true}, + "feat:weaponproficiencycommoner_club.description": {ID: 11, Retired: true}, + "feat:weaponfocus_club.name": {ID: 12, Retired: false}, + "feat:specialattacks.name": {ID: 13, Retired: true}, + "masterfeats:skillfocus.name": {ID: 14, Retired: true}, + } + + pruneRetiredGeneratedFeatTLKEntries(filepath.Join(root, "topdata"), entries) + + if _, ok := entries["feat:skillfocus_craftalchemy.name"]; ok { + t.Fatal("expected retired generated skill focus TLK key to be pruned") + } + if _, ok := entries["feat:weaponproficiencycommoner_club.description"]; !ok { + t.Fatal("expected module-authored proficiency TLK key to be preserved") + } + if _, ok := entries["feat:weaponfocus_club.name"]; !ok { + t.Fatal("expected active generated feat TLK key to be preserved") + } + if _, ok := entries["feat:specialattacks.name"]; !ok { + t.Fatal("expected manual specialattacks TLK key to be preserved") + } + if _, ok := entries["masterfeats:skillfocus.name"]; !ok { + t.Fatal("expected non-feat TLK key to be preserved") + } +} + +func TestBuildSupportsGeneratedFeatFamilies(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "skill_focus.json": `{"family":"skill_family_from_data","family_key":"skillfocus","template":"masterfeats:skillfocus","name_prefix":"Skill Focus","label_prefix":"FEAT_SKILL_FOCUS","constant_prefix":"FEAT_SKILL_FOCUS","template_fields":["DESCRIPTION","ICON","CRValue","ReqSkillMinRanks","ALLCLASSESCANUSE","PreReqEpic","ReqAction"],"default_fields":{"TOOLSCATEGORIES":"2"},"child_ref_field":"REQSKILL","allow_existing_only":true,"child_source":{"dataset":"skills","predicate":"accessible"},"overrides":{"skills:concentration":{"ICON":"ife_skfoc_override"}}}` + "\n", + "greater_skill_focus.json": `{"family":"greater_skill_family_from_data","family_key":"greaterskillfocus","template":"masterfeats:greaterskillfocus","name_prefix":"Greater Skill Focus","label_prefix":"FEAT_GREATER_SKILL_FOCUS","constant_prefix":"FEAT_GREATER_SKILL_FOCUS","template_fields":["DESCRIPTION","ICON","CRValue","ReqSkillMinRanks","ALLCLASSESCANUSE","PreReqEpic","ReqAction"],"default_fields":{"TOOLSCATEGORIES":"2"},"child_ref_field":"REQSKILL","allow_existing_only":true,"child_source":{"dataset":"skills","predicate":"accessible"}}` + "\n", + "weapon_focus.json": `{"family":"custom_weapon_mastery","family_key":"weaponfocus","template":"masterfeats:weaponfocus","name_prefix":"Weapon Focus","label_prefix":"FEAT_WEAPON_FOCUS","constant_prefix":"FEAT_WEAPON_FOCUS","template_fields":["DESCRIPTION","ICON","MINATTACKBONUS","ALLCLASSESCANUSE","PreReqEpic","ReqAction","CRValue","MinLevel","MinLevelClass","MINSTR"],"default_fields":{"TOOLSCATEGORIES":"1"},"identity_source":"child_source_value","child_source":{"dataset":"baseitems","column":"WeaponFocusFeat"},"overrides":{"baseitems:testclub":{"ICON":"ife_wepfoc_override"}}}` + "\n", + "improved_critical.json": `{"family":"custom_critical_chain","family_key":"improvedcritical","template":"masterfeats:improvedcritical","name_prefix":"Improved Critical","label_prefix":"FEAT_IMPROVED_CRITICAL","constant_prefix":"FEAT_IMPROVED_CRITICAL","template_fields":["DESCRIPTION","ICON","MINATTACKBONUS","ALLCLASSESCANUSE","PreReqEpic","ReqAction","CRValue","MinLevel","MinLevelClass","MINSTR"],"default_fields":{"TOOLSCATEGORIES":"1"},"identity_source":"child_source_value","auto_prereq_fields":{"PREREQFEAT1":"weaponfocus"},"child_source":{"dataset":"baseitems","column":"WeaponImprovedCriticalFeat"}}` + "\n", + }, map[string]int{"feat:skillfocus_concentration": 173, "feat:greaterskillfocus_concentration": 589, "feat:skillaffinityathletics": 1253, "feat:weaponfocus_testclub": 3003, "feat:improvedcritical_testclub": 3004, "feat:weaponproficiencycommoner_club": 1123, "feat:specialattacks": 1187, "feat:improvedfeint": 1194}) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules", "activecombat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules", "proficiency")) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "activecombat", "specialattacks.json"), `{"entries":{"feat:specialattacks":{"LABEL":"FEAT_SPECIAL_ATTACKS","FEAT":{"tlk":{"key":"feat:specialattacks.name","text":"Special Attacks"}},"DESCRIPTION":{"tlk":{"key":"feat:specialattacks.description","text":"Access special attacks."}},"ICON":"ife_specialatk","CATEGORY":"22","HostileFeat":"1","MAXCR":"0","SPELLID":{"id":"spells:specialattacks"},"Constant":"FEAT_SPECIAL_ATTACKS","TOOLSCATEGORIES":"2","PreReqEpic":"0","ReqAction":"0"},"feat:improvedfeint":{"LABEL":"FEAT_IMPROVED_FEINT","FEAT":{"tlk":{"key":"feat:improvedfeint.name","text":"Improved Feint"}},"DESCRIPTION":{"tlk":{"key":"feat:improvedfeint.description","text":"Improve your feint."}},"ICON":"ife_feint","PREREQFEAT1":{"id":"feat:specialattacks"},"Constant":"FEAT_IMPROVED_FEINT","TOOLSCATEGORIES":{"field":"TOOLSCATEGORIES","ref":"feat:specialattacks"},"PreReqEpic":"0","ReqAction":"1"}}}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "proficiency", "standalone.json"), `{"entries":{"feat:towershieldproficiency":{"LABEL":"FEAT_TOWER_SHIELD_PROFICIENCY","FEAT":{"tlk":{"key":"feat:towershieldproficiency.name","text":"Tower Shield Proficiency"}},"DESCRIPTION":{"tlk":{"key":"feat:towershieldproficiency.description","text":"Use tower shields."}},"ICON":"ife_towshprof","PREREQFEAT1":{"id":"feat:shieldproficiency"},"Constant":"FEAT_TOWER_SHIELD_PROFICIENCY","ReqAction":"1","PreReqEpic":"0"}},"overrides":[{"key":"feat:shieldproficiency","id":32,"DESCRIPTION":{"tlk":{"key":"feat:shieldproficiency.description","text":"Use shields."}}},{"key":"feat:weaponproficiencycommoner_club","id":1123,"LABEL":"CommonerWeaponProficiencyCLUB","Constant":"FEAT_WEAPON_PROFICIENCY_CLUB","FEAT":{"tlk":{"key":"feat:weaponproficiencycommoner_club.name","text":"Commoner Weapon Proficiency (club)"}},"DESCRIPTION":{"field":"DESCRIPTION","ref":"masterfeats:weaponproficiencycommoner"},"ICON":{"field":"ICON","ref":"masterfeats:weaponproficiencycommoner"},"MASTERFEAT":{"id":"masterfeats:weaponproficiencycommoner"},"SUCCESSOR":{"field":"SUCCESSOR","ref":"masterfeats:weaponproficiencycommoner"},"ALLCLASSESCANUSE":{"field":"ALLCLASSESCANUSE","ref":"masterfeats:weaponproficiencycommoner"},"PreReqEpic":{"field":"PreReqEpic","ref":"masterfeats:weaponproficiencycommoner"},"ReqAction":{"field":"ReqAction","ref":"masterfeats:weaponproficiencycommoner"},"CRValue":{"field":"CRValue","ref":"masterfeats:weaponproficiencycommoner"}}]}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + text := string(got) + for _, want := range []string{"FEAT_SKILL_FOCUS_CONCENTRATION", "FEAT_GREATER_SKILL_FOCUS_CONCENTRATION", "FEAT_SKILL_AFFINITY_ATHLETICS", "FEAT_WEAPON_FOCUS_TEST_CLUB", "FEAT_IMPROVED_CRITICAL_TEST_CLUB", "CommonerWeaponProficiencyCLUB", "FEAT_SPECIAL_ATTACKS", "FEAT_IMPROVED_FEINT", "FEAT_TOWER_SHIELD_PROFICIENCY", "ife_wepfoc_override"} { + if !strings.Contains(text, want) { + t.Fatalf("expected generated feat output to contain %q, got:\n%s", want, text) + } + } +} + +func TestBuildFamilyExpansionUsesAuthoredSpecForCoverageAndBaselines(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "skill_focus.json": `{"family":"whatever_the_author_wants","family_key":"skillfocus","template":"masterfeats:skillfocus","name_prefix":"Skill Focus","label_prefix":"FEAT_SKILL_FOCUS","constant_prefix":"FEAT_SKILL_FOCUS","template_fields":["DESCRIPTION","ICON","CRValue","ReqSkillMinRanks","ALLCLASSESCANUSE","PreReqEpic","ReqAction"],"default_fields":{"TOOLSCATEGORIES":"2"},"child_ref_field":"REQSKILL","child_source":{"dataset":"skills","predicate":"accessible"}}` + "\n", + "weapon_focus.json": `{"family":"another_authored_name","family_key":"weaponfocus","template":"masterfeats:weaponfocus","name_prefix":"Weapon Focus","label_prefix":"FEAT_WEAPON_FOCUS","constant_prefix":"FEAT_WEAPON_FOCUS","template_fields":["DESCRIPTION","ICON","MINATTACKBONUS","ALLCLASSESCANUSE","PreReqEpic","ReqAction","CRValue","MinLevel","MinLevelClass","MINSTR"],"default_fields":{"TOOLSCATEGORIES":"9","ALLCLASSESCANUSE":"0"},"identity_source":"child_source_value","child_source":{"dataset":"baseitems","column":"WeaponFocusFeat"}}` + "\n", + }, map[string]int{"feat:skillfocus_concentration": 173, "feat:skillfocus_athletics": 174, "feat:weaponfocus_testclub": 3003}) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules", "weapons")) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "weapons", "testclub.json"), `{"overrides":[{"key":"feat:weaponfocus_testclub","id":3003,"LABEL":"FEAT_WEAPON_FOCUS_TEST_CLUB","Constant":"FEAT_WEAPON_FOCUS_TEST_CLUB","FEAT":{"tlk":{"key":"feat:weaponfocus_testclub.name","text":"Weapon Focus (Test Club)"}}}]}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + text := string(got) + + for _, want := range []string{ + "FEAT_SKILL_FOCUS_CONCENTRATION", + "FEAT_SKILL_FOCUS_ATHLETICS", + "FEAT_WEAPON_FOCUS_TEST_CLUB", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected feat output to contain %q, got:\n%s", want, text) + } + } + + type rowData map[string]string + parse2DA := func(data string) map[string]rowData { + lines := strings.Split(data, "\n") + header := strings.Split(lines[2], "\t") + rows := map[string]rowData{} + for _, line := range lines[3:] { + if strings.TrimSpace(line) == "" { + continue + } + parts := strings.Split(line, "\t") + if len(parts) < len(header)+1 { + continue + } + row := rowData{} + for i, col := range header { + row[col] = parts[i+1] + } + rows[parts[0]] = row + } + return rows + } + + featRows := parse2DA(text) + masterRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "masterfeats.2da")) + if err != nil { + t.Fatalf("read masterfeats.2da: %v", err) + } + masterRows := parse2DA(string(masterRaw)) + + assertFieldEquals := func(featID, masterID, field string) { + got := featRows[featID][field] + want := masterRows[masterID][field] + if got != want { + t.Fatalf("expected %s field %s=%q from masterfeat %s, got %q", featID, field, want, masterID, got) + } + } + + assertFieldEquals("173", "4", "DESCRIPTION") + assertFieldEquals("173", "4", "CRValue") + assertFieldEquals("173", "4", "ReqSkillMinRanks") + assertFieldEquals("173", "4", "ALLCLASSESCANUSE") + assertFieldEquals("173", "4", "PreReqEpic") + assertFieldEquals("173", "4", "ReqAction") + if got := featRows["173"]["REQSKILL"]; got != "1" { + t.Fatalf("expected skill child ref to bind to REQSKILL=1, got %q", got) + } + if got := featRows["173"]["TOOLSCATEGORIES"]; got != "2" { + t.Fatalf("expected skill default TOOLSCATEGORIES=2, got %q", got) + } + + if got := featRows["3003"]["TOOLSCATEGORIES"]; got != "9" { + t.Fatalf("expected weapon shared default TOOLSCATEGORIES=9, got %q", got) + } + if got := featRows["3003"]["ALLCLASSESCANUSE"]; got != "0" { + t.Fatalf("expected weapon shared default ALLCLASSESCANUSE=0 on existing row, got %q", got) + } +} + +func TestBuildFamilyExpansionCanApplySharedAuthoredFieldsAfterModules(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "skill_focus.json": `{"family":"skill_focus","family_key":"skillfocus","template":"masterfeats:skillfocus","name_prefix":"Skill Focus","label_prefix":"FEAT_SKILL_FOCUS","constant_prefix":"FEAT_SKILL_FOCUS","default_fields":{"DESCRIPTION":{"tlk":{"key":"feat:skillfocus.shared.description","text":"Family-owned description."}},"CRValue":"0.5","ReqSkillMinRanks":"1","ALLCLASSESCANUSE":"1","TOOLSCATEGORIES":"6","PreReqEpic":"0","ReqAction":"1"},"apply_after_modules":true,"child_ref_field":"REQSKILL","allow_existing_only":true,"child_source":{"dataset":"skills","predicate":"accessible"},"overrides":{"skills:concentration":{"ICON":"ife_family_owned_conc"}}}` + "\n", + }, map[string]int{"feat:skillfocus_concentration": 173}) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules", "skillfeat")) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "skillfeat", "focus.json"), `{"overrides":[{"key":"feat:skillfocus_concentration","id":173,"LABEL":"FEAT_SKILL_FOCUS_CONCENTRATION","FEAT":{"tlk":{"key":"feat:skillfocus_concentration.name","text":"Skill Focus (Concentration)"}},"DESCRIPTION":{"tlk":{"key":"feat:skillfocus_concentration.description","text":"Wrong module description."}},"ICON":"ife_wrong_module","CRValue":"9.9","MASTERFEAT":{"id":"masterfeats:skillfocus"},"REQSKILL":{"id":"skills:concentration"},"ReqSkillMinRanks":"99","TOOLSCATEGORIES":"2","ALLCLASSESCANUSE":"0","PreReqEpic":"1","ReqAction":"0","Constant":"FEAT_SKILL_FOCUS_CONCENTRATION"}]}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + lines := strings.Split(string(got), "\n") + header := strings.Split(lines[2], "\t") + var row map[string]string + for _, line := range lines[3:] { + if !strings.HasPrefix(line, "173\t") { + continue + } + parts := strings.Split(line, "\t") + row = map[string]string{} + for i, col := range header { + row[col] = parts[i+1] + } + break + } + if row == nil { + t.Fatal("expected feat row 173 to exist") + } + if got := row["ICON"]; got != "ife_family_owned_conc" { + t.Fatalf("expected family override ICON after modules, got %q", got) + } + if got := row["CRValue"]; got != "0.5" { + t.Fatalf("expected family shared CRValue after modules, got %q", got) + } + if got := row["ReqSkillMinRanks"]; got != "1" { + t.Fatalf("expected family shared ReqSkillMinRanks after modules, got %q", got) + } + if got := row["ALLCLASSESCANUSE"]; got != "1" { + t.Fatalf("expected family shared ALLCLASSESCANUSE after modules, got %q", got) + } + if got := row["TOOLSCATEGORIES"]; got != "6" { + t.Fatalf("expected family shared TOOLSCATEGORIES after modules, got %q", got) + } + if got := row["PreReqEpic"]; got != "0" { + t.Fatalf("expected family shared PreReqEpic after modules, got %q", got) + } + if got := row["ReqAction"]; got != "1" { + t.Fatalf("expected family shared ReqAction after modules, got %q", got) + } +} + +func TestBuildGeneratedFeatSpecialAttacksRequiresSpellsTarget(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{}, map[string]int{"feat:specialattacks": 1187}) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules", "activecombat")) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "activecombat", "specialattacks.json"), `{"entries":{"feat:specialattacks":{"LABEL":"FEAT_SPECIAL_ATTACKS","FEAT":{"tlk":{"key":"feat:specialattacks.name","text":"Special Attacks"}},"DESCRIPTION":{"tlk":{"key":"feat:specialattacks.description","text":"Access special attacks."}},"ICON":"ife_specialatk","SPELLID":{"id":"spells:missing"},"Constant":"FEAT_SPECIAL_ATTACKS"}}}`+"\n") + _, err := BuildNative(testProject(root), nil) + if err == nil || !strings.Contains(err.Error(), "spells:missing") { + t.Fatalf("expected missing spells target error, got %v", err) + } +} + +func TestBuildGeneratedFeatSpecialAttacksRequiresSelfRefTarget(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{}, map[string]int{"feat:improvedfeint": 1194}) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules", "activecombat")) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "activecombat", "specialattacks.json"), `{"entries":{"feat:improvedfeint":{"LABEL":"FEAT_IMPROVED_FEINT","FEAT":{"tlk":{"key":"feat:improvedfeint.name","text":"Improved Feint"}},"DESCRIPTION":{"tlk":{"key":"feat:improvedfeint.description","text":"Improve your feint."}},"ICON":"ife_feint","PREREQFEAT1":{"id":"feat:specialattacks"},"Constant":"FEAT_IMPROVED_FEINT"}}}`+"\n") + _, err := BuildNative(testProject(root), nil) + if err == nil || !strings.Contains(err.Error(), "feat:specialattacks") { + t.Fatalf("expected missing feat self-ref error, got %v", err) + } +} + +func TestBuildGeneratedFeatRequiresMasterfeatAndSkillTargets(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "skill_focus.json": `{"family":"skill_focus","family_key":"skillfocus","template":"masterfeats:missing","name_prefix":"Skill Focus","label_prefix":"FEAT_SKILL_FOCUS","constant_prefix":"FEAT_SKILL_FOCUS","template_fields":["DESCRIPTION","ICON"],"default_fields":{"TOOLSCATEGORIES":"2"},"child_ref_field":"REQSKILL","allow_existing_only":true,"child_source":{"dataset":"skills","predicate":"accessible"}}` + "\n", + }, map[string]int{"feat:skillfocus_concentration": 173}) + _, err := BuildNative(testProject(root), nil) + if err == nil { + t.Fatal("expected missing dependency error") + } + if !strings.Contains(err.Error(), "masterfeats:missing") { + t.Fatalf("expected error containing missing masterfeat target, got %v", err) + } +} + +func TestDiscoverNativeOutputCatalogSkipsCompareDisabledDataset(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "data", "feat")) + writeFile(t, filepath.Join(root, "data", "feat", "base.json"), `{ + "output": "feat.2da", + "compare_reference": false, + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "data", "feat", "lock.json"), `{"feat:test":0}`+"\n") + + catalog, err := discoverNativeOutputCatalog(filepath.Join(root, "data")) + if err != nil { + t.Fatalf("discoverNativeOutputCatalog: %v", err) + } + if _, ok := catalog["feat.2da"]; ok { + t.Fatalf("expected compare-disabled feat.2da to be excluded from coverage catalog") + } +} + +func TestNormalizeProjectImportsLegacyMasterfeats(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + + mkdirAll(t, filepath.Join(root, "reference", "data", "feat", "masterfeats")) + writeFile(t, filepath.Join(root, "reference", "data", "feat", "masterfeats", "masterfeats.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON", "SUCCESSOR"], + "rows": [ + {"id": 0, "key": "masterfeats:weaponfocus", "LABEL": "WeaponFocus", "STRREF": "6490", "DESCRIPTION": "436", "ICON": "ife_wepfoc"}, + {"key": "masterfeats:skillfocus", "LABEL": "SkillFocus", "STRREF": {"tlk": "masterfeats:skillfocus.name"}, "DESCRIPTION": {"tlk": "masterfeats:skillfocus.description"}, "ICON": "ife_skfoc"}, + {"id": 2, "LABEL": "DevastatingCrit", "STRREF": "83482", "DESCRIPTION": "3909", "ICON": "ife_X2DevCrit"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "feat", "masterfeats", "lock.json"), `{ + "masterfeats:weaponfocus": 0, + "masterfeats:skillfocus": 1 +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference", "tlk", "modules", "feat", "masterfeats")) + writeFile(t, filepath.Join(root, "reference", "tlk", "lock.json"), `{ + "masterfeats:skillfocus.name": 100, + "masterfeats:skillfocus.description": 101 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "tlk", "modules", "feat", "masterfeats", "skillfocus.json"), `{ + "entries": { + "masterfeats:skillfocus": { + "name": {"text": "Skill Focus"}, + "description": {"text": "Choose a skill."} + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 2 { + t.Fatalf("expected imported masterfeats files, got %#v", result) + } + + raw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "masterfeats", "base.json")) + if err != nil { + t.Fatalf("read imported masterfeats: %v", err) + } + text := string(raw) + for _, want := range []string{ + `"STRREF": "6490"`, + `"DESCRIPTION": "436"`, + `"key": "masterfeats:skillfocus.name"`, + `"text": "Skill Focus"`, + `"key": "masterfeats:skillfocus.description"`, + `"text": "Choose a skill."`, + `"id": 1`, + } { + if !strings.Contains(text, want) { + t.Fatalf("expected imported masterfeats content %q, got:\n%s", want, text) + } + } + + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "masterfeats", "lock.json")) + if err != nil { + t.Fatalf("read imported lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"masterfeats:skillfocus": 1`) { + t.Fatalf("expected imported lock data, got:\n%s", string(lockRaw)) + } + + stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", tlkStateFile)) + if err != nil { + t.Fatalf("read tlk state: %v", err) + } + if !strings.Contains(string(stateRaw), `"masterfeats:skillfocus.name"`) || !strings.Contains(string(stateRaw), `"id": 100`) { + t.Fatalf("expected seeded masterfeats tlk state, got:\n%s", string(stateRaw)) + } +} + +func TestBuildSupportsCanonicalMasterfeats(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "ICON", "SUCCESSOR"], + "rows": [ + {"id": 0, "key": "masterfeats:weaponfocus", "LABEL": "WeaponFocus", "STRREF": "6490", "DESCRIPTION": "436", "ICON": "ife_wepfoc"}, + {"id": 1, "key": "masterfeats:skillfocus", "LABEL": "SkillFocus", "STRREF": {"tlk": {"key": "masterfeats:skillfocus.name", "text": "Skill Focus"}}, "DESCRIPTION": {"tlk": {"key": "masterfeats:skillfocus.description", "text": "Choose a skill."}}, "ICON": "ife_skfoc"}, + {"id": 2, "key": "masterfeats:weaponproficiencycommoner", "LABEL": "CommonerWeaponProficiency", "STRREF": {"tlk": {"key": "masterfeats:weaponproficiencycommoner.name", "text": "Commoner Weapon Proficiency"}}, "DESCRIPTION": {"tlk": {"key": "masterfeats:weaponproficiencycommoner.description", "text": "Simple weapon training."}}, "SUCCESSOR": {"id": "feat:weaponproficiencysimple"}, "ICON": "ife_weppro_sim"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{ + "masterfeats:weaponfocus": 0, + "masterfeats:skillfocus": 1, + "masterfeats:weaponproficiencycommoner": 2 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:weaponproficiencysimple":7}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 7, "key": "feat:weaponproficiencysimple", "LABEL": "WEAPON_PROFICIENCY_SIMPLE"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "masterfeats.2da")) + if err != nil { + t.Fatalf("read masterfeats.2da: %v", err) + } + want := "2DA V2.0\n\nLABEL\tSTRREF\tDESCRIPTION\tICON\tSUCCESSOR\n0\tWeaponFocus\t6490\t436\tife_wepfoc\t****\n1\tSkillFocus\t16777216\t16777217\tife_skfoc\t****\n2\tCommonerWeaponProficiency\t16777218\t16777219\tife_weppro_sim\t7\n" + if string(got) != want { + t.Fatalf("unexpected masterfeats.2da output:\n%s", string(got)) + } +} + +func TestNormalizeProjectImportsLegacyCreaturespeed(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "creaturespeed", "modules")) + writeFile(t, filepath.Join(root, "reference", "data", "creaturespeed", "base.json"), `{ + "output": "creaturespeed.2da", + "columns": ["Label", "Name", "2DAName", "WALKRATE", "RUNRATE"], + "rows": [ + {"key": "creaturespeed:pcmovement", "id": 0, "Label": "PC_Movement", "Name": "****", "2DAName": "PLAYER", "WALKRATE": "2.00", "RUNRATE": "4.00"}, + {"key": "creaturespeed:normal", "id": 4, "Label": "Normal", "Name": "2322", "2DAName": "NORM", "WALKRATE": "1.75", "RUNRATE": "3.50"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "creaturespeed", "lock.json"), `{"creaturespeed:normal":4}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "creaturespeed", "modules", "ovr_mobspeed.json"), `{ + "overrides": [ + {"key": "creaturespeed:normal", "id": 4, "WALKRATE": "2.00", "RUNRATE": "4.00"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 1 { + t.Fatalf("expected creaturespeed file to be imported, got %#v", result) + } + + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "creaturespeed", "creaturespeed.json")) + if err != nil { + t.Fatalf("read creaturespeed table: %v", err) + } + text := string(baseRaw) + for _, want := range []string{`"output": "creaturespeed.2da"`, `"Label": "Normal"`, `"WALKRATE": "2.00"`, `"RUNRATE": "4.00"`} { + if !strings.Contains(text, want) { + t.Fatalf("expected imported creaturespeed content %q, got:\n%s", want, text) + } + } + if strings.Contains(text, `"key": "creaturespeed:normal"`) { + t.Fatalf("did not expect creaturespeed keys in plain-table import, got:\n%s", text) + } + if _, err := os.Stat(filepath.Join(root, "topdata", "data", "creaturespeed", "lock.json")); !os.IsNotExist(err) { + t.Fatalf("expected no creaturespeed lock file, got err=%v", err) + } + if _, err := os.Stat(filepath.Join(root, "topdata", "data", "creaturespeed", "modules", "ovr_mobspeed.json")); !os.IsNotExist(err) { + t.Fatalf("expected no creaturespeed module file, got err=%v", err) + } +} + +func TestBuildSupportsCanonicalCreaturespeed(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "creaturespeed")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "creaturespeed", "creaturespeed.json"), `{ + "output": "creaturespeed.2da", + "columns": ["Label", "Name", "2DAName", "WALKRATE", "RUNRATE"], + "rows": [ + {"id": 0, "Label": "PC_Movement", "Name": "****", "2DAName": "PLAYER", "WALKRATE": "2.00", "RUNRATE": "4.00"}, + {"id": 4, "Label": "Normal", "Name": "2322", "2DAName": "NORM", "WALKRATE": "2.00", "RUNRATE": "4.00"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "creaturespeed.2da")) + if err != nil { + t.Fatalf("read creaturespeed.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "Label\tName\t2DAName\tWALKRATE\tRUNRATE\n") || + !strings.Contains(text, "0\tPC_Movement\t****\tPLAYER\t2.00\t4.00\n") || + !strings.Contains(text, "4\tNormal\t2322\tNORM\t2.00\t4.00\n") { + t.Fatalf("unexpected creaturespeed.2da output:\n%s", string(got)) + } +} + +func TestNormalizeProjectImportsLegacyArmor(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "armor", "modules")) + writeFile(t, filepath.Join(root, "reference", "data", "armor", "base.json"), `{ + "columns": ["ACBONUS", "DEXBONUS", "ACCHECK", "ARCANEFAILURE%", "WEIGHT", "COST", "DESCRIPTIONS", "BASEITEMSTATREF"], + "rows": [ + {"id": 0, "ACBONUS": "0", "DEXBONUS": "100", "ACCHECK": "0", "ARCANEFAILURE%": "0", "WEIGHT": "10", "COST": "1", "DESCRIPTIONS": "1727", "BASEITEMSTATREF": "5411"}, + {"id": 8, "ACBONUS": "8", "DEXBONUS": "1", "ACCHECK": "-8", "ARCANEFAILURE%": "45", "WEIGHT": "500", "COST": "1500", "DESCRIPTIONS": "1736", "BASEITEMSTATREF": "5441"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "armor", "lock.json"), `{"armor:ac9":9}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "armor", "modules", "add_12ac.json"), `{ + "entries": { + "armor:ac9": {"id": 9, "ACBONUS": "9", "DEXBONUS": "0", "ACCHECK": "-8", "ARCANEFAILURE%": "45", "WEIGHT": "650", "COST": "2400", "DESCRIPTIONS": "1736", "BASEITEMSTATREF": "****"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "armor", "modules", "ovr_statref.json"), `{ + "overrides": [ + {"id": 0, "BASEITEMSTATREF": "****"}, + {"id": 8, "BASEITEMSTATREF": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 1 { + t.Fatalf("expected armor file to be imported, got %#v", result) + } + + tableRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "armor", "armor.json")) + if err != nil { + t.Fatalf("read armor table: %v", err) + } + text := string(tableRaw) + for _, want := range []string{`"output": "armor.2da"`, `"id": 0`, `"BASEITEMSTATREF": "****"`, `"id": 9`, `"ACBONUS": "9"`} { + if !strings.Contains(text, want) { + t.Fatalf("expected imported armor content %q, got:\n%s", want, text) + } + } + if _, err := os.Stat(filepath.Join(root, "topdata", "data", "armor", "lock.json")); !os.IsNotExist(err) { + t.Fatalf("expected no armor lock file, got err=%v", err) + } + if _, err := os.Stat(filepath.Join(root, "topdata", "data", "armor", "modules")); !os.IsNotExist(err) { + t.Fatalf("expected no armor modules dir, got err=%v", err) + } +} + +func TestBuildSupportsCanonicalArmor(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "armor")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "armor", "armor.json"), `{ + "output": "armor.2da", + "columns": ["ACBONUS", "DEXBONUS", "ACCHECK", "ARCANEFAILURE%", "WEIGHT", "COST", "DESCRIPTIONS", "BASEITEMSTATREF"], + "rows": [ + {"id": 0, "ACBONUS": "0", "DEXBONUS": "100", "ACCHECK": "0", "ARCANEFAILURE%": "0", "WEIGHT": "10", "COST": "1", "DESCRIPTIONS": "1727", "BASEITEMSTATREF": "****"}, + {"id": 9, "ACBONUS": "9", "DEXBONUS": "0", "ACCHECK": "-8", "ARCANEFAILURE%": "45", "WEIGHT": "650", "COST": "2400", "DESCRIPTIONS": "1736", "BASEITEMSTATREF": "****"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "armor.2da")) + if err != nil { + t.Fatalf("read armor.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "ACBONUS\tDEXBONUS\tACCHECK\tARCANEFAILURE%\tWEIGHT\tCOST\tDESCRIPTIONS\tBASEITEMSTATREF\n") || + !strings.Contains(text, "0\t0\t100\t0\t0\t10\t1\t1727\t****\n") || + !strings.Contains(text, "9\t9\t0\t-8\t45\t650\t2400\t1736\t****\n") { + t.Fatalf("unexpected armor.2da output:\n%s", text) + } +} + +func TestBuildCanonicalArmorMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "armor", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "armor.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("ACBONUS\tDEXBONUS\tACCHECK\tARCANEFAILURE%\tWEIGHT\tCOST\tDESCRIPTIONS\tBASEITEMSTATREF\n") + f.write("0\t0\t100\t0\t0\t10\t1\t1727\t****\n") + f.write("1\t1\t8\t0\t5\t50\t5\t1728\t****\n") + f.write("2\t2\t6\t0\t10\t100\t10\t1729\t****\n") + f.write("3\t3\t4\t-1\t20\t150\t15\t1730\t****\n") + f.write("4\t4\t4\t-2\t20\t300\t100\t1731\t****\n") + f.write("5\t5\t2\t-5\t30\t400\t150\t1732\t****\n") + f.write("6\t6\t1\t-7\t40\t450\t200\t1733\t****\n") + f.write("7\t7\t1\t-7\t40\t500\t600\t1734\t****\n") + f.write("8\t8\t1\t-8\t45\t500\t1500\t1736\t****\n") + f.write("9\t9\t0\t-8\t45\t650\t2400\t1736\t****\n") + f.write("10\t10\t0\t-9\t50\t800\t5000\t1736\t****\n") + f.write("11\t11\t0\t-10\t55\t800\t7500\t1736\t****\n") + f.write("12\t12\t0\t-11\t60\t1250\t10000\t1736\t****\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "armor", "base.json"), `{ + "columns": ["ACBONUS", "DEXBONUS", "ACCHECK", "ARCANEFAILURE%", "WEIGHT", "COST", "DESCRIPTIONS", "BASEITEMSTATREF"], + "rows": [ + {"id": 0, "ACBONUS": "0", "DEXBONUS": "100", "ACCHECK": "0", "ARCANEFAILURE%": "0", "WEIGHT": "10", "COST": "1", "DESCRIPTIONS": "1727", "BASEITEMSTATREF": "5411"}, + {"id": 1, "ACBONUS": "1", "DEXBONUS": "8", "ACCHECK": "0", "ARCANEFAILURE%": "5", "WEIGHT": "50", "COST": "5", "DESCRIPTIONS": "1728", "BASEITEMSTATREF": "5432"}, + {"id": 2, "ACBONUS": "2", "DEXBONUS": "6", "ACCHECK": "0", "ARCANEFAILURE%": "10", "WEIGHT": "100", "COST": "10", "DESCRIPTIONS": "1729", "BASEITEMSTATREF": "5435"}, + {"id": 3, "ACBONUS": "3", "DEXBONUS": "4", "ACCHECK": "-1", "ARCANEFAILURE%": "20", "WEIGHT": "150", "COST": "15", "DESCRIPTIONS": "1730", "BASEITEMSTATREF": "5436"}, + {"id": 4, "ACBONUS": "4", "DEXBONUS": "4", "ACCHECK": "-2", "ARCANEFAILURE%": "20", "WEIGHT": "300", "COST": "100", "DESCRIPTIONS": "1731", "BASEITEMSTATREF": "5437"}, + {"id": 5, "ACBONUS": "5", "DEXBONUS": "2", "ACCHECK": "-5", "ARCANEFAILURE%": "30", "WEIGHT": "400", "COST": "150", "DESCRIPTIONS": "1732", "BASEITEMSTATREF": "5438"}, + {"id": 6, "ACBONUS": "6", "DEXBONUS": "1", "ACCHECK": "-7", "ARCANEFAILURE%": "40", "WEIGHT": "450", "COST": "200", "DESCRIPTIONS": "1733", "BASEITEMSTATREF": "5439"}, + {"id": 7, "ACBONUS": "7", "DEXBONUS": "1", "ACCHECK": "-7", "ARCANEFAILURE%": "40", "WEIGHT": "500", "COST": "600", "DESCRIPTIONS": "1734", "BASEITEMSTATREF": "5440"}, + {"id": 8, "ACBONUS": "8", "DEXBONUS": "1", "ACCHECK": "-8", "ARCANEFAILURE%": "45", "WEIGHT": "500", "COST": "1500", "DESCRIPTIONS": "1736", "BASEITEMSTATREF": "5441"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "armor", "lock.json"), `{"armor:ac9":9,"armor:ac10":10,"armor:ac11":11,"armor:ac12":12}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "armor", "modules", "add_12ac.json"), `{ + "entries": { + "armor:ac9": {"id": 9, "ACBONUS": "9", "DEXBONUS": "0", "ACCHECK": "-8", "ARCANEFAILURE%": "45", "WEIGHT": "650", "COST": "2400", "DESCRIPTIONS": "1736", "BASEITEMSTATREF": "****"}, + "armor:ac10": {"id": 10, "ACBONUS": "10", "DEXBONUS": "0", "ACCHECK": "-9", "ARCANEFAILURE%": "50", "WEIGHT": "800", "COST": "5000", "DESCRIPTIONS": "1736", "BASEITEMSTATREF": "****"}, + "armor:ac11": {"id": 11, "ACBONUS": "11", "DEXBONUS": "0", "ACCHECK": "-10", "ARCANEFAILURE%": "55", "WEIGHT": "800", "COST": "7500", "DESCRIPTIONS": "1736", "BASEITEMSTATREF": "****"}, + "armor:ac12": {"id": 12, "ACBONUS": "12", "DEXBONUS": "0", "ACCHECK": "-11", "ARCANEFAILURE%": "60", "WEIGHT": "1250", "COST": "10000", "DESCRIPTIONS": "1736", "BASEITEMSTATREF": "****"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "armor", "modules", "ovr_statref.json"), `{ + "overrides": [ + {"id": 0, "BASEITEMSTATREF": "****"}, + {"id": 1, "BASEITEMSTATREF": "****"}, + {"id": 2, "BASEITEMSTATREF": "****"}, + {"id": 3, "BASEITEMSTATREF": "****"}, + {"id": 4, "BASEITEMSTATREF": "****"}, + {"id": 5, "BASEITEMSTATREF": "****"}, + {"id": 6, "BASEITEMSTATREF": "****"}, + {"id": 7, "BASEITEMSTATREF": "****"}, + {"id": 8, "BASEITEMSTATREF": "****"} + ] +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "armor.2da")) + if err != nil { + t.Fatalf("read native armor.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "armor.2da")) + if err != nil { + t.Fatalf("read reference armor.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("armor output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsLegacyDamagetypesRegistry(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + for _, part := range []string{"core", "groups", "hitvisual"} { + mkdirAll(t, filepath.Join(root, "reference", "data", "damagetypes", part, "modules")) + } + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "core", "base.json"), `{ + "output": "damagetypes.2da", + "columns": ["Label", "CharsheetStrref", "DamageTypeGroup", "DamageRangedProjectile"], + "rows": [ + {"id": 0, "Label": "Bludgeoning", "CharsheetStrref": "58345", "DamageTypeGroup": "0", "DamageRangedProjectile": "0"}, + {"id": 12, "Label": "Base", "CharsheetStrref": "58301", "DamageTypeGroup": "0", "DamageRangedProjectile": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "groups", "base.json"), `{ + "output": "damagetypegroups.2da", + "columns": ["Label", "FeedbackStrref", "ColorR", "ColorG", "ColorB"], + "rows": [ + {"id": 0, "Label": "Physical", "FeedbackStrref": "5594", "ColorR": "255", "ColorG": "102", "ColorB": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "hitvisual", "base.json"), `{ + "output": "damagehitvisual.2da", + "columns": ["Label", "VisualEffectID", "RangedEffectID"], + "rows": [ + {"id": 0, "Label": "Bludgeoning", "VisualEffectID": "****", "RangedEffectID": "****"}, + {"id": 12, "Label": "Base", "VisualEffectID": "****", "RangedEffectID": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "core", "lock.json"), `{"damagetypes:psychic":13}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "groups", "lock.json"), `{"damagetypegroups:psychic":10}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "hitvisual", "lock.json"), `{"damagehitvisual:psychic":13}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "core", "modules", "add_eos.json"), `{ + "entries": { + "damagetypes:psychic": {"Label": "PsychicDamage", "CharsheetStrref": "70001", "DamageTypeGroup": "10", "DamageRangedProjectile": "0"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "groups", "modules", "add_eos.json"), `{ + "entries": { + "damagetypegroups:psychic": {"Label": "Psychic", "FeedbackStrref": "70011", "ColorR": "225", "ColorG": "117", "ColorB": "255"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "hitvisual", "modules", "add_eos.json"), `{ + "entries": { + "damagehitvisual:psychic": {"Label": "PsychicDamage", "VisualEffectID": "202", "RangedEffectID": "****"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 2 { + t.Fatalf("expected damagetypes registry files to be imported, got %#v", result) + } + + typesRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "damagetypes", "registry", "types.json")) + if err != nil { + t.Fatalf("read damagetypes registry: %v", err) + } + text := string(typesRaw) + for _, want := range []string{`"key": "damagetype:bludgeoning"`, `"group_label": "Physical"`, `"key": "damagetype:psychic"`, `"feedback_strref": "70011"`} { + if !strings.Contains(text, want) { + t.Fatalf("expected imported damagetypes content %q, got:\n%s", want, text) + } + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "damagetypes", "registry", "lock.json")) + if err != nil { + t.Fatalf("read damagetypes lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"damagetype:psychic": 13`) { + t.Fatalf("expected imported damagetypes lock ids, got:\n%s", string(lockRaw)) + } + for _, oldPath := range []string{ + filepath.Join(root, "topdata", "data", "damagetypes", "core"), + filepath.Join(root, "topdata", "data", "damagetypes", "groups"), + filepath.Join(root, "topdata", "data", "damagetypes", "hitvisual"), + } { + if _, err := os.Stat(oldPath); !os.IsNotExist(err) { + t.Fatalf("expected no legacy canonical damagetypes path %s, got err=%v", oldPath, err) + } + } +} + +func TestBuildSupportsCanonicalDamagetypesRegistry(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "damagetypes", "registry")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "damagetypes", "registry", "types.json"), `{ + "rows": [ + {"key": "damagetype:bludgeoning", "id": 0, "label": "Bludgeoning", "charsheet_strref": "58345", "damage_type_group": "0", "damage_ranged_projectile": "0", "group_label": "Physical", "feedback_strref": "5594", "color_r": "255", "color_g": "102", "color_b": "0", "visual_effect_id": "****", "ranged_effect_id": "****"}, + {"key": "damagetype:psychic", "id": 13, "label": "PsychicDamage", "charsheet_strref": "70001", "damage_type_group": "10", "damage_ranged_projectile": "0", "group_label": "Psychic", "feedback_strref": "70011", "color_r": "225", "color_g": "117", "color_b": "255", "visual_effect_id": "202", "ranged_effect_id": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "damagetypes", "registry", "lock.json"), `{"damagetype:bludgeoning":0,"damagetype:psychic":13}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + coreRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "damagetypes.2da")) + if err != nil { + t.Fatalf("read damagetypes.2da: %v", err) + } + groupRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "damagetypegroups.2da")) + if err != nil { + t.Fatalf("read damagetypegroups.2da: %v", err) + } + hitRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "damagehitvisual.2da")) + if err != nil { + t.Fatalf("read damagehitvisual.2da: %v", err) + } + if !strings.Contains(string(coreRaw), "0\tBludgeoning\t58345\t0\t0\n") || !strings.Contains(string(coreRaw), "13\tPsychicDamage\t70001\t10\t0\n") { + t.Fatalf("unexpected damagetypes.2da output:\n%s", string(coreRaw)) + } + if !strings.Contains(string(groupRaw), "0\tPhysical\t5594\t255\t102\t0\n") || !strings.Contains(string(groupRaw), "10\tPsychic\t70011\t225\t117\t255\n") { + t.Fatalf("unexpected damagetypegroups.2da output:\n%s", string(groupRaw)) + } + if !strings.Contains(string(hitRaw), "0\tBludgeoning\t****\t****\n") || !strings.Contains(string(hitRaw), "13\tPsychicDamage\t202\t****\n") { + t.Fatalf("unexpected damagehitvisual.2da output:\n%s", string(hitRaw)) + } +} + +func TestBuildCanonicalDamagetypesRegistryMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + for _, part := range []string{"core", "groups", "hitvisual"} { + mkdirAll(t, filepath.Join(root, "reference", "data", "damagetypes", part, "modules")) + } + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "damagetypes.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tCharsheetStrref\tDamageTypeGroup\tDamageRangedProjectile\n") + f.write("0\tBludgeoning\t58345\t0\t0\n") + f.write("1\tPiercing\t58341\t0\t0\n") + f.write("13\tPsychicDamage\t70001\t10\t0\n") +with open(os.path.join(out, "damagetypegroups.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tFeedbackStrref\tColorR\tColorG\tColorB\n") + f.write("0\tPhysical\t5594\t255\t102\t0\n") + f.write("1\tMagical\t5593\t204\t119\t255\n") + f.write("10\tPsychic\t70011\t225\t117\t255\n") +with open(os.path.join(out, "damagehitvisual.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tVisualEffectID\tRangedEffectID\n") + f.write("0\tBludgeoning\t****\t****\n") + f.write("1\tPiercing\t****\t****\n") + f.write("13\tPsychicDamage\t202\t****\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "core", "base.json"), `{ + "output": "damagetypes.2da", + "columns": ["Label", "CharsheetStrref", "DamageTypeGroup", "DamageRangedProjectile"], + "rows": [ + {"id": 0, "Label": "Bludgeoning", "CharsheetStrref": "58345", "DamageTypeGroup": "0", "DamageRangedProjectile": "0"}, + {"id": 1, "Label": "Piercing", "CharsheetStrref": "58341", "DamageTypeGroup": "0", "DamageRangedProjectile": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "groups", "base.json"), `{ + "output": "damagetypegroups.2da", + "columns": ["Label", "FeedbackStrref", "ColorR", "ColorG", "ColorB"], + "rows": [ + {"id": 0, "Label": "Physical", "FeedbackStrref": "5594", "ColorR": "255", "ColorG": "102", "ColorB": "0"}, + {"id": 1, "Label": "Magical", "FeedbackStrref": "5593", "ColorR": "204", "ColorG": "119", "ColorB": "255"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "hitvisual", "base.json"), `{ + "output": "damagehitvisual.2da", + "columns": ["Label", "VisualEffectID", "RangedEffectID"], + "rows": [ + {"id": 0, "Label": "Bludgeoning", "VisualEffectID": "****", "RangedEffectID": "****"}, + {"id": 1, "Label": "Piercing", "VisualEffectID": "****", "RangedEffectID": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "core", "lock.json"), `{"damagetypes:psychic":13}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "groups", "lock.json"), `{"damagetypegroups:psychic":10}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "hitvisual", "lock.json"), `{"damagehitvisual:psychic":13}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "core", "modules", "add_eos.json"), `{"entries":{"damagetypes:psychic":{"Label":"PsychicDamage","CharsheetStrref":"70001","DamageTypeGroup":"10","DamageRangedProjectile":"0"}}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "groups", "modules", "add_eos.json"), `{"entries":{"damagetypegroups:psychic":{"Label":"Psychic","FeedbackStrref":"70011","ColorR":"225","ColorG":"117","ColorB":"255"}}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "damagetypes", "hitvisual", "modules", "add_eos.json"), `{"entries":{"damagehitvisual:psychic":{"Label":"PsychicDamage","VisualEffectID":"202","RangedEffectID":"****"}}}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + for _, name := range []string{"damagetypes.2da", "damagetypegroups.2da", "damagehitvisual.2da"} { + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, name)) + if err != nil { + t.Fatalf("read native %s: %v", name, err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, name)) + if err != nil { + t.Fatalf("read reference %s: %v", name, err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("%s output mismatch\nnative:\n%s\nreference:\n%s", name, string(nativeBytes), string(referenceBytes)) + } + } +} + +func TestBuildCanonicalCreaturespeedMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "creaturespeed", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "creaturespeed.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tName\t2DAName\tWALKRATE\tRUNRATE\n") + f.write("0\tPC_Movement\t****\tPLAYER\t2.00\t4.00\n") + f.write("1\t****\t****\t****\t****\t****\n") + f.write("2\t****\t****\t****\t****\t****\n") + f.write("3\t****\t****\t****\t****\t****\n") + f.write("4\tNormal\t2322\tNORM\t2.00\t4.00\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "creaturespeed", "base.json"), `{ + "output": "creaturespeed.2da", + "columns": ["Label", "Name", "2DAName", "WALKRATE", "RUNRATE"], + "rows": [ + {"key": "creaturespeed:pcmovement", "id": 0, "Label": "PC_Movement", "Name": "****", "2DAName": "PLAYER", "WALKRATE": "2.00", "RUNRATE": "4.00"}, + {"key": "creaturespeed:normal", "id": 4, "Label": "Normal", "Name": "2322", "2DAName": "NORM", "WALKRATE": "1.75", "RUNRATE": "3.50"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "creaturespeed", "lock.json"), `{"creaturespeed:normal":4}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "creaturespeed", "modules", "ovr_mobspeed.json"), `{ + "overrides": [ + {"key": "creaturespeed:normal", "id": 4, "WALKRATE": "2.00", "RUNRATE": "4.00"} + ] +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "creaturespeed.2da")) + if err != nil { + t.Fatalf("read native creaturespeed.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "creaturespeed.2da")) + if err != nil { + t.Fatalf("read reference creaturespeed.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("creaturespeed output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsLegacySkyboxes(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "skyboxes", "modules")) + writeFile(t, filepath.Join(root, "reference", "data", "skyboxes", "base.json"), `{ + "output": "skyboxes.2da", + "columns": ["LABEL", "STRING_REF", "CYCLICAL", "DAWN", "DAY", "DUSK", "NIGHT"], + "rows": [ + {"id": 0, "LABEL": "(None)", "STRING_REF": "****", "CYCLICAL": "****", "DAWN": "****", "DAY": "****", "DUSK": "****", "NIGHT": "****"}, + {"id": 1, "LABEL": "Grass_Clear", "STRING_REF": "****", "CYCLICAL": "1", "DAWN": "Skyda_001", "DAY": "Sky_001", "DUSK": "Skyd_001", "NIGHT": "Skyn_001"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skyboxes", "lock.json"), `{ + "festerpot:id_7": 7 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skyboxes", "modules", "add_fpskies.json"), `{ + "entries": { + "festerpot:id_7": { + "LABEL": "Mountains", + "CYCLICAL": 1, + "DAWN": "CCS_01a", + "DAY": "CCS_01b", + "DUSK": "CCS_01c", + "NIGHT": "CCS_01d" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 3 { + t.Fatalf("expected skyboxes files to be imported, got %#v", result) + } + + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "skyboxes", "base.json"), + filepath.Join(root, "topdata", "data", "skyboxes", "lock.json"), + filepath.Join(root, "topdata", "data", "skyboxes", "modules", "add_fpskies.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported skyboxes path %s: %v", path, err) + } + } + + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "skyboxes", "lock.json")) + if err != nil { + t.Fatalf("read skyboxes lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"festerpot:id_7": 7`) { + t.Fatalf("expected imported skyboxes lock IDs, got:\n%s", string(lockRaw)) + } +} + +func TestBuildSupportsCanonicalSkyboxes(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skyboxes", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skyboxes", "base.json"), `{ + "output": "skyboxes.2da", + "columns": ["LABEL", "STRING_REF", "CYCLICAL", "DAWN", "DAY", "DUSK", "NIGHT"], + "rows": [ + {"id": 0, "LABEL": "(None)", "STRING_REF": "****", "CYCLICAL": "****", "DAWN": "****", "DAY": "****", "DUSK": "****", "NIGHT": "****"}, + {"id": 1, "LABEL": "Grass_Clear", "STRING_REF": "****", "CYCLICAL": "1", "DAWN": "Skyda_001", "DAY": "Sky_001", "DUSK": "Skyd_001", "NIGHT": "Skyn_001"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skyboxes", "lock.json"), `{ + "festerpot:id_7": 7 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skyboxes", "modules", "add_fpskies.json"), `{ + "entries": { + "festerpot:id_7": { + "LABEL": "Mountains", + "CYCLICAL": 1, + "DAWN": "CCS_01a", + "DAY": "CCS_01b", + "DUSK": "CCS_01c", + "NIGHT": "CCS_01d" + } + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "skyboxes.2da")) + if err != nil { + t.Fatalf("read skyboxes.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "LABEL\tSTRING_REF\tCYCLICAL\tDAWN\tDAY\tDUSK\tNIGHT\n") || + !strings.Contains(text, "0\t(None)\t****\t****\t****\t****\t****\t****\n") || + !strings.Contains(text, "1\tGrass_Clear\t****\t1\tSkyda_001\tSky_001\tSkyd_001\tSkyn_001\n") || + !strings.Contains(text, "7\tMountains\t****\t1\tCCS_01a\tCCS_01b\tCCS_01c\tCCS_01d\n") { + t.Fatalf("unexpected skyboxes.2da output:\n%s", string(got)) + } +} + +func TestBuildCanonicalSkyboxesMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "skyboxes", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "skyboxes.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("LABEL\tSTRING_REF\tCYCLICAL\tDAWN\tDAY\tDUSK\tNIGHT\n") + f.write("0\t(None)\t****\t****\t****\t****\t****\t****\n") + f.write("1\tGrass_Clear\t****\t1\tSkyda_001\tSky_001\tSkyd_001\tSkyn_001\n") + for i in range(2, 7): + f.write(f"{i}\t****\t****\t****\t****\t****\t****\t****\n") + f.write("7\tMountains\t****\t1\tCCS_01a\tCCS_01b\tCCS_01c\tCCS_01d\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "skyboxes", "base.json"), `{ + "output": "skyboxes.2da", + "columns": ["LABEL", "STRING_REF", "CYCLICAL", "DAWN", "DAY", "DUSK", "NIGHT"], + "rows": [ + {"id": 0, "LABEL": "(None)", "STRING_REF": "****", "CYCLICAL": "****", "DAWN": "****", "DAY": "****", "DUSK": "****", "NIGHT": "****"}, + {"id": 1, "LABEL": "Grass_Clear", "STRING_REF": "****", "CYCLICAL": "1", "DAWN": "Skyda_001", "DAY": "Sky_001", "DUSK": "Skyd_001", "NIGHT": "Skyn_001"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skyboxes", "lock.json"), `{ + "festerpot:id_7": 7 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skyboxes", "modules", "add_fpskies.json"), `{ + "entries": { + "festerpot:id_7": { + "LABEL": "Mountains", + "CYCLICAL": 1, + "DAWN": "CCS_01a", + "DAY": "CCS_01b", + "DUSK": "CCS_01c", + "NIGHT": "CCS_01d" + } + } +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "skyboxes.2da")) + if err != nil { + t.Fatalf("read native skyboxes.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "skyboxes.2da")) + if err != nil { + t.Fatalf("read reference skyboxes.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("skyboxes output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestCompareNativeCoverageCountsPlainAndBaseDatasets(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "topdata", "data", "creaturespeed")) + writeFile(t, filepath.Join(root, "topdata", "data", "creaturespeed", "creaturespeed.json"), `{ + "output": "creaturespeed.2da", + "columns": ["Label", "Name", "2DAName", "WALKRATE", "RUNRATE"], + "rows": [ + {"id": 0, "Label": "PC_Movement", "Name": "****", "2DAName": "PLAYER", "WALKRATE": "2.00", "RUNRATE": "4.00"}, + {"id": 4, "Label": "Normal", "Name": "2322", "2DAName": "NORM", "WALKRATE": "2.00", "RUNRATE": "4.00"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "topdata", "data", "skyboxes", "modules")) + writeFile(t, filepath.Join(root, "topdata", "data", "skyboxes", "base.json"), `{ + "output": "skyboxes.2da", + "columns": ["LABEL", "STRING_REF", "CYCLICAL", "DAWN", "DAY", "DUSK", "NIGHT"], + "rows": [ + {"id": 0, "LABEL": "(None)", "STRING_REF": "****", "CYCLICAL": "****", "DAWN": "****", "DAY": "****", "DUSK": "****", "NIGHT": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skyboxes", "lock.json"), `{ + "festerpot:id_7": 7 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skyboxes", "modules", "add_fpskies.json"), `{ + "entries": { + "festerpot:id_7": { + "LABEL": "Mountains", + "CYCLICAL": 1, + "DAWN": "CCS_01a", + "DAY": "CCS_01b", + "DUSK": "CCS_01c", + "NIGHT": "CCS_01d" + } + } +}`+"\n") + + mkdirAll(t, filepath.Join(root, "reference", "data", "creaturespeed", "modules")) + writeFile(t, filepath.Join(root, "reference", "data", "creaturespeed", "base.json"), `{ + "output": "creaturespeed.2da", + "columns": ["Label", "Name", "2DAName", "WALKRATE", "RUNRATE"], + "rows": [ + {"key": "creaturespeed:pcmovement", "id": 0, "Label": "PC_Movement", "Name": "****", "2DAName": "PLAYER", "WALKRATE": "2.00", "RUNRATE": "4.00"}, + {"key": "creaturespeed:normal", "id": 4, "Label": "Normal", "Name": "2322", "2DAName": "NORM", "WALKRATE": "1.75", "RUNRATE": "3.50"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "creaturespeed", "lock.json"), `{"creaturespeed:normal":4}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "creaturespeed", "modules", "ovr_mobspeed.json"), `{ + "overrides": [ + {"key": "creaturespeed:normal", "id": 4, "WALKRATE": "2.00", "RUNRATE": "4.00"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "skyboxes", "modules")) + writeFile(t, filepath.Join(root, "reference", "data", "skyboxes", "base.json"), `{ + "output": "skyboxes.2da", + "columns": ["LABEL", "STRING_REF", "CYCLICAL", "DAWN", "DAY", "DUSK", "NIGHT"], + "rows": [ + {"id": 0, "LABEL": "(None)", "STRING_REF": "****", "CYCLICAL": "****", "DAWN": "****", "DAY": "****", "DUSK": "****", "NIGHT": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skyboxes", "lock.json"), `{ + "festerpot:id_7": 7 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skyboxes", "modules", "add_fpskies.json"), `{ + "entries": { + "festerpot:id_7": { + "LABEL": "Mountains", + "CYCLICAL": 1, + "DAWN": "CCS_01a", + "DAY": "CCS_01b", + "DUSK": "CCS_01c", + "NIGHT": "CCS_01d" + } + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + pass, notYetCanonical, mismatch, err := compareNativeCoverage(testProject(root), nativeResult.Output2DADir, nil) + if err != nil { + t.Fatalf("compareNativeCoverage failed: %v", err) + } + if pass != 2 || notYetCanonical != 0 || mismatch != 0 { + t.Fatalf("unexpected coverage counts: pass=%d notYetCanonical=%d mismatch=%d", pass, notYetCanonical, mismatch) + } +} + +func TestNormalizeProjectImportsLegacyVFXPersistent(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "vfx_persistent", "modules")) + writeFile(t, filepath.Join(root, "reference", "data", "vfx_persistent", "base.json"), `{ + "output": "vfx_persistent.2da", + "columns": ["LABEL", "SHAPE", "RADIUS"], + "rows": [ + {"id": 0, "LABEL": "VFX_PER_FOGACID", "SHAPE": "C", "RADIUS": "5"}, + {"id": 46, "LABEL": "VFX_MOB_NIGHTMARE_SMOKE", "SHAPE": "C", "RADIUS": "1.5"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "vfx_persistent", "lock.json"), `{ + "vfx_persistent:blankradius5": 47, + "vfx_persistent:blankradius10": 48, + "vfx_persistent:blankradius15": 49, + "vfx_persistent:blankradius20": 50, + "vfx_persistent:blankradius25": 51, + "vfx_persistent:blankradius30": 52 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "vfx_persistent", "modules", "freeformaoes.json"), `{ + "entries": { + "vfx_persistent:blankradius5": {"LABEL": "VFX_PER_BLANK_RADIUS_5", "SHAPE": "C", "RADIUS": "5"}, + "vfx_persistent:blankradius10": {"LABEL": "VFX_PER_BLANK_RADIUS_10", "SHAPE": "C", "RADIUS": "5"}, + "vfx_persistent:blankradius15": {"LABEL": "VFX_PER_BLANK_RADIUS_15", "SHAPE": "C", "RADIUS": "15"}, + "vfx_persistent:blankradius20": {"LABEL": "VFX_PER_BLANK_RADIUS_20", "SHAPE": "C", "RADIUS": "20"}, + "vfx_persistent:blankradius25": {"LABEL": "VFX_PER_BLANK_RADIUS_25", "SHAPE": "C", "RADIUS": "25"}, + "vfx_persistent:blankradius30": {"LABEL": "VFX_PER_BLANK_RADIUS_30", "SHAPE": "C", "RADIUS": "30"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 2 { + t.Fatalf("expected vfx_persistent files to be imported, got %#v", result) + } + + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "vfx_persistent", "base.json"), + filepath.Join(root, "topdata", "data", "vfx_persistent", "lock.json"), + filepath.Join(root, "topdata", "data", "vfx_persistent", "modules", "freeformaoes.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported vfx_persistent path %s: %v", path, err) + } + } + + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "vfx_persistent", "lock.json")) + if err != nil { + t.Fatalf("read vfx_persistent lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"vfx_persistent:vfx_per_fogacid": 0`) || + !strings.Contains(lockText, `"vfx_persistent:vfx_mob_nightmare_smoke": 46`) || + !strings.Contains(lockText, `"vfx_persistent:blankradius5": 47`) || + !strings.Contains(lockText, `"vfx_persistent:blankradius10": 48`) || + !strings.Contains(lockText, `"vfx_persistent:blankradius30": 52`) { + t.Fatalf("expected imported vfx_persistent lock IDs, got:\n%s", lockText) + } + + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "vfx_persistent", "base.json")) + if err != nil { + t.Fatalf("read vfx_persistent base: %v", err) + } + baseText := string(baseRaw) + if !strings.Contains(baseText, `"key": "vfx_persistent:vfx_per_fogacid"`) || !strings.Contains(baseText, `"key": "vfx_persistent:vfx_mob_nightmare_smoke"`) { + t.Fatalf("expected imported vfx_persistent keys, got:\n%s", baseText) + } +} + +func TestNormalizeProjectRefreshesLegacyVFXPersistentLockFromSnapshot(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "vfx_persistent", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "migration_snapshot", "data", "vfx_persistent", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "vfx_persistent", "base.json"), `{ + "output": "vfx_persistent.2da", + "columns": ["LABEL", "SHAPE", "RADIUS"], + "rows": [ + {"id": 0, "key": "vfx_persistent:vfx_per_fogacid", "LABEL": "VFX_PER_FOGACID", "SHAPE": "C", "RADIUS": "5"}, + {"id": 46, "key": "vfx_persistent:vfx_mob_nightmare_smoke", "LABEL": "VFX_MOB_NIGHTMARE_SMOKE", "SHAPE": "C", "RADIUS": "1.5"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "vfx_persistent", "lock.json"), `{ + "vfx_persistent:blankradius10": 47, + "vfx_persistent:blankradius15": 48, + "vfx_persistent:blankradius20": 49, + "vfx_persistent:blankradius25": 50, + "vfx_persistent:blankradius30": 51, + "vfx_persistent:blankradius5": 52 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "vfx_persistent", "modules", "freeformaoes.json"), `{"entries": {}}`+"\n") + + writeFile(t, filepath.Join(root, "topdata", "migration_snapshot", "data", "vfx_persistent", "base.json"), `{ + "output": "vfx_persistent.2da", + "columns": ["LABEL", "SHAPE", "RADIUS"], + "rows": [ + {"id": 0, "LABEL": "VFX_PER_FOGACID", "SHAPE": "C", "RADIUS": "5"}, + {"id": 46, "LABEL": "VFX_MOB_NIGHTMARE_SMOKE", "SHAPE": "C", "RADIUS": "1.5"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "migration_snapshot", "data", "vfx_persistent", "lock.json"), `{ + "vfx_persistent:blankradius5": 47, + "vfx_persistent:blankradius10": 48, + "vfx_persistent:blankradius15": 49, + "vfx_persistent:blankradius20": 50, + "vfx_persistent:blankradius25": 51, + "vfx_persistent:blankradius30": 52 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "migration_snapshot", "data", "vfx_persistent", "modules", "freeformaoes.json"), `{ + "entries": { + "vfx_persistent:blankradius5": {"LABEL": "VFX_PER_BLANK_RADIUS_5", "SHAPE": "C", "RADIUS": "5"} + } +}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + result, err := NormalizeProject(p) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 3 { + t.Fatalf("expected snapshot-backed vfx_persistent refresh, got %#v", result) + } + + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "vfx_persistent", "lock.json")) + if err != nil { + t.Fatalf("read vfx_persistent lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"vfx_persistent:blankradius5": 47`) || + !strings.Contains(lockText, `"vfx_persistent:blankradius10": 48`) || + !strings.Contains(lockText, `"vfx_persistent:blankradius30": 52`) { + t.Fatalf("expected refreshed snapshot lock IDs, got:\n%s", lockText) + } +} + +func TestBuildSupportsCanonicalVFXPersistent(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "vfx_persistent")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "vfx_persistent", "base.json"), `{ + "output": "vfx_persistent.2da", + "columns": ["LABEL", "SHAPE", "RADIUS"], + "rows": [ + {"id": 0, "key": "vfx_persistent:vfx_per_fogacid", "LABEL": "VFX_PER_FOGACID", "SHAPE": "C", "RADIUS": "5"}, + {"id": 46, "key": "vfx_persistent:vfx_mob_nightmare_smoke", "LABEL": "VFX_MOB_NIGHTMARE_SMOKE", "SHAPE": "C", "RADIUS": "1.5"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "vfx_persistent", "lock.json"), `{ + "vfx_persistent:vfx_per_fogacid": 0, + "vfx_persistent:vfx_mob_nightmare_smoke": 46 +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "vfx_persistent.2da")) + if err != nil { + t.Fatalf("read vfx_persistent.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "LABEL\tSHAPE\tRADIUS\n") || + !strings.Contains(text, "0\tVFX_PER_FOGACID\tC\t5\n") || + !strings.Contains(text, "46\tVFX_MOB_NIGHTMARE_SMOKE\tC\t1.5\n") { + t.Fatalf("unexpected vfx_persistent.2da output:\n%s", string(got)) + } +} + +func TestBuildCanonicalVFXPersistentMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "vfx_persistent", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "vfx_persistent.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("LABEL\tSHAPE\tRADIUS\n") + f.write("0\tVFX_PER_FOGACID\tC\t5\n") + for i in range(1, 46): + f.write(f"{i}\t****\t****\t****\n") + f.write("46\tVFX_MOB_NIGHTMARE_SMOKE\tC\t1.5\n") + f.write("47\tVFX_PER_BLANK_RADIUS_5\tC\t5\n") + f.write("48\tVFX_PER_BLANK_RADIUS_10\tC\t5\n") + f.write("49\tVFX_PER_BLANK_RADIUS_15\tC\t15\n") + f.write("50\tVFX_PER_BLANK_RADIUS_20\tC\t20\n") + f.write("51\tVFX_PER_BLANK_RADIUS_25\tC\t25\n") + f.write("52\tVFX_PER_BLANK_RADIUS_30\tC\t30\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "vfx_persistent", "base.json"), `{ + "output": "vfx_persistent.2da", + "columns": ["LABEL", "SHAPE", "RADIUS"], + "rows": [ + {"id": 0, "LABEL": "VFX_PER_FOGACID", "SHAPE": "C", "RADIUS": "5"}, + {"id": 46, "LABEL": "VFX_MOB_NIGHTMARE_SMOKE", "SHAPE": "C", "RADIUS": "1.5"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "vfx_persistent", "lock.json"), `{ + "vfx_persistent:blankradius5": 47, + "vfx_persistent:blankradius10": 48, + "vfx_persistent:blankradius15": 49, + "vfx_persistent:blankradius20": 50, + "vfx_persistent:blankradius25": 51, + "vfx_persistent:blankradius30": 52 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "vfx_persistent", "modules", "freeformaoes.json"), `{ + "entries": { + "vfx_persistent:blankradius5": {"LABEL": "VFX_PER_BLANK_RADIUS_5", "SHAPE": "C", "RADIUS": "5"}, + "vfx_persistent:blankradius10": {"LABEL": "VFX_PER_BLANK_RADIUS_10", "SHAPE": "C", "RADIUS": "5"}, + "vfx_persistent:blankradius15": {"LABEL": "VFX_PER_BLANK_RADIUS_15", "SHAPE": "C", "RADIUS": "15"}, + "vfx_persistent:blankradius20": {"LABEL": "VFX_PER_BLANK_RADIUS_20", "SHAPE": "C", "RADIUS": "20"}, + "vfx_persistent:blankradius25": {"LABEL": "VFX_PER_BLANK_RADIUS_25", "SHAPE": "C", "RADIUS": "25"}, + "vfx_persistent:blankradius30": {"LABEL": "VFX_PER_BLANK_RADIUS_30", "SHAPE": "C", "RADIUS": "30"} + } +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "vfx_persistent.2da")) + if err != nil { + t.Fatalf("read native vfx_persistent.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "vfx_persistent.2da")) + if err != nil { + t.Fatalf("read reference vfx_persistent.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("vfx_persistent output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestBuildNativeRemovesStaleTopdataOutputs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 0, "LABEL": "TEST"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "lock.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "topdata", "assets", "2da")) + writeFile(t, filepath.Join(root, "topdata", "assets", "2da", "stale.2da"), "stale\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + result, err := BuildNative(p, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + if got, want := result.Output2DADir, filepath.Join(root, ".cache", "2da"); got != want { + t.Fatalf("unexpected cached 2da output dir: got %q want %q", got, want) + } + if _, err := os.Stat(filepath.Join(result.Output2DADir, "stale.2da")); !os.IsNotExist(err) { + t.Fatalf("expected stale legacy topdata output to be ignored, stat err=%v", err) + } +} + +func TestBuildNativeWritesCompiledOutputsToCache(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 0, "LABEL": "TEST"} + ] +}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + result, err := BuildNative(p, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + if got, want := result.Output2DADir, filepath.Join(root, ".cache", "2da"); got != want { + t.Fatalf("unexpected 2da output dir: got %q want %q", got, want) + } + if got, want := result.OutputTLKDir, filepath.Join(root, "build"); got != want { + t.Fatalf("unexpected tlk output dir: got %q want %q", got, want) + } + if _, err := os.Stat(filepath.Join(result.Output2DADir, "repadjust.2da")); err != nil { + t.Fatalf("expected compiled 2da output: %v", err) + } + if _, err := os.Stat(filepath.Join(result.OutputTLKDir, defaultTLKName)); err != nil { + t.Fatalf("expected compiled tlk output: %v", err) + } +} + +func TestValidateAndBuildTopdataDoNotRequireReferenceBuilder(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 0, "LABEL": "TEST"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "lock.json"), "{}\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + + report := ValidateProject(p) + if report.HasErrors() { + t.Fatalf("expected validate to succeed without reference builder, got %#v", report.Diagnostics) + } + if strings.Contains(diagnosticsText(report.Diagnostics), "reference_builder") { + t.Fatalf("did not expect reference_builder validation diagnostic, got %#v", report.Diagnostics) + } + if _, err := BuildNative(p, nil); err != nil { + t.Fatalf("BuildNative failed without reference builder: %v", err) + } +} + +func TestCompareReferenceFailsClearlyWithoutReferenceBuilder(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 0, "LABEL": "TEST"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "lock.json"), "{}\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + if _, err := BuildNative(p, nil); err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + compare, err := CompareReference(p, nil) + if err != nil { + t.Fatalf("CompareReference failed: %v", err) + } + if compare.Mode != "native" || compare.NativeMismatch != 0 || compare.NativePass != 1 { + t.Fatalf("unexpected compare result without reference builder: %#v", compare) + } +} + +func TestNormalizeProjectImportsLegacyProgfx(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "progfx")) + writeFile(t, filepath.Join(root, "reference", "data", "progfx", "base.json"), `{ + "output": "progfx.2da", + "columns": ["Label", "Type", "Param1", "Param2", "Param3", "Param4", "Param5", "Param6", "Param7", "Param8"], + "rows": [ + {"id": 0, "Label": "****", "Type": "****", "Param1": "****", "Param2": "****", "Param3": "****", "Param4": "****", "Param5": "****", "Param6": "****", "Param7": "****", "Param8": "****"}, + {"id": 1300, "Label": "FreezeAnimations", "Type": "13", "Param1": "****", "Param2": "****", "Param3": "****", "Param4": "****", "Param5": "****", "Param6": "****", "Param7": "****", "Param8": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "progfx", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 2 { + t.Fatalf("expected progfx files to be imported, got %#v", result) + } + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "progfx", "base.json"), + filepath.Join(root, "topdata", "data", "progfx", "lock.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported progfx path %s: %v", path, err) + } + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "progfx", "lock.json")) + if err != nil { + t.Fatalf("read progfx lock: %v", err) + } + if strings.TrimSpace(string(lockRaw)) != "{}" { + t.Fatalf("expected empty imported progfx lock, got:\n%s", string(lockRaw)) + } +} + +func TestBuildSupportsCanonicalProgfx(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "progfx")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "progfx", "base.json"), `{ + "output": "progfx.2da", + "columns": ["Label", "Type", "Param1", "Param2", "Param3", "Param4", "Param5", "Param6", "Param7", "Param8"], + "rows": [ + {"id": 0, "Label": "****", "Type": "****", "Param1": "****", "Param2": "****", "Param3": "****", "Param4": "****", "Param5": "****", "Param6": "****", "Param7": "****", "Param8": "****"}, + {"id": 1300, "Label": "FreezeAnimations", "Type": "13", "Param1": "****", "Param2": "****", "Param3": "****", "Param4": "****", "Param5": "****", "Param6": "****", "Param7": "****", "Param8": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "progfx", "lock.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "progfx.2da")) + if err != nil { + t.Fatalf("read progfx.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "Label\tType\tParam1\tParam2\tParam3\tParam4\tParam5\tParam6\tParam7\tParam8\n") || + !strings.Contains(text, "0\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\n") || + !strings.Contains(text, "1300\tFreezeAnimations\t13\t****\t****\t****\t****\t****\t****\t****\t****\n") { + t.Fatalf("unexpected progfx.2da output:\n%s", string(got)) + } +} + +func TestBuildCanonicalProgfxMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "progfx")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "progfx.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tType\tParam1\tParam2\tParam3\tParam4\tParam5\tParam6\tParam7\tParam8\n") + f.write("0\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\n") + for i in range(1, 1300): + f.write(f"{i}\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\n") + f.write("1300\tFreezeAnimations\t13\t****\t****\t****\t****\t****\t****\t****\t****\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "progfx", "base.json"), `{ + "output": "progfx.2da", + "columns": ["Label", "Type", "Param1", "Param2", "Param3", "Param4", "Param5", "Param6", "Param7", "Param8"], + "rows": [ + {"id": 0, "Label": "****", "Type": "****", "Param1": "****", "Param2": "****", "Param3": "****", "Param4": "****", "Param5": "****", "Param6": "****", "Param7": "****", "Param8": "****"}, + {"id": 1300, "Label": "FreezeAnimations", "Type": "13", "Param1": "****", "Param2": "****", "Param3": "****", "Param4": "****", "Param5": "****", "Param6": "****", "Param7": "****", "Param8": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "progfx", "lock.json"), "{}\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "progfx.2da")) + if err != nil { + t.Fatalf("read native progfx.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "progfx.2da")) + if err != nil { + t.Fatalf("read reference progfx.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("progfx output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsLegacyRepadjust(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "repadjust")) + writeFile(t, filepath.Join(root, "reference", "data", "repadjust", "base.json"), `{ + "columns": ["LABEL", "PERSONALREP", "FACTIONREP", "WITFRIA", "WITFRIB", "WITFRIC", "WITNEUA", "WITNEUB", "WITNEUC", "WITENEA", "WITENEB", "WITENEC"], + "rows": [ + {"id": 0, "LABEL": "Attack", "PERSONALREP": "-100", "FACTIONREP": "-5", "WITFRIA": "-100", "WITFRIB": "-100", "WITFRIC": "0", "WITNEUA": "-5", "WITNEUB": "0", "WITNEUC": "0", "WITENEA": "0", "WITENEB": "0", "WITENEC": "0"}, + {"id": 3, "LABEL": "Help", "PERSONALREP": "0", "FACTIONREP": "0", "WITFRIA": "0", "WITFRIB": "0", "WITFRIC": "0", "WITNEUA": "0", "WITNEUB": "0", "WITNEUC": "0", "WITENEA": "0", "WITENEB": "0", "WITENEC": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "repadjust", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 2 { + t.Fatalf("expected repadjust files to be imported, got %#v", result) + } + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "repadjust", "base.json"), + filepath.Join(root, "topdata", "data", "repadjust", "lock.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported repadjust path %s: %v", path, err) + } + } + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "repadjust", "base.json")) + if err != nil { + t.Fatalf("read repadjust base: %v", err) + } + if !strings.Contains(string(baseRaw), `"output": "repadjust.2da"`) { + t.Fatalf("expected imported repadjust output name, got:\n%s", string(baseRaw)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "repadjust", "lock.json")) + if err != nil { + t.Fatalf("read repadjust lock: %v", err) + } + if strings.TrimSpace(string(lockRaw)) != "{}" { + t.Fatalf("expected empty imported repadjust lock, got:\n%s", string(lockRaw)) + } +} + +func TestBuildSupportsCanonicalRepadjust(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["LABEL", "PERSONALREP", "FACTIONREP", "WITFRIA", "WITFRIB", "WITFRIC", "WITNEUA", "WITNEUB", "WITNEUC", "WITENEA", "WITENEB", "WITENEC"], + "rows": [ + {"id": 0, "LABEL": "Attack", "PERSONALREP": "-100", "FACTIONREP": "-5", "WITFRIA": "-100", "WITFRIB": "-100", "WITFRIC": "0", "WITNEUA": "-5", "WITNEUB": "0", "WITNEUC": "0", "WITENEA": "0", "WITENEB": "0", "WITENEC": "0"}, + {"id": 3, "LABEL": "Help", "PERSONALREP": "0", "FACTIONREP": "0", "WITFRIA": "0", "WITFRIB": "0", "WITFRIC": "0", "WITNEUA": "0", "WITNEUB": "0", "WITNEUC": "0", "WITENEA": "0", "WITENEB": "0", "WITENEC": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "lock.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "repadjust.2da")) + if err != nil { + t.Fatalf("read repadjust.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "LABEL\tPERSONALREP\tFACTIONREP\tWITFRIA\tWITFRIB\tWITFRIC\tWITNEUA\tWITNEUB\tWITNEUC\tWITENEA\tWITENEB\tWITENEC\n") || + !strings.Contains(text, "0\tAttack\t-100\t-5\t-100\t-100\t0\t-5\t0\t0\t0\t0\t0\n") || + !strings.Contains(text, "3\tHelp\t0\t0\t0\t0\t0\t0\t0\t0\t0\t0\t0\n") { + t.Fatalf("unexpected repadjust.2da output:\n%s", string(got)) + } +} + +func TestBuildCanonicalRepadjustMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "repadjust.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("LABEL\tPERSONALREP\tFACTIONREP\tWITFRIA\tWITFRIB\tWITFRIC\tWITNEUA\tWITNEUB\tWITNEUC\tWITENEA\tWITENEB\tWITENEC\n") + f.write("0\tAttack\t-100\t-5\t-100\t-100\t0\t-5\t0\t0\t0\t0\t0\n") + f.write("1\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\n") + f.write("2\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\n") + f.write("3\tHelp\t0\t0\t0\t0\t0\t0\t0\t0\t0\t0\t0\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["LABEL", "PERSONALREP", "FACTIONREP", "WITFRIA", "WITFRIB", "WITFRIC", "WITNEUA", "WITNEUB", "WITNEUC", "WITENEA", "WITENEB", "WITENEC"], + "rows": [ + {"id": 0, "LABEL": "Attack", "PERSONALREP": "-100", "FACTIONREP": "-5", "WITFRIA": "-100", "WITFRIB": "-100", "WITFRIC": "0", "WITNEUA": "-5", "WITNEUB": "0", "WITNEUC": "0", "WITENEA": "0", "WITENEB": "0", "WITENEC": "0"}, + {"id": 3, "LABEL": "Help", "PERSONALREP": "0", "FACTIONREP": "0", "WITFRIA": "0", "WITFRIB": "0", "WITFRIC": "0", "WITNEUA": "0", "WITNEUB": "0", "WITNEUC": "0", "WITENEA": "0", "WITENEB": "0", "WITENEC": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "repadjust", "lock.json"), "{}\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "repadjust.2da")) + if err != nil { + t.Fatalf("read native repadjust.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "repadjust.2da")) + if err != nil { + t.Fatalf("read reference repadjust.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("repadjust output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsLegacyWingmodel(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "wingmodel", "modules")) + writeFile(t, filepath.Join(root, "reference", "data", "wingmodel", "base.json"), `{ + "columns": ["LABEL", "MODEL", "ENVMAP"], + "rows": [ + {"key": "wingmodel:none", "id": 0, "LABEL": "(None)", "MODEL": "****", "ENVMAP": "****"}, + {"key": "wingmodel:angel", "id": 2, "LABEL": "Angel", "MODEL": "c_angel", "ENVMAP": "default"}, + {"key": "wingmodel:greatsword", "id": 89, "LABEL": "Greatsword", "MODEL": "c_greatsword", "ENVMAP": "default"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "wingmodel", "lock.json"), `{ + "wings:armoredangel": 90, + "wings:fallenangel": 91, + "wings:mephisto": 92, + "wingmodel:demon": 1, + "wingmodel:angel": 2 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "wingmodel", "modules", "add.json"), `{ + "entries": { + "wings:armoredangel": { + "LABEL": "Armored Angel", + "MODEL": "c_armoredangel", + "ENVMAP": "default" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "wingmodel", "modules", "ovr_wingprefixes.json"), `{ + "overrides": [ + {"key": "wingmodel:angel", "id": 2, "LABEL": "Wings: Angel"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 4 { + t.Fatalf("expected wingmodel files to be imported, got %#v", result) + } + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "wingmodel", "base.json"), + filepath.Join(root, "topdata", "data", "wingmodel", "lock.json"), + filepath.Join(root, "topdata", "data", "wingmodel", "modules", "add.json"), + filepath.Join(root, "topdata", "data", "wingmodel", "modules", "ovr_wingprefixes.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported wingmodel path %s: %v", path, err) + } + } + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "wingmodel", "base.json")) + if err != nil { + t.Fatalf("read wingmodel base: %v", err) + } + if !strings.Contains(string(baseRaw), `"output": "wingmodel.2da"`) { + t.Fatalf("expected imported wingmodel output name, got:\n%s", string(baseRaw)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "wingmodel", "lock.json")) + if err != nil { + t.Fatalf("read wingmodel lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"wings:mephisto": 92`) || !strings.Contains(lockText, `"wingmodel:angel": 2`) { + t.Fatalf("expected imported wingmodel lock IDs, got:\n%s", lockText) + } +} + +func TestBuildSupportsCanonicalWingmodel(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "wingmodel", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "wingmodel", "base.json"), `{ + "output": "wingmodel.2da", + "columns": ["LABEL", "MODEL", "ENVMAP"], + "rows": [ + {"key": "wingmodel:none", "id": 0, "LABEL": "(None)", "MODEL": "****", "ENVMAP": "****"}, + {"key": "wingmodel:angel", "id": 2, "LABEL": "Angel", "MODEL": "c_angel", "ENVMAP": "default"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "wingmodel", "lock.json"), `{ + "wings:armoredangel": 90, + "wingmodel:angel": 2 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "wingmodel", "modules", "add.json"), `{ + "entries": { + "wings:armoredangel": { + "LABEL": "Armored Angel", + "MODEL": "c_armoredangel", + "ENVMAP": "default" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "wingmodel", "modules", "ovr_wingprefixes.json"), `{ + "overrides": [ + {"key": "wingmodel:angel", "id": 2, "LABEL": "Wings: Angel"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "wingmodel.2da")) + if err != nil { + t.Fatalf("read wingmodel.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "LABEL\tMODEL\tENVMAP\n") || + !strings.Contains(text, "0\t(None)\t****\t****\n") || + !strings.Contains(text, "2\t\"Wings: Angel\"\tc_angel\tdefault\n") || + !strings.Contains(text, "90\t\"Armored Angel\"\tc_armoredangel\tdefault\n") { + t.Fatalf("unexpected wingmodel.2da output:\n%s", string(got)) + } +} + +func TestBuildCanonicalWingmodelMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "wingmodel", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "wingmodel.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("LABEL\tMODEL\tENVMAP\n") + f.write("0\t(None)\t****\t****\n") + f.write("1\t****\t****\t****\n") + f.write("2\tWings: Angel\tc_angel\tdefault\n") + for i in range(3, 90): + f.write(f"{i}\t****\t****\t****\n") + f.write("90\tArmored Angel\tc_armoredangel\tdefault\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "wingmodel", "base.json"), `{ + "output": "wingmodel.2da", + "columns": ["LABEL", "MODEL", "ENVMAP"], + "rows": [ + {"key": "wingmodel:none", "id": 0, "LABEL": "(None)", "MODEL": "****", "ENVMAP": "****"}, + {"key": "wingmodel:angel", "id": 2, "LABEL": "Angel", "MODEL": "c_angel", "ENVMAP": "default"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "wingmodel", "lock.json"), `{ + "wings:armoredangel": 90, + "wingmodel:angel": 2 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "wingmodel", "modules", "add.json"), `{ + "entries": { + "wings:armoredangel": { + "LABEL": "Armored Angel", + "MODEL": "c_armoredangel", + "ENVMAP": "default" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "wingmodel", "modules", "ovr_wingprefixes.json"), `{ + "overrides": [ + {"key": "wingmodel:angel", "id": 2, "LABEL": "Wings: Angel"} + ] +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "wingmodel.2da")) + if err != nil { + t.Fatalf("read native wingmodel.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "wingmodel.2da")) + if err != nil { + t.Fatalf("read reference wingmodel.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("wingmodel output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsLegacyTailmodel(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + + mkdirAll(t, filepath.Join(root, "reference", "data", "tailmodel", "modules")) + writeFile(t, filepath.Join(root, "reference", "data", "tailmodel", "base.json"), `{ + "columns": ["LABEL", "MODEL", "ENVMAP"], + "rows": [ + {"key": "tailmodel:none", "id": 0, "LABEL": "(None)", "MODEL": "****", "ENVMAP": "****"}, + {"key": "tailmodel:lizard", "id": 1, "LABEL": "Lizard", "MODEL": "c_tailliz", "ENVMAP": "default"}, + {"key": "tailmodel:dragonwhite", "id": 13, "LABEL": "Dragon, White", "MODEL": "c_tail_whi", "ENVMAP": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "tailmodel", "lock.json"), `{ + "tails:taildemon": 5050, + "tails:taildemon2": 5051, + "tails:taildemon4": 5052, + "tails:tailfluff": 5053, + "tailmodel:lizard": 1, + "tailmodel:dragonwhite": 13 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "tailmodel", "modules", "add.json"), `{ + "entries": { + "tails:taildemon": {"LABEL": "Tail: Demon", "MODEL": "c_taildemon", "ENVMAP": "default"}, + "tails:tailfluff": {"LABEL": "Tail: Demon, Fluffy", "MODEL": "c_tailfluff", "ENVMAP": "default"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "tailmodel", "modules", "ovr_tailprefixes.json"), `{ + "overrides": [ + {"key": "tailmodel:lizard", "id": 1, "LABEL": "Tail: Lizard"}, + {"key": "tailmodel:dragonwhite", "id": 13, "LABEL": "Tail: Dragon, White"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 4 { + t.Fatalf("expected tailmodel files to be imported, got %#v", result) + } + + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "tailmodel", "base.json"), + filepath.Join(root, "topdata", "data", "tailmodel", "lock.json"), + filepath.Join(root, "topdata", "data", "tailmodel", "modules", "add.json"), + filepath.Join(root, "topdata", "data", "tailmodel", "modules", "ovr_tailprefixes.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported tailmodel path %s: %v", path, err) + } + } + + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "tailmodel", "base.json")) + if err != nil { + t.Fatalf("read tailmodel base: %v", err) + } + if !strings.Contains(string(baseRaw), `"output": "tailmodel.2da"`) { + t.Fatalf("expected imported tailmodel output name, got:\n%s", string(baseRaw)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "tailmodel", "lock.json")) + if err != nil { + t.Fatalf("read tailmodel lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"tails:taildemon": 5050`) || !strings.Contains(lockText, `"tailmodel:dragonwhite": 13`) { + t.Fatalf("expected imported tailmodel lock IDs, got:\n%s", lockText) + } +} + +func TestBuildSupportsCanonicalTailmodel(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "tailmodel", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "tailmodel", "base.json"), `{ + "output": "tailmodel.2da", + "columns": ["LABEL", "MODEL", "ENVMAP"], + "rows": [ + {"key": "tailmodel:none", "id": 0, "LABEL": "(None)", "MODEL": "****", "ENVMAP": "****"}, + {"key": "tailmodel:lizard", "id": 1, "LABEL": "Lizard", "MODEL": "c_tailliz", "ENVMAP": "default"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "tailmodel", "lock.json"), `{ + "tails:taildemon": 5050, + "tailmodel:lizard": 1 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "tailmodel", "modules", "add.json"), `{ + "entries": { + "tails:taildemon": {"LABEL": "Tail: Demon", "MODEL": "c_taildemon", "ENVMAP": "default"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "tailmodel", "modules", "ovr_tailprefixes.json"), `{ + "overrides": [ + {"key": "tailmodel:lizard", "id": 1, "LABEL": "Tail: Lizard"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "tailmodel.2da")) + if err != nil { + t.Fatalf("read tailmodel.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "LABEL\tMODEL\tENVMAP\n") || + !strings.Contains(text, "0\t(None)\t****\t****\n") || + !strings.Contains(text, "1\t\"Tail: Lizard\"\tc_tailliz\tdefault\n") || + !strings.Contains(text, "5050\t\"Tail: Demon\"\tc_taildemon\tdefault\n") { + t.Fatalf("unexpected tailmodel.2da output:\n%s", string(got)) + } +} + +func TestBuildCanonicalTailmodelMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "tailmodel", "modules")) + mkdirAll(t, filepath.Join(root, "reference", "data", "tailmodel", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "tailmodel.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("LABEL\tMODEL\tENVMAP\n") + f.write("0\t(None)\t****\t****\n") + f.write("1\t\"Tail: Lizard\"\tc_tailliz\tdefault\n") + for i in range(2, 5050): + f.write(f"{i}\t****\t****\t****\n") + f.write("5050\t\"Tail: Demon\"\tc_taildemon\tdefault\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "tailmodel", "base.json"), `{ + "output": "tailmodel.2da", + "columns": ["LABEL", "MODEL", "ENVMAP"], + "rows": [ + {"key": "tailmodel:none", "id": 0, "LABEL": "(None)", "MODEL": "****", "ENVMAP": "****"}, + {"key": "tailmodel:lizard", "id": 1, "LABEL": "Lizard", "MODEL": "c_tailliz", "ENVMAP": "default"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "tailmodel", "lock.json"), `{ + "tails:taildemon": 5050, + "tailmodel:lizard": 1 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "tailmodel", "modules", "add.json"), `{ + "entries": { + "tails:taildemon": {"LABEL": "Tail: Demon", "MODEL": "c_taildemon", "ENVMAP": "default"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "tailmodel", "modules", "ovr_tailprefixes.json"), `{ + "overrides": [ + {"key": "tailmodel:lizard", "id": 1, "LABEL": "Tail: Lizard"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "tailmodel", "base.json"), `{ + "output": "tailmodel.2da", + "columns": ["LABEL", "MODEL", "ENVMAP"], + "rows": [ + {"key": "tailmodel:none", "id": 0, "LABEL": "(None)", "MODEL": "****", "ENVMAP": "****"}, + {"key": "tailmodel:lizard", "id": 1, "LABEL": "Lizard", "MODEL": "c_tailliz", "ENVMAP": "default"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "tailmodel", "lock.json"), `{ + "tails:taildemon": 5050, + "tailmodel:lizard": 1 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "tailmodel", "modules", "add.json"), `{ + "entries": { + "tails:taildemon": {"LABEL": "Tail: Demon", "MODEL": "c_taildemon", "ENVMAP": "default"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "tailmodel", "modules", "ovr_tailprefixes.json"), `{ + "overrides": [ + {"key": "tailmodel:lizard", "id": 1, "LABEL": "Tail: Lizard"} + ] +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "tailmodel.2da")) + if err != nil { + t.Fatalf("read native tailmodel.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "tailmodel.2da")) + if err != nil { + t.Fatalf("read reference tailmodel.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("tailmodel output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsLegacyDoortypes(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + + mkdirAll(t, filepath.Join(root, "reference", "data", "doortypes", "modules")) + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "base.json"), `{ + "columns": ["Label", "Model", "TileSet", "TemplateResRef", "StringRefGame", "BlockSight", "VisibleModel", "SoundAppType"], + "rows": [ + {"id": 0, "Label": "Generic", "Model": "****", "TileSet": "****", "TemplateResRef": "****", "StringRefGame": "66717", "BlockSight": "****", "VisibleModel": "****", "SoundAppType": "****"}, + {"id": 1, "Label": "Wall1Door", "Model": "TTR_UDoor_01", "TileSet": "TTR01", "TemplateResRef": "nw_door_ttr_01", "StringRefGame": "63491", "BlockSight": "1", "VisibleModel": "1", "SoundAppType": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "lock.json"), `{ + "txpple:id_6000": 5200, + "ancarion:id_125": 5226, + "babayaga:id_1046": 5244 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "modules", "add_pld01.json"), `{ + "entries": { + "ancarion:id_125": {"Label": "CastleTrans", "Model": "pld_door_01", "TileSet": "PLD01", "TemplateResRef": "nw_door_ttr_15", "StringRefGame": 63520, "BlockSight": 0, "VisibleModel": 0} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "modules", "add_sic11.json"), `{ + "entries": { + "sens:id_1351": {"Label": "MulsantirGreat01", "Model": "door_mul01", "TileSet": "TNO01", "TemplateResRef": "nw_door_ttr_01", "StringRefGame": 5349, "BlockSight": 1, "VisibleModel": 1, "SoundAppType": 1} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "modules", "add_tapr.json"), `{ + "entries": { + "txpple:id_6000": {"Label": "Necropolis: Main Door", "Model": "mst01_door_01", "TileSet": "mst01", "TemplateResRef": "nw_door_metal", "StringRefGame": 66717, "BlockSight": 1, "VisibleModel": 1, "SoundAppType": 9} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "modules", "add_tdm01.json"), `{ + "entries": { + "projectq:id_5225": {"Label": "DungeonGate", "Model": "tdm02_sdoor_01", "TileSet": "TDM01", "TemplateResRef": "nw_door_ttr_01", "StringRefGame": 63491, "BlockSight": 1, "VisibleModel": 1, "SoundAppType": 1} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "modules", "add_tdx01.json"), `{ + "entries": { + "sixandzwerkules:id_1167": {"Label": "DungeonMechGate1", "Model": "tdx_door04", "TileSet": "TDX01", "TemplateResRef": "nw_door_ttr_03", "StringRefGame": 63537, "BlockSight": 1, "VisibleModel": 1, "SoundAppType": 3} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "modules", "add_tei01.json"), `{ + "entries": { + "zwerkules:id_1451": {"Label": "TEI01_StairTrans1", "Model": "tei_udoor01", "TileSet": "TEI01", "TemplateResRef": "nw_door_ttr_34", "StringRefGame": 5349, "BlockSight": 0, "VisibleModel": 0} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "modules", "add_tfm01.json"), `{ + "entries": { + "babayaga:id_1046": {"Label": "TFM01_BarnDoor", "Model": "tfm_udoor_06", "TileSet": "TFM01", "TemplateResRef": "nw_door_ttr_05", "StringRefGame": 5350, "BlockSight": 1, "VisibleModel": 1, "SoundAppType": 1} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 9 { + t.Fatalf("expected doortypes files to be imported, got %#v", result) + } + + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "doortypes", "base.json"), + filepath.Join(root, "topdata", "data", "doortypes", "lock.json"), + filepath.Join(root, "topdata", "data", "doortypes", "modules", "add_pld01.json"), + filepath.Join(root, "topdata", "data", "doortypes", "modules", "add_sic11.json"), + filepath.Join(root, "topdata", "data", "doortypes", "modules", "add_tapr.json"), + filepath.Join(root, "topdata", "data", "doortypes", "modules", "add_tdm01.json"), + filepath.Join(root, "topdata", "data", "doortypes", "modules", "add_tdx01.json"), + filepath.Join(root, "topdata", "data", "doortypes", "modules", "add_tei01.json"), + filepath.Join(root, "topdata", "data", "doortypes", "modules", "add_tfm01.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported doortypes path %s: %v", path, err) + } + } + + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "doortypes", "base.json")) + if err != nil { + t.Fatalf("read doortypes base: %v", err) + } + if !strings.Contains(string(baseRaw), `"output": "doortypes.2da"`) { + t.Fatalf("expected imported doortypes output name, got:\n%s", string(baseRaw)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "doortypes", "lock.json")) + if err != nil { + t.Fatalf("read doortypes lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"txpple:id_6000": 5200`) || !strings.Contains(lockText, `"babayaga:id_1046": 5244`) { + t.Fatalf("expected imported doortypes lock IDs, got:\n%s", lockText) + } +} + +func TestBuildSupportsCanonicalDoortypes(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "doortypes", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "doortypes", "base.json"), `{ + "output": "doortypes.2da", + "columns": ["Label", "Model", "TileSet", "TemplateResRef", "StringRefGame", "BlockSight", "VisibleModel", "SoundAppType"], + "rows": [ + {"id": 0, "Label": "Generic", "Model": "****", "TileSet": "****", "TemplateResRef": "****", "StringRefGame": "66717", "BlockSight": "****", "VisibleModel": "****", "SoundAppType": "****"}, + {"id": 1, "Label": "Wall1Door", "Model": "TTR_UDoor_01", "TileSet": "TTR01", "TemplateResRef": "nw_door_ttr_01", "StringRefGame": "63491", "BlockSight": "1", "VisibleModel": "1", "SoundAppType": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "doortypes", "lock.json"), `{ + "txpple:id_6000": 5200 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "doortypes", "modules", "add_tapr.json"), `{ + "entries": { + "txpple:id_6000": {"Label": "Necropolis: Main Door", "Model": "mst01_door_01", "TileSet": "mst01", "TemplateResRef": "nw_door_metal", "StringRefGame": 66717, "BlockSight": 1, "VisibleModel": 1, "SoundAppType": 9} + } +}`+"\n") + for _, name := range []string{"add_pld01.json", "add_sic11.json", "add_tdm01.json", "add_tdx01.json", "add_tei01.json", "add_tfm01.json"} { + writeFile(t, filepath.Join(root, "topdata", "data", "doortypes", "modules", name), `{"entries": {}}`+"\n") + } + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "doortypes.2da")) + if err != nil { + t.Fatalf("read doortypes.2da: %v", err) + } + if !strings.Contains(string(got), "Label\tModel\tTileSet\tTemplateResRef\tStringRefGame\tBlockSight\tVisibleModel\tSoundAppType\n") || + !strings.Contains(string(got), "0\tGeneric\t****\t****\t****\t66717\t****\t****\t****\n") || + !strings.Contains(string(got), "1\tWall1Door\tTTR_UDoor_01\tTTR01\tnw_door_ttr_01\t63491\t1\t1\t1\n") || + !strings.Contains(string(got), "5200\t\"Necropolis: Main Door\"\tmst01_door_01\tmst01\tnw_door_metal\t66717\t1\t1\t9\n") { + t.Fatalf("unexpected doortypes.2da output:\n%s", string(got)) + } +} + +func TestBuildCanonicalDoortypesMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "doortypes", "modules")) + mkdirAll(t, filepath.Join(root, "reference", "data", "doortypes", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "doortypes.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tModel\tTileSet\tTemplateResRef\tStringRefGame\tBlockSight\tVisibleModel\tSoundAppType\n") + f.write("0\tGeneric\t****\t****\t****\t66717\t****\t****\t****\n") + f.write("1\tWall1Door\tTTR_UDoor_01\tTTR01\tnw_door_ttr_01\t63491\t1\t1\t1\n") + for row_id in range(2, 5200): + f.write(f"{row_id}\t****\t****\t****\t****\t****\t****\t****\t****\n") + f.write("5200\t\"Necropolis: Main Door\"\tmst01_door_01\tmst01\tnw_door_metal\t66717\t1\t1\t9\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "base.json"), `{ + "output": "doortypes.2da", + "columns": ["Label", "Model", "TileSet", "TemplateResRef", "StringRefGame", "BlockSight", "VisibleModel", "SoundAppType"], + "rows": [ + {"id": 0, "Label": "Generic", "Model": "****", "TileSet": "****", "TemplateResRef": "****", "StringRefGame": "66717", "BlockSight": "****", "VisibleModel": "****", "SoundAppType": "****"}, + {"id": 1, "Label": "Wall1Door", "Model": "TTR_UDoor_01", "TileSet": "TTR01", "TemplateResRef": "nw_door_ttr_01", "StringRefGame": "63491", "BlockSight": "1", "VisibleModel": "1", "SoundAppType": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "lock.json"), `{ + "txpple:id_6000": 5200 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "modules", "add_tapr.json"), `{ + "entries": { + "txpple:id_6000": {"Label": "Necropolis: Main Door", "Model": "mst01_door_01", "TileSet": "mst01", "TemplateResRef": "nw_door_metal", "StringRefGame": 66717, "BlockSight": 1, "VisibleModel": 1, "SoundAppType": 9} + } +}`+"\n") + for _, name := range []string{"add_pld01.json", "add_sic11.json", "add_tdm01.json", "add_tdx01.json", "add_tei01.json", "add_tfm01.json"} { + writeFile(t, filepath.Join(root, "reference", "data", "doortypes", "modules", name), `{"entries": {}}`+"\n") + } + writeFile(t, filepath.Join(root, "topdata", "data", "doortypes", "base.json"), `{ + "output": "doortypes.2da", + "columns": ["Label", "Model", "TileSet", "TemplateResRef", "StringRefGame", "BlockSight", "VisibleModel", "SoundAppType"], + "rows": [ + {"id": 0, "Label": "Generic", "Model": "****", "TileSet": "****", "TemplateResRef": "****", "StringRefGame": "66717", "BlockSight": "****", "VisibleModel": "****", "SoundAppType": "****"}, + {"id": 1, "Label": "Wall1Door", "Model": "TTR_UDoor_01", "TileSet": "TTR01", "TemplateResRef": "nw_door_ttr_01", "StringRefGame": "63491", "BlockSight": "1", "VisibleModel": "1", "SoundAppType": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "doortypes", "lock.json"), `{ + "txpple:id_6000": 5200 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "doortypes", "modules", "add_tapr.json"), `{ + "entries": { + "txpple:id_6000": {"Label": "Necropolis: Main Door", "Model": "mst01_door_01", "TileSet": "mst01", "TemplateResRef": "nw_door_metal", "StringRefGame": 66717, "BlockSight": 1, "VisibleModel": 1, "SoundAppType": 9} + } +}`+"\n") + for _, name := range []string{"add_pld01.json", "add_sic11.json", "add_tdm01.json", "add_tdx01.json", "add_tei01.json", "add_tfm01.json"} { + writeFile(t, filepath.Join(root, "topdata", "data", "doortypes", "modules", name), `{"entries": {}}`+"\n") + } + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "doortypes.2da")) + if err != nil { + t.Fatalf("read native doortypes.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "doortypes.2da")) + if err != nil { + t.Fatalf("read reference doortypes.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("doortypes output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsLegacyVisualeffects(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + + mkdirAll(t, filepath.Join(root, "reference", "data", "visualeffects", "modules", "accessories")) + writeFile(t, filepath.Join(root, "reference", "data", "visualeffects", "base.json"), `{ + "columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "Imp_Impact_Node", "Imp_Root_S_Node", "Imp_Root_M_Node", "Imp_Root_L_Node", "Imp_Root_H_Node", "ProgFX_Impact", "SoundImpact", "ProgFX_Duration", "SoundDuration", "ProgFX_Cessation", "SoundCessastion", "Ces_HeadCon_Node", "Ces_Impact_Node", "Ces_Root_S_Node", "Ces_Root_M_Node", "Ces_Root_L_Node", "Ces_Root_H_Node", "ShakeType", "ShakeDelay", "ShakeDuration", "LowViolence", "LowQuality", "OrientWithObject"], + "rows": [ + {"key": "visualeffects:fnf_blinddeaf", "id": 18, "Label": "VFX_FNF_BLINDDEAF", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "Imp_Impact_Node": "****", "Imp_Root_S_Node": "****", "Imp_Root_M_Node": "****", "Imp_Root_L_Node": "****", "Imp_Root_H_Node": "****", "ProgFX_Impact": "****", "SoundImpact": "****", "ProgFX_Duration": "****", "SoundDuration": "****", "ProgFX_Cessation": "****", "SoundCessastion": "****", "Ces_HeadCon_Node": "****", "Ces_Impact_Node": "****", "Ces_Root_S_Node": "****", "Ces_Root_M_Node": "****", "Ces_Root_L_Node": "****", "Ces_Root_H_Node": "****", "ShakeType": "****", "ShakeDelay": "****", "ShakeDuration": "****", "LowViolence": "****", "LowQuality": "****", "OrientWithObject": "0"}, + {"key": "visualeffects:com_hitnegative", "id": 288, "Label": "VFX_COM_HIT_NEGATIVE", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "Imp_Impact_Node": "****", "Imp_Root_S_Node": "****", "Imp_Root_M_Node": "****", "Imp_Root_L_Node": "****", "Imp_Root_H_Node": "****", "ProgFX_Impact": "****", "SoundImpact": "old_sound", "ProgFX_Duration": "****", "SoundDuration": "****", "ProgFX_Cessation": "****", "SoundCessastion": "****", "Ces_HeadCon_Node": "****", "Ces_Impact_Node": "****", "Ces_Root_S_Node": "****", "Ces_Root_M_Node": "****", "Ces_Root_L_Node": "****", "Ces_Root_H_Node": "****", "ShakeType": "****", "ShakeDelay": "****", "ShakeDuration": "****", "LowViolence": "****", "LowQuality": "****", "OrientWithObject": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "visualeffects", "lock.json"), `{ + "visualeffects:chatbubble": 10100, + "visualeffects:head_accessories_antlers_l": 10101, + "visualeffects:vcm_hitmasscrit": 10125, + "visualeffects:com_hitnegative": 288, + "visualeffects:fnf_blinddeaf": 18 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "visualeffects", "modules", "chatbubble.json"), `{ + "entries": { + "visualeffects:chatbubble": { + "Label": "VFX_DUR_CHAT_BUBBLE", + "Type_FD": "D", + "OrientWithGround": "0", + "Imp_HeadCon_Node": "vdr_chatbubble", + "Imp_Impact_Node": "****", + "Imp_Root_S_Node": "****", + "Imp_Root_M_Node": "****", + "Imp_Root_L_Node": "****", + "Imp_Root_H_Node": "****", + "ProgFX_Impact": "****", + "SoundImpact": "****", + "ProgFX_Duration": "****", + "SoundDuration": "****", + "ProgFX_Cessation": "****", + "SoundCessastion": "****", + "Ces_HeadCon_Node": "****", + "Ces_Impact_Node": "****", + "Ces_Root_S_Node": "****", + "Ces_Root_M_Node": "****", + "Ces_Root_L_Node": "****", + "Ces_Root_H_Node": "****", + "ShakeType": "****", + "ShakeDelay": "****", + "ShakeDuration": "****", + "LowViolence": "****", + "LowQuality": "****", + "OrientWithObject": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "visualeffects", "modules", "add_onhitvfx.json"), `{ + "entries": { + "visualeffects:vcm_hitmasscrit": { + "Label": "VFX_COM_HIT_MASSIVE_CRITICALS", + "Type_FD": "F", + "OrientWithGround": 0, + "Imp_Impact_Node": "vcm_hitmasscrit", + "OrientWithObject": 0 + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "visualeffects", "modules", "fix_comhitnegative.json"), `{ + "overrides": [ + {"key": "visualeffects:com_hitnegative", "id": 288, "SoundImpact": "scm_hitsmite"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "visualeffects", "modules", "ovr_lowqualitymodels.json"), `{ + "overrides": [ + {"key": "visualeffects:fnf_blinddeaf", "id": 18, "LowQuality": "vff_explblind_l"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "visualeffects", "modules", "accessories", "add_eos.json"), `{ + "entries": { + "visualeffects:head_accessories_antlers_l": { + "Label": "head_accessories_ANTLERS_L", + "Type_FD": "D", + "OrientWithGround": "0", + "Imp_HeadCon_Node": "hfx_antlers_l", + "Imp_Impact_Node": "****", + "Imp_Root_S_Node": "****", + "Imp_Root_M_Node": "****", + "Imp_Root_L_Node": "****", + "Imp_Root_H_Node": "****", + "ProgFX_Impact": "****", + "SoundImpact": "****", + "ProgFX_Duration": "****", + "SoundDuration": "****", + "ProgFX_Cessation": "****", + "SoundCessastion": "****", + "Ces_HeadCon_Node": "****", + "Ces_Impact_Node": "****", + "Ces_Root_S_Node": "****", + "Ces_Root_M_Node": "****", + "Ces_Root_L_Node": "****", + "Ces_Root_H_Node": "****", + "ShakeType": "****", + "ShakeDelay": "****", + "ShakeDuration": "****", + "LowViolence": "****", + "LowQuality": "****", + "OrientWithObject": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 6 { + t.Fatalf("expected visualeffects files to be imported, got %#v", result) + } + + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "visualeffects", "base.json"), + filepath.Join(root, "topdata", "data", "visualeffects", "lock.json"), + filepath.Join(root, "topdata", "data", "visualeffects", "modules", "add_onhitvfx.json"), + filepath.Join(root, "topdata", "data", "visualeffects", "modules", "chatbubble.json"), + filepath.Join(root, "topdata", "data", "visualeffects", "modules", "fix_comhitnegative.json"), + filepath.Join(root, "topdata", "data", "visualeffects", "modules", "ovr_lowqualitymodels.json"), + filepath.Join(root, "topdata", "data", "visualeffects", "modules", "accessories", "add_eos.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported visualeffects path %s: %v", path, err) + } + } + + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "visualeffects", "base.json")) + if err != nil { + t.Fatalf("read visualeffects base: %v", err) + } + if !strings.Contains(string(baseRaw), `"output": "visualeffects.2da"`) { + t.Fatalf("expected imported visualeffects output name, got:\n%s", string(baseRaw)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "visualeffects", "lock.json")) + if err != nil { + t.Fatalf("read visualeffects lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"visualeffects:chatbubble": 10100`) || !strings.Contains(lockText, `"visualeffects:head_accessories_antlers_l": 10101`) { + t.Fatalf("expected imported visualeffects lock IDs, got:\n%s", lockText) + } +} + +func TestBuildSupportsCanonicalVisualeffects(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "visualeffects", "modules", "accessories")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "visualeffects", "base.json"), `{ + "output": "visualeffects.2da", + "columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "Imp_Impact_Node", "Imp_Root_S_Node", "Imp_Root_M_Node", "Imp_Root_L_Node", "Imp_Root_H_Node", "ProgFX_Impact", "SoundImpact", "ProgFX_Duration", "SoundDuration", "ProgFX_Cessation", "SoundCessastion", "Ces_HeadCon_Node", "Ces_Impact_Node", "Ces_Root_S_Node", "Ces_Root_M_Node", "Ces_Root_L_Node", "Ces_Root_H_Node", "ShakeType", "ShakeDelay", "ShakeDuration", "LowViolence", "LowQuality", "OrientWithObject"], + "rows": [ + {"key": "visualeffects:fnf_blinddeaf", "id": 18, "Label": "VFX_FNF_BLINDDEAF", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "Imp_Impact_Node": "****", "Imp_Root_S_Node": "****", "Imp_Root_M_Node": "****", "Imp_Root_L_Node": "****", "Imp_Root_H_Node": "****", "ProgFX_Impact": "****", "SoundImpact": "****", "ProgFX_Duration": "****", "SoundDuration": "****", "ProgFX_Cessation": "****", "SoundCessastion": "****", "Ces_HeadCon_Node": "****", "Ces_Impact_Node": "****", "Ces_Root_S_Node": "****", "Ces_Root_M_Node": "****", "Ces_Root_L_Node": "****", "Ces_Root_H_Node": "****", "ShakeType": "****", "ShakeDelay": "****", "ShakeDuration": "****", "LowViolence": "****", "LowQuality": "****", "OrientWithObject": "0"}, + {"key": "visualeffects:com_hitnegative", "id": 288, "Label": "VFX_COM_HIT_NEGATIVE", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "Imp_Impact_Node": "****", "Imp_Root_S_Node": "****", "Imp_Root_M_Node": "****", "Imp_Root_L_Node": "****", "Imp_Root_H_Node": "****", "ProgFX_Impact": "****", "SoundImpact": "old_sound", "ProgFX_Duration": "****", "SoundDuration": "****", "ProgFX_Cessation": "****", "SoundCessastion": "****", "Ces_HeadCon_Node": "****", "Ces_Impact_Node": "****", "Ces_Root_S_Node": "****", "Ces_Root_M_Node": "****", "Ces_Root_L_Node": "****", "Ces_Root_H_Node": "****", "ShakeType": "****", "ShakeDelay": "****", "ShakeDuration": "****", "LowViolence": "****", "LowQuality": "****", "OrientWithObject": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "visualeffects", "lock.json"), `{ + "visualeffects:chatbubble": 10100, + "visualeffects:head_accessories_antlers_l": 10101, + "visualeffects:vcm_hitmasscrit": 10125, + "visualeffects:com_hitnegative": 288, + "visualeffects:fnf_blinddeaf": 18 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "visualeffects", "modules", "chatbubble.json"), `{ + "entries": { + "visualeffects:chatbubble": { + "Label": "VFX_DUR_CHAT_BUBBLE", + "Type_FD": "D", + "OrientWithGround": "0", + "Imp_HeadCon_Node": "vdr_chatbubble", + "Imp_Impact_Node": "****", + "Imp_Root_S_Node": "****", + "Imp_Root_M_Node": "****", + "Imp_Root_L_Node": "****", + "Imp_Root_H_Node": "****", + "ProgFX_Impact": "****", + "SoundImpact": "****", + "ProgFX_Duration": "****", + "SoundDuration": "****", + "ProgFX_Cessation": "****", + "SoundCessastion": "****", + "Ces_HeadCon_Node": "****", + "Ces_Impact_Node": "****", + "Ces_Root_S_Node": "****", + "Ces_Root_M_Node": "****", + "Ces_Root_L_Node": "****", + "Ces_Root_H_Node": "****", + "ShakeType": "****", + "ShakeDelay": "****", + "ShakeDuration": "****", + "LowViolence": "****", + "LowQuality": "****", + "OrientWithObject": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "visualeffects", "modules", "add_onhitvfx.json"), `{ + "entries": { + "visualeffects:vcm_hitmasscrit": { + "Label": "VFX_COM_HIT_MASSIVE_CRITICALS", + "Type_FD": "F", + "OrientWithGround": 0, + "Imp_Impact_Node": "vcm_hitmasscrit", + "OrientWithObject": 0 + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "visualeffects", "modules", "fix_comhitnegative.json"), `{ + "overrides": [ + {"key": "visualeffects:com_hitnegative", "id": 288, "SoundImpact": "scm_hitsmite"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "visualeffects", "modules", "ovr_lowqualitymodels.json"), `{ + "overrides": [ + {"key": "visualeffects:fnf_blinddeaf", "id": 18, "LowQuality": "vff_explblind_l"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "visualeffects", "modules", "accessories", "add_eos.json"), `{ + "entries": { + "visualeffects:head_accessories_antlers_l": { + "Label": "head_accessories_ANTLERS_L", + "Type_FD": "D", + "OrientWithGround": "0", + "Imp_HeadCon_Node": "hfx_antlers_l", + "Imp_Impact_Node": "****", + "Imp_Root_S_Node": "****", + "Imp_Root_M_Node": "****", + "Imp_Root_L_Node": "****", + "Imp_Root_H_Node": "****", + "ProgFX_Impact": "****", + "SoundImpact": "****", + "ProgFX_Duration": "****", + "SoundDuration": "****", + "ProgFX_Cessation": "****", + "SoundCessastion": "****", + "Ces_HeadCon_Node": "****", + "Ces_Impact_Node": "****", + "Ces_Root_S_Node": "****", + "Ces_Root_M_Node": "****", + "Ces_Root_L_Node": "****", + "Ces_Root_H_Node": "****", + "ShakeType": "****", + "ShakeDelay": "****", + "ShakeDuration": "****", + "LowViolence": "****", + "LowQuality": "****", + "OrientWithObject": "0" + } + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "visualeffects.2da")) + if err != nil { + t.Fatalf("read visualeffects.2da: %v", err) + } + text := string(got) + for _, want := range []string{ + "18\tVFX_FNF_BLINDDEAF\tF\t0", + "288\tVFX_COM_HIT_NEGATIVE\tF\t0", + "10100\tVFX_DUR_CHAT_BUBBLE\tD\t0", + "10101\thead_accessories_ANTLERS_L\tD\t0", + "10125\tVFX_COM_HIT_MASSIVE_CRITICALS\tF\t0", + "scm_hitsmite", + "vff_explblind_l", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected visualeffects.2da to contain %q, got:\n%s", want, text) + } + } +} + +func TestBuildCanonicalVisualeffectsMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "visualeffects", "modules", "accessories")) + mkdirAll(t, filepath.Join(root, "reference", "data", "visualeffects", "modules", "accessories")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "visualeffects.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tType_FD\tOrientWithGround\tImp_HeadCon_Node\tImp_Impact_Node\tImp_Root_S_Node\tImp_Root_M_Node\tImp_Root_L_Node\tImp_Root_H_Node\tProgFX_Impact\tSoundImpact\tProgFX_Duration\tSoundDuration\tProgFX_Cessation\tSoundCessastion\tCes_HeadCon_Node\tCes_Impact_Node\tCes_Root_S_Node\tCes_Root_M_Node\tCes_Root_L_Node\tCes_Root_H_Node\tShakeType\tShakeDelay\tShakeDuration\tLowViolence\tLowQuality\tOrientWithObject\n") + f.write("18\tVFX_FNF_BLINDDEAF\tF\t0\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\tvff_explblind_l\t0\n") + f.write("288\tVFX_COM_HIT_NEGATIVE\tF\t0\t****\t****\t****\t****\t****\t****\tscm_hitsmite\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t0\n") + f.write("10100\tVFX_DUR_CHAT_BUBBLE\tD\t0\tvdr_chatbubble\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t0\n") + f.write("10101\thead_accessories_ANTLERS_L\tD\t0\thfx_antlers_l\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t0\n") + f.write("10125\tVFX_COM_HIT_MASSIVE_CRITICALS\tF\t0\t****\tvcm_hitmasscrit\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t0\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`+"\n") + for _, target := range []string{"topdata", "reference"} { + writeFile(t, filepath.Join(root, target, "data", "visualeffects", "base.json"), `{ + "output": "visualeffects.2da", + "columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "Imp_Impact_Node", "Imp_Root_S_Node", "Imp_Root_M_Node", "Imp_Root_L_Node", "Imp_Root_H_Node", "ProgFX_Impact", "SoundImpact", "ProgFX_Duration", "SoundDuration", "ProgFX_Cessation", "SoundCessastion", "Ces_HeadCon_Node", "Ces_Impact_Node", "Ces_Root_S_Node", "Ces_Root_M_Node", "Ces_Root_L_Node", "Ces_Root_H_Node", "ShakeType", "ShakeDelay", "ShakeDuration", "LowViolence", "LowQuality", "OrientWithObject"], + "rows": [ + {"key": "visualeffects:fnf_blinddeaf", "id": 18, "Label": "VFX_FNF_BLINDDEAF", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "Imp_Impact_Node": "****", "Imp_Root_S_Node": "****", "Imp_Root_M_Node": "****", "Imp_Root_L_Node": "****", "Imp_Root_H_Node": "****", "ProgFX_Impact": "****", "SoundImpact": "****", "ProgFX_Duration": "****", "SoundDuration": "****", "ProgFX_Cessation": "****", "SoundCessastion": "****", "Ces_HeadCon_Node": "****", "Ces_Impact_Node": "****", "Ces_Root_S_Node": "****", "Ces_Root_M_Node": "****", "Ces_Root_L_Node": "****", "Ces_Root_H_Node": "****", "ShakeType": "****", "ShakeDelay": "****", "ShakeDuration": "****", "LowViolence": "****", "LowQuality": "****", "OrientWithObject": "0"}, + {"key": "visualeffects:com_hitnegative", "id": 288, "Label": "VFX_COM_HIT_NEGATIVE", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "Imp_Impact_Node": "****", "Imp_Root_S_Node": "****", "Imp_Root_M_Node": "****", "Imp_Root_L_Node": "****", "Imp_Root_H_Node": "****", "ProgFX_Impact": "****", "SoundImpact": "old_sound", "ProgFX_Duration": "****", "SoundDuration": "****", "ProgFX_Cessation": "****", "SoundCessastion": "****", "Ces_HeadCon_Node": "****", "Ces_Impact_Node": "****", "Ces_Root_S_Node": "****", "Ces_Root_M_Node": "****", "Ces_Root_L_Node": "****", "Ces_Root_H_Node": "****", "ShakeType": "****", "ShakeDelay": "****", "ShakeDuration": "****", "LowViolence": "****", "LowQuality": "****", "OrientWithObject": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, target, "data", "visualeffects", "lock.json"), `{ + "visualeffects:chatbubble": 10100, + "visualeffects:head_accessories_antlers_l": 10101, + "visualeffects:vcm_hitmasscrit": 10125, + "visualeffects:com_hitnegative": 288, + "visualeffects:fnf_blinddeaf": 18 +}`+"\n") + writeFile(t, filepath.Join(root, target, "data", "visualeffects", "modules", "chatbubble.json"), `{ + "entries": { + "visualeffects:chatbubble": { + "Label": "VFX_DUR_CHAT_BUBBLE", + "Type_FD": "D", + "OrientWithGround": "0", + "Imp_HeadCon_Node": "vdr_chatbubble", + "Imp_Impact_Node": "****", + "Imp_Root_S_Node": "****", + "Imp_Root_M_Node": "****", + "Imp_Root_L_Node": "****", + "Imp_Root_H_Node": "****", + "ProgFX_Impact": "****", + "SoundImpact": "****", + "ProgFX_Duration": "****", + "SoundDuration": "****", + "ProgFX_Cessation": "****", + "SoundCessastion": "****", + "Ces_HeadCon_Node": "****", + "Ces_Impact_Node": "****", + "Ces_Root_S_Node": "****", + "Ces_Root_M_Node": "****", + "Ces_Root_L_Node": "****", + "Ces_Root_H_Node": "****", + "ShakeType": "****", + "ShakeDelay": "****", + "ShakeDuration": "****", + "LowViolence": "****", + "LowQuality": "****", + "OrientWithObject": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, target, "data", "visualeffects", "modules", "add_onhitvfx.json"), `{ + "entries": { + "visualeffects:vcm_hitmasscrit": { + "Label": "VFX_COM_HIT_MASSIVE_CRITICALS", + "Type_FD": "F", + "OrientWithGround": 0, + "Imp_Impact_Node": "vcm_hitmasscrit", + "OrientWithObject": 0 + } + } +}`+"\n") + writeFile(t, filepath.Join(root, target, "data", "visualeffects", "modules", "fix_comhitnegative.json"), `{ + "overrides": [ + {"key": "visualeffects:com_hitnegative", "id": 288, "SoundImpact": "scm_hitsmite"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, target, "data", "visualeffects", "modules", "ovr_lowqualitymodels.json"), `{ + "overrides": [ + {"key": "visualeffects:fnf_blinddeaf", "id": 18, "LowQuality": "vff_explblind_l"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, target, "data", "visualeffects", "modules", "accessories", "add_eos.json"), `{ + "entries": { + "visualeffects:head_accessories_antlers_l": { + "Label": "head_accessories_ANTLERS_L", + "Type_FD": "D", + "OrientWithGround": "0", + "Imp_HeadCon_Node": "hfx_antlers_l", + "Imp_Impact_Node": "****", + "Imp_Root_S_Node": "****", + "Imp_Root_M_Node": "****", + "Imp_Root_L_Node": "****", + "Imp_Root_H_Node": "****", + "ProgFX_Impact": "****", + "SoundImpact": "****", + "ProgFX_Duration": "****", + "SoundDuration": "****", + "ProgFX_Cessation": "****", + "SoundCessastion": "****", + "Ces_HeadCon_Node": "****", + "Ces_Impact_Node": "****", + "Ces_Root_S_Node": "****", + "Ces_Root_M_Node": "****", + "Ces_Root_L_Node": "****", + "Ces_Root_H_Node": "****", + "ShakeType": "****", + "ShakeDelay": "****", + "ShakeDuration": "****", + "LowViolence": "****", + "LowQuality": "****", + "OrientWithObject": "0" + } + } +}`+"\n") + } + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "visualeffects.2da")) + if err != nil { + t.Fatalf("read native visualeffects.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "visualeffects.2da")) + if err != nil { + t.Fatalf("read reference visualeffects.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("visualeffects output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestApplyAutogenConsumersAugmentsPartsFromLocalOverride(t *testing.T) { + root := testProjectRoot(t) + overrideRoot := filepath.Join(root, "autogen-assets") + mkdirAll(t, filepath.Join(overrideRoot, "part", "belt")) + writeFile(t, filepath.Join(overrideRoot, "part", "belt", "pfa0_belt018.mdl"), "mdl\n") + + p := testProject(root) + p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{ + { + ID: "parts", + Producer: "parts", + Dataset: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: project.AutogenDeriveConfig{Kind: "trailing_numeric_suffix", GroupFrom: "first_path_segment"}, + Manifest: project.AutogenManifestConfig{ReleaseTag: "parts-manifest-current", AssetName: "sow-parts-manifest.json", CacheName: "sow-parts-manifest.json"}, + LocalOverrideRoot: "autogen-assets", + }, + } + + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{Name: "parts/belt"}, + Columns: []string{"COSTMODIFIER", "ACBONUS"}, + Rows: []map[string]any{ + {"id": 0, "COSTMODIFIER": "0", "ACBONUS": "0.00"}, + }, + }, + } + + got, err := applyAutogenConsumers(p, collected, nil) + if err != nil { + t.Fatalf("applyAutogenConsumers failed: %v", err) + } + if len(got) != 1 || len(got[0].Rows) != 2 { + t.Fatalf("expected autogenerated part row, got %#v", got) + } + row := got[0].Rows[1] + if row["id"] != 18 || row["COSTMODIFIER"] != "0" || row["ACBONUS"] != "0.00" { + t.Fatalf("unexpected autogenerated part row: %#v", row) + } +} + +func TestApplyAutogenConsumersAugmentsHeadVisualeffectsFromLocalOverride(t *testing.T) { + root := testProjectRoot(t) + overrideRoot := filepath.Join(root, "autogen-assets") + mkdirAll(t, filepath.Join(overrideRoot, "vfxs", "head_accessories", "hat")) + mkdirAll(t, filepath.Join(overrideRoot, "vfxs", "head_decorations", "laurel")) + mkdirAll(t, filepath.Join(overrideRoot, "vfxs", "head_features", "hair")) + writeFile(t, filepath.Join(overrideRoot, "vfxs", "head_accessories", "hat", "hfx_bandana.mdl"), "mdl\n") + writeFile(t, filepath.Join(overrideRoot, "vfxs", "head_decorations", "laurel", "hfx_laurel.mdl"), "mdl\n") + writeFile(t, filepath.Join(overrideRoot, "vfxs", "head_features", "hair", "hfx_hair_bangs.mdl"), "mdl\n") + + p := testProject(root) + p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{ + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "accessory_visualeffects", + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_decorations/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"}, + Manifest: project.AutogenManifestConfig{ReleaseTag: "head-vfx-manifest-current", AssetName: "sow-accessory-vfx-manifest.json", CacheName: "sow-accessory-vfx-manifest.json"}, + LocalOverrideRoot: "autogen-assets", + }, + } + + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase}, + Columns: []string{"Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "OrientWithObject"}, + Rows: nil, + LockData: map[string]int{ + "visualeffects:existing": 17, + }, + }, + } + + got, err := applyAutogenConsumers(p, collected, nil) + if err != nil { + t.Fatalf("applyAutogenConsumers failed: %v", err) + } + if len(got) != 1 || len(got[0].Rows) != 3 { + t.Fatalf("expected autogenerated visualeffects rows, got %#v", got) + } + + rowByKey := map[string]map[string]any{} + for _, row := range got[0].Rows { + rowByKey[row["key"].(string)] = row + } + + bandana := rowByKey["visualeffects:head_accessories/hat/bandana"] + if bandana == nil { + t.Fatalf("expected head accessory row, got %#v", got[0].Rows) + } + if bandana["Label"] != "head_accessories/hat/bandana" || bandana["Imp_HeadCon_Node"] != "hfx_bandana" || bandana["OrientWithObject"] != "1" { + t.Fatalf("unexpected head accessory defaults: %#v", bandana) + } + + laurel := rowByKey["visualeffects:head_decorations/laurel/laurel"] + if laurel == nil { + t.Fatalf("expected head decoration row, got %#v", got[0].Rows) + } + if laurel["Label"] != "head_decorations/laurel/laurel" || laurel["Imp_HeadCon_Node"] != "hfx_laurel" || laurel["Type_FD"] != "D" { + t.Fatalf("unexpected head decoration defaults: %#v", laurel) + } + + hair := rowByKey["visualeffects:head_features/hair/hair_bangs"] + if hair == nil { + t.Fatalf("expected head feature row, got %#v", got[0].Rows) + } + if hair["Label"] != "head_features/hair/hair_bangs" || hair["Imp_HeadCon_Node"] != "hfx_hair_bangs" || hair["Type_FD"] != "D" { + t.Fatalf("unexpected head feature defaults: %#v", hair) + } + + if _, ok := got[0].LockData["visualeffects:head_accessories/hat/bandana"]; !ok { + t.Fatalf("expected head accessory lock id, got %#v", got[0].LockData) + } + if _, ok := got[0].LockData["visualeffects:head_decorations/laurel/laurel"]; !ok { + t.Fatalf("expected head decoration lock id, got %#v", got[0].LockData) + } + if _, ok := got[0].LockData["visualeffects:head_features/hair/hair_bangs"]; !ok { + t.Fatalf("expected head feature lock id, got %#v", got[0].LockData) + } +} + +func TestApplyAutogenConsumersUsesFolderDrivenSlashHeadVisualeffectsNames(t *testing.T) { + root := testProjectRoot(t) + overrideRoot := filepath.Join(root, "autogen-assets") + mkdirAll(t, filepath.Join(overrideRoot, "vfxs", "head_features", "sinfar", "ears_plt")) + writeFile(t, filepath.Join(overrideRoot, "vfxs", "head_features", "sinfar", "ears_plt", "hfx_Ear_EL_L.mdl"), "mdl\n") + + p := testProject(root) + p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{ + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "accessory_visualeffects", + Root: "vfxs", + Include: []string{"head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"}, + Manifest: project.AutogenManifestConfig{ReleaseTag: "head-vfx-manifest-current", AssetName: "sow-accessory-vfx-manifest.json", CacheName: "sow-accessory-vfx-manifest.json"}, + LocalOverrideRoot: "autogen-assets", + HeadVisualeffects: project.HeadVisualeffectsConfig{ + Groups: map[string]project.HeadVisualeffectGroupConfig{ + "head_features": { + ModelColumns: []string{"Imp_HeadCon_Node", "Imp_Root_H_Node"}, + }, + }, + GroupTokenSource: "folder_name", + CategoryFrom: "immediate_parent", + Delimiter: "/", + Case: "preserve", + StripModelPrefixes: []string{"hfx_"}, + KeyFormat: "{dataset}:{group}{delimiter}{category_segment}{stem}", + LabelFormat: "{group}{delimiter}{category_segment}{stem}", + RowDefaults: map[string]string{ + "Type_FD": "D", + "OrientWithGround": "0", + "OrientWithObject": "1", + }, + }, + }, + } + + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase}, + Columns: []string{"Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "Imp_Root_H_Node", "OrientWithObject"}, + Rows: nil, + LockData: map[string]int{ + "visualeffects:head_features/ears_plt/Ear_EL_L": 11030, + }, + }, + } + + got, err := applyAutogenConsumers(p, collected, nil) + if err != nil { + t.Fatalf("applyAutogenConsumers failed: %v", err) + } + if len(got) != 1 || len(got[0].Rows) != 1 { + t.Fatalf("expected one autogenerated visualeffects row, got %#v", got) + } + row := got[0].Rows[0] + if row["key"] != "visualeffects:head_features/ears_plt/Ear_EL_L" { + t.Fatalf("expected folder-driven key, got %#v", row) + } + if row["id"] != 11030 { + t.Fatalf("expected existing lock id to be preserved, got %#v", row) + } + if row["Label"] != "head_features/ears_plt/Ear_EL_L" { + t.Fatalf("expected preserve-case slash label, got %#v", row) + } + if row["Imp_HeadCon_Node"] != "hfx_Ear_EL_L" || row["Imp_Root_H_Node"] != "hfx_Ear_EL_L" { + t.Fatalf("expected configured model columns, got %#v", row) + } +} + +func TestBuildNativeUsesConfiguredHeadVisualeffectsGroups(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "visualeffects")) + mkdirAll(t, filepath.Join(root, "autogen-assets", "vfxs", "chest_accessories")) + mkdirAll(t, filepath.Join(root, "autogen-assets", "vfxs", "head_jewels")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "visualeffects", "base.json"), `{ + "output": "visualeffects.2da", + "columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "Imp_Root_S_Node", "Imp_Root_M_Node", "Imp_Root_L_Node", "Imp_Root_H_Node", "OrientWithObject"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "autogen-assets", "vfxs", "chest_accessories", "tfx_sash.mdl"), "mdl\n") + writeFile(t, filepath.Join(root, "autogen-assets", "vfxs", "head_jewels", "hfx_circlet.mdl"), "mdl\n") + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test + resref: test +paths: + source: src + build: build +topdata: + source: topdata + build: build/topdata +autogen: + consumers: + - id: accessory_visualeffects + producer: accessory_visualeffects + dataset: visualeffects + mode: accessory_visualeffects + root: vfxs + include: + - chest_accessories/**/*.mdl + - head_jewels/**/*.mdl + derive: + kind: model_stem + group_from: first_path_segment + manifest: + release_tag: head-vfx-manifest-current + asset_name: sow-accessory-vfx-manifest.json + cache_name: sow-accessory-vfx-manifest.json + local_override_root: autogen-assets + accessory_visualeffects: + groups: + chest_accessories: + prefix: chest_accessories + model_columns: + - Imp_Root_S_Node + - Imp_Root_M_Node + - Imp_Root_L_Node + - Imp_Root_H_Node + head_jewels: + prefix: headjewel + label_format: "{prefix}_{stem}" + strip_model_prefixes: + - hfx_ + - tfx_ + row_defaults: + Type_FD: F + OrientWithGround: "0" + OrientWithObject: "0" +`) + + proj, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + visualeffectsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "visualeffects.2da")) + if err != nil { + t.Fatalf("read visualeffects.2da: %v", err) + } + text := string(visualeffectsBytes) + for _, want := range []string{ + "0\tchest_accessories_sash\tF\t0\t****\ttfx_sash\ttfx_sash\ttfx_sash\ttfx_sash\t0", + "1\theadjewel_circlet\tF\t0\thfx_circlet\t****\t****\t****\t****\t0", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected configured visualeffects output to contain %q, got:\n%s", want, text) + } + } +} + +func TestBuildGenerated2DAAssetsAppendsCachedModelsFromHeadVisualeffects(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "assets", "vfxs", "chest_accessories")) + mkdirAll(t, filepath.Join(root, "assets", "vfxs", "head_accessories")) + mkdirAll(t, filepath.Join(root, "assets", "vfxs", "head_decorations")) + mkdirAll(t, filepath.Join(root, "assets", "vfxs", "head_features", "hair")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "cachedmodels")) + writeFile(t, filepath.Join(root, "assets", "vfxs", "chest_accessories", "tfx_sash.mdl"), "mdl\n") + writeFile(t, filepath.Join(root, "assets", "vfxs", "head_accessories", "hfx_bandana.mdl"), "mdl\n") + writeFile(t, filepath.Join(root, "assets", "vfxs", "head_accessories", "hfx_existing.mdl"), "mdl\n") + writeFile(t, filepath.Join(root, "assets", "vfxs", "head_decorations", "hfx_laurel.mdl"), "mdl\n") + writeFile(t, filepath.Join(root, "assets", "vfxs", "head_features", "hair", "hfx_hair_bangs.mdl"), "mdl\n") + writeFile(t, filepath.Join(root, "topdata", "data", "cachedmodels", "base.json"), `{ + "columns": ["Model"], + "rows": [ + {"id": 0, "Model": "grn_m_bone"}, + {"id": 1, "Model": "hfx_existing"} + ] +}`+"\n") + + p := testProject(root) + p.Config.Paths.Assets = "assets" + p.Config.Generated.TopData2DA = []project.GeneratedTopData2DAConfig{ + { + ID: "cachedmodels", + Source: "topdata", + Output: ".cache/generated-assets/cachedmodels-2da", + IncludeDatasets: []string{"cachedmodels"}, + PackageRoot: "vfxs", + Autogen: project.AutogenConsumerConfig{ + ID: "cachedmodels", + Mode: "cachedmodels_rows", + Root: "vfxs", + Include: []string{"chest_accessories/**/*.mdl", "head_accessories/**/*.mdl", "head_decorations/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"}, + }, + }, + } + + assets, err := BuildGenerated2DAAssets(p, nil) + if err != nil { + t.Fatalf("BuildGenerated2DAAssets failed: %v", err) + } + if len(assets) != 1 { + t.Fatalf("expected one generated asset, got %#v", assets) + } + if assets[0].Rel != "vfxs/cachedmodels.2da" { + t.Fatalf("expected vfxs/cachedmodels.2da rel, got %q", assets[0].Rel) + } + bytes, err := os.ReadFile(assets[0].SourcePath) + if err != nil { + t.Fatalf("read generated cachedmodels.2da: %v", err) + } + text := string(bytes) + for _, want := range []string{ + "0\tgrn_m_bone", + "1\thfx_existing", + "2\ttfx_sash", + "3\thfx_bandana", + "4\thfx_laurel", + "5\thfx_hair_bangs", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected generated cachedmodels output to contain %q, got:\n%s", want, text) + } + } + if strings.Contains(text, ".mdl") { + t.Fatalf("expected cached model rows without .mdl extension, got:\n%s", text) + } + if strings.Count(text, "hfx_existing") != 1 { + t.Fatalf("expected duplicate discovered model to be skipped, got:\n%s", text) + } +} + +func TestBuildGenerated2DAAssetsWritesLockfilesForSelectedDatasets(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "cachedmodels", "modules")) + writeFile(t, filepath.Join(root, "topdata", "data", "cachedmodels", "base.json"), `{ + "columns": ["Model"], + "rows": [ + {"id": 0, "key": "cachedmodels:base_model", "Model": "base_model"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "cachedmodels", "modules", "add_chatbubble.json"), `{ + "entries": { + "cachedmodels:vdr_chatbubble": { + "Model": "vdr_chatbubble" + } + } +}`+"\n") + + p := testProject(root) + p.Config.Paths.Assets = "assets" + p.Config.Generated.TopData2DA = []project.GeneratedTopData2DAConfig{ + { + ID: "cachedmodels", + Source: "topdata", + Output: ".cache/generated-assets/cachedmodels-2da", + IncludeDatasets: []string{"cachedmodels"}, + PackageRoot: "vfxs", + }, + } + + if _, err := BuildGenerated2DAAssets(p, nil); err != nil { + t.Fatalf("BuildGenerated2DAAssets failed: %v", err) + } + + lockBytes, err := os.ReadFile(filepath.Join(root, "topdata", "data", "cachedmodels", "lock.json")) + if err != nil { + t.Fatalf("read generated cachedmodels lockfile: %v", err) + } + lockText := string(lockBytes) + for _, want := range []string{ + `"cachedmodels:base_model": 0`, + `"cachedmodels:vdr_chatbubble": 1`, + } { + if !strings.Contains(lockText, want) { + t.Fatalf("expected generated asset lockfile to contain %q, got:\n%s", want, lockText) + } + } +} + +func TestNormalizeProjectImportsLegacyLoadscreens(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference", "data", "loadscreens", "modules")) + writeFile(t, filepath.Join(root, "reference", "data", "loadscreens", "base.json"), `{ + "columns": ["Label", "ScriptingName", "BMPResRef", "TileSet", "StrRef"], + "rows": [ + {"id": 0, "Label": "Random", "ScriptingName": "****", "BMPResRef": "****", "TileSet": "****", "StrRef": "63402"}, + {"id": 333, "Label": "Daggerford12", "ScriptingName": "AREA_TRANSITION_DAG_12", "BMPResRef": "LS_DAG_12", "TileSet": "TMS01", "StrRef": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "loadscreens", "lock.json"), `{ + "loadscreens:vdungeon01": 334, + "loadscreens:tcd_01": 335, + "loadscreens:ori_01": 336, + "loadscreens:ori_02": 337, + "loadscreens:ori_03": 338, + "loadscreens:ori_04": 339, + "loadscreens:ori_05": 340 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "loadscreens", "modules", "add_lsvd.json"), `{ + "entries": { + "loadscreens:vdungeon01": { + "Label": "VDungeon01", + "ScriptingName": "AREA_TRANSITION_VDUNGEON_01", + "BMPResRef": "LS_VD_01", + "TileSet": "TVD" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "loadscreens", "modules", "add_ori.json"), `{ + "entries": { + "loadscreens:ori_01": { + "Label": "Bamboo", + "ScriptingName": "AREA_TRANSITION_RURAL_06", + "BMPResRef": "LS_Ori_01", + "TileSet": "TTR01" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "loadscreens", "modules", "add_tcd01.json"), `{ + "entries": { + "loadscreens:tcd_01": { + "Label": "cd01", + "ScriptingName": "AREA_TRANSITION_CDUNGEON_01", + "BMPResRef": "ls_tcd_01", + "TileSet": "TCD01", + "StrRef": 68555 + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 5 { + t.Fatalf("expected loadscreens files to be imported, got %#v", result) + } + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "loadscreens", "base.json"), + filepath.Join(root, "topdata", "data", "loadscreens", "lock.json"), + filepath.Join(root, "topdata", "data", "loadscreens", "modules", "add_lsvd.json"), + filepath.Join(root, "topdata", "data", "loadscreens", "modules", "add_ori.json"), + filepath.Join(root, "topdata", "data", "loadscreens", "modules", "add_tcd01.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported loadscreens path %s: %v", path, err) + } + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "loadscreens", "lock.json")) + if err != nil { + t.Fatalf("read loadscreens lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"loadscreens:ori_05": 340`) { + t.Fatalf("expected imported loadscreens lock IDs, got:\n%s", string(lockRaw)) + } +} + +func TestBuildSupportsCanonicalLoadscreens(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "loadscreens", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "loadscreens", "base.json"), `{ + "columns": ["Label", "ScriptingName", "BMPResRef", "TileSet", "StrRef"], + "rows": [ + {"id": 0, "Label": "Random", "ScriptingName": "****", "BMPResRef": "****", "TileSet": "****", "StrRef": "63402"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "loadscreens", "lock.json"), `{ + "loadscreens:vdungeon01": 334 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "loadscreens", "modules", "add_lsvd.json"), `{ + "entries": { + "loadscreens:vdungeon01": { + "Label": "VDungeon01", + "ScriptingName": "AREA_TRANSITION_VDUNGEON_01", + "BMPResRef": "LS_VD_01", + "TileSet": "TVD" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "loadscreens", "modules", "add_ori.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "loadscreens", "modules", "add_tcd01.json"), `{"entries": {}}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "loadscreens.2da")) + if err != nil { + t.Fatalf("read loadscreens.2da: %v", err) + } + if !strings.Contains(string(got), "Label\tScriptingName\tBMPResRef\tTileSet\tStrRef\n") || + !strings.Contains(string(got), "0\tRandom\t****\t****\t****\t63402\n") || + !strings.Contains(string(got), "334\tVDungeon01\tAREA_TRANSITION_VDUNGEON_01\tLS_VD_01\tTVD\t****\n") { + t.Fatalf("unexpected loadscreens.2da output:\n%s", string(got)) + } +} + +func TestBuildCanonicalLoadscreensMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "loadscreens", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "loadscreens.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tScriptingName\tBMPResRef\tTileSet\tStrRef\n") + f.write("0\tRandom\t****\t****\t****\t63402\n") + for row_id in range(1, 334): + f.write(f"{row_id}\t****\t****\t****\t****\t****\n") + f.write("334\tVDungeon01\tAREA_TRANSITION_VDUNGEON_01\tLS_VD_01\tTVD\t****\n") + f.write("335\tcd01\tAREA_TRANSITION_CDUNGEON_01\tls_tcd_01\tTCD01\t68555\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "loadscreens", "base.json"), `{ + "columns": ["Label", "ScriptingName", "BMPResRef", "TileSet", "StrRef"], + "rows": [ + {"id": 0, "Label": "Random", "ScriptingName": "****", "BMPResRef": "****", "TileSet": "****", "StrRef": "63402"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "loadscreens", "lock.json"), `{ + "loadscreens:vdungeon01": 334, + "loadscreens:tcd_01": 335 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "loadscreens", "modules", "add_lsvd.json"), `{ + "entries": { + "loadscreens:vdungeon01": { + "Label": "VDungeon01", + "ScriptingName": "AREA_TRANSITION_VDUNGEON_01", + "BMPResRef": "LS_VD_01", + "TileSet": "TVD" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "loadscreens", "modules", "add_ori.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "loadscreens", "modules", "add_tcd01.json"), `{ + "entries": { + "loadscreens:tcd_01": { + "Label": "cd01", + "ScriptingName": "AREA_TRANSITION_CDUNGEON_01", + "BMPResRef": "ls_tcd_01", + "TileSet": "TCD01", + "StrRef": 68555 + } + } +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "loadscreens.2da")) + if err != nil { + t.Fatalf("read native loadscreens.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "loadscreens.2da")) + if err != nil { + t.Fatalf("read reference loadscreens.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("loadscreens output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeImportsLegacyGenericdoors(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata")) + mkdirAll(t, filepath.Join(root, "reference", "data", "genericdoors", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "data", "genericdoors", "base.json"), `{ + "columns": ["Label", "StrRef", "ModelName", "BlockSight", "VisibleModel", "SoundAppType", "Name"], + "rows": [ + {"id": 0, "Label": "Wood_strong", "StrRef": "5349", "ModelName": "T_DOOR01", "BlockSight": "1", "VisibleModel": "1", "SoundAppType": "0", "Name": "63750"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "genericdoors", "lock.json"), `{ + "babayaga:hs_door60": 3200, + "orientaltileset:t_Door12": 3202 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "genericdoors", "modules", "add_mb017.json"), `{ + "entries": { + "babayaga:hs_door60": { + "Label": "Door60", + "StrRef": "****", + "ModelName": "hs_door60", + "BlockSight": "1", + "VisibleModel": "1", + "SoundAppType": "0", + "Name": "****" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "genericdoors", "modules", "add_ori.json"), `{ + "entries": { + "orientaltileset:t_Door12": { + "Label": "Door12", + "StrRef": "5349", + "ModelName": "t_Door12", + "BlockSight": "1", + "VisibleModel": "1", + "SoundAppType": "0", + "Name": "14445" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 4 { + t.Fatalf("expected genericdoors files to be imported, got %#v", result) + } + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "genericdoors", "base.json"), + filepath.Join(root, "topdata", "data", "genericdoors", "lock.json"), + filepath.Join(root, "topdata", "data", "genericdoors", "modules", "add_mb017.json"), + filepath.Join(root, "topdata", "data", "genericdoors", "modules", "add_ori.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported genericdoors path %s: %v", path, err) + } + } + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "genericdoors", "base.json")) + if err != nil { + t.Fatalf("read genericdoors base: %v", err) + } + if !strings.Contains(string(baseRaw), `"output": "genericdoors.2da"`) { + t.Fatalf("expected imported genericdoors output name, got:\n%s", string(baseRaw)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "genericdoors", "lock.json")) + if err != nil { + t.Fatalf("read genericdoors lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"orientaltileset:t_Door12": 3202`) { + t.Fatalf("expected imported genericdoors lock IDs, got:\n%s", string(lockRaw)) + } +} + +func TestBuildSupportsCanonicalGenericdoors(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "genericdoors", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "genericdoors", "base.json"), `{ + "output": "genericdoors.2da", + "columns": ["Label", "StrRef", "ModelName", "BlockSight", "VisibleModel", "SoundAppType", "Name"], + "rows": [ + {"id": 0, "Label": "Wood_strong", "StrRef": "5349", "ModelName": "T_DOOR01", "BlockSight": "1", "VisibleModel": "1", "SoundAppType": "0", "Name": "63750"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "genericdoors", "lock.json"), `{ + "babayaga:hs_door60": 3200 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "genericdoors", "modules", "add_mb017.json"), `{ + "entries": { + "babayaga:hs_door60": { + "Label": "Door60", + "StrRef": "****", + "ModelName": "hs_door60", + "BlockSight": "1", + "VisibleModel": "1", + "SoundAppType": "0", + "Name": "****" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "genericdoors", "modules", "add_ori.json"), `{"entries": {}}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "genericdoors.2da")) + if err != nil { + t.Fatalf("read genericdoors.2da: %v", err) + } + if !strings.Contains(string(got), "Label\tStrRef\tModelName\tBlockSight\tVisibleModel\tSoundAppType\tName\n") || + !strings.Contains(string(got), "0\tWood_strong\t5349\tT_DOOR01\t1\t1\t0\t63750\n") || + !strings.Contains(string(got), "3200\tDoor60\t****\ths_door60\t1\t1\t0\t****\n") { + t.Fatalf("unexpected genericdoors.2da output:\n%s", string(got)) + } +} + +func TestBuildCanonicalGenericdoorsMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "genericdoors", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "genericdoors.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tStrRef\tModelName\tBlockSight\tVisibleModel\tSoundAppType\tName\n") + f.write("0\tWood_strong\t5349\tT_DOOR01\t1\t1\t0\t63750\n") + for row_id in range(1, 3200): + f.write(f"{row_id}\t****\t****\t****\t****\t****\t****\t****\n") + f.write("3200\tDoor60\t****\ths_door60\t1\t1\t0\t****\n") + f.write("3201\t****\t****\t****\t****\t****\t****\t****\n") + f.write("3202\tDoor12\t5349\tt_Door12\t1\t1\t0\t14445\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "genericdoors", "base.json"), `{ + "columns": ["Label", "StrRef", "ModelName", "BlockSight", "VisibleModel", "SoundAppType", "Name"], + "rows": [ + {"id": 0, "Label": "Wood_strong", "StrRef": "5349", "ModelName": "T_DOOR01", "BlockSight": "1", "VisibleModel": "1", "SoundAppType": "0", "Name": "63750"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "genericdoors", "lock.json"), `{ + "babayaga:hs_door60": 3200, + "orientaltileset:t_Door12": 3202 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "genericdoors", "modules", "add_mb017.json"), `{ + "entries": { + "babayaga:hs_door60": { + "Label": "Door60", + "StrRef": "****", + "ModelName": "hs_door60", + "BlockSight": "1", + "VisibleModel": "1", + "SoundAppType": "0", + "Name": "****" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "genericdoors", "modules", "add_ori.json"), `{ + "entries": { + "orientaltileset:t_Door12": { + "Label": "Door12", + "StrRef": "5349", + "ModelName": "t_Door12", + "BlockSight": "1", + "VisibleModel": "1", + "SoundAppType": "0", + "Name": "14445" + } + } +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "genericdoors.2da")) + if err != nil { + t.Fatalf("read native genericdoors.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "genericdoors.2da")) + if err != nil { + t.Fatalf("read reference genericdoors.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("genericdoors output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsLegacyBaseitems(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata")) + mkdirAll(t, filepath.Join(root, "reference", "data", "baseitems", "modules")) + mkdirAll(t, filepath.Join(root, "reference", "tlk")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "base.json"), `{ + "columns": ["Name", "label", "MaxRange"], + "rows": [ + {"key": "baseitems:shortsword", "id": 0, "Name": "106", "label": "shortsword", "MaxRange": "100"}, + {"key": "baseitems:helmet", "id": 17, "Name": "500", "label": "helmet", "MaxRange": "100"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "lock.json"), `{ + "baseitems:shortsword": 0, + "baseitems:helmet": 17, + "baseitems:coin": 113 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "blunderbuss.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "coin.json"), `{ + "entries": { + "baseitems:coin": { + "Name": { "tlk": "baseitems:coin.name" }, + "label": "Coin", + "MaxRange": "100" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "estoc.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "heavymace.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "ovr_baseitems.json"), `{ + "overrides": [ + {"key": "baseitems:shortsword", "id": 0, "label": "shortsword"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "ovr_cloak255.json"), `{"overrides": []}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "ovr_helmet255.json"), `{ + "overrides": [ + {"key": "baseitems:helmet", "id": 17, "MaxRange": "255"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "maulandfalchion.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "shortspear.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "tlk", "base.json"), `{ + "language": "en", + "entries": { + "baseitems:coin.name": { "text": "Coin" } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "tlk", "lock.json"), `{ + "baseitems:coin.name": 70000 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 10 { + t.Fatalf("expected baseitems files to be imported, got %#v", result) + } + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "baseitems", "base.json"), + filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), + filepath.Join(root, "topdata", "data", "baseitems", "modules", "blunderbuss.json"), + filepath.Join(root, "topdata", "data", "baseitems", "modules", "coin.json"), + filepath.Join(root, "topdata", "data", "baseitems", "modules", "estoc.json"), + filepath.Join(root, "topdata", "data", "baseitems", "modules", "heavymace.json"), + filepath.Join(root, "topdata", "data", "baseitems", "modules", "maulandfalchion.json"), + filepath.Join(root, "topdata", "data", "baseitems", "modules", "ovr_baseitems.json"), + filepath.Join(root, "topdata", "data", "baseitems", "modules", "ovr_cloak255.json"), + filepath.Join(root, "topdata", "data", "baseitems", "modules", "ovr_helmet255.json"), + filepath.Join(root, "topdata", "data", "baseitems", "modules", "shortspear.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported baseitems path %s: %v", path, err) + } + } + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "baseitems", "base.json")) + if err != nil { + t.Fatalf("read baseitems base: %v", err) + } + if !strings.Contains(string(baseRaw), `"output": "baseitems.2da"`) { + t.Fatalf("expected imported baseitems output name, got:\n%s", string(baseRaw)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "baseitems", "lock.json")) + if err != nil { + t.Fatalf("read baseitems lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"baseitems:coin": 113`) { + t.Fatalf("expected imported baseitems lock IDs, got:\n%s", string(lockRaw)) + } + moduleRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "baseitems", "modules", "coin.json")) + if err != nil { + t.Fatalf("read baseitems coin module: %v", err) + } + if !strings.Contains(string(moduleRaw), `"text": "Coin"`) { + t.Fatalf("expected imported baseitems module tlk to be inlined, got:\n%s", string(moduleRaw)) + } +} + +func TestBuildSupportsCanonicalBaseitems(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["Name", "label", "MaxRange"], + "rows": [ + {"key": "baseitems:shortsword", "id": 0, "Name": "106", "label": "shortsword", "MaxRange": "100"}, + {"key": "baseitems:helmet", "id": 17, "Name": "500", "label": "helmet", "MaxRange": "100"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{ + "baseitems:shortsword": 0, + "baseitems:helmet": 17, + "baseitems:coin": 113 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "blunderbuss.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "coin.json"), `{ + "entries": { + "baseitems:coin": { + "Name": "Coin", + "label": "Coin", + "MaxRange": "100", + "meta": { + "wiki": { + "reqfeats": [{"id": "feat:weaponproficiencysimple"}] + } + } + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "estoc.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "heavymace.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "ovr_baseitems.json"), `{ + "overrides": [ + {"key": "baseitems:shortsword", "id": 0, "label": "shortsword"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "ovr_cloak255.json"), `{"overrides": []}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "ovr_helmet255.json"), `{ + "overrides": [ + {"key": "baseitems:helmet", "id": 17, "MaxRange": "255"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "shortspear.json"), `{"entries": {}}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "baseitems.2da")) + if err != nil { + t.Fatalf("read baseitems.2da: %v", err) + } + if !strings.Contains(string(got), "Name\tlabel\tMaxRange\n") || + !strings.Contains(string(got), "0\t106\tshortsword\t100\n") || + !strings.Contains(string(got), "17\t500\thelmet\t255\n") || + !strings.Contains(string(got), "113\tCoin\tCoin\t100\n") { + t.Fatalf("unexpected baseitems.2da output:\n%s", string(got)) + } + if strings.Contains(string(got), "WIKIREQFEATS") { + t.Fatalf("authoring-only WIKI fields leaked into baseitems.2da:\n%s", string(got)) + } +} + +func TestBuildUsesBaseDialogLegacyEntries(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), `{ + "language": "en", + "entries": { + "baseitems:coin.name": {"text": "Coin"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["Name", "label"], + "rows": [ + {"key": "baseitems:coin", "id": 113, "Name": {"tlk": "baseitems:coin.name"}, "label": "Coin"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "baseitems.2da")) + if err != nil { + t.Fatalf("read baseitems.2da: %v", err) + } + if !strings.Contains(string(got), "113\t16777216\tCoin\n") { + t.Fatalf("expected base_dialog-backed strref in output, got:\n%s", string(got)) + } + stateRaw, err := os.ReadFile(filepath.Join(root, "topdata", ".tlk_state.json")) + if err != nil { + t.Fatalf("read tlk state: %v", err) + } + if !strings.Contains(string(stateRaw), `"baseitems:coin.name"`) { + t.Fatalf("expected base_dialog key to be tracked in tlk state, got:\n%s", string(stateRaw)) + } +} + +func TestNormalize2DAForCompareDropsDeprecatedWikiColumns(t *testing.T) { + input := []byte("2DA V2.0\n\nName\tWIKIGENERATE\tValue\n0\tCoin\t****\t100\n") + got := normalize2DAForCompare("test.2da", input) + expected := "2DA V2.0\n\nName\tValue\n0\tCoin\t100\n" + if string(got) != expected { + t.Fatalf("unexpected normalized 2da:\n%s", string(got)) + } +} + +func TestBuildOverridePrecedenceSuppressesBaseAndEntries(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ + "output": "dense.2da", + "columns": ["Label", "Value"], + "rows": [ + {"key": "dense:base", "id": 0, "Label": "Base", "Value": "base"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "lock.json"), `{ + "dense:entry": 1 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "entries.json"), `{ + "entries": { + "dense:entry": { + "Label": "Entry", + "Value": "entry" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "overrides.json"), `{ + "overrides": [ + {"key": "dense:base", "Label": "Override", "Value": null}, + {"key": "dense:entry", "Label": "EntryOverride", "Value": null} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "dense.2da")) + if err != nil { + t.Fatalf("read dense.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "0\tOverride\t****\n") { + t.Fatalf("expected base row override to win and null out Value, got:\n%s", text) + } + if !strings.Contains(text, "1\tEntryOverride\t****\n") { + t.Fatalf("expected entry row override to win and null out Value, got:\n%s", text) + } +} + +func TestBuildNullRowRetiresLockIdentityGlobally(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ + "output": "dense.2da", + "columns": ["Label", "Value"], + "rows": [ + {"id": 0, "key": "dense:one", "Label": "One", "Value": "one"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "lock.json"), `{ + "dense:one": 0 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "rmv_one.json"), `{ + "overrides": [ + {"id": 0, "key": null, "null": true} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "dense", "lock.json")) + if err != nil { + t.Fatalf("read dense lock: %v", err) + } + if strings.Contains(string(lockRaw), `"dense:one"`) { + t.Fatalf("expected nulled row to retire lock identity, got:\n%s", string(lockRaw)) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "dense.2da")) + if err != nil { + t.Fatalf("read dense.2da: %v", err) + } + if !strings.Contains(string(got), "0\t****\t****\n") { + t.Fatalf("expected nulled dense row in output, got:\n%s", string(got)) + } +} + +func TestResolveOverrideTargetPrefersIDOverKey(t *testing.T) { + keyRow := map[string]any{"id": 192, "key": "feat:skillfocus_intimidate"} + idRow := map[string]any{"id": 916, "key": "feat:taunt"} + rowByID := map[int]map[string]any{ + 192: keyRow, + 916: idRow, + } + rowByKey := map[string]map[string]any{ + "feat:skillfocus_intimidate": keyRow, + "feat:taunt": idRow, + } + override := map[string]any{ + "key": "feat:skillfocus_intimidate", + "id": 916, + } + got, err := resolveOverrideTarget("feat", override, rowByID, rowByKey) + if err != nil { + t.Fatalf("resolveOverrideTarget failed: %v", err) + } + if got["id"] != 916 { + t.Fatalf("expected id-targeted row id 916, got %v", got["id"]) + } +} + +func TestUpdateOverrideRowKeyRetiresOldKeyEvenIfReferenced(t *testing.T) { + row := map[string]any{"id": 443, "key": "spells:etherealness"} + rowByKey := map[string]map[string]any{ + "spells:etherealness": row, + } + lockData := map[string]int{ + "spells:etherealness": 443, + } + expanded := map[string]any{ + "key": "spells:greater_sanctuary", + } + + changed, retiredKey, err := updateOverrideRowKey("spells", row, expanded, rowByKey, lockData) + if err != nil { + t.Fatalf("updateOverrideRowKey failed: %v", err) + } + if !changed { + t.Fatal("expected key reassignment to mark the row changed") + } + if retiredKey != "spells:etherealness" { + t.Fatalf("expected retired key spells:etherealness, got %q", retiredKey) + } + if row["key"] != "spells:greater_sanctuary" { + t.Fatalf("expected row key to be updated, got %v", row["key"]) + } + if _, ok := lockData["spells:etherealness"]; ok { + t.Fatalf("expected old key to be removed from lock data, got %#v", lockData) + } + if lockData["spells:greater_sanctuary"] != 443 { + t.Fatalf("expected new key to be locked to row 443, got %#v", lockData) + } + + referencedKeys := map[string]struct{}{ + "spells:etherealness": {}, + } + retiredKeys := map[string]struct{}{ + retiredKey: {}, + } + rows := []map[string]any{row} + pruned, updated := pruneLockDataToActiveRows(lockData, rows, referencedKeys, retiredKeys) + if updated != 0 { + t.Fatalf("expected no lock id updates after prune, got %d", updated) + } + if pruned != 0 { + t.Fatalf("expected no additional prune once old key was already retired, got %d", pruned) + } + if _, ok := lockData["spells:etherealness"]; ok { + t.Fatalf("expected retired key to stay removed even when referenced, got %#v", lockData) + } +} + +func TestUpdateOverrideRowKeyDoesNotDeleteRelocatedReplacementLock(t *testing.T) { + row := map[string]any{"id": 58, "key": "baseitems:shortspear"} + rowByKey := map[string]map[string]any{ + "baseitems:shortspear": row, + } + lockData := map[string]int{ + "baseitems:shortspear": 126, + } + expanded := map[string]any{ + "key": "baseitems:spear", + } + + changed, retiredKey, err := updateOverrideRowKey("baseitems", row, expanded, rowByKey, lockData) + if err != nil { + t.Fatalf("updateOverrideRowKey failed: %v", err) + } + if !changed { + t.Fatal("expected key reassignment to mark the row changed") + } + if retiredKey != "baseitems:shortspear" { + t.Fatalf("expected retired key baseitems:shortspear, got %q", retiredKey) + } + if lockData["baseitems:shortspear"] != 126 { + t.Fatalf("expected relocated replacement lock to survive old row retirement, got %#v", lockData) + } + if lockData["baseitems:spear"] != 58 { + t.Fatalf("expected new row key to lock to row 58, got %#v", lockData) + } +} + +func TestRetireRowIdentityDoesNotDeleteRelocatedReplacementLock(t *testing.T) { + row := map[string]any{"id": 916, "key": "feat:skill_focus_intimidate"} + lockData := map[string]int{ + "feat:skill_focus_intimidate": 1333, + } + retiredKeys := map[string]struct{}{} + + retireRowIdentity(row, lockData, retiredKeys) + + if lockData["feat:skill_focus_intimidate"] != 1333 { + t.Fatalf("expected replacement lock to survive nulled old row, got %#v", lockData) + } + if _, ok := retiredKeys["feat:skill_focus_intimidate"]; !ok { + t.Fatalf("expected old row key to be recorded as retired") + } +} + +func TestBuildPrunesLockedIDThatTargetsManualBaseSpace(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ + "output": "dense.2da", + "columns": ["Label", "Value"], + "rows": [ + {"id": 0, "Label": "****", "Value": "****"}, + {"id": 1, "key": "dense:base", "Label": "Base", "Value": "base"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "lock.json"), `{ + "dense:new": 0, + "dense:base": 1 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "entries.json"), `{ + "entries": { + "dense:new": { + "Label": "New", + "Value": "new" + } + } +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "dense.2da")) + if err != nil { + t.Fatalf("read dense.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "0\t****\t****\n") { + t.Fatalf("expected manual null row 0 to be preserved, got:\n%s", text) + } + if !strings.Contains(text, "2\tNew\tnew\n") { + t.Fatalf("expected generated entry to be regenerated after base boundary, got:\n%s", text) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "dense", "lock.json")) + if err != nil { + t.Fatalf("read lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"dense:new": 2`) { + t.Fatalf("expected invalid low lock id to be regenerated at row 2, got:\n%s", string(lockRaw)) + } +} + +func TestBuildPrunesBaseKeyAfterOverrideRenamesRow(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ + "output": "dense.2da", + "columns": ["Label", "Value"], + "rows": [ + {"id": 0, "key": "dense:old", "Label": "Old", "Value": "old"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "lock.json"), `{ + "dense:old": 0 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "rename.json"), `{ + "overrides": [ + { + "id": 0, + "key": "dense:new", + "Label": "New", + "Value": "new" + } + ] +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "dense.2da")) + if err != nil { + t.Fatalf("read dense.2da: %v", err) + } + if !strings.Contains(string(got), "0\tNew\tnew\n") { + t.Fatalf("expected renamed row override to win, got:\n%s", string(got)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "dense", "lock.json")) + if err != nil { + t.Fatalf("read lock: %v", err) + } + lockText := string(lockRaw) + if strings.Contains(lockText, `"dense:old"`) { + t.Fatalf("expected old base key to be pruned after row rename, got:\n%s", lockText) + } + if !strings.Contains(lockText, `"dense:new": 0`) { + t.Fatalf("expected new override key to be locked to row 0, got:\n%s", lockText) + } +} + +func TestBuildDuplicateBaseKeyUsesSurvivingRowAfterOverrideRename(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ + "output": "dense.2da", + "columns": ["Label", "Value"], + "rows": [ + {"id": 3, "key": "dense:duplicate", "Label": "Renamed", "Value": "renamed"}, + {"id": 7, "key": "dense:duplicate", "Label": "Survivor", "Value": "survivor"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "lock.json"), `{ + "dense:duplicate": 3 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "rename.json"), `{ + "overrides": [ + { + "id": 3, + "key": "dense:renamed", + "Label": "Renamed", + "Value": "renamed" + } + ] +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "dense.2da")) + if err != nil { + t.Fatalf("read dense.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "3\tRenamed\trenamed\n") || !strings.Contains(text, "7\tSurvivor\tsurvivor\n") { + t.Fatalf("expected renamed and surviving duplicate-key rows to keep distinct ids, got:\n%s", text) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "dense", "lock.json")) + if err != nil { + t.Fatalf("read lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"dense:renamed": 3`) { + t.Fatalf("expected renamed key to keep original row id, got:\n%s", lockText) + } + if !strings.Contains(lockText, `"dense:duplicate": 7`) { + t.Fatalf("expected surviving duplicate key to lock to surviving row id, got:\n%s", lockText) + } +} + +func TestBuildDuplicateBaseKeyUsesEarlierSurvivingRowAfterOverrideRename(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ + "output": "dense.2da", + "columns": ["Label", "Value"], + "rows": [ + {"id": 3, "key": "dense:duplicate", "Label": "Survivor", "Value": "survivor"}, + {"id": 7, "key": "dense:duplicate", "Label": "Renamed", "Value": "renamed"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "lock.json"), `{ + "dense:duplicate": 7 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "rename.json"), `{ + "overrides": [ + { + "id": 7, + "key": "dense:renamed", + "Label": "Renamed", + "Value": "renamed" + } + ] +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "dense.2da")) + if err != nil { + t.Fatalf("read dense.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "3\tSurvivor\tsurvivor\n") || !strings.Contains(text, "7\tRenamed\trenamed\n") { + t.Fatalf("expected surviving and renamed duplicate-key rows to keep distinct ids, got:\n%s", text) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "dense", "lock.json")) + if err != nil { + t.Fatalf("read lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"dense:renamed": 7`) { + t.Fatalf("expected renamed key to keep original row id, got:\n%s", lockText) + } + if !strings.Contains(lockText, `"dense:duplicate": 3`) { + t.Fatalf("expected surviving duplicate key to lock to surviving row id, got:\n%s", lockText) + } +} + +func TestBuildAutoAllocatedRowsReserveExplicitModuleIDs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ + "output": "dense.2da", + "columns": ["Label", "Value"], + "rows": [ + {"id": 0, "key": "dense:base", "Label": "Base", "Value": "base"}, + {"id": 1, "Label": "****", "Value": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "lock.json"), `{ + "dense:base": 0 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "a_entries.json"), `{ + "entries": { + "dense:auto": { + "Label": "Auto", + "Value": "auto" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "z_explicit.json"), `{ + "overrides": [ + { + "key": "dense:explicit", + "id": 2, + "Label": "Explicit", + "Value": "explicit" + } + ] +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "dense.2da")) + if err != nil { + t.Fatalf("read dense.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "2\tExplicit\texplicit\n") { + t.Fatalf("expected explicit module id 2 to be preserved, got:\n%s", text) + } + if !strings.Contains(text, "3\tAuto\tauto\n") { + t.Fatalf("expected auto entry to skip explicit module id 2, got:\n%s", text) + } +} + +func TestBuildGeneratedApplyAfterPrunesLowLockedID(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "generated")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["Label", "STRREF", "DESCRIPTION"], + "rows": [ + {"id": 10, "key": "masterfeats:skillfocus", "Label": "SkillFocus", "STRREF": "100", "DESCRIPTION": "1000"}, + {"id": 11, "key": "masterfeats:greaterskillfocus", "Label": "GreaterSkillFocus", "STRREF": "101", "DESCRIPTION": "1001"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{ + "masterfeats:skillfocus": 10, + "masterfeats:greaterskillfocus": 11 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "HideFromLevelUp"], + "rows": [ + {"id": 0, "key": "skills:profession_herbalist", "Label": "Profession Herbalist", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{ + "skills:profession_herbalist": 0 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["Label", "FEAT", "MASTERFEAT", "REQSKILL", "Constant"], + "rows": [ + {"id": 0, "Label": "****", "FEAT": "****", "MASTERFEAT": "****", "REQSKILL": "****", "Constant": "****"}, + {"id": 10, "key": "feat:skillfocus", "Label": "SkillFocus", "FEAT": "100", "MASTERFEAT": "10", "REQSKILL": "****", "Constant": "FEAT_SKILL_FOCUS"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:skillfocus": 10, + "feat:skillfocus_professionherbalist": 0, + "feat:greaterskillfocus_professionherbalist": 0 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "removed.json"), `{ + "overrides": [ + { + "key": null, + "id": 0, + "Label": "****", + "FEAT": "****", + "MASTERFEAT": "****", + "REQSKILL": "****", + "Constant": "****" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "generated", "greater_skill_focus.json"), `{ + "family": "greater_skill_focus", + "family_key": "greaterskillfocus", + "template": "masterfeats:greaterskillfocus", + "apply_after_modules": true, + "child_ref_field": "REQSKILL", + "child_source": { + "dataset": "skills", + "predicate": "accessible" + }, + "label_prefix": "GREATER_SKILL_FOCUS", + "name_prefix": "Greater Skill Focus", + "constant_prefix": "FEAT_GREATER_SKILL_FOCUS" +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "generated", "skill_focus.json"), `{ + "family": "skill_focus", + "family_key": "skillfocus", + "template": "masterfeats:skillfocus", + "apply_after_modules": true, + "child_ref_field": "REQSKILL", + "child_source": { + "dataset": "skills", + "predicate": "accessible" + }, + "label_prefix": "SKILL_FOCUS", + "name_prefix": "Skill Focus", + "constant_prefix": "FEAT_SKILL_FOCUS", + "template_fields": ["FEAT", "MASTERFEAT"] +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "0\t****\t****\t****\t****\t****\n") { + t.Fatalf("expected manual null row 0 to be preserved, got:\n%s", text) + } + if !strings.Contains(text, "11\tGREATER_SKILL_FOCUS_PROFESSION_HERBALIST\t16777216\t11\t0\tFEAT_GREATER_SKILL_FOCUS_PROFESSION_HERBALIST\n") { + t.Fatalf("expected greater skill focus to regenerate after base boundary, got:\n%s", text) + } + if !strings.Contains(text, "12\tSKILL_FOCUS_PROFESSION_HERBALIST\t****\t****\t0\tFEAT_SKILL_FOCUS_PROFESSION_HERBALIST\n") { + t.Fatalf("expected skill focus to regenerate after base boundary, got:\n%s", text) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "feat", "lock.json")) + if err != nil { + t.Fatalf("read lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"feat:skillfocus_profession_herbalist": 12`) { + t.Fatalf("expected generated skill focus lock id to be regenerated at row 12, got:\n%s", string(lockRaw)) + } + if !strings.Contains(string(lockRaw), `"feat:greaterskillfocus_profession_herbalist": 11`) { + t.Fatalf("expected generated greater skill focus lock id to be regenerated at row 11, got:\n%s", string(lockRaw)) + } +} + +func TestBuildOverrideAsteriskKeyRemovesIdentityWithoutNullingRow(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ + "output": "dense.2da", + "columns": ["Label", "Value"], + "rows": [ + {"id": 7, "key": "dense:old", "Label": "Old", "Value": "old"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "lock.json"), `{ + "dense:old": 7 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "nullify.json"), `{ + "overrides": [ + { + "id": 7, + "key": "****", + "Label": "ShouldNotLeak", + "Value": "still-not-kept" + } + ] +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "dense.2da")) + if err != nil { + t.Fatalf("read dense.2da: %v", err) + } + if !strings.Contains(string(got), "7\tShouldNotLeak\tstill-not-kept\n") { + t.Fatalf("expected override key \"****\" to remove identity without blanking row 7, got:\n%s", string(got)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "dense", "lock.json")) + if err != nil { + t.Fatalf("read lock: %v", err) + } + if strings.Contains(string(lockRaw), `"dense:old"`) { + t.Fatalf("expected nullified row key to be pruned from lock data, got:\n%s", string(lockRaw)) + } +} + +func TestBuildCanonicalBaseitemsMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "baseitems", "modules")) + mkdirAll(t, filepath.Join(root, "reference", "tlk")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "baseitems.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\\n\\n") + f.write("Name\\tlabel\\tMaxRange\\n") + f.write("0\\t106\\tshortsword\\t100\\n") + for row_id in range(1, 17): + f.write(f"{row_id}\\t****\\t****\\t****\\n") + f.write("17\\t500\\thelmet\\t255\\n") + for row_id in range(18, 113): + f.write(f"{row_id}\\t****\\t****\\t****\\n") + f.write("113\\tCoin\\tCoin\\t100\\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "base.json"), `{ + "columns": ["Name", "label", "MaxRange"], + "rows": [ + {"key": "baseitems:shortsword", "id": 0, "Name": "106", "label": "shortsword", "MaxRange": "100"}, + {"key": "baseitems:helmet", "id": 17, "Name": "500", "label": "helmet", "MaxRange": "100"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "lock.json"), `{ + "baseitems:shortsword": 0, + "baseitems:helmet": 17, + "baseitems:coin": 113 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "blunderbuss.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "coin.json"), `{ + "entries": { + "baseitems:coin": { + "Name": { "tlk": "baseitems:coin.name" }, + "label": "Coin", + "MaxRange": "100" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "estoc.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "heavymace.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "ovr_baseitems.json"), `{ + "overrides": [ + {"key": "baseitems:shortsword", "id": 0, "label": "shortsword"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "ovr_cloak255.json"), `{"overrides": []}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "ovr_helmet255.json"), `{ + "overrides": [ + {"key": "baseitems:helmet", "id": 17, "MaxRange": "255"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "maulandfalchion.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "baseitems", "modules", "shortspear.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "tlk", "base.json"), `{ + "language": "en", + "entries": { + "baseitems:coin.name": { "text": "Coin" } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "tlk", "lock.json"), `{ + "baseitems:coin.name": 70000 +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "baseitems.2da")) + if err != nil { + t.Fatalf("read native baseitems.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "baseitems.2da")) + if err != nil { + t.Fatalf("read reference baseitems.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("baseitems output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestBuildUsesReferenceLockIDsForUnmigratedKeyRefs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "masterfeats")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "base.json"), `{ + "output": "masterfeats.2da", + "columns": ["LABEL", "STRREF", "DESCRIPTION", "MinLevelClass"], + "rows": [ + {"id": 0, "key": "masterfeats:weaponspecialization", "LABEL": "WeaponSpecialization", "STRREF": "****", "DESCRIPTION": "****", "MinLevelClass": {"id": "classes:fighter"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "masterfeats", "lock.json"), `{"masterfeats:weaponspecialization":0}`+"\n") + mkdirAll(t, filepath.Join(root, "topdata", "migration_snapshot", "data", "classes")) + writeFile(t, filepath.Join(root, "topdata", "migration_snapshot", "data", "classes", "lock.json"), `{"classes:fighter":4}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "masterfeats.2da")) + if err != nil { + t.Fatalf("read masterfeats.2da: %v", err) + } + want := "2DA V2.0\n\nLABEL\tSTRREF\tDESCRIPTION\tMinLevelClass\n0\tWeaponSpecialization\t****\t****\t4\n" + if string(got) != want { + t.Fatalf("unexpected masterfeats.2da output:\n%s", string(got)) + } +} + +func TestBuildBaseDatasetAllowsModuleDeclaredColumns(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "custom", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "base.json"), `{ + "output": "custom.2da", + "columns": ["Label"], + "rows": [ + {"id": 0, "key": "custom:a", "Label": "A"}, + {"id": 1, "key": "custom:b", "Label": "B"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "custom", "modules", "extend.json"), `{ + "columns": ["Value"], + "overrides": [ + {"id": 1, "Value": "2"} + ], + "entries": { + "custom:c": { + "Label": "C", + "Value": "3" + } + } +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "custom.2da")) + if err != nil { + t.Fatalf("read custom.2da: %v", err) + } + want := "2DA V2.0\n\nLabel\tValue\n0\tA\t****\n1\tB\t2\n2\tC\t3\n" + if string(got) != want { + t.Fatalf("unexpected custom.2da output:\n%s", string(got)) + } +} + +func TestBuildInjectsExpansionDataIntoTargetDataset(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ + "output": "portraits.2da", + "columns": ["BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"], + "rows": [ + {"id": 0, "BaseResRef": "****", "Sex": 4} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), "{}") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "base.json"), `{ + "output": "appearance.2da", + "columns": ["LABEL", "PORTRAIT"], + "rows": [ + {"key": "appearance:crawlingclaw", "id": 0, "LABEL": "Crawling Claw", "PORTRAIT": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "lock.json"), `{"appearance:crawlingclaw":0}`) + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "modules", "add_crawlingclaw.json"), `{ + "entries": { + "appearance:crawlingclaw": { + "LABEL": "Crawling Claw", + "PORTRAIT": { + "value": "po_cclaw_", + "data": { + "portraits": { + "key": "portraits:cclaw_", + "BaseResRef": "cclaw_", + "Sex": 4, + "Race": 10, + "Plot": 0 + } + } + } + } + } +}`+"\n") + + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + appearanceBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "appearance.2da")) + if err != nil { + t.Fatalf("read appearance.2da: %v", err) + } + if !strings.Contains(string(appearanceBytes), "po_cclaw_") { + t.Fatalf("expected PORTRAIT value in appearance.2da, got:\n%s", string(appearanceBytes)) + } + + portraitsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da")) + if err != nil { + t.Fatalf("read portraits.2da: %v", err) + } + text := string(portraitsBytes) + if !strings.Contains(text, "cclaw_") || !strings.Contains(text, "Sex\t") || !strings.Contains(text, "Race\t") { + t.Fatalf("expected injected portrait data in portraits.2da, got:\n%s", text) + } + if !strings.Contains(text, "4\t10\t") { + t.Fatalf("expected Sex=4 Race=10 in portraits.2da, got:\n%s", text) + } + + lockBytes, err := os.ReadFile(filepath.Join(root, "topdata", "data", "portraits", "lock.json")) + if err != nil { + t.Fatalf("read portraits lock.json: %v", err) + } + lockText := string(lockBytes) + if !strings.Contains(lockText, "portraits:cclaw_") { + t.Fatalf("expected lockfile entry for portraits:cclaw_, got:\n%s", lockText) + } +} + +func TestBuildExpansionDataDoesNotDuplicatePortraitRows(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ + "output": "portraits.2da", + "columns": ["BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"], + "rows": [ + {"id": 0, "BaseResRef": "****", "Sex": 4} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), "{}") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "base.json"), `{ + "output": "appearance.2da", + "columns": ["LABEL", "PORTRAIT"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "lock.json"), "{}") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "modules", "add_zombieknight.json"), `{ + "entries": { + "appearance:zombieknight3": { + "LABEL": "Zombie Knight 3", + "PORTRAIT": { + "value": "po_zk_", + "data": { + "portraits": { + "key": "portraits:zk_", + "BaseResRef": "zk_", + "Sex": 4, + "Race": 23, + "Plot": 0 + } + } + } + }, + "appearance:zombieknight4": { + "LABEL": "Zombie Knight 4", + "PORTRAIT": { + "value": "po_zk_", + "data": { + "portraits": { + "key": "portraits:zk_", + "BaseResRef": "zk_", + "Sex": 4, + "Race": 23, + "Plot": 0 + } + } + } + } + } +}`+"\n") + + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + portraitsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da")) + if err != nil { + t.Fatalf("read portraits.2da: %v", err) + } + text := string(portraitsBytes) + lines := strings.Split(text, "\n") + zkCount := 0 + for _, line := range lines { + if strings.Contains(line, "zk_") { + zkCount++ + } + } + if zkCount != 1 { + t.Fatalf("expected exactly 1 portrait row with zk_, got %d:\n%s", zkCount, text) + } +} + +func TestBuildExpansionDataIsGlobalAndPrunesStaleInjectedLocks(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "source", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "target")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + + writeFile(t, filepath.Join(root, "topdata", "data", "source", "base.json"), `{ + "output": "source.2da", + "columns": ["Label", "Link"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "source", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "target", "base.json"), `{ + "output": "target.2da", + "columns": ["Label", "Value"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "target", "lock.json"), "{}\n") + + modulePath := filepath.Join(root, "topdata", "data", "source", "modules", "inject.json") + writeFile(t, modulePath, `{ + "entries": { + "source:one": { + "Label": "One", + "Link": { + "value": "target-one", + "data": { + "target": { + "key": "target:one", + "Label": "Target One", + "Value": "one" + } + } + } + } + } +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + targetBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "target.2da")) + if err != nil { + t.Fatalf("read target.2da: %v", err) + } + if !strings.Contains(string(targetBytes), "0\t\"Target One\"\tone\n") { + t.Fatalf("expected injected target row, got:\n%s", string(targetBytes)) + } + lockBytes, err := os.ReadFile(filepath.Join(root, "topdata", "data", "target", "lock.json")) + if err != nil { + t.Fatalf("read target lock.json: %v", err) + } + if !strings.Contains(string(lockBytes), `"target:one": 0`) { + t.Fatalf("expected lock entry for injected target row, got:\n%s", string(lockBytes)) + } + + writeFile(t, modulePath, `{ + "entries": { + "source:one": { + "Label": "One", + "Link": { + "value": "target-two", + "data": { + "target": { + "key": "target:two", + "Label": "Target Two", + "Value": "two" + } + } + } + } + } +}`+"\n") + result, err = BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative after alteration failed: %v", err) + } + targetBytes, err = os.ReadFile(filepath.Join(result.Output2DADir, "target.2da")) + if err != nil { + t.Fatalf("read altered target.2da: %v", err) + } + targetText := string(targetBytes) + if strings.Contains(targetText, "Target One") || !strings.Contains(targetText, "0\t\"Target Two\"\ttwo\n") { + t.Fatalf("expected altered injected target row only, got:\n%s", targetText) + } + lockBytes, err = os.ReadFile(filepath.Join(root, "topdata", "data", "target", "lock.json")) + if err != nil { + t.Fatalf("read altered target lock.json: %v", err) + } + lockText := string(lockBytes) + if strings.Contains(lockText, "target:one") || !strings.Contains(lockText, `"target:two": 0`) { + t.Fatalf("expected stale injected lock to be pruned after alteration, got:\n%s", lockText) + } + + if err := os.Remove(modulePath); err != nil { + t.Fatalf("remove injection module: %v", err) + } + result, err = BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative after deletion failed: %v", err) + } + targetBytes, err = os.ReadFile(filepath.Join(result.Output2DADir, "target.2da")) + if err != nil { + t.Fatalf("read deleted target.2da: %v", err) + } + if strings.Contains(string(targetBytes), "Target Two") { + t.Fatalf("expected injected target row to be removed after module deletion, got:\n%s", string(targetBytes)) + } + lockBytes, err = os.ReadFile(filepath.Join(root, "topdata", "data", "target", "lock.json")) + if err != nil { + t.Fatalf("read deleted target lock.json: %v", err) + } + if strings.Contains(string(lockBytes), "target:two") { + t.Fatalf("expected injected target lock to be pruned after module deletion, got:\n%s", string(lockBytes)) + } +} + +func TestBuildExpansionDataPreservesExistingLockfileIDs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ + "output": "portraits.2da", + "columns": ["BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"], + "rows": [ + {"id": 0, "BaseResRef": "****", "Sex": 4}, + {"id": 1, "BaseResRef": "existing_", "Sex": 2, "key": "portraits:existing"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "modules", "placeables.json"), `{ + "entries": { + "portraits:plc_scala": { + "BaseResRef": "plc_scala", + "Sex": 4, + "Race": 10, + "Plot": 0 + }, + "portraits:plc_scalb": { + "BaseResRef": "plc_scalb", + "Sex": 4, + "Race": 10, + "Plot": 0 + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), `{ + "portraits:existing": 1, + "portraits:grue_air_": 16632, + "portraits:grue_fire_": 16633, + "portraits:eyeball_": 16634, + "portraits:zk_": 16635, + "portraits:plc_scala": 16636, + "portraits:plc_scalb": 16637 +}`) + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "base.json"), `{ + "output": "appearance.2da", + "columns": ["LABEL", "PORTRAIT"], + "rows": [ + {"key": "appearance:grue_air", "id": 0, "LABEL": "Air Grue", "PORTRAIT": "****"}, + {"key": "appearance:grue_fire", "id": 1, "LABEL": "Fire Grue", "PORTRAIT": "****"}, + {"key": "appearance:eyeball", "id": 2, "LABEL": "Eyeball", "PORTRAIT": "****"}, + {"key": "appearance:zk", "id": 3, "LABEL": "Zombie Knight", "PORTRAIT": "****"}, + {"key": "appearance:displacer", "id": 4, "LABEL": "Displacer", "PORTRAIT": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "lock.json"), `{ + "appearance:grue_air": 0, + "appearance:grue_fire": 1, + "appearance:eyeball": 2, + "appearance:zk": 3, + "appearance:displacer": 4 +}`) + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "modules", "add_grues.json"), `{ + "entries": { + "appearance:grue_air": { + "LABEL": "Air Grue", + "PORTRAIT": { + "value": "po_grue_air_", + "data": { + "portraits": { + "key": "portraits:grue_air_", + "BaseResRef": "grue_air_", + "Sex": 4, + "Race": 10, + "Plot": 0 + } + } + } + }, + "appearance:grue_fire": { + "LABEL": "Fire Grue", + "PORTRAIT": { + "value": "po_grue_fire_", + "data": { + "portraits": { + "key": "portraits:grue_fire_", + "BaseResRef": "grue_fire_", + "Sex": 4, + "Race": 10, + "Plot": 0 + } + } + } + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "modules", "add_eyeball.json"), `{ + "entries": { + "appearance:eyeball": { + "LABEL": "Eyeball", + "PORTRAIT": { + "value": "po_eyeball_", + "data": { + "portraits": { + "key": "portraits:eyeball_", + "BaseResRef": "eyeball_", + "Sex": 4, + "Race": 10, + "Plot": 0 + } + } + } + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "modules", "add_zk.json"), `{ + "entries": { + "appearance:zk": { + "LABEL": "Zombie Knight", + "PORTRAIT": { + "value": "po_zk_", + "data": { + "portraits": { + "key": "portraits:zk_", + "BaseResRef": "zk_", + "Sex": 4, + "Race": 10, + "Plot": 0 + } + } + } + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "modules", "add_displacer.json"), `{ + "entries": { + "appearance:displacer": { + "LABEL": "Displacer", + "PORTRAIT": { + "value": "po_displacer1_", + "data": { + "portraits": { + "key": "portraits:displacer1_", + "BaseResRef": "displacer1_", + "Sex": 4, + "Race": 19, + "Plot": 0 + } + } + } + } + } +}`+"\n") + + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + lockBytes, err := os.ReadFile(filepath.Join(root, "topdata", "data", "portraits", "lock.json")) + if err != nil { + t.Fatalf("read portraits lock.json: %v", err) + } + lockText := string(lockBytes) + + // All existing lockfile entries must preserve their IDs — no shifting + expected := map[string]string{ + "portraits:grue_air_": "16632", + "portraits:grue_fire_": "16633", + "portraits:eyeball_": "16634", + "portraits:zk_": "16635", + "portraits:plc_scala": "16636", + "portraits:plc_scalb": "16637", + } + for key, wantID := range expected { + entry := fmt.Sprintf(`"%s": %s`, key, wantID) + if !strings.Contains(lockText, entry) { + t.Fatalf("lockfile entry %s was not preserved:\n%s", entry, lockText) + } + } + + // New entry must NOT conflict with any existing ID + var lockData map[string]int + if err := json.Unmarshal([]byte(lockText), &lockData); err != nil { + t.Fatalf("unmarshal lock.json: %v", err) + } + newID, ok := lockData["portraits:displacer1_"] + if !ok { + t.Fatalf("expected lockfile entry for portraits:displacer1_, got:\n%s", lockText) + } + for key, id := range lockData { + if key != "portraits:displacer1_" && id == newID { + t.Fatalf("new portrait displacer1_ got ID %d which conflicts with existing %s=%d", newID, key, id) + } + } + if newID < 2 { + t.Fatalf("new portrait displacer1_ got unreasonably low ID %d", newID) + } + + portraitsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da")) + if err != nil { + t.Fatalf("read portraits.2da: %v", err) + } + text := string(portraitsBytes) + if !strings.Contains(text, "displacer1_") { + t.Fatalf("expected injected portrait in portraits.2da, got:\n%s", text) + } +} + +func TestValidateProjectRejectsUnknownMetadataKey(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "base.json"), `{ + "output": "placeables.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "Shelf", "meta": {"unknown": {}}} + ] +}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatalf("expected metadata validation error, got %#v", report.Diagnostics) + } + if !strings.Contains(diagnosticsText(report.Diagnostics), `unknown metadata key "unknown"`) { + t.Fatalf("expected unknown metadata diagnostic, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + +func TestValidateProjectRejectsUnknownMetadataKeyCaseInsensitive(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "base.json"), `{ + "output": "placeables.2da", + "columns": ["Label"], + "rows": [ + {"id": 0, "Label": "Shelf", "Meta": {"unknown": {}}} + ] +}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatalf("expected metadata validation error, got %#v", report.Diagnostics) + } + if !strings.Contains(diagnosticsText(report.Diagnostics), `unknown metadata key "unknown"`) { + t.Fatalf("expected unknown metadata diagnostic, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + +func TestBuildNativeTreatsMetaCaseInsensitive(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ + "output": "dense.2da", + "columns": ["Label"], + "rows": [ + {"id": 0, "Label": "Base", "Meta": {"wiki": {"status": "base"}}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "lock.json"), `{"dense:entry":1}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "entries.json"), `{ + "entries": { + "dense:entry": {"Label": "Entry", "Meta": {"wiki": {"status": "entry"}}} + }, + "overrides": [ + {"id": 0, "key": "dense:base", "Label": "BaseOverride", "Meta": {"wiki": {"status": "override"}}} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "dense.2da")) + if err != nil { + t.Fatalf("read dense.2da: %v", err) + } + if string(got) != "2DA V2.0\n\nLabel\n0\tBaseOverride\n1\tEntry\n" { + t.Fatalf("unexpected dense.2da output:\n%s", string(got)) + } + if strings.Contains(string(got), "meta") { + t.Fatalf("expected metadata to stay out of dense.2da, got:\n%s", string(got)) + } +} + +func TestBuildCanonicalAppearanceMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "appearance", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "appearance.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("LABEL\tPORTRAIT\n") + f.write("0\tDwarf\t****\n") + f.write("15105\tBlack Lagoon Reaver\tpo_cotbl\n") + f.write("15111\tBlack Lagoon Reaver, Swimming\tpo_cotbl\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "appearance", "base.json"), `{ + "columns": ["LABEL", "PORTRAIT"], + "rows": [ + {"key": "appearance:dwarf", "id": 0, "LABEL": "Dwarf", "PORTRAIT": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "appearance", "lock.json"), `{ + "appearance:blacklagoonreaver": 15105, + "appearance:blacklagoonreaverswimming": 15111 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "appearance", "modules", "cotblreaver.json"), `{ + "entries": { + "appearance:blacklagoonreaver": {"LABEL": "Black Lagoon Reaver", "PORTRAIT": "po_cotbl"}, + "appearance:blacklagoonreaverswimming": {"LABEL": "Black Lagoon Reaver, Swimming", "PORTRAIT": "po_cotbl"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "appearance", "modules", "crawlingclaw.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "appearance", "modules", "halfogre.json"), `{"entries": {}}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "appearance", "modules", "zombieknight.json"), `{"entries": {}}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "appearance.2da")) + if err != nil { + t.Fatalf("read native appearance.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "appearance.2da")) + if err != nil { + t.Fatalf("read reference appearance.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("appearance output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsLegacyPlaceables(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "placeables", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "data", "placeables", "base.json"), `{ + "columns": ["Label", "StrRef", "ModelName", "LightColor", "LightOffsetX", "LightOffsetY", "LightOffsetZ", "SoundAppType", "ShadowSize", "BodyBag", "LowGore", "Reflection", "Static"], + "rows": [ + {"key": "placeables:armoire", "id": 0, "Label": "Armoire", "StrRef": "5645", "ModelName": "PLC_A01", "SoundAppType": "13", "ShadowSize": "1", "BodyBag": "0", "Static": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "placeables", "lock.json"), `{ + "placeables:armoire": 0, + "placeables:tmp_crappile01": 16500 +}`+"\n") + for _, name := range []string{ + "add_ccfeb2019plcs.json", + "add_cepcarpets.json", + "add_cepfloors.json", + "add_cepnwn2library.json", + "add_nwic.json", + "add_ori.json", + "add_sic11.json", + "add_tapr.json", + "add_tdfloors.json", + "add_undecals.json", + "add_witcher1.json", + "add_witcher2.json", + } { + writeFile(t, filepath.Join(root, "reference", "data", "placeables", "modules", name), `{"entries": {}}`+"\n") + } + writeFile(t, filepath.Join(root, "reference", "data", "placeables", "modules", "add_avernostragarbage.json"), `{ + "entries": { + "placeables:tmp_crappile01": { + "Label": "Decoration: Pile 1 (Avernostra)", + "ModelName": "tmp_crappile01", + "Static": 1 + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "placeables", "modules", "ovr_vanillaplcsorts.json"), `{ + "overrides": [ + {"key": "placeables:armoire", "id": 0, "Label": "Furniture: Armoire (Bioware)", "StrRef": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 16 { + t.Fatalf("expected placeables files to be imported, got %#v", result) + } + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "placeables", "base.json"), + filepath.Join(root, "topdata", "data", "placeables", "lock.json"), + filepath.Join(root, "topdata", "data", "placeables", "modules", "add_avernostragarbage.json"), + filepath.Join(root, "topdata", "data", "placeables", "modules", "ovr_vanillaplcsorts.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported placeables path %s: %v", path, err) + } + } + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "placeables", "base.json")) + if err != nil { + t.Fatalf("read placeables base: %v", err) + } + if !strings.Contains(string(baseRaw), `"output": "placeables.2da"`) { + t.Fatalf("expected imported placeables output name, got:\n%s", string(baseRaw)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "placeables", "lock.json")) + if err != nil { + t.Fatalf("read placeables lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"placeables:tmp_crappile01": 16500`) { + t.Fatalf("expected imported placeables lock IDs, got:\n%s", string(lockRaw)) + } +} + +func TestNormalizeProjectImportsLegacyRuleset(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "ruleset", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "data", "ruleset", "base.json"), `{ + "columns": ["Label", "Value"], + "rows": [ + {"id": 0, "Label": "****", "Value": "****"}, + {"id": 7, "Label": "CALLED_SHOT_TO_HIT_MODIFIER", "Value": "-4"}, + {"id": 21, "Label": "GOOD_AIM_MODIFIER", "Value": "1"}, + {"id": 100, "Label": "HASTE_DODGE_AC_INCREASE_AMOUNT", "Value": "0"}, + {"id": 101, "Label": "HASTED_SPELL_CONJURE_TIME_MODIFIER", "Value": "0.5f"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "ruleset", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "data", "ruleset", "modules", "ovr_single.json"), `{ + "overrides": [{"match": {"Label": "GOOD_AIM_MODIFIER"}, "Value": "2"}] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "ruleset", "modules", "ovr_array.json"), `{ + "overrides": [{"match": {"Label": ["HASTE_DODGE_AC_INCREASE_AMOUNT", "HASTED_SPELL_CONJURE_TIME_MODIFIER"]}, "Value": "1.0f"}] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 1 { + t.Fatalf("expected ruleset import, got %#v", result) + } + rulesetPath := filepath.Join(root, "topdata", "data", "ruleset", "ruleset.json") + if _, err := os.Stat(rulesetPath); err != nil { + t.Fatalf("expected imported ruleset file: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "topdata", "data", "ruleset", "lock.json")); !os.IsNotExist(err) { + t.Fatalf("expected no ruleset lock file, got %v", err) + } + if _, err := os.Stat(filepath.Join(root, "topdata", "data", "ruleset", "modules")); !os.IsNotExist(err) { + t.Fatalf("expected no ruleset modules dir, got %v", err) + } + raw, err := os.ReadFile(rulesetPath) + if err != nil { + t.Fatalf("read ruleset: %v", err) + } + text := string(raw) + if !strings.Contains(text, `"output": "ruleset.2da"`) || + !strings.Contains(text, `"Label": "GOOD_AIM_MODIFIER"`) || + !strings.Contains(text, `"Value": "2"`) || + !strings.Contains(text, `"Label": "HASTED_SPELL_CONJURE_TIME_MODIFIER"`) || + !strings.Contains(text, `"Value": "1.0f"`) { + t.Fatalf("unexpected imported ruleset:\n%s", text) + } +} + +func TestBuildSupportsCanonicalRuleset(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "ruleset")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "ruleset", "ruleset.json"), `{ + "output": "ruleset.2da", + "columns": ["Label", "Value"], + "rows": [ + {"id": 0, "Label": "****", "Value": "****"}, + {"id": 21, "Label": "GOOD_AIM_MODIFIER", "Value": "2"}, + {"id": 100, "Label": "HASTE_DODGE_AC_INCREASE_AMOUNT", "Value": "1.0f"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "ruleset.2da")) + if err != nil { + t.Fatalf("read ruleset.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "Label\tValue\n") || + !strings.Contains(text, "21\tGOOD_AIM_MODIFIER\t2\n") || + !strings.Contains(text, "100\tHASTE_DODGE_AC_INCREASE_AMOUNT\t1.0f\n") { + t.Fatalf("unexpected ruleset.2da output:\n%s", text) + } +} + +func TestBuildCanonicalRulesetMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "ruleset", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "ruleset.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tValue\n") + f.write("0\t****\t****\n") + f.write("21\tGOOD_AIM_MODIFIER\t2\n") + f.write("100\tHASTE_DODGE_AC_INCREASE_AMOUNT\t1.0f\n") + f.write("101\tHASTED_SPELL_CONJURE_TIME_MODIFIER\t1.0f\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "ruleset", "base.json"), `{ + "columns": ["Label", "Value"], + "rows": [ + {"id": 0, "Label": "****", "Value": "****"}, + {"id": 21, "Label": "GOOD_AIM_MODIFIER", "Value": "1"}, + {"id": 100, "Label": "HASTE_DODGE_AC_INCREASE_AMOUNT", "Value": "0"}, + {"id": 101, "Label": "HASTED_SPELL_CONJURE_TIME_MODIFIER", "Value": "0.5f"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "ruleset", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "data", "ruleset", "modules", "ovr_single.json"), `{ + "overrides": [{"match": {"Label": "GOOD_AIM_MODIFIER"}, "Value": "2"}] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "ruleset", "modules", "ovr_array.json"), `{ + "overrides": [{"match": {"Label": ["HASTE_DODGE_AC_INCREASE_AMOUNT", "HASTED_SPELL_CONJURE_TIME_MODIFIER"]}, "Value": "1.0f"}] +}`+"\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 1 { + t.Fatalf("expected ruleset import, got %#v", result) + } + if _, err := Build(testProject(root), nil); err != nil { + t.Fatalf("Build failed: %v", err) + } + compare, err := CompareReference(testProject(root), nil) + if err != nil { + t.Fatalf("CompareReference failed: %v", err) + } + if compare.NativeMismatch != 0 || compare.NativePass != 1 { + t.Fatalf("unexpected compare result: %#v", compare) + } +} + +func TestNormalizeProjectImportsLegacySkills(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules", "skills")) + mkdirAll(t, filepath.Join(root, "reference", "data", "skills", "modules", "craft")) + mkdirAll(t, filepath.Join(root, "reference", "data", "skills", "modules", "knowledge")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "lock.json"), `{ + "skills:athletics.name": 1000, + "skills:athletics.description": 1001, + "skills:craft_armorsmithing.name": 1002, + "skills:craft_armorsmithing.description": 1003, + "skills:knowledge_arcana.name": 1004 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "modules", "skills", "skills.json"), `{ + "entries": { + "skills:athletics.name": {"text": "Athletics"}, + "skills:athletics.description": {"text": "Athletics description"}, + "skills:craft_armorsmithing.name": {"text": "Craft Armor Smithing"}, + "skills:craft_armorsmithing.description": {"text": "Craft armor description"}, + "skills:knowledge_arcana.name": {"text": "Knowledge Arcana"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skills", "base.json"), `{ + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"key": "skills:animalempathy", "id": 0, "Label": "AnimalEmpathy", "Name": "269", "Description": "344", "Icon": "isk_aniemp", "Untrained": "0", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "0", "Category": "****", "MaxCR": "1", "Constant": "SKILL_ANIMAL_EMPATHY", "HostileSkill": "1", "HideFromLevelUp": "0"}, + {"key": "skills:concentration", "id": 1, "Label": "Concentration", "Name": "270", "Description": "345", "Icon": "isk_concen", "Untrained": "1", "KeyAbility": "CON", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_CONCENTRATION", "HostileSkill": "0", "HideFromLevelUp": "0"}, + {"key": "skills:discipline", "id": 3, "Label": "Discipline", "Name": "343", "Description": "347", "Icon": "isk_discipline", "Untrained": "1", "KeyAbility": "STR", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_DISCIPLINE", "HostileSkill": "0", "HideFromLevelUp": "0"}, + {"key": "skills:tumble", "id": 25, "Label": "Tumble", "Name": "280", "Description": "356", "Icon": "isk_tumble", "Untrained": "1", "KeyAbility": "DEX", "ArmorCheckPenalty": "1", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_TUMBLE", "HostileSkill": "0", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skills", "lock.json"), `{ + "skills:craft_armorsmithing": 25, + "skills:athletics": 28, + "skills:knowledge_arcana": 40 +}`+"\n") + for _, name := range []string{ + "add_disguise.json", + "add_linguistics.json", + "add_perception.json", + "add_scriptcraft.json", + "add_sensemotive.json", + "add_stealth.json", + "add_survival.json", + "add_userope.json", + "ovr_acrobatics.json", + "ovr_animalhandling.json", + "ovr_appraise.json", + "ovr_concentration.json", + "ovr_disabledevice.json", + "ovr_heal.json", + "ovr_hiddenskills.json", + "ovr_influence.json", + "ovr_openlock.json", + "ovr_parry.json", + "ovr_searchtowis.json", + "ovr_sleightofhand.json", + "ovr_taunttointimidate.json", + } { + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", name), `{"overrides": []}`+"\n") + } + for _, name := range []string{ + "add_craftalchemy.json", + "add_craftcooking.json", + "add_craftjewelry.json", + "add_craftleatherworking.json", + "add_craftstonework.json", + "add_crafttextiles.json", + "ovr_crafttinkering.json", + "ovr_craftweaponsmithing.json", + "ovr_craftwoodworking.json", + } { + payload := `{"entries": {}}` + if strings.HasPrefix(name, "ovr_") { + payload = `{"overrides": []}` + } + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "craft", name), payload+"\n") + } + for _, name := range []string{ + "add_knowledgearchitecture.json", + "add_knowledgedungeoneering.json", + "add_knowledgegeography.json", + "add_knowledgehistory.json", + "add_knowledgelocal.json", + "add_knowledgenature.json", + "add_knowledgenobility.json", + "add_knowledgeplanar.json", + "add_knowledgereligion.json", + } { + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "knowledge", name), `{"entries": {}}`+"\n") + } + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "add_athletics.json"), `{ + "entries": { + "skills:athletics": { + "Label": "Athletics", + "Name": {"tlk": "skills:athletics.name"}, + "Description": {"tlk": "skills:athletics.description"}, + "Icon": "isk_athletics", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "AllClassesCanUse": "1", + "Constant": "SKILL_ATHLETICS", + "HostileSkill": "0", + "HideFromLevelUp": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "craft", "ovr_craftarmorsmithing.json"), `{ + "overrides": [ + { + "key": "skills:craft_armorsmithing", + "id": 25, + "Label": "Craftarmorsmithing", + "Name": {"tlk": "skills:craft_armorsmithing.name"}, + "Description": {"tlk": "skills:craft_armorsmithing.description"}, + "Icon": "isk_x2carm", + "Untrained": "0", + "Constant": "SKILL_CRAFT_ARMORSMITHING", + "meta": {"wiki": {"status": "new"}}, + "HideFromLevelUp": "0" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "knowledge", "add_knowledgearcana.json"), `{ + "entries": { + "skills:knowledge_arcana": { + "Label": "Knowledgearcana", + "Name": {"tlk": "skills:knowledge_arcana.name"}, + "Icon": "isk_knarcana", + "Untrained": "1", + "KeyAbility": "INT", + "ArmorCheckPenalty": "0", + "AllClassesCanUse": "1", + "Constant": "SKILL_KNOWLEDGE_ARCANA", + "HostileSkill": "0", + "HideFromLevelUp": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "rmv_removedskills.json"), `{ + "overrides": [ + { + "key": null, + "id": 3, + "Label": "Discipline_REMOVED", + "Constant": "****", + "AllClassesCanUse": "0", + "HideFromLevelUp": "1" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 6 { + t.Fatalf("expected skills files to be imported, got %#v", result) + } + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "skills", "base.json"), + filepath.Join(root, "topdata", "data", "skills", "lock.json"), + filepath.Join(root, "topdata", "data", "skills", "modules", "add_athletics.json"), + filepath.Join(root, "topdata", "data", "skills", "modules", "craft", "ovr_craftarmorsmithing.json"), + filepath.Join(root, "topdata", "data", "skills", "modules", "knowledge", "add_knowledgearcana.json"), + filepath.Join(root, "topdata", "data", "skills", "modules", "rmv_removedskills.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported skills path %s: %v", path, err) + } + } + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "skills", "base.json")) + if err != nil { + t.Fatalf("read skills base: %v", err) + } + if !strings.Contains(string(baseRaw), `"output": "skills.2da"`) { + t.Fatalf("expected imported skills output name, got:\n%s", string(baseRaw)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "skills", "lock.json")) + if err != nil { + t.Fatalf("read skills lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"skills:athletics": 28`) || !strings.Contains(string(lockRaw), `"skills:knowledge_arcana": 40`) { + t.Fatalf("expected imported skills lock ids, got:\n%s", string(lockRaw)) + } + moduleRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "skills", "modules", "add_athletics.json")) + if err != nil { + t.Fatalf("read skills athletics module: %v", err) + } + if !strings.Contains(string(moduleRaw), `"text": "Athletics"`) || !strings.Contains(string(moduleRaw), `"text": "Athletics description"`) { + t.Fatalf("expected normalized inline TLK payloads in athletics module, got:\n%s", string(moduleRaw)) + } + craftRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "skills", "modules", "craft", "ovr_craftarmorsmithing.json")) + if err != nil { + t.Fatalf("read skills craft override: %v", err) + } + if !strings.Contains(string(craftRaw), `"text": "Craft Armor Smithing"`) || !strings.Contains(string(craftRaw), `"text": "Craft armor description"`) { + t.Fatalf("expected normalized inline TLK payloads in craft override, got:\n%s", string(craftRaw)) + } +} + +func TestBuildSupportsCanonicalSkills(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills", "modules", "craft")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills", "modules", "knowledge")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"key": "skills:animalempathy", "id": 0, "Label": "AnimalEmpathy", "Name": "269", "Description": "344", "Icon": "isk_aniemp", "Untrained": "0", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "0", "Category": "****", "MaxCR": "1", "Constant": "SKILL_ANIMAL_EMPATHY", "HostileSkill": "1", "HideFromLevelUp": "0"}, + {"key": "skills:concentration", "id": 1, "Label": "Concentration", "Name": "270", "Description": "345", "Icon": "isk_concen", "Untrained": "1", "KeyAbility": "CON", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_CONCENTRATION", "HostileSkill": "0", "HideFromLevelUp": "0"}, + {"key": "skills:discipline", "id": 3, "Label": "Discipline", "Name": "343", "Description": "347", "Icon": "isk_discipline", "Untrained": "1", "KeyAbility": "STR", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_DISCIPLINE", "HostileSkill": "0", "HideFromLevelUp": "0"}, + {"key": "skills:craft_armorsmithing", "id": 25, "Label": "Tumble", "Name": "280", "Description": "356", "Icon": "isk_tumble", "Untrained": "1", "KeyAbility": "DEX", "ArmorCheckPenalty": "1", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_TUMBLE", "HostileSkill": "0", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{ + "skills:craft_armorsmithing": 25, + "skills:athletics": 28, + "skills:knowledge_arcana": 40 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "modules", "add_athletics.json"), `{ + "entries": { + "skills:athletics": { + "Label": "Athletics", + "Name": "1000", + "Description": "1001", + "Icon": "isk_athletics", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "AllClassesCanUse": "1", + "Constant": "SKILL_ATHLETICS", + "HostileSkill": "0", + "HideFromLevelUp": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "modules", "craft", "ovr_craftarmorsmithing.json"), `{ + "overrides": [ + { + "key": "skills:craft_armorsmithing", + "id": 25, + "Label": "Craftarmorsmithing", + "Name": "1002", + "Description": "1003", + "Icon": "isk_x2carm", + "Untrained": "0", + "Constant": "SKILL_CRAFT_ARMORSMITHING", + "meta": {"wiki": {"status": "new"}}, + "HideFromLevelUp": "0" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "modules", "knowledge", "add_knowledgearcana.json"), `{ + "entries": { + "skills:knowledge_arcana": { + "Label": "Knowledgearcana", + "Name": "1004", + "Icon": "isk_knarcana", + "Untrained": "1", + "KeyAbility": "INT", + "ArmorCheckPenalty": "0", + "AllClassesCanUse": "1", + "Constant": "SKILL_KNOWLEDGE_ARCANA", + "HostileSkill": "0", + "HideFromLevelUp": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "modules", "rmv_removedskills.json"), `{ + "overrides": [ + { + "key": null, + "id": 3, + "Label": "Discipline_REMOVED", + "Constant": "****", + "AllClassesCanUse": "0", + "HideFromLevelUp": "1" + } + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "skills.2da")) + if err != nil { + t.Fatalf("read skills.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "Label\tName\tDescription\tIcon") || + !strings.Contains(text, "3\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\n") || + !strings.Contains(text, "25\tCraftarmorsmithing\t1002\t1003\tisk_x2carm\t0\tDEX\t1\t1\t****\t****\tSKILL_CRAFT_ARMORSMITHING\t0\t0\n") || + !strings.Contains(text, "28\tAthletics\t1000\t1001\tisk_athletics\t1\tSTR\t1\t1\t****\t****\tSKILL_ATHLETICS\t0\t0\n") || + !strings.Contains(text, "40\tKnowledgearcana\t1004\t****\tisk_knarcana\t1\tINT\t0\t1\t****\t****\tSKILL_KNOWLEDGE_ARCANA\t0\t0\n") { + t.Fatalf("unexpected skills.2da output:\n%s", text) + } +} + +func TestBuildGeneratedSkillFocusUsesOverriddenCanonicalSkillKey(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "skill_focus.json": `{ + "family": "skill_focus", + "family_key": "skillfocus", + "template": "masterfeats:skillfocus", + "name_prefix": "Skill Focus", + "label_prefix": "FEAT_SKILL_FOCUS", + "constant_prefix": "FEAT_SKILL_FOCUS", + "default_fields": { + "CRValue": "0.5", + "ReqSkillMinRanks": "1", + "ALLCLASSESCANUSE": "1", + "TOOLSCATEGORIES": "6", + "PreReqEpic": "0", + "ReqAction": "1" + }, + "child_ref_field": "REQSKILL", + "child_source": { + "dataset": "skills", + "predicate": "accessible" + } +}` + "\n", + "greater_skill_focus.json": `{ + "family": "greater_skill_focus", + "family_key": "greaterskillfocus", + "template": "masterfeats:greaterskillfocus", + "name_prefix": "Greater Skill Focus", + "label_prefix": "FEAT_GREATER_SKILL_FOCUS", + "constant_prefix": "FEAT_GREATER_SKILL_FOCUS", + "default_fields": { + "CRValue": "0.2", + "ReqSkillMinRanks": "15", + "ALLCLASSESCANUSE": "1", + "TOOLSCATEGORIES": "6", + "PreReqEpic": "0", + "ReqAction": "1" + }, + "child_ref_field": "REQSKILL", + "child_source": { + "dataset": "skills", + "predicate": "accessible" + } +}` + "\n", + }, map[string]int{ + "feat:skillfocus_intimidate": 192, + "feat:greaterskillfocus_intimidate": 1305, + }) + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"id": 18, "key": "skills:taunt", "Label": "Taunt", "Name": "280", "Description": "356", "Icon": "isk_taunt", "Untrained": "1", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_TAUNT", "HostileSkill": "1", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{ + "skills:intimidate": 18 +}`+"\n") + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills", "modules")) + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "modules", "ovr_taunttointimidate.json"), `{ + "overrides": [ + { + "id": 18, + "key": "skills:intimidate", + "Label": "Intimidate", + "Constant": "SKILL_INTIMIDATE", + "HideFromLevelUp": "0" + } + ] +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "192\tFEAT_SKILL_FOCUS_INTIMIDATE\t") { + t.Fatalf("expected generated skill focus intimidate row, got:\n%s", text) + } + if !strings.Contains(text, "1305\tFEAT_GREATER_SKILL_FOCUS_INTIMIDATE\t") { + t.Fatalf("expected generated greater skill focus intimidate row, got:\n%s", text) + } + if strings.Contains(text, "FEAT_SKILL_FOCUS_TAUNT") || strings.Contains(text, "FEAT_GREATER_SKILL_FOCUS_TAUNT") { + t.Fatalf("expected generated feat families to follow overridden canonical skill key, got:\n%s", text) + } +} + +func TestBuildGeneratedSkillFocusReusesHardcodedSkillFeatRowsForRenamedSkills(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "skill_focus.json": `{ + "family": "skill_focus", + "family_key": "skill_focus", + "template": "masterfeats:skillfocus", + "name_prefix": "Skill Focus", + "label_prefix": "FEAT_SKILL_FOCUS", + "constant_prefix": "FEAT_SKILL_FOCUS", + "default_fields": { + "CRValue": "0.5", + "ReqSkillMinRanks": "1", + "ALLCLASSESCANUSE": "1", + "TOOLSCATEGORIES": "6", + "PreReqEpic": "0", + "ReqAction": "1" + }, + "apply_after_modules": true, + "child_ref_field": "REQSKILL", + "child_source": { + "dataset": "skills", + "predicate": "accessible" + }, + "overrides": { + "skills:craft_woodworking": { + "ICON": "ife_X1SFCrTrap" + } + } +}` + "\n", + "greater_skill_focus.json": `{ + "family": "greater_skill_focus", + "family_key": "greater_skill_focus", + "template": "masterfeats:greaterskillfocus", + "name_prefix": "Greater Skill Focus", + "label_prefix": "FEAT_GREATER_SKILL_FOCUS", + "constant_prefix": "FEAT_GREATER_SKILL_FOCUS", + "legacy_family_keys": ["epic_skill_focus"], + "default_fields": { + "CRValue": "0.2", + "ReqSkillMinRanks": "15", + "ALLCLASSESCANUSE": "1", + "TOOLSCATEGORIES": "6", + "PreReqEpic": "0", + "ReqAction": "1" + }, + "apply_after_modules": true, + "child_ref_field": "REQSKILL", + "child_source": { + "dataset": "skills", + "predicate": "accessible" + }, + "overrides": { + "skills:craft_woodworking": { + "ICON": "ife_X2EpSkFCrTr" + } + } +}` + "\n", + }, map[string]int{ + "feat:skill_focus_craft_trap": 407, + "feat:skill_focus_craft_woodworking": 1372, + "feat:greater_skill_focus_craft_trap": 590, + "feat:greater_skill_focus_craft_woodworking": 1314, + }) + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"id": 22, "key": "skills:craft_woodworking", "Label": "CraftWoodworking", "Name": "2000", "Description": "2001", "Icon": "isk_craftwood", "Untrained": "1", "KeyAbility": "INT", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_CRAFT_WOODWORKING", "HostileSkill": "0", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:craft_woodworking":22}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "compare_reference": false, + "columns": [ + "LABEL","FEAT","DESCRIPTION","ICON","MINATTACKBONUS","MINSTR","MINDEX","MININT","MINWIS","MINCON","MINCHA", + "MINSPELLLVL","PREREQFEAT1","PREREQFEAT2","GAINMULTIPLE","EFFECTSSTACK","ALLCLASSESCANUSE","CATEGORY","MAXCR", + "SPELLID","SUCCESSOR","CRValue","USESPERDAY","MASTERFEAT","TARGETSELF","OrReqFeat0","OrReqFeat1","OrReqFeat2", + "OrReqFeat3","OrReqFeat4","REQSKILL","ReqSkillMinRanks","REQSKILL2","ReqSkillMinRanks2","Constant","TOOLSCATEGORIES", + "HostileFeat","MinLevel","MinLevelClass","MaxLevel","MinFortSave","PreReqEpic","ReqAction" + ], + "rows": [ + {"id": 407, "key": "feat:skill_focus_craft_trap", "LABEL": "FEAT_SKILL_FOCUS_CRAFT_TRAP", "REQSKILL": 22, "MASTERFEAT": 4, "Constant": "FEAT_SKILL_FOCUS_CRAFT_TRAP"}, + {"id": 590, "key": "feat:epic_skill_focus_craft_trap", "LABEL": "FEAT_EPIC_SKILL_FOCUS_CRAFT_TRAP", "REQSKILL": 22, "MASTERFEAT": 5, "Constant": "FEAT_EPIC_SKILL_FOCUS_CRAFT_TRAP"} + ] +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "407\tFEAT_SKILL_FOCUS_CRAFT_WOODWORKING\t") { + t.Fatalf("expected generated skill focus woodworking to reuse hardcoded Craft Trap row 407, got:\n%s", text) + } + if !strings.Contains(text, "590\tFEAT_GREATER_SKILL_FOCUS_CRAFT_WOODWORKING\t") { + t.Fatalf("expected generated greater skill focus woodworking to reuse hardcoded epic Craft Trap row 590, got:\n%s", text) + } + for _, unexpected := range []string{ + "1372\tFEAT_SKILL_FOCUS_CRAFT_WOODWORKING\t", + "1314\tFEAT_GREATER_SKILL_FOCUS_CRAFT_WOODWORKING\t", + "FEAT_SKILL_FOCUS_CRAFT_TRAP\t", + "FEAT_EPIC_SKILL_FOCUS_CRAFT_TRAP\t", + } { + if strings.Contains(text, unexpected) { + t.Fatalf("expected no duplicate stale skill focus row %q, got:\n%s", unexpected, text) + } + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "feat", "lock.json")) + if err != nil { + t.Fatalf("read feat lock: %v", err) + } + lockText := string(lockRaw) + for _, expected := range []string{ + `"feat:skill_focus_craft_woodworking": 407`, + `"feat:greater_skill_focus_craft_woodworking": 590`, + } { + if !strings.Contains(lockText, expected) { + t.Fatalf("expected lock to contain %s, got:\n%s", expected, lockText) + } + } + for _, stale := range []string{ + "feat:skill_focus_craft_trap", + "feat:greater_skill_focus_craft_trap", + "feat:epic_skill_focus_craft_trap", + } { + if strings.Contains(lockText, stale) { + t.Fatalf("expected stale lock %s to be pruned, got:\n%s", stale, lockText) + } + } +} + +func TestFeatGeneratedContextIgnoresExplicitlyRetiredFeatKeys(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "skill_focus.json": `{ + "family": "skill_focus", + "family_key": "skill_focus", + "template": "masterfeats:skillfocus", + "name_prefix": "Skill Focus", + "label_prefix": "FEAT_SKILL_FOCUS", + "constant_prefix": "FEAT_SKILL_FOCUS", + "default_fields": { + "CRValue": "0.5", + "ReqSkillMinRanks": "1", + "ALLCLASSESCANUSE": "1", + "TOOLSCATEGORIES": "6", + "PreReqEpic": "0", + "ReqAction": "1" + }, + "apply_after_modules": true, + "child_ref_field": "REQSKILL", + "child_source": { + "dataset": "skills", + "predicate": "accessible" + } +}` + "\n", + }, map[string]int{ + "feat:skill_focus_intimidate": 916, + "feat:epic_skill_focus_intimidate": 918, + }) + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"id": 18, "key": "skills:intimidate", "Label": "Intimidate", "Name": "8756", "Description": "8786", "Icon": "isk_x2inti", "Untrained": "1", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_INTIMIDATE", "HostileSkill": "0", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:intimidate":18}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "compare_reference": false, + "columns": [ + "LABEL","FEAT","DESCRIPTION","ICON","MINATTACKBONUS","MINSTR","MINDEX","MININT","MINWIS","MINCON","MINCHA", + "MINSPELLLVL","PREREQFEAT1","PREREQFEAT2","GAINMULTIPLE","EFFECTSSTACK","ALLCLASSESCANUSE","CATEGORY","MAXCR", + "SPELLID","SUCCESSOR","CRValue","USESPERDAY","MASTERFEAT","TARGETSELF","OrReqFeat0","OrReqFeat1","OrReqFeat2", + "OrReqFeat3","OrReqFeat4","REQSKILL","ReqSkillMinRanks","REQSKILL2","ReqSkillMinRanks2","Constant","TOOLSCATEGORIES", + "HostileFeat","MinLevel","MinLevelClass","MaxLevel","MinFortSave","PreReqEpic","ReqAction" + ], + "rows": [ + {"id": 916, "key": "feat:skill_focus_intimidate", "LABEL": "FEAT_SKILL_FOCUS_INTIMIDATE", "REQSKILL": 24, "MASTERFEAT": 4, "Constant": "FEAT_SKILL_FOCUS_INTIMIDATE"}, + {"id": 918, "key": "feat:epic_skill_focus_intimidate", "LABEL": "FEAT_EPIC_SKILL_FOCUS_INTIMIDATE", "REQSKILL": 24, "MASTERFEAT": 15, "Constant": "FEAT_EPIC_SKILL_FOCUS_INTIMIDATE"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules", "removedandhidden")) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "removedandhidden", "rmv_feat_intimidate.json"), `{ + "overrides": [ + {"id": 916, "null": true, "key": null}, + {"id": 918, "null": true, "key": null} + ] +}`+"\n") + + datasets, err := discoverNativeDatasets(filepath.Join(root, "topdata", "data")) + if err != nil { + t.Fatalf("discoverNativeDatasets failed: %v", err) + } + var featDataset nativeDataset + found := false + for _, dataset := range datasets { + if dataset.Name == "feat" { + featDataset = dataset + found = true + break + } + } + if !found { + t.Fatal("expected feat dataset to be discovered") + } + lockData, err := loadLockfile(featDataset.LockPath) + if err != nil { + t.Fatalf("load feat lock failed: %v", err) + } + ctx, err := newFeatGeneratedContext(featDataset, lockData) + if err != nil { + t.Fatalf("newFeatGeneratedContext failed: %v", err) + } + if ctx.featKeyExists("feat:skill_focus_intimidate") { + t.Fatal("expected explicitly retired feat key to be absent from existing feat set") + } + if _, ok := ctx.lockData["feat:skill_focus_intimidate"]; ok { + t.Fatalf("expected explicitly retired feat key to be absent from generated lock view, got %#v", ctx.lockData) + } +} + +func TestBuildGeneratedSkillFocusDoesNotReuseNulledFeatRow(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "skill_focus.json": `{ + "family": "skill_focus", + "family_key": "skill_focus", + "template": "masterfeats:skillfocus", + "name_prefix": "Skill Focus", + "label_prefix": "FEAT_SKILL_FOCUS", + "constant_prefix": "FEAT_SKILL_FOCUS", + "default_fields": { + "CRValue": "0.5", + "ReqSkillMinRanks": "1", + "ALLCLASSESCANUSE": "1", + "TOOLSCATEGORIES": "6", + "PreReqEpic": "0", + "ReqAction": "1" + }, + "apply_after_modules": true, + "child_ref_field": "REQSKILL", + "child_source": { + "dataset": "skills", + "predicate": "accessible" + }, + "overrides": { + "skills:intimidate": { + "ICON": "ife_X2SkFInti" + } + } +}` + "\n", + }, map[string]int{ + "feat:skill_focus_intimidate": 916, + }) + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"id": 18, "key": "skills:intimidate", "Label": "Intimidate", "Name": "8756", "Description": "8786", "Icon": "isk_x2inti", "Untrained": "1", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_INTIMIDATE", "HostileSkill": "1", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:intimidate":18}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "compare_reference": false, + "columns": [ + "LABEL","FEAT","DESCRIPTION","ICON","MINATTACKBONUS","MINSTR","MINDEX","MININT","MINWIS","MINCON","MINCHA", + "MINSPELLLVL","PREREQFEAT1","PREREQFEAT2","GAINMULTIPLE","EFFECTSSTACK","ALLCLASSESCANUSE","CATEGORY","MAXCR", + "SPELLID","SUCCESSOR","CRValue","USESPERDAY","MASTERFEAT","TARGETSELF","OrReqFeat0","OrReqFeat1","OrReqFeat2", + "OrReqFeat3","OrReqFeat4","REQSKILL","ReqSkillMinRanks","REQSKILL2","ReqSkillMinRanks2","Constant","TOOLSCATEGORIES", + "HostileFeat","MinLevel","MinLevelClass","MaxLevel","MinFortSave","PreReqEpic","ReqAction" + ], + "rows": [ + {"id": 192, "key": "feat:skill_focus_taunt", "LABEL": "FEAT_SKILL_FOCUS_TAUNT", "REQSKILL": 18, "MASTERFEAT": 4, "Constant": "FEAT_SKILL_FOCUS_TAUNT"}, + {"id": 916, "key": "feat:skill_focus_intimidate", "LABEL": "FEAT_SKILL_FOCUS_INTIMIDATE", "REQSKILL": 24, "MASTERFEAT": 4, "Constant": "FEAT_SKILL_FOCUS_INTIMIDATE"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules", "removedandhidden")) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "removedandhidden", "rmv_feat_intimidate.json"), `{ + "overrides": [ + {"id": 916, "null": true, "key": null} + ] +}`+"\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "feat", "lock.json")) + if err != nil { + t.Fatalf("read feat lock: %v", err) + } + lockText := string(lockRaw) + if strings.Contains(lockText, `"feat:skill_focus_intimidate": 916`) { + t.Fatalf("expected generated skill focus intimidate to avoid nulled row 916, got:\n%s", lockText) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "feat.2da")) + if err != nil { + t.Fatalf("read feat.2da: %v", err) + } + text := string(got) + if strings.Contains(text, "916\tFEAT_SKILL_FOCUS_INTIMIDATE\t") { + t.Fatalf("expected generated skill focus intimidate to avoid row 916, got:\n%s", text) + } + if !strings.Contains(text, "FEAT_SKILL_FOCUS_INTIMIDATE") { + t.Fatalf("expected generated skill focus intimidate row, got:\n%s", text) + } +} + +func TestOverrideRequestsNullRowUsesExplicitNullFlag(t *testing.T) { + if overrideRequestsNullRow(map[string]any{"key": nil}) { + t.Fatal("key: null should not blank a row") + } + if overrideRequestsNullRow(map[string]any{"key": "****"}) { + t.Fatal(`key: "****" should not blank a row`) + } + if !overrideRequestsNullRow(map[string]any{"null": true}) { + t.Fatal(`null: true should blank a row`) + } +} + +func TestBuildSkillsKeyNullRemovesIdentityWithoutNullingRow(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"id": 18, "key": "skills:taunt", "Label": "Taunt", "Name": "280", "Description": "356", "Icon": "isk_taunt", "Untrained": "1", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_TAUNT", "HostileSkill": "1", "HideFromLevelUp": "0"}, + {"id": 24, "key": "skills:intimidate", "Label": "Intimidate", "Name": "8756", "Description": "8786", "Icon": "isk_X2Inti", "Untrained": "1", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_INTIMIDATE", "HostileSkill": "0", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "modules", "ovr_taunttointimidate.json"), `{ + "overrides": [ + { + "id": 18, + "key": "skills:intimidate", + "Label": "Intimidate", + "Constant": "SKILL_INTIMIDATE", + "HideFromLevelUp": "0" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "modules", "rmv_removedskills.json"), `{ + "overrides": [ + { + "id": 24, + "key": null, + "Label": null, + "Constant": null, + "HideFromLevelUp": "1" + } + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "skills", "lock.json")) + if err != nil { + t.Fatalf("read skills lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"skills:intimidate": 18`) { + t.Fatalf("expected remapped intimidate lock entry, got:\n%s", lockText) + } + if strings.Contains(lockText, `"skills:taunt"`) { + t.Fatalf("expected taunt to be removed from lock, got:\n%s", lockText) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "skills.2da")) + if err != nil { + t.Fatalf("read skills.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "24\t****\t8756\t8786\tisk_X2Inti\t1\tCHA\t0\t1\t****\t****\t****\t0\t1\n") { + t.Fatalf("expected id 24 row to remain present but de-identified, got:\n%s", text) + } +} + +func TestBuildSpellsAllowsDuplicateBaseKeyToBeSplitAcrossTwoOverrideRenames(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "base.json"), `{ + "output": "spells.2da", + "columns": ["Label", "Name", "SpellDesc"], + "rows": [ + {"id": 443, "key": "spells:etherealness", "Label": "GreaterSanctuary", "Name": "2364", "SpellDesc": "2371"}, + {"id": 724, "key": "spells:etherealness_83893", "Label": "Etherealness", "Name": "83893", "SpellDesc": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "ovr_fixduplicates.json"), `{ + "overrides": [ + { + "id": 443, + "key": "spells:greater_sanctuary" + }, + { + "id": 724, + "key": "spells:etherealness" + } + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + datasets, err := discoverNativeDatasets(filepath.Join(root, "topdata", "data")) + if err != nil { + t.Fatalf("discoverNativeDatasets failed: %v", err) + } + if len(datasets) != 1 { + t.Fatalf("expected one dataset, got %d", len(datasets)) + } + collectedDataset, err := collectNativeDataset(datasets[0]) + if err != nil { + t.Fatalf("collectNativeDataset failed: %v", err) + } + if collectedDataset.LockData["spells:greater_sanctuary"] != 443 { + t.Fatalf("expected collected greater_sanctuary lock entry, got %#v", collectedDataset.LockData) + } + if collectedDataset.LockData["spells:etherealness"] != 724 { + t.Fatalf("expected collected etherealness to be remapped to row 724, got lock=%#v rows=%#v", collectedDataset.LockData, collectedDataset.Rows) + } + if _, ok := collectedDataset.LockData["spells:etherealness_83893"]; ok { + t.Fatalf("expected collected etherealness_83893 to be retired, got %#v", collectedDataset.LockData) + } + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "spells", "lock.json")) + if err != nil { + t.Fatalf("read spells lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"spells:greater_sanctuary": 443`) { + t.Fatalf("expected greater_sanctuary lock entry, got:\n%s", lockText) + } + if !strings.Contains(lockText, `"spells:etherealness": 724`) { + t.Fatalf("expected etherealness to be remapped to row 724, got:\n%s", lockText) + } + if strings.Contains(lockText, `"spells:etherealness_83893"`) { + t.Fatalf("expected etherealness_83893 to be retired, got:\n%s", lockText) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "spells.2da")) + if err != nil { + t.Fatalf("read spells.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "GreaterSanctuary") || !strings.Contains(text, "Etherealness") { + t.Fatalf("expected both renamed rows to remain present, got:\n%s", text) + } +} + +func TestBuildBaseitemsSplitReaddsRetiredKeyToLockfile(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["label", "MaxRange"], + "rows": [ + {"id": 58, "key": "baseitems:shortspear", "label": "shortspear", "MaxRange": "100"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "00_ovr_baseitems_spear.json"), `{ + "overrides": [ + { + "id": 58, + "key": "baseitems:spear", + "label": "spear" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_add_baseitems_shortspear.json"), `{ + "entries": { + "baseitems:shortspear": { + "label": "shortspear", + "MaxRange": "255" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "20_ovr_baseitems_maxranges.json"), `{ + "overrides": [ + { + "key": "baseitems:shortspear", + "MaxRange": "255" + } + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "baseitems", "lock.json")) + if err != nil { + t.Fatalf("read baseitems lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"baseitems:spear": 58`) { + t.Fatalf("expected spear lock entry, got:\n%s", lockText) + } + if !strings.Contains(lockText, `"baseitems:shortspear": 59`) { + t.Fatalf("expected shortspear lock entry on new row, got:\n%s", lockText) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "baseitems.2da")) + if err != nil { + t.Fatalf("read baseitems.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "58\tspear") || !strings.Contains(text, "59\tshortspear") { + t.Fatalf("expected split spear rows in output, got:\n%s", text) + } +} + +func TestBuildBaseitemsAllowsKeyOnlyOverrideBeforeSameKeyIsRelocatedAndReadded(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_vanillaoverrides")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["label", "MaxRange"], + "rows": [ + {"id": 58, "key": "baseitems:shortspear", "label": "shortspear", "MaxRange": "100"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:shortspear":59}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_vanillaoverrides", "ovr_baseitems_maxranges.json"), `{ + "overrides": [ + { + "key": "baseitems:shortspear", + "MaxRange": "255" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_vanillaoverrides", "ovr_baseitems_spear.json"), `{ + "overrides": [ + { + "id": 58, + "key": "baseitems:spear", + "label": "spear" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "add_baseitems_shortspear.json"), `{ + "entries": { + "baseitems:shortspear": { + "label": "shortspear", + "MaxRange": "255" + } + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "baseitems", "lock.json")) + if err != nil { + t.Fatalf("read baseitems lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"baseitems:spear": 58`) { + t.Fatalf("expected spear lock entry, got:\n%s", lockText) + } + lockData := map[string]int{} + if err := json.Unmarshal(lockRaw, &lockData); err != nil { + t.Fatalf("parse baseitems lock: %v", err) + } + shortspearID, ok := lockData["baseitems:shortspear"] + if !ok { + t.Fatalf("expected relocated shortspear lock entry to survive, got:\n%s", lockText) + } + if shortspearID == 58 { + t.Fatalf("expected relocated shortspear to use a fresh row id, got lock:\n%s", lockText) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "baseitems.2da")) + if err != nil { + t.Fatalf("read baseitems.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "58\tspear") || !strings.Contains(text, fmt.Sprintf("%d\tshortspear", shortspearID)) { + t.Fatalf("expected split spear rows in output, got:\n%s", text) + } +} + +func TestBuildNativeKeepsGeneratedIDsStableAcrossRepeatedBuilds(t *testing.T) { + root := testProjectRoot(t) + writeFeatGeneratedHarness(t, root, map[string]string{ + "skill_focus.json": `{ + "family": "skill_focus", + "family_key": "skill_focus", + "template": "masterfeats:skillfocus", + "name_prefix": "Skill Focus", + "label_prefix": "FEAT_SKILL_FOCUS", + "constant_prefix": "FEAT_SKILL_FOCUS", + "default_fields": { + "CRValue": "0.5", + "ReqSkillMinRanks": "1", + "ALLCLASSESCANUSE": "1", + "TOOLSCATEGORIES": "6", + "PreReqEpic": "0", + "ReqAction": "1" + }, + "apply_after_modules": true, + "child_ref_field": "REQSKILL", + "child_source": { + "dataset": "skills", + "predicate": "accessible" + } +}` + "\n", + }, map[string]int{ + "feat:skill_focus_intimidate": 1333, + }) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_vanillaoverrides")) + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["label", "MaxRange"], + "rows": [ + {"id": 58, "key": "baseitems:shortspear", "label": "shortspear", "MaxRange": "100"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{ + "baseitems:shortspear": 126 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "10_vanillaoverrides", "ovr_baseitems_spear.json"), `{ + "overrides": [ + { + "id": 58, + "key": "baseitems:spear", + "label": "spear" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "add_baseitems_shortspear.json"), `{ + "entries": { + "baseitems:shortspear": { + "label": "shortspear", + "MaxRange": "255" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"id": 18, "key": "skills:intimidate", "Label": "Intimidate", "Name": "8756", "Description": "8786", "Icon": "isk_x2inti", "Untrained": "1", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_INTIMIDATE", "HostileSkill": "1", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{ + "skills:intimidate": 18 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "compare_reference": false, + "columns": [ + "LABEL","FEAT","DESCRIPTION","ICON","MINATTACKBONUS","MINSTR","MINDEX","MININT","MINWIS","MINCON","MINCHA", + "MINSPELLLVL","PREREQFEAT1","PREREQFEAT2","GAINMULTIPLE","EFFECTSSTACK","ALLCLASSESCANUSE","CATEGORY","MAXCR", + "SPELLID","SUCCESSOR","CRValue","USESPERDAY","MASTERFEAT","TARGETSELF","OrReqFeat0","OrReqFeat1","OrReqFeat2", + "OrReqFeat3","OrReqFeat4","REQSKILL","ReqSkillMinRanks","REQSKILL2","ReqSkillMinRanks2","Constant","TOOLSCATEGORIES", + "HostileFeat","MinLevel","MinLevelClass","MaxLevel","MinFortSave","PreReqEpic","ReqAction" + ], + "rows": [ + {"id": 39, "key": "feat:stunning_fist", "LABEL": "FEAT_STUNNING_FIST", "MASTERFEAT": "****", "Constant": "FEAT_STUNNING_FIST"}, + {"id": 916, "key": "feat:skill_focus_intimidate", "LABEL": "FEAT_SKILL_FOCUS_INTIMIDATE", "REQSKILL": 24, "MASTERFEAT": 4, "Constant": "FEAT_SKILL_FOCUS_INTIMIDATE"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:skill_focus_intimidate": 1333, + "feat:stunning_fist": 1116 +}`+"\n") + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules", "00_removed")) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "00_removed", "rmv_feat_intimidate.json"), `{ + "overrides": [ + {"id": 916, "null": true, "key": null} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "00_removed", "rmv_feat_stunningfist.json"), `{ + "overrides": [ + {"id": 39, "null": true, "key": null} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules", "activecombat")) + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "activecombat", "add_feat_stunning_fist.json"), `{ + "entries": { + "feat:stunning_fist": { + "LABEL": "FEAT_STUNNING_FIST", + "Constant": "FEAT_STUNNING_FIST" + } + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + expectedBaseitems := map[string]int{ + "baseitems:spear": 58, + "baseitems:shortspear": 126, + } + expectedFeat := map[string]int{ + "feat:skill_focus_intimidate": 1333, + "feat:stunning_fist": 1116, + } + for run := 1; run <= 10; run++ { + if _, err := BuildNative(testProject(root), nil); err != nil { + t.Fatalf("BuildNative run %d failed: %v", run, err) + } + assertLockIDs(t, run, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), expectedBaseitems) + assertLockIDs(t, run, filepath.Join(root, "topdata", "data", "feat", "lock.json"), expectedFeat) + } +} + +func assertLockIDs(t *testing.T, run int, path string, expected map[string]int) { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("run %d: read lockfile %s: %v", run, path, err) + } + lockData := map[string]int{} + if err := json.Unmarshal(raw, &lockData); err != nil { + t.Fatalf("run %d: parse lockfile %s: %v\n%s", run, path, err, string(raw)) + } + for key, want := range expected { + if got := lockData[key]; got != want { + t.Fatalf("run %d: expected %s to stay locked to id %d, got %d in %s:\n%s", run, key, want, got, path, string(raw)) + } + } +} + +func TestBuildSkillsIgnoresStaleBaseSpanLockDuringInitialLoad(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"id": 18, "key": "skills:taunt", "Label": "Taunt", "Name": "342", "Description": "366", "Icon": "isk_taunt", "Untrained": "1", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "1", "Constant": "SKILL_TAUNT", "HostileSkill": "1", "HideFromLevelUp": "0"}, + {"id": 24, "key": "skills:intimidate", "Label": "Intimidate", "Name": "8756", "Description": "8786", "Icon": "isk_X2Inti", "Untrained": "1", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_INTIMIDATE", "HostileSkill": "0", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{ + "skills:intimidate": 18, + "skills:taunt": 18 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "modules", "ovr_taunttointimidate.json"), `{ + "overrides": [ + { + "id": 18, + "key": "skills:intimidate", + "Label": "Intimidate", + "Constant": "SKILL_INTIMIDATE", + "HideFromLevelUp": "0" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "modules", "rmv_removedskills.json"), `{ + "overrides": [ + { + "id": 24, + "key": null, + "Label": null, + "Constant": null, + "HideFromLevelUp": "1" + } + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + if _, err := buildNativeUnchecked(testProject(root), NativeBuildOptions{BuildWiki: false}, nil, nil); err != nil { + t.Fatalf("buildNativeUnchecked failed: %v", err) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "skills", "lock.json")) + if err != nil { + t.Fatalf("read skills lock: %v", err) + } + lockText := string(lockRaw) + if !strings.Contains(lockText, `"skills:intimidate": 18`) { + t.Fatalf("expected intimidate to remain locked to row 18, got:\n%s", lockText) + } + if strings.Contains(lockText, `"skills:taunt"`) { + t.Fatalf("expected stale taunt alias to be pruned, got:\n%s", lockText) + } +} + +func TestBuildCanonicalSkillsMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules", "skills")) + mkdirAll(t, filepath.Join(root, "reference", "data", "skills", "modules", "craft")) + mkdirAll(t, filepath.Join(root, "reference", "data", "skills", "modules", "knowledge")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "lock.json"), `{ + "skills:athletics.name": 1000, + "skills:athletics.description": 1001, + "skills:craft_armorsmithing.name": 1002, + "skills:craft_armorsmithing.description": 1003, + "skills:knowledge_arcana.name": 1004 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "modules", "skills", "skills.json"), `{ + "entries": { + "skills:athletics.name": {"text": "Athletics"}, + "skills:athletics.description": {"text": "Athletics description"}, + "skills:craft_armorsmithing.name": {"text": "Craft Armor Smithing"}, + "skills:craft_armorsmithing.description": {"text": "Craft armor description"}, + "skills:knowledge_arcana.name": {"text": "Knowledge Arcana"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +cols = ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"] +with open(os.path.join(out, "skills.2da"), "w", encoding="utf-8") as f: + def emit(row_id, values): + row = {c: "****" for c in cols} + row.update(values) + f.write(str(row_id) + "\t" + "\t".join(row[c] for c in cols) + "\n") + f.write("2DA V2.0\n\n") + f.write("\t".join(cols) + "\n") + emit(0, {"Label": "AnimalEmpathy", "Name": "269", "Description": "344", "Icon": "isk_aniemp", "Untrained": "0", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "0", "Category": "****", "MaxCR": "1", "Constant": "SKILL_ANIMAL_EMPATHY", "HostileSkill": "1", "HideFromLevelUp": "0"}) + emit(1, {"Label": "Concentration", "Name": "270", "Description": "345", "Icon": "isk_concen", "Untrained": "1", "KeyAbility": "CON", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_CONCENTRATION", "HostileSkill": "0", "HideFromLevelUp": "0"}) + emit(2, {}) + emit(3, {}) + for row_id in range(4, 25): + emit(row_id, {}) + emit(25, {"Label": "Craftarmorsmithing", "Name": "1002", "Description": "1003", "Icon": "isk_x2carm", "Untrained": "0", "KeyAbility": "DEX", "ArmorCheckPenalty": "1", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_CRAFT_ARMORSMITHING", "HostileSkill": "0", "HideFromLevelUp": "0"}) + emit(26, {}) + emit(27, {}) + emit(28, {"Label": "Athletics", "Name": "1000", "Description": "1001", "Icon": "isk_athletics", "Untrained": "1", "KeyAbility": "STR", "ArmorCheckPenalty": "1", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_ATHLETICS", "HostileSkill": "0", "HideFromLevelUp": "0"}) + for row_id in range(29, 40): + emit(row_id, {}) + emit(40, {"Label": "Knowledgearcana", "Name": "1004", "Description": "****", "Icon": "isk_knarcana", "Untrained": "1", "KeyAbility": "INT", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_KNOWLEDGE_ARCANA", "HostileSkill": "0", "HideFromLevelUp": "0"}) +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "skills", "base.json"), `{ + "columns": ["Label", "Name", "Description", "Icon", "Untrained", "KeyAbility", "ArmorCheckPenalty", "AllClassesCanUse", "Category", "MaxCR", "Constant", "HostileSkill", "HideFromLevelUp"], + "rows": [ + {"key": "skills:animalempathy", "id": 0, "Label": "AnimalEmpathy", "Name": "269", "Description": "344", "Icon": "isk_aniemp", "Untrained": "0", "KeyAbility": "CHA", "ArmorCheckPenalty": "0", "AllClassesCanUse": "0", "Category": "****", "MaxCR": "1", "Constant": "SKILL_ANIMAL_EMPATHY", "HostileSkill": "1", "HideFromLevelUp": "0"}, + {"key": "skills:concentration", "id": 1, "Label": "Concentration", "Name": "270", "Description": "345", "Icon": "isk_concen", "Untrained": "1", "KeyAbility": "CON", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_CONCENTRATION", "HostileSkill": "0", "HideFromLevelUp": "0"}, + {"key": "skills:discipline", "id": 3, "Label": "Discipline", "Name": "343", "Description": "347", "Icon": "isk_discipline", "Untrained": "1", "KeyAbility": "STR", "ArmorCheckPenalty": "0", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_DISCIPLINE", "HostileSkill": "0", "HideFromLevelUp": "0"}, + {"key": "skills:craft_armorsmithing", "id": 25, "Label": "Tumble", "Name": "280", "Description": "356", "Icon": "isk_tumble", "Untrained": "1", "KeyAbility": "DEX", "ArmorCheckPenalty": "1", "AllClassesCanUse": "1", "Category": "****", "MaxCR": "****", "Constant": "SKILL_TUMBLE", "HostileSkill": "0", "HideFromLevelUp": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skills", "lock.json"), `{ + "skills:craft_armorsmithing": 25, + "skills:athletics": 28, + "skills:knowledge_arcana": 40 +}`+"\n") + for _, name := range []string{ + "add_disguise.json", + "add_linguistics.json", + "add_perception.json", + "add_scriptcraft.json", + "add_sensemotive.json", + "add_stealth.json", + "add_survival.json", + "add_userope.json", + "ovr_acrobatics.json", + "ovr_animalhandling.json", + "ovr_appraise.json", + "ovr_concentration.json", + "ovr_disabledevice.json", + "ovr_heal.json", + "ovr_hiddenskills.json", + "ovr_influence.json", + "ovr_openlock.json", + "ovr_parry.json", + "ovr_searchtowis.json", + "ovr_sleightofhand.json", + "ovr_taunttointimidate.json", + } { + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", name), `{"overrides": []}`+"\n") + } + for _, name := range []string{ + "add_craftalchemy.json", + "add_craftcooking.json", + "add_craftjewelry.json", + "add_craftleatherworking.json", + "add_craftstonework.json", + "add_crafttextiles.json", + "ovr_crafttinkering.json", + "ovr_craftweaponsmithing.json", + "ovr_craftwoodworking.json", + } { + payload := `{"entries": {}}` + if strings.HasPrefix(name, "ovr_") { + payload = `{"overrides": []}` + } + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "craft", name), payload+"\n") + } + for _, name := range []string{ + "add_knowledgearchitecture.json", + "add_knowledgedungeoneering.json", + "add_knowledgegeography.json", + "add_knowledgehistory.json", + "add_knowledgelocal.json", + "add_knowledgenature.json", + "add_knowledgenobility.json", + "add_knowledgeplanar.json", + "add_knowledgereligion.json", + } { + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "knowledge", name), `{"entries": {}}`+"\n") + } + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "add_athletics.json"), `{ + "entries": { + "skills:athletics": { + "Label": "Athletics", + "Name": {"tlk": "skills:athletics.name"}, + "Description": {"tlk": "skills:athletics.description"}, + "Icon": "isk_athletics", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "AllClassesCanUse": "1", + "Constant": "SKILL_ATHLETICS", + "HostileSkill": "0", + "HideFromLevelUp": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "craft", "ovr_craftarmorsmithing.json"), `{ + "overrides": [ + { + "key": "skills:craft_armorsmithing", + "id": 25, + "Label": "Craftarmorsmithing", + "Name": {"tlk": "skills:craft_armorsmithing.name"}, + "Description": {"tlk": "skills:craft_armorsmithing.description"}, + "Icon": "isk_x2carm", + "Untrained": "0", + "Constant": "SKILL_CRAFT_ARMORSMITHING", + "meta": {"wiki": {"status": "new"}}, + "HideFromLevelUp": "0" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "knowledge", "add_knowledgearcana.json"), `{ + "entries": { + "skills:knowledge_arcana": { + "Label": "Knowledgearcana", + "Name": {"tlk": "skills:knowledge_arcana.name"}, + "Icon": "isk_knarcana", + "Untrained": "1", + "KeyAbility": "INT", + "ArmorCheckPenalty": "0", + "AllClassesCanUse": "1", + "Constant": "SKILL_KNOWLEDGE_ARCANA", + "HostileSkill": "0", + "HideFromLevelUp": "0" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "skills", "modules", "rmv_removedskills.json"), `{ + "overrides": [ + { + "key": null, + "id": 3, + "Label": "Discipline_REMOVED", + "Constant": "****", + "AllClassesCanUse": "0", + "HideFromLevelUp": "1" + } + ] +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "skills.2da")) + if err != nil { + t.Fatalf("read native skills.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "skills.2da")) + if err != nil { + t.Fatalf("read reference skills.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("skills output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsLegacySpells(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules", "spells")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules", "feat")) + mkdirAll(t, filepath.Join(root, "reference", "data", "spells", "modules", "bardicmusic")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "lock.json"), `{ + "spells:specialattacks.name": 525, + "feat:bardfascinate.name": 700, + "feat:bardfascinate.description": 701, + "feat:bardfascinate.alt": 702 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "modules", "spells", "specialattacks.json"), `{ + "entries": { + "spells:specialattacks.name": {"text": "Special Attacks"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "modules", "feat", "bardfascinate.json"), `{ + "entries": { + "feat:bardfascinate.name": {"text": "Fascinate"}, + "feat:bardfascinate.description": {"text": "Fascinate description"}, + "feat:bardfascinate.alt": {"text": "Alt message"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "spells", "base.json"), `{ + "columns": ["Label", "Name", "IconResRef", "School", "Range", "VS", "MetaMagic", "TargetType", "ImpactScript", "Bard", "Cleric", "Druid", "Paladin", "Ranger", "Wiz_Sorc", "Innate", "ConjTime", "ConjAnim", "ConjHeadVisual", "ConjHandVisual", "ConjGrndVisual", "ConjSoundVFX", "ConjSoundMale", "ConjSoundFemale", "CastAnim", "CastTime", "CastHeadVisual", "CastHandVisual", "CastGrndVisual", "CastSound", "Proj", "ProjModel", "ProjType", "ProjSpwnPoint", "ProjSound", "ProjOrientation", "ImmunityType", "ItemImmunity", "SubRadSpell1", "SubRadSpell2", "SubRadSpell3", "SubRadSpell4", "SubRadSpell5", "Category", "Master", "UserType", "SpellDesc", "UseConcentration", "SpontaneouslyCast", "AltMessage", "HostileSetting", "FeatID", "Counter1", "Counter2", "HasProjectile", "TargetShape", "TargetSizeX", "TargetSizeY", "TargetFlags", "SubRadSpell6", "SubRadSpell7", "SubRadSpell8"], + "rows": [ + {"key": "spells:blessweapon", "id": 7, "Label": "Bless_Weapon", "Name": "123", "Category": "8"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "spells", "lock.json"), `{ + "spells:specialattacks": 840, + "spells:disarm": 841, + "spells:bardfascinate": 849 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "spells", "modules", "specialattacks.json"), `{ + "entries": { + "spells:specialattacks": { + "Label": "SpecialAttacks", + "Name": {"tlk": "spells:specialattacks.name"}, + "Category": "22", + "FeatID": {"id": "feat:specialattacks"}, + "SubRadSpell1": {"id": "spells:disarm"} + }, + "spells:disarm": { + "Label": "Disarm", + "Name": {"tlk": "spells:disarm.name"}, + "FeatID": {"id": "feat:disarm"}, + "Master": {"id": "spells:specialattacks"}, + "Category": {"ref": "spells:specialattacks", "field": "Category"} + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "spells", "modules", "ovr_fixduplicates.json"), `{ + "overrides": [ + {"key": null, "id": 7, "Label": "Bless_Weapon_OLD"} + ] +}`+"\n") + for _, name := range []string{ + "inspirecompetence.json", + "inspirecourage.json", + "inspiregreatness.json", + "inspireheroics.json", + "songoffreedom.json", + } { + writeFile(t, filepath.Join(root, "reference", "data", "spells", "modules", "bardicmusic", name), `{"entries": {}}`+"\n") + } + writeFile(t, filepath.Join(root, "reference", "data", "spells", "modules", "bardicmusic", "fascinate.json"), `{ + "entries": { + "spells:bardfascinate": { + "Label": "Bard_Fascinate", + "Name": {"tlk": "feat:bardfascinate.name"}, + "SpellDesc": {"tlk": "feat:bardfascinate.description"}, + "FeatID": {"id": "feat:bardfascinate"}, + "AltMessage": {"tlk": "feat:bardfascinate.alt"}, + "Category": "2" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := NormalizeProject(testProject(root)) + if err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + if result.UpdatedFiles < 10 { + t.Fatalf("expected spells files to be imported, got %#v", result) + } + for _, path := range []string{ + filepath.Join(root, "topdata", "data", "spells", "base.json"), + filepath.Join(root, "topdata", "data", "spells", "lock.json"), + filepath.Join(root, "topdata", "data", "spells", "modules", "specialattacks.json"), + filepath.Join(root, "topdata", "data", "spells", "modules", "ovr_fixduplicates.json"), + filepath.Join(root, "topdata", "data", "spells", "modules", "bardicmusic", "fascinate.json"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected imported spells path %s: %v", path, err) + } + } + baseRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "spells", "base.json")) + if err != nil { + t.Fatalf("read spells base: %v", err) + } + if !strings.Contains(string(baseRaw), `"output": "spells.2da"`) { + t.Fatalf("expected imported spells output name, got:\n%s", string(baseRaw)) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "spells", "lock.json")) + if err != nil { + t.Fatalf("read spells lock: %v", err) + } + if !strings.Contains(string(lockRaw), `"spells:bardfascinate": 849`) || !strings.Contains(string(lockRaw), `"spells:specialattacks": 840`) { + t.Fatalf("expected imported spells lock ids, got:\n%s", string(lockRaw)) + } + moduleRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "spells", "modules", "bardicmusic", "fascinate.json")) + if err != nil { + t.Fatalf("read spells bardicmusic module: %v", err) + } + if !strings.Contains(string(moduleRaw), `"text": "Fascinate"`) || !strings.Contains(string(moduleRaw), `"text": "Alt message"`) { + t.Fatalf("expected normalized inline TLK payloads in bardicmusic module, got:\n%s", string(moduleRaw)) + } +} + +func TestBuildSupportsCanonicalSpells(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells", "modules", "bardicmusic")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 1000, "key": "feat:specialattacks", "LABEL": "Special Attacks"}, + {"id": 1001, "key": "feat:disarm", "LABEL": "Disarm"}, + {"id": 1002, "key": "feat:bardfascinate", "LABEL": "Fascinate"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:specialattacks":1000,"feat:disarm":1001,"feat:bardfascinate":1002}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "base.json"), `{ + "output": "spells.2da", + "columns": ["Label", "Name", "IconResRef", "School", "Range", "VS", "MetaMagic", "TargetType", "ImpactScript", "Bard", "Cleric", "Druid", "Paladin", "Ranger", "Wiz_Sorc", "Innate", "ConjTime", "ConjAnim", "ConjHeadVisual", "ConjHandVisual", "ConjGrndVisual", "ConjSoundVFX", "ConjSoundMale", "ConjSoundFemale", "CastAnim", "CastTime", "CastHeadVisual", "CastHandVisual", "CastGrndVisual", "CastSound", "Proj", "ProjModel", "ProjType", "ProjSpwnPoint", "ProjSound", "ProjOrientation", "ImmunityType", "ItemImmunity", "SubRadSpell1", "SubRadSpell2", "SubRadSpell3", "SubRadSpell4", "SubRadSpell5", "Category", "Master", "UserType", "SpellDesc", "UseConcentration", "SpontaneouslyCast", "AltMessage", "HostileSetting", "FeatID", "Counter1", "Counter2", "HasProjectile", "TargetShape", "TargetSizeX", "TargetSizeY", "TargetFlags", "SubRadSpell6", "SubRadSpell7", "SubRadSpell8"], + "rows": [ + {"key": "spells:blessweapon", "id": 7, "Label": "Bless_Weapon", "Name": "123", "Category": "8"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "lock.json"), `{ + "spells:specialattacks": 840, + "spells:disarm": 841, + "spells:bardfascinate": 849 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "specialattacks.json"), `{ + "entries": { + "spells:specialattacks": { + "Label": "SpecialAttacks", + "Name": "500", + "Category": "22", + "FeatID": {"id": "feat:specialattacks"}, + "SubRadSpell1": {"id": "spells:disarm"} + }, + "spells:disarm": { + "Label": "Disarm", + "Name": "501", + "FeatID": {"id": "feat:disarm"}, + "Master": {"id": "spells:specialattacks"}, + "Category": {"ref": "spells:specialattacks", "field": "Category"} + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "ovr_fixduplicates.json"), `{ + "overrides": [ + {"key": null, "id": 7, "Label": "Bless_Weapon_OLD"} + ] +}`+"\n") + for _, name := range []string{ + "inspirecompetence.json", + "inspirecourage.json", + "inspiregreatness.json", + "inspireheroics.json", + "songoffreedom.json", + } { + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "bardicmusic", name), `{"entries": {}}`+"\n") + } + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "bardicmusic", "fascinate.json"), `{ + "entries": { + "spells:bardfascinate": { + "Label": "Bard_Fascinate", + "Name": "600", + "SpellDesc": "601", + "FeatID": {"id": "feat:bardfascinate"}, + "AltMessage": "602", + "Category": "2" + } + } +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "spells.2da")) + if err != nil { + t.Fatalf("read spells.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "Label\tName\tIconResRef") || + !strings.Contains(text, "7\t****\t****\t****") || + !strings.Contains(text, "840\tSpecialAttacks\t500") || + !strings.Contains(text, "\t841\t") || + !strings.Contains(text, "\t22\t") || + !strings.Contains(text, "\t1000\t") || + !strings.Contains(text, "841\tDisarm\t501") || + !strings.Contains(text, "\t840\t") || + !strings.Contains(text, "\t1001\t") || + !strings.Contains(text, "849\tBard_Fascinate\t600") || + !strings.Contains(text, "\t601\t") || + !strings.Contains(text, "\t602\t") || + !strings.Contains(text, "\t1002\t") { + t.Fatalf("unexpected spells.2da output:\n%s", text) + } +} + +func TestBuildCanonicalSpellsImpactScriptsComeFromGlobalDefaults(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "base.json"), `{ + "output": "spells.2da", + "columns": ["Label", "ImpactScript"], + "rows": [ + {"id": 0, "key": "spells:acid_fog", "Label": "AcidFog", "ImpactScript": "NW_S0_AcidFog"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "lock.json"), `{ + "spells:acid_fog": 0, + "spells:special_attacks": 840 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "specialattacks.json"), `{ + "entries": { + "spells:special_attacks": { + "Label": "SpecialAttacks" + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "global.json"), `{ + "defaults": [ + { + "match": {"source": "entries"}, + "values": { + "ImpactScript": {"format": "ss_{id}"} + } + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "spells.2da")) + if err != nil { + t.Fatalf("read spells.2da: %v", err) + } + text := string(raw) + if !strings.Contains(text, "0\tAcidFog\tNW_S0_AcidFog\n") { + t.Fatalf("expected base ImpactScript to be preserved, got:\n%s", text) + } + if !strings.Contains(text, "840\tSpecialAttacks\tss_840\n") { + t.Fatalf("expected global default generated ImpactScript, got:\n%s", text) + } +} + +func TestBuildCanonicalSpellsMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules", "spells")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules", "feat")) + mkdirAll(t, filepath.Join(root, "reference", "data", "spells", "modules", "bardicmusic")) + mkdirAll(t, filepath.Join(root, "reference", "data", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "lock.json"), `{ + "spells:specialattacks.name": 525, + "spells:disarm.name": 526, + "feat:bardfascinate.name": 700, + "feat:bardfascinate.description": 701, + "feat:bardfascinate.alt": 702 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "modules", "spells", "specialattacks.json"), `{ + "entries": { + "spells:specialattacks.name": {"text": "Special Attacks"}, + "spells:disarm.name": {"text": "Disarm"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "modules", "feat", "bardfascinate.json"), `{ + "entries": { + "feat:bardfascinate.name": {"text": "Fascinate"}, + "feat:bardfascinate.description": {"text": "Fascinate description"}, + "feat:bardfascinate.alt": {"text": "Alt message"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "spells.2da"), "w", encoding="utf-8") as f: + cols = ["Label", "Name", "IconResRef", "School", "Range", "VS", "MetaMagic", "TargetType", "ImpactScript", "Bard", "Cleric", "Druid", "Paladin", "Ranger", "Wiz_Sorc", "Innate", "ConjTime", "ConjAnim", "ConjHeadVisual", "ConjHandVisual", "ConjGrndVisual", "ConjSoundVFX", "ConjSoundMale", "ConjSoundFemale", "CastAnim", "CastTime", "CastHeadVisual", "CastHandVisual", "CastGrndVisual", "CastSound", "Proj", "ProjModel", "ProjType", "ProjSpwnPoint", "ProjSound", "ProjOrientation", "ImmunityType", "ItemImmunity", "SubRadSpell1", "SubRadSpell2", "SubRadSpell3", "SubRadSpell4", "SubRadSpell5", "Category", "Master", "UserType", "SpellDesc", "UseConcentration", "SpontaneouslyCast", "AltMessage", "HostileSetting", "FeatID", "Counter1", "Counter2", "HasProjectile", "TargetShape", "TargetSizeX", "TargetSizeY", "TargetFlags", "SubRadSpell6", "SubRadSpell7", "SubRadSpell8"] + def emit(row_id, values): + row = {c: "****" for c in cols} + row.update(values) + f.write(str(row_id) + "\t" + "\t".join(row[c] for c in cols) + "\n") + f.write("2DA V2.0\n\n") + f.write("\t".join(cols) + "\n") + for row_id in range(0, 7): + emit(row_id, {}) + emit(7, {}) + for row_id in range(8, 840): + emit(row_id, {}) + emit(840, {"Label": "SpecialAttacks", "Name": "16777741", "SubRadSpell1": "841", "Category": "22", "FeatID": "1000"}) + emit(841, {"Label": "Disarm", "Name": "16777742", "Category": "22", "Master": "840", "FeatID": "1001"}) + for row_id in range(842, 849): + emit(row_id, {}) + emit(849, {"Label": "Bard_Fascinate", "Name": "16777916", "Category": "2", "SpellDesc": "16777917", "AltMessage": "16777918", "FeatID": "1002"}) +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 1000, "key": "feat:specialattacks", "LABEL": "Special Attacks"}, + {"id": 1001, "key": "feat:disarm", "LABEL": "Disarm"}, + {"id": 1002, "key": "feat:bardfascinate", "LABEL": "Fascinate"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "feat", "lock.json"), `{"feat:specialattacks":1000,"feat:disarm":1001,"feat:bardfascinate":1002}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "spells", "base.json"), `{ + "columns": ["Label", "Name", "IconResRef", "School", "Range", "VS", "MetaMagic", "TargetType", "ImpactScript", "Bard", "Cleric", "Druid", "Paladin", "Ranger", "Wiz_Sorc", "Innate", "ConjTime", "ConjAnim", "ConjHeadVisual", "ConjHandVisual", "ConjGrndVisual", "ConjSoundVFX", "ConjSoundMale", "ConjSoundFemale", "CastAnim", "CastTime", "CastHeadVisual", "CastHandVisual", "CastGrndVisual", "CastSound", "Proj", "ProjModel", "ProjType", "ProjSpwnPoint", "ProjSound", "ProjOrientation", "ImmunityType", "ItemImmunity", "SubRadSpell1", "SubRadSpell2", "SubRadSpell3", "SubRadSpell4", "SubRadSpell5", "Category", "Master", "UserType", "SpellDesc", "UseConcentration", "SpontaneouslyCast", "AltMessage", "HostileSetting", "FeatID", "Counter1", "Counter2", "HasProjectile", "TargetShape", "TargetSizeX", "TargetSizeY", "TargetFlags", "SubRadSpell6", "SubRadSpell7", "SubRadSpell8"], + "rows": [ + {"key": "spells:blessweapon", "id": 7, "Label": "Bless_Weapon", "Name": "123", "Category": "8"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "spells", "lock.json"), `{ + "spells:specialattacks": 840, + "spells:disarm": 841, + "spells:bardfascinate": 849 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "spells", "modules", "specialattacks.json"), `{ + "entries": { + "spells:specialattacks": { + "Label": "SpecialAttacks", + "Name": {"tlk": "spells:specialattacks.name"}, + "Category": "22", + "UserType": "3", + "FeatID": {"id": "feat:specialattacks"}, + "SubRadSpell1": {"id": "spells:disarm"} + }, + "spells:disarm": { + "Label": "Disarm", + "Name": {"tlk": "spells:disarm.name"}, + "FeatID": {"id": "feat:disarm"}, + "Master": {"id": "spells:specialattacks"}, + "Category": {"ref": "spells:specialattacks", "field": "Category"}, + "UserType": {"ref": "spells:specialattacks", "field": "UserType"} + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "spells", "modules", "ovr_fixduplicates.json"), `{ + "overrides": [ + {"key": null, "id": 7, "Label": "Bless_Weapon_OLD"} + ] +}`+"\n") + for _, name := range []string{ + "inspirecompetence.json", + "inspirecourage.json", + "inspiregreatness.json", + "inspireheroics.json", + "songoffreedom.json", + } { + writeFile(t, filepath.Join(root, "reference", "data", "spells", "modules", "bardicmusic", name), `{"entries": {}}`+"\n") + } + writeFile(t, filepath.Join(root, "reference", "data", "spells", "modules", "bardicmusic", "fascinate.json"), `{ + "entries": { + "spells:bardfascinate": { + "Label": "Bard_Fascinate", + "Name": {"tlk": "feat:bardfascinate.name"}, + "SpellDesc": {"tlk": "feat:bardfascinate.description"}, + "FeatID": {"id": "feat:bardfascinate"}, + "AltMessage": {"tlk": "feat:bardfascinate.alt"}, + "Category": "2" + } + } +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "spells.2da")) + if err != nil { + t.Fatalf("read native spells.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "spells.2da")) + if err != nil { + t.Fatalf("read reference spells.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("spells output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestBuildSupportsCanonicalPlaceables(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "base.json"), `{ + "output": "placeables.2da", + "columns": ["Label", "StrRef", "ModelName", "LightColor", "LightOffsetX", "LightOffsetY", "LightOffsetZ", "SoundAppType", "ShadowSize", "BodyBag", "LowGore", "Reflection", "Static"], + "rows": [ + {"key": "placeables:armoire", "id": 0, "Label": "Armoire", "StrRef": "5645", "ModelName": "PLC_A01", "SoundAppType": "13", "ShadowSize": "1", "BodyBag": "0", "Static": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "lock.json"), `{ + "placeables:armoire": 0, + "placeables:tmp_crappile01": 16500 +}`+"\n") + for _, name := range []string{ + "add_ccfeb2019plcs.json", + "add_cepcarpets.json", + "add_cepfloors.json", + "add_cepnwn2library.json", + "add_nwic.json", + "add_ori.json", + "add_sic11.json", + "add_tapr.json", + "add_tdfloors.json", + "add_undecals.json", + "add_witcher1.json", + "add_witcher2.json", + } { + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "modules", name), `{"entries": {}}`+"\n") + } + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "modules", "add_avernostragarbage.json"), `{ + "entries": { + "placeables:tmp_crappile01": { + "Label": "Decoration: Pile 1 (Avernostra)", + "ModelName": "tmp_crappile01", + "Static": 1 + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "modules", "ovr_vanillaplcsorts.json"), `{ + "overrides": [ + {"key": "placeables:armoire", "id": 0, "Label": "Furniture: Armoire (Bioware)", "StrRef": "****"} + ] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "placeables.2da")) + if err != nil { + t.Fatalf("read placeables.2da: %v", err) + } + if !strings.Contains(string(got), "Label\tStrRef\tModelName\tLightColor\tLightOffsetX\tLightOffsetY\tLightOffsetZ\tSoundAppType\tShadowSize\tBodyBag\tLowGore\tReflection\tStatic\n") || + !strings.Contains(string(got), "0\t\"Furniture: Armoire (Bioware)\"\t****\tPLC_A01") || + !strings.Contains(string(got), "\t13\t1\t0\t****\t****\t1\n") || + !strings.Contains(string(got), "16500\t\"Decoration: Pile 1 (Avernostra)\"\t****\ttmp_crappile01") { + t.Fatalf("unexpected placeables.2da output:\n%s", string(got)) + } +} + +func TestBuildCanonicalPlaceablesMatchesReference(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "placeables", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "placeables.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") + f.write("Label\tStrRef\tModelName\tLightColor\tLightOffsetX\tLightOffsetY\tLightOffsetZ\tSoundAppType\tShadowSize\tBodyBag\tLowGore\tReflection\tStatic\n") + f.write("0\t\"Furniture: Armoire (Bioware)\"\t****\tPLC_A01\t****\t****\t****\t****\t13\t1\t0\t****\t****\t1\n") + for row_id in range(1, 16500): + f.write(f"{row_id}\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\t****\n") + f.write("16500\t\"Decoration: Pile 1 (Avernostra)\"\t****\ttmp_crappile01\t****\t****\t****\t****\t****\t****\t****\t****\t1\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + writeFile(t, filepath.Join(root, "reference", "data", "placeables", "base.json"), `{ + "columns": ["Label", "StrRef", "ModelName", "LightColor", "LightOffsetX", "LightOffsetY", "LightOffsetZ", "SoundAppType", "ShadowSize", "BodyBag", "LowGore", "Reflection", "Static"], + "rows": [ + {"key": "placeables:armoire", "id": 0, "Label": "Armoire", "StrRef": "5645", "ModelName": "PLC_A01", "SoundAppType": "13", "ShadowSize": "1", "BodyBag": "0", "Static": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "placeables", "lock.json"), `{ + "placeables:armoire": 0, + "placeables:tmp_crappile01": 16500 +}`+"\n") + for _, name := range []string{ + "add_ccfeb2019plcs.json", + "add_cepcarpets.json", + "add_cepfloors.json", + "add_cepnwn2library.json", + "add_nwic.json", + "add_ori.json", + "add_sic11.json", + "add_tapr.json", + "add_tdfloors.json", + "add_undecals.json", + "add_witcher1.json", + "add_witcher2.json", + } { + writeFile(t, filepath.Join(root, "reference", "data", "placeables", "modules", name), `{"entries": {}}`+"\n") + } + writeFile(t, filepath.Join(root, "reference", "data", "placeables", "modules", "add_avernostragarbage.json"), `{ + "entries": { + "placeables:tmp_crappile01": { + "Label": "Decoration: Pile 1 (Avernostra)", + "ModelName": "tmp_crappile01", + "Static": 1 + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "placeables", "modules", "ovr_vanillaplcsorts.json"), `{ + "overrides": [ + {"key": "placeables:armoire", "id": 0, "Label": "Furniture: Armoire (Bioware)", "StrRef": "****"} + ] +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + nativeResult, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + referenceResult, err := BuildReference(testProject(root), nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + nativeBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, "placeables.2da")) + if err != nil { + t.Fatalf("read native placeables.2da: %v", err) + } + referenceBytes, err := os.ReadFile(filepath.Join(referenceResult.Output2DADir, "placeables.2da")) + if err != nil { + t.Fatalf("read reference placeables.2da: %v", err) + } + if string(nativeBytes) != string(referenceBytes) { + t.Fatalf("placeables output mismatch\nnative:\n%s\nreference:\n%s", string(nativeBytes), string(referenceBytes)) + } +} + +func TestNormalizeProjectImportsRacialtypesRegistry(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "reference", "data", "racialtypes", "core", "modules")) + mkdirAll(t, filepath.Join(root, "reference", "data", "racialtypes", "feats")) + mkdirAll(t, filepath.Join(root, "reference", "tlk", "modules", "racialtypes")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + writeFile(t, filepath.Join(root, "reference", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "Name", "Appearance", "ToolsetDefaultClass", "FeatsTable", "Description"], + "rows": [ + {"id": 6, "Label": "Human", "Name": "34", "Appearance": "0", "ToolsetDefaultClass": "4", "FeatsTable": "RACE_FEAT_HUMAN", "Description": "****"}, + {"id": 7, "Label": "Beast", "Name": "527", "Appearance": "****", "ToolsetDefaultClass": "****", "FeatsTable": "****", "Description": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "racialtypes", "core", "lock.json"), `{ + "racialtypes:human": 6, + "racialtypes:tiefling": 31 +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "racialtypes", "core", "modules", "ovr_human.json"), `{ + "overrides": [ + {"key": "racialtypes:human", "id": 6, "Label": "Human", "Description": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "racialtypes", "core", "modules", "tiefling.json"), `{ + "entries": { + "racialtypes:tiefling": { + "Label": "Tiefling", + "Appearance": {"id": "appearance:human"}, + "ToolsetDefaultClass": {"id": "classes:rogue"}, + "FeatsTable": {"table": "racialtypes/feats:tiefling"}, + "Description": {"tlk": "racialtypes:tiefling.description"} + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "data", "racialtypes", "feats", "tiefling.json"), `{ + "key": "racialtypes/feats:tiefling", + "output": "race_feat_tief.2da", + "columns": ["FeatLabel", "FeatIndex"], + "rows": [ + {"FeatLabel": "Darkvision", "FeatIndex": {"id": "feat:darkvision"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "reference", "tlk", "lock.json"), `{"racialtypes:tiefling.description": 100}`+"\n") + writeFile(t, filepath.Join(root, "reference", "tlk", "modules", "racialtypes", "tiefling.json"), `{ + "entries": { + "racialtypes:tiefling": { + "description": {"text": "Tiefling description"} + } + } +}`+"\n") + + if _, err := NormalizeProject(testProject(root)); err != nil { + t.Fatalf("NormalizeProject failed: %v", err) + } + + baseRowsRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "racialtypes", "registry", "base.json")) + if err != nil { + t.Fatalf("read base.json: %v", err) + } + baseText := string(baseRowsRaw) + if !strings.Contains(baseText, `"Label": "Beast"`) || !strings.Contains(baseText, `"key": "racialtypes:beast"`) { + t.Fatalf("expected keyed static core row scaffold, got:\n%s", baseText) + } + + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "racialtypes", "registry", "lock.json")) + if err != nil { + t.Fatalf("read lock.json: %v", err) + } + if !strings.Contains(string(lockRaw), `"racialtypes:human": 6`) || !strings.Contains(string(lockRaw), `"racialtypes:tiefling": 31`) || !strings.Contains(string(lockRaw), `"racialtypes:beast": 7`) { + t.Fatalf("expected preserved lock ids, got:\n%s", string(lockRaw)) + } + + tieflingRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "racialtypes", "registry", "races", "tiefling.json")) + if err != nil { + t.Fatalf("read tiefling.json: %v", err) + } + tieflingText := string(tieflingRaw) + if !strings.Contains(tieflingText, `"feat_output": "race_feat_tief.2da"`) || + !strings.Contains(tieflingText, `"text": "Tiefling description"`) || + !strings.Contains(tieflingText, `"FeatLabel": "Darkvision"`) { + t.Fatalf("unexpected tiefling registry row:\n%s", tieflingText) + } +} + +func TestBuildSupportsRacialtypesRegistryWithSnapshotClassIDs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "registry", "races")) + mkdirAll(t, filepath.Join(root, "topdata", "migration_snapshot", "data", "classes", "core")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "base.json"), `{ + "output": "appearance.2da", + "columns": ["LABEL"], + "rows": [ + {"id": 0, "key": "appearance:human", "LABEL": "Human"}, + {"id": 1, "key": "appearance:elf", "LABEL": "Elf"}, + {"id": 2, "key": "appearance:halfogre", "LABEL": "Half-Ogre"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "lock.json"), `{ + "appearance:human": 0, + "appearance:elf": 1, + "appearance:halfogre": 2 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:darkvision": 77 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "migration_snapshot", "data", "classes", "core", "lock.json"), `{ + "classes:fighter": 4, + "classes:rogue": 8 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "registry", "base.json"), `{ + "columns": ["Label", "Appearance", "ToolsetDefaultClass", "FeatsTable"], + "rows": [ + {"id": 7, "key": "racialtypes:beast", "Label": "Beast", "Appearance": "****", "ToolsetDefaultClass": "****", "FeatsTable": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "registry", "lock.json"), `{ + "racialtypes:beast": 7, + "racialtypes:human": 6, + "racialtypes:tiefling": 31 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "registry", "races", "human.json"), `{ + "key": "racialtypes:human", + "core": { + "Label": "Human", + "Appearance": {"id": "appearance:human"}, + "ToolsetDefaultClass": {"id": "classes:fighter"}, + "FeatsTable": "RACE_FEAT_HUMAN" + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "registry", "races", "tiefling.json"), `{ + "key": "racialtypes:tiefling", + "core": { + "Label": "Tiefling", + "Appearance": {"id": "appearance:human"}, + "ToolsetDefaultClass": {"id": "classes:rogue"} + }, + "feat_output": "race_feat_tief.2da", + "feats": [ + {"FeatLabel": "Darkvision", "FeatIndex": {"id": "feat:darkvision"}} + ] +}`+"\n") + + p := testProject(root) + p.Config.TopData.ReferenceBuilder = "" + result, err := BuildNative(p, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + coreBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "racialtypes.2da")) + if err != nil { + t.Fatalf("read racialtypes.2da: %v", err) + } + coreText := string(coreBytes) + if !strings.Contains(coreText, "Label\tAppearance\tToolsetDefaultClass\tFeatsTable\n") || + !strings.Contains(coreText, "6\tHuman\t0\t4\tRACE_FEAT_HUMAN\n") || + !strings.Contains(coreText, "31\tTiefling\t0\t8\trace_feat_tief\n") { + t.Fatalf("unexpected racialtypes.2da output:\n%s", coreText) + } + + featBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "race_feat_tief.2da")) + if err != nil { + t.Fatalf("read race_feat_tief.2da: %v", err) + } + if !strings.Contains(string(featBytes), "FeatLabel\tFeatIndex\n0\tDarkvision\t77\n") { + t.Fatalf("unexpected race_feat_tief.2da output:\n%s", string(featBytes)) + } +} + +func TestScanAutogeneratedParts(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets", "part", "belt")) + mkdirAll(t, filepath.Join(root, "assets", "part", "chest")) + writeFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt018.mdl"), "") + writeFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt171.mdl"), "") + writeFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt172.mdl"), "") + writeFile(t, filepath.Join(root, "assets", "part", "chest", "pfa0_chest081.mdl"), "") + writeFile(t, filepath.Join(root, "assets", "part", "chest", "pfa0_chest095.mdl"), "") + + result, err := scanAutogeneratedParts(filepath.Join(root, "assets")) + if err != nil { + t.Fatalf("scanAutogeneratedParts failed: %v", err) + } + + beltIDs, ok := result["belt"] + if !ok { + t.Fatal("expected belt category in result") + } + if len(beltIDs) != 3 { + t.Fatalf("expected 3 belt IDs, got %d", len(beltIDs)) + } + for _, id := range []int{18, 171, 172} { + if _, exists := beltIDs[id]; !exists { + t.Errorf("expected belt ID %d to exist", id) + } + } + + chestIDs, ok := result["chest"] + if !ok { + t.Fatal("expected chest category in result") + } + if len(chestIDs) != 2 { + t.Fatalf("expected 2 chest IDs, got %d", len(chestIDs)) + } + for _, id := range []int{81, 95} { + if _, exists := chestIDs[id]; !exists { + t.Errorf("expected chest ID %d to exist", id) + } + } +} + +func TestScanAutogeneratedPartsSkipsNonNumericSuffix(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets", "part", "belt")) + writeFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_beltABC.mdl"), "") + writeFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt_test.mdl"), "") + + result, err := scanAutogeneratedParts(root) + if err != nil { + t.Fatalf("scanAutogeneratedParts failed: %v", err) + } + + beltIDs, ok := result["belt"] + if !ok { + t.Fatal("expected belt category in result") + } + if len(beltIDs) != 0 { + t.Fatalf("expected 0 belt IDs (non-numeric suffixes), got %d", len(beltIDs)) + } +} + +func TestScanAutogeneratedPartsReturnsNilForEmptyPath(t *testing.T) { + result, err := scanAutogeneratedParts("") + if err != nil { + t.Fatalf("scanAutogeneratedParts failed: %v", err) + } + if result != nil { + t.Fatalf("expected nil result for empty path, got %v", result) + } +} + +func TestAugmentWithAutogeneratedParts(t *testing.T) { + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Name: "parts/belt", + OutputName: "parts_belt.2da", + }, + Columns: []string{"COSTMODIFIER", "ACBONUS"}, + Rows: []map[string]any{ + {"id": 0, "COSTMODIFIER": "1", "ACBONUS": "0.25"}, + }, + LockData: map[string]int{}, + }, + } + + autogenerated := map[string]map[int]struct{}{ + "belt": {18: {}, 171: {}, 172: {}}, + "chest": {81: {}}, + } + + result := augmentWithAutogeneratedParts(collected, autogenerated) + + if len(result) != 1 { + t.Fatalf("expected 1 dataset, got %d", len(result)) + } + beltDataset := result[0] + if len(beltDataset.Rows) != 4 { + t.Fatalf("expected 4 rows (1 existing + 3 autogenerated), got %d", len(beltDataset.Rows)) + } + + rowByID := make(map[int]map[string]any) + for _, row := range beltDataset.Rows { + id := row["id"].(int) + rowByID[id] = row + } + + if cost, ok := rowByID[0]["COSTMODIFIER"].(string); !ok || cost != "1" { + t.Errorf("expected existing row 0 to keep COSTMODIFIER=1, got %v", rowByID[0]["COSTMODIFIER"]) + } + + for _, id := range []int{18, 171, 172} { + row := rowByID[id] + if cost, ok := row["COSTMODIFIER"].(string); !ok || cost != "0" { + t.Errorf("expected autogenerated row %d to have COSTMODIFIER=0, got %v", id, row["COSTMODIFIER"]) + } + if ac, ok := row["ACBONUS"].(string); !ok || ac != "0.00" { + t.Errorf("expected autogenerated row %d to have ACBONUS=0.00, got %v", id, row["ACBONUS"]) + } + } +} + +func TestAugmentWithAutogeneratedPartsRespectsExistingIDs(t *testing.T) { + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Name: "parts/belt", + OutputName: "parts_belt.2da", + }, + Columns: []string{"COSTMODIFIER", "ACBONUS"}, + Rows: []map[string]any{ + {"id": 18, "COSTMODIFIER": "5", "ACBONUS": "1.00"}, + }, + LockData: map[string]int{}, + }, + } + + autogenerated := map[string]map[int]struct{}{ + "belt": {18: {}, 171: {}}, + } + + result := augmentWithAutogeneratedParts(collected, autogenerated) + + beltDataset := result[0] + if len(beltDataset.Rows) != 2 { + t.Fatalf("expected 2 rows (1 existing with ID 18, 1 autogenerated with ID 171), got %d", len(beltDataset.Rows)) + } + + rowByID := make(map[int]map[string]any) + for _, row := range beltDataset.Rows { + id := row["id"].(int) + rowByID[id] = row + } + + if cost, ok := rowByID[18]["COSTMODIFIER"].(string); !ok || cost != "5" { + t.Errorf("expected existing row 18 to keep COSTMODIFIER=5, got %v", rowByID[18]["COSTMODIFIER"]) + } + + if cost, ok := rowByID[171]["COSTMODIFIER"].(string); !ok || cost != "0" { + t.Errorf("expected autogenerated row 171 to have COSTMODIFIER=0, got %v", rowByID[171]["COSTMODIFIER"]) + } +} + +func TestNormalizePartsRowsACBonusUsesConfiguredPolicyBeforeOverrides(t *testing.T) { + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{Name: "parts/belt", OutputName: "parts_belt.2da"}, + Columns: []string{"COSTMODIFIER", "ACBONUS"}, + Rows: []map[string]any{ + {"id": 1, "COSTMODIFIER": "5", "ACBONUS": "0.90"}, + {"id": 999, "COSTMODIFIER": "0", "ACBONUS": "****"}, + {"id": 1000, "COSTMODIFIER": "0", "ACBONUS": "0.00"}, + }, + }, + { + Dataset: nativeDataset{Name: "parts/chest", OutputName: "parts_chest.2da"}, + Columns: []string{"COSTMODIFIER", "ACBONUS"}, + Rows: []map[string]any{ + {"id": 14, "COSTMODIFIER": "0", "ACBONUS": "8.00"}, + }, + }, + { + Dataset: nativeDataset{Name: "parts/robe", OutputName: "parts_robe.2da"}, + Columns: []string{"COSTMODIFIER", "ACBONUS"}, + Rows: []map[string]any{ + {"id": 95, "COSTMODIFIER": "0", "ACBONUS": "0.00"}, + }, + }, + } + cfg := project.PartsRowsConfig{ + RowDefaults: map[string]string{"COSTMODIFIER": "0"}, + ACBonus: project.PartsRowsACBonusConfig{ + Default: project.PartsRowsACBonusPolicy{Strategy: "descending_row_id_sort_key", MaxRowID: 999, Divisor: 100, Format: "%.2f"}, + Datasets: map[string]project.PartsRowsACBonusPolicy{ + "chest": {Strategy: "fixed", Value: "0.00"}, + }, + }, + } + + got, err := normalizePartsRowsACBonus(collected, cfg) + if err != nil { + t.Fatalf("normalizePartsRowsACBonus failed: %v", err) + } + + beltRows := rowsByID(got[0].Rows) + if got := beltRows[1]["ACBONUS"]; got != "9.98" { + t.Fatalf("expected authored belt ACBONUS to be replaced with computed sort key 9.98, got %v", got) + } + if got := beltRows[999]["ACBONUS"]; got != "****" { + t.Fatalf("expected authored null belt row to remain ****, got %v", got) + } + if got := beltRows[1000]["ACBONUS"]; got != "-0.01" { + t.Fatalf("expected non-null high belt row to use computed sort key, got %v", got) + } + chestRows := rowsByID(got[1].Rows) + if got := chestRows[14]["ACBONUS"]; got != "0.00" { + t.Fatalf("expected chest ACBONUS to be forced to 0.00, got %v", got) + } + robeRows := rowsByID(got[2].Rows) + if got := robeRows[95]["ACBONUS"]; got != "9.04" { + t.Fatalf("expected robe to use computed non-chest sort key, got %v", got) + } +} + +func TestNormalizePartsRowsACBonusSupportsAscendingRowIDSortKey(t *testing.T) { + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{Name: "parts/belt", OutputName: "parts_belt.2da"}, + Columns: []string{"COSTMODIFIER", "ACBONUS"}, + Rows: []map[string]any{ + {"id": 1, "COSTMODIFIER": "0", "ACBONUS": "0.00"}, + {"id": 117, "COSTMODIFIER": "0", "ACBONUS": "0.00"}, + {"id": 118, "COSTMODIFIER": "0", "ACBONUS": "****"}, + }, + }, + } + cfg := project.PartsRowsConfig{ + ACBonus: project.PartsRowsACBonusConfig{ + Default: project.PartsRowsACBonusPolicy{Strategy: "ascending_row_id_sort_key", Divisor: 100, Format: "%.2f"}, + }, + } + + got, err := normalizePartsRowsACBonus(collected, cfg) + if err != nil { + t.Fatalf("normalizePartsRowsACBonus failed: %v", err) + } + + beltRows := rowsByID(got[0].Rows) + if got := beltRows[1]["ACBONUS"]; got != "0.01" { + t.Fatalf("expected low belt row to get lower sort key 0.01, got %v", got) + } + if got := beltRows[117]["ACBONUS"]; got != "1.17" { + t.Fatalf("expected high belt row to get higher sort key 1.17, got %v", got) + } + if got := beltRows[118]["ACBONUS"]; got != "****" { + t.Fatalf("expected authored null belt row to remain ****, got %v", got) + } +} + +func rowsByID(rows []map[string]any) map[int]map[string]any { + out := map[int]map[string]any{} + for _, row := range rows { + id, _ := row["id"].(int) + out[id] = row + } + return out +} + +func TestAugmentWithAutogeneratedPartsSortsRows(t *testing.T) { + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Name: "parts/belt", + OutputName: "parts_belt.2da", + }, + Columns: []string{"COSTMODIFIER"}, + Rows: []map[string]any{ + {"id": 100, "COSTMODIFIER": "1"}, + }, + LockData: map[string]int{}, + }, + } + + autogenerated := map[string]map[int]struct{}{ + "belt": {18: {}, 171: {}}, + } + + result := augmentWithAutogeneratedParts(collected, autogenerated) + + beltDataset := result[0] + if len(beltDataset.Rows) != 3 { + t.Fatalf("expected 3 rows, got %d", len(beltDataset.Rows)) + } + + for i, row := range beltDataset.Rows { + expectedID := []int{18, 100, 171}[i] + if id := row["id"].(int); id != expectedID { + t.Errorf("expected row at index %d to have ID %d, got %d", i, expectedID, id) + } + } +} + +func TestAugmentWithAutogeneratedRobePartsDefaultsHideColumns(t *testing.T) { + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Name: "parts/robe", + OutputName: "parts_robe.2da", + }, + Columns: []string{ + "COSTMODIFIER", "ACBONUS", "HIDEFOOTR", "HIDEFOOTL", "HIDECHEST", + }, + Rows: []map[string]any{}, + LockData: map[string]int{}, + }, + } + + autogenerated := map[string]map[int]struct{}{ + "robe": {95: {}}, + } + + result := augmentWithAutogeneratedParts(collected, autogenerated) + robeDataset := result[0] + if len(robeDataset.Rows) != 1 { + t.Fatalf("expected 1 autogenerated robe row, got %d", len(robeDataset.Rows)) + } + + row := robeDataset.Rows[0] + if got := row["id"]; got != 95 { + t.Fatalf("expected autogenerated robe row id 95, got %v", got) + } + if got := row["COSTMODIFIER"]; got != "0" { + t.Fatalf("expected COSTMODIFIER=0, got %v", got) + } + if got := row["ACBONUS"]; got != "0.00" { + t.Fatalf("expected ACBONUS=0.00, got %v", got) + } + for _, column := range []string{"HIDEFOOTR", "HIDEFOOTL", "HIDECHEST"} { + if got := row[column]; got != "0" { + t.Fatalf("expected %s=0, got %v", column, got) + } + } +} + +func TestAugmentWithAutogeneratedRobePartsActivatesUnsetHideColumns(t *testing.T) { + collected := []nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Name: "parts/robe", + OutputName: "parts_robe.2da", + }, + Columns: []string{ + "COSTMODIFIER", "ACBONUS", "HIDEFOOTR", "HIDEFOOTL", "HIDECHEST", + }, + Rows: []map[string]any{ + { + "id": 95, + "COSTMODIFIER": "****", + "ACBONUS": "****", + "HIDEFOOTR": "****", + "HIDEFOOTL": "****", + "HIDECHEST": "****", + }, + }, + LockData: map[string]int{}, + }, + } + + autogenerated := map[string]map[int]struct{}{ + "robe": {95: {}}, + } + + result := augmentWithAutogeneratedParts(collected, autogenerated) + row := result[0].Rows[0] + if got := row["COSTMODIFIER"]; got != "0" { + t.Fatalf("expected COSTMODIFIER=0, got %v", got) + } + if got := row["ACBONUS"]; got != "0.00" { + t.Fatalf("expected ACBONUS=0.00, got %v", got) + } + for _, column := range []string{"HIDEFOOTR", "HIDEFOOTL", "HIDECHEST"} { + if got := row[column]; got != "0" { + t.Fatalf("expected %s=0, got %v", column, got) + } + } +} + +func TestBuildNativeWithAutogeneratedParts(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "parts")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER", "ACBONUS"], + "rows": [{"id": 0, "COSTMODIFIER": "1", "ACBONUS": "0.25"}] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + mkdirAll(t, filepath.Join(root, "assets", "part", "belt")) + writeFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt018.mdl"), "") + writeFile(t, filepath.Join(root, "assets", "part", "belt", "pfa0_belt171.mdl"), "") + + proj := testProject(root) + proj.Config.TopData.Assets = filepath.Join(root, "assets") + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + partsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "parts_belt.2da")) + if err != nil { + t.Fatalf("read parts_belt.2da: %v", err) + } + partsText := string(partsBytes) + + if !strings.Contains(partsText, "0\t1\t0.25") { + t.Errorf("expected existing row 0 to be preserved, got:\n%s", partsText) + } + if !strings.Contains(partsText, "18\t0\t0.00") { + t.Errorf("expected autogenerated row 18, got:\n%s", partsText) + } + if !strings.Contains(partsText, "171\t0\t0.00") { + t.Errorf("expected autogenerated row 171, got:\n%s", partsText) + } +} + +func TestBuildNativeWithAutogeneratedRobeParts(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "parts")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "parts", "robe.json"), `{ + "output": "parts_robe.2da", + "columns": ["COSTMODIFIER", "ACBONUS", "HIDEFOOTR", "HIDEFOOTL", "HIDECHEST"], + "rows": [{"id": 0, "COSTMODIFIER": "1", "ACBONUS": "0.25", "HIDEFOOTR": "1", "HIDEFOOTL": "1", "HIDECHEST": "1"}] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + mkdirAll(t, filepath.Join(root, "assets", "part", "robe")) + writeFile(t, filepath.Join(root, "assets", "part", "robe", "pfa0_robe095.mdl"), "") + + proj := testProject(root) + proj.Config.TopData.Assets = filepath.Join(root, "assets") + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + partsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "parts_robe.2da")) + if err != nil { + t.Fatalf("read parts_robe.2da: %v", err) + } + partsText := string(partsBytes) + + if !strings.Contains(partsText, "95\t0\t0.00\t0\t0\t0") { + t.Fatalf("expected autogenerated robe row 95 with zeroed hide defaults, got:\n%s", partsText) + } +} + +func TestBuildNativeWithReleasedPartsManifest(t *testing.T) { + root := t.TempDir() + projRoot := filepath.Join(root, "project") + mkdirAll(t, filepath.Join(projRoot, "src")) + mkdirAll(t, filepath.Join(projRoot, "assets")) + mkdirAll(t, filepath.Join(projRoot, "build")) + mkdirAll(t, filepath.Join(projRoot, "topdata", "data", "parts")) + writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(projRoot, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER", "ACBONUS"], + "rows": [{"id": 0, "COSTMODIFIER": "1", "ACBONUS": "0.25"}] +}`+"\n") + mkdirAll(t, filepath.Join(projRoot, "reference")) + writeFile(t, filepath.Join(projRoot, "reference", "build.py"), "print('ok')\n") + + manifestJSON := `{ + "repo": "ShadowsOverWestgate/sow-assets", + "ref": "test-manifest", + "generated_at": "2026-04-09T00:00:00Z", + "categories": { + "belt": [18, 171] + } +}` + "\n" + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/repos/ShadowsOverWestgate/sow-assets/releases/tags/parts-manifest-current": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"assets":[{"name":"sow-parts-manifest.json","browser_download_url":"` + server.URL + `/downloads/sow-parts-manifest.json"}]}`)) + case "/downloads/sow-parts-manifest.json": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(manifestJSON)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + runGitTest(t, "", "init", "-b", "main", projRoot) + runGitTest(t, projRoot, "remote", "add", "origin", server.URL+"/ShadowsOverWestgate/sow-module.git") + + proj := &project.Project{ + Root: projRoot, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: project.TopDataConfig{ + Source: "topdata", + Build: "build/topdata", + }, + Autogen: project.AutogenConfig{ + Consumers: []project.AutogenConsumerConfig{ + { + ID: "parts", + Producer: "parts", + Dataset: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "trailing_numeric_suffix", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "parts-manifest-current", + AssetName: "sow-parts-manifest.json", + CacheName: "sow-parts-manifest.json", + }, + }, + }, + }, + }, + } + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + partsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "parts_belt.2da")) + if err != nil { + t.Fatalf("read parts_belt.2da: %v", err) + } + partsText := string(partsBytes) + + if !strings.Contains(partsText, "0\t1\t0.25") { + t.Errorf("expected existing row 0 to be preserved, got:\n%s", partsText) + } + if !strings.Contains(partsText, "18\t0\t0.00") { + t.Errorf("expected autogenerated row 18 from released manifest, got:\n%s", partsText) + } + if !strings.Contains(partsText, "171\t0\t0.00") { + t.Errorf("expected autogenerated row 171 from released manifest, got:\n%s", partsText) + } +} + +func TestBuildNativeWithReleasedPartsManifestUsesAssetsServerOverride(t *testing.T) { + root := t.TempDir() + projRoot := filepath.Join(root, "project") + mkdirAll(t, filepath.Join(projRoot, "src")) + mkdirAll(t, filepath.Join(projRoot, "assets")) + mkdirAll(t, filepath.Join(projRoot, "build")) + mkdirAll(t, filepath.Join(projRoot, "topdata", "data", "parts")) + writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(projRoot, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER", "ACBONUS"], + "rows": [] +}`+"\n") + mkdirAll(t, filepath.Join(projRoot, "reference")) + writeFile(t, filepath.Join(projRoot, "reference", "build.py"), "print('ok')\n") + + manifestJSON := `{ + "repo": "ShadowsOverWestgate/sow-assets", + "ref": "test-manifest", + "generated_at": "2026-04-09T00:00:00Z", + "categories": { + "belt": [18] + } +}` + "\n" + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/repos/ShadowsOverWestgate/sow-assets/releases/tags/parts-manifest-current": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"assets":[{"name":"sow-parts-manifest.json","browser_download_url":"` + server.URL + `/downloads/sow-parts-manifest.json"}]}`)) + case "/downloads/sow-parts-manifest.json": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(manifestJSON)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + runGitTest(t, "", "init", "-b", "main", projRoot) + runGitTest(t, projRoot, "remote", "add", "origin", "git@git-ssh.westgate.pw:ShadowsOverWestgate/sow-module.git") + t.Setenv("SOW_ASSETS_SERVER_URL", server.URL) + t.Setenv("SOW_ASSETS_REPO", "ShadowsOverWestgate/sow-assets") + + proj := &project.Project{ + Root: projRoot, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: project.TopDataConfig{ + Source: "topdata", + Build: "build/topdata", + }, + Autogen: project.AutogenConfig{ + Consumers: []project.AutogenConsumerConfig{ + { + ID: "parts", + Producer: "parts", + Dataset: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "trailing_numeric_suffix", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "parts-manifest-current", + AssetName: "sow-parts-manifest.json", + CacheName: "sow-parts-manifest.json", + }, + }, + }, + }, + }, + } + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + partsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "parts_belt.2da")) + if err != nil { + t.Fatalf("read parts_belt.2da: %v", err) + } + if partsText := string(partsBytes); !strings.Contains(partsText, "18\t0\t0.00") { + t.Errorf("expected autogenerated row 18 from released manifest, got:\n%s", partsText) + } +} + +func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) { + root := t.TempDir() + projRoot := filepath.Join(root, "project") + mkdirAll(t, filepath.Join(projRoot, "src")) + mkdirAll(t, filepath.Join(projRoot, "assets")) + mkdirAll(t, filepath.Join(projRoot, "build")) + mkdirAll(t, filepath.Join(projRoot, ".cache")) + mkdirAll(t, filepath.Join(projRoot, "topdata", "data", "visualeffects")) + writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "base.json"), `{ + "output": "visualeffects.2da", + "columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "Imp_Root_H_Node", "OrientWithObject"], + "rows": [ + {"key": "visualeffects:existing", "id": 17, "Label": "EXISTING", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "Imp_Root_H_Node": "****", "OrientWithObject": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "lock.json"), `{ + "visualeffects:existing": 17 +}`+"\n") + mkdirAll(t, filepath.Join(projRoot, "reference")) + writeFile(t, filepath.Join(projRoot, "reference", "build.py"), "print('ok')\n") + + manifestJSON := `{ + "id": "accessory_visualeffects", + "repo": "ShadowsOverWestgate/sow-assets", + "ref": "test-manifest", + "generated_at": "2026-04-25T00:00:00Z", + "accessory_visualeffects": { + "groups": { + "head_accessories": { + "model_columns": ["Imp_HeadCon_Node", "Imp_Root_H_Node"] + }, + "head_features": { + "model_columns": ["Imp_HeadCon_Node", "Imp_Root_H_Node"] + } + }, + "group_token_source": "folder_name", + "category_from": "immediate_parent", + "delimiter": "/", + "case": "preserve", + "strip_model_prefixes": ["hfx_"], + "key_format": "{dataset}:{group}{delimiter}{category_segment}{stem}", + "label_format": "{group}{delimiter}{category_segment}{stem}", + "row_defaults": { + "Type_FD": "D", + "OrientWithGround": "0", + "OrientWithObject": "1" + } + }, + "entries": [ + { + "group": "head_accessories", + "model_stem": "hfx_bandana", + "source": "head_accessories/hfx_bandana.mdl" + }, + { + "group": "head_features", + "model_stem": "hfx_hair_bangs", + "source": "head_features/hair/hfx_hair_bangs.mdl" + } + ] +}` + "\n" + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/repos/ShadowsOverWestgate/sow-assets/releases/tags/head-vfx-manifest-current": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"assets":[{"name":"sow-accessory-vfx-manifest.json","browser_download_url":"` + server.URL + `/downloads/sow-accessory-vfx-manifest.json"}]}`)) + case "/downloads/sow-accessory-vfx-manifest.json": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(manifestJSON)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + runGitTest(t, "", "init", "-b", "main", projRoot) + runGitTest(t, projRoot, "remote", "add", "origin", server.URL+"/ShadowsOverWestgate/sow-module.git") + + proj := &project.Project{ + Root: projRoot, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: project.TopDataConfig{ + Source: "topdata", + Build: "build/topdata", + }, + Autogen: project.AutogenConfig{ + Consumers: []project.AutogenConsumerConfig{ + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "accessory_visualeffects", + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "model_stem", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "sow-accessory-vfx-manifest.json", + CacheName: "sow-accessory-vfx-manifest.json", + }, + }, + }, + }, + }, + } + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + visualeffectsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "visualeffects.2da")) + if err != nil { + t.Fatalf("read visualeffects.2da: %v", err) + } + text := string(visualeffectsBytes) + for _, want := range []string{ + "0\thead_accessories/bandana\tD\t0\thfx_bandana\thfx_bandana\t1", + "1\thead_features/hair/hair_bangs\tD\t0\thfx_hair_bangs\thfx_hair_bangs\t1", + "17\tEXISTING\tF\t0\t****\t****\t0", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected visualeffects output to contain %q, got:\n%s", want, text) + } + } +} + +func TestBuildNativePrefersRemoteReleasedHeadVisualeffectsManifestOverFreshCache(t *testing.T) { + root := t.TempDir() + projRoot := filepath.Join(root, "project") + mkdirAll(t, filepath.Join(projRoot, "src")) + mkdirAll(t, filepath.Join(projRoot, "assets")) + mkdirAll(t, filepath.Join(projRoot, "build")) + mkdirAll(t, filepath.Join(projRoot, ".cache")) + mkdirAll(t, filepath.Join(projRoot, "topdata", "data", "visualeffects")) + writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "base.json"), `{ + "output": "visualeffects.2da", + "columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "OrientWithObject"], + "rows": [ + {"key": "visualeffects:existing", "id": 17, "Label": "EXISTING", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "OrientWithObject": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "lock.json"), `{ + "visualeffects:existing": 17, + "visualeffects:head_accessories/bandana": 42 +}`+"\n") + mkdirAll(t, filepath.Join(projRoot, "reference")) + writeFile(t, filepath.Join(projRoot, "reference", "build.py"), "print('ok')\n") + + cachePath := filepath.Join(projRoot, ".cache", "sow-accessory-vfx-manifest.json") + writeFile(t, cachePath, `{ + "id": "accessory_visualeffects", + "generated_at": "2026-04-25T00:00:00Z", + "entries": [ + { + "group": "head_features", + "model_stem": "hfx_hair_bangs", + "source": "head_features/hair/hfx_hair_bangs.mdl" + } + ] +}`+"\n") + cacheTime := time.Date(2026, 4, 25, 10, 0, 0, 0, time.UTC) + setFileTime(t, cachePath, cacheTime) + + manifestJSON := `{ + "id": "accessory_visualeffects", + "repo": "ShadowsOverWestgate/sow-assets", + "ref": "remote-manifest", + "generated_at": "2026-04-25T12:00:00Z", + "entries": [ + { + "group": "head_accessories", + "model_stem": "hfx_bandana", + "source": "head_accessories/hfx_bandana.mdl" + } + ] +}` + "\n" + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/repos/ShadowsOverWestgate/sow-assets/releases/tags/head-vfx-manifest-current": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"assets":[{"name":"sow-accessory-vfx-manifest.json","updated_at":"2026-04-25T12:00:00Z","browser_download_url":"` + server.URL + `/downloads/sow-accessory-vfx-manifest.json"}]}`)) + case "/downloads/sow-accessory-vfx-manifest.json": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(manifestJSON)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + runGitTest(t, "", "init", "-b", "main", projRoot) + runGitTest(t, projRoot, "remote", "add", "origin", server.URL+"/ShadowsOverWestgate/sow-module.git") + + proj := &project.Project{ + Root: projRoot, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: project.TopDataConfig{ + Source: "topdata", + Build: "build/topdata", + }, + Autogen: project.AutogenConfig{ + Consumers: []project.AutogenConsumerConfig{ + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "accessory_visualeffects", + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "model_stem", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "sow-accessory-vfx-manifest.json", + CacheName: "sow-accessory-vfx-manifest.json", + }, + }, + }, + }, + }, + } + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + visualeffectsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "visualeffects.2da")) + if err != nil { + t.Fatalf("read visualeffects.2da: %v", err) + } + text := string(visualeffectsBytes) + if !strings.Contains(text, "42\thead_accessories/bandana\tD\t0\thfx_bandana\t1") { + t.Fatalf("expected remote manifest entry with preserved lock id, got:\n%s", text) + } + if strings.Contains(text, "HAIR_BANGS") { + t.Fatalf("expected fresh remote manifest to replace cached manifest contents, got:\n%s", text) + } +} + +func TestBuildPackageInvalidatesFreshAutogenCacheWhenReleaseAssetIsNewer(t *testing.T) { + root := topPackageTestProject(t) + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.Autogen.Consumers = nil + + result, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("initial BuildAndPackage failed: %v", err) + } + + sourceTime := time.Now().Add(-6 * time.Hour) + outputTime := time.Now().Add(-4 * time.Hour) + setTopDataSourceTimes(t, root, sourceTime) + setCompiledOutputTimes(t, result, outputTime) + + cachePath := filepath.Join(root, ".cache", "sow-accessory-vfx-manifest.json") + writeFile(t, cachePath, `{"id":"accessory_visualeffects","generated_at":"2026-04-25T10:00:00Z","entries":[{"group":"head_accessories","model_stem":"hfx_bandana","source":"head_accessories/hfx_bandana.mdl"}]}`+"\n") + setFileTime(t, cachePath, time.Now()) + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/repos/ShadowsOverWestgate/sow-assets/releases/tags/head-vfx-manifest-current": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"assets":[{"name":"sow-accessory-vfx-manifest.json","updated_at":"3026-04-25T12:00:00Z","browser_download_url":"` + server.URL + `/downloads/sow-accessory-vfx-manifest.json"}]}`)) + case "/downloads/sow-accessory-vfx-manifest.json": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"accessory_visualeffects","entries":[{"group":"head_accessories","model_stem":"hfx_bandana","source":"head_accessories/hfx_bandana.mdl"}]}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + runGitTest(t, "", "init", "-b", "main", root) + runGitTest(t, root, "remote", "add", "origin", server.URL+"/ShadowsOverWestgate/sow-module.git") + + proj.Config.Autogen.Consumers = []project.AutogenConsumerConfig{ + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "accessory_visualeffects", + Optional: true, + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "model_stem", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "sow-accessory-vfx-manifest.json", + CacheName: "sow-accessory-vfx-manifest.json", + }, + }, + } + + if _, err := BuildPackage(proj, nil); err == nil || !strings.Contains(err.Error(), "older than source file") { + t.Fatalf("expected newer remote autogen manifest to invalidate compiled output, got %v", err) + } +} + +func TestBuildNativeRejectsGitRepoTopDataAssetsOverride(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "parts")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER", "ACBONUS"], + "rows": [] +}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + proj := testProject(root) + proj.Config.TopData.Assets = "ssh://git@gitea.westgate.pw:2222/ShadowsOverWestgate/sow-assets.git" + + _, err := BuildNative(proj, nil) + if err == nil || !strings.Contains(err.Error(), "no longer supports git repo specs") { + t.Fatalf("expected git repo topdata.assets override to be rejected, got %v", err) + } +} + +func TestBuildNativeFailsWhenReleasedPartsManifestUnavailable(t *testing.T) { + root := t.TempDir() + projRoot := filepath.Join(root, "project") + mkdirAll(t, filepath.Join(projRoot, "src")) + mkdirAll(t, filepath.Join(projRoot, "assets")) + mkdirAll(t, filepath.Join(projRoot, "build")) + mkdirAll(t, filepath.Join(projRoot, "topdata", "data", "parts")) + writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(projRoot, "topdata", "data", "parts", "belt.json"), `{ + "output": "parts_belt.2da", + "columns": ["COSTMODIFIER", "ACBONUS"], + "rows": [] +}`+"\n") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.NotFound(w, r) + })) + defer server.Close() + + runGitTest(t, "", "init", "-b", "main", projRoot) + runGitTest(t, projRoot, "remote", "add", "origin", server.URL+"/ShadowsOverWestgate/sow-module.git") + + proj := &project.Project{ + Root: projRoot, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: project.TopDataConfig{ + Source: "topdata", + Build: "build/topdata", + }, + Autogen: project.AutogenConfig{ + Consumers: []project.AutogenConsumerConfig{ + { + ID: "parts", + Producer: "parts", + Dataset: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "trailing_numeric_suffix", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "parts-manifest-current", + AssetName: "sow-parts-manifest.json", + CacheName: "sow-parts-manifest.json", + }, + }, + }, + }, + }, + } + + _, err := BuildNative(proj, nil) + if err == nil || !strings.Contains(err.Error(), "fetch parts manifest release metadata") { + t.Fatalf("expected missing released parts manifest to fail, got %v", err) + } +} + +func TestBuildAndPackageIncludesCompiled2DAAndTopAssets(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + pngPayload := bytes.Repeat([]byte("png-payload-"), 4096) + writeBytes(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.png"), pngPayload) + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + result, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("BuildAndPackage failed: %v", err) + } + if result.OutputHAKPath != filepath.Join(root, "build", PackageHAKFileName) { + t.Fatalf("unexpected hak path: %s", result.OutputHAKPath) + } + if result.OutputTLKPath != filepath.Join(root, "build", PackageTLKFileName) { + t.Fatalf("unexpected tlk path: %s", result.OutputTLKPath) + } + + input, err := os.Open(result.OutputHAKPath) + if err != nil { + t.Fatalf("open hak: %v", err) + } + defer input.Close() + archive, err := erf.Read(input) + if err != nil { + t.Fatalf("read hak: %v", err) + } + found2DA := false + foundAsset := false + foundTLK := false + for _, resource := range archive.Resources { + ext, _ := erf.ExtensionForResourceType(resource.Type) + key := strings.ToLower(resource.Name) + "." + ext + if key == "repadjust.2da" { + if resource.Type != 0x07E1 { + t.Fatalf("expected repadjust.2da to keep resource type 0x07E1, got 0x%04X", resource.Type) + } + found2DA = true + } + if key == "testicon.png" { + if !bytes.Equal(resource.Data, pngPayload) { + t.Fatalf("expected testicon.png payload to be preserved, got %d bytes", len(resource.Data)) + } + foundAsset = true + } + if ext == "tlk" { + foundTLK = true + } + } + if !found2DA { + t.Fatalf("expected repadjust.2da in top package") + } + if !foundAsset { + t.Fatalf("expected testicon.png in top package") + } + if foundTLK { + t.Fatalf("expected top package hak to exclude tlk resources") + } + if _, err := os.Stat(result.OutputTLKPath); err != nil { + t.Fatalf("expected packaged tlk output: %v", err) + } +} + +func TestCollectTopPackageResourcesRejectsDuplicateTopAssetKeys(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, ".cache", "2da")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui", "icons")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui", "portraits")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, ".cache", "2da", "repadjust.2da"), "2DA V2.0\n\n Label\n0 TEST_LABEL\n") + writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "icons", "duplicate.png"), "icon-a") + writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "portraits", "duplicate.png"), "icon-b") + + proj := testProject(root) + if _, _, err := collectTopPackageResources(proj, filepath.Join(root, ".cache", "2da")); err == nil || !strings.Contains(err.Error(), "collides with") { + t.Fatalf("expected duplicate topdata asset collision, got %v", err) + } +} + +func TestBuildAndPackageHonorsConfiguredTopPackageOutputNames(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.png"), "icon-data") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.TopData.PackageHAK = "custom_top.hak" + proj.Config.TopData.PackageTLK = "custom_dialog.tlk" + + result, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("BuildAndPackage failed: %v", err) + } + if result.OutputHAKPath != proj.TopDataPackageHAKPath() { + t.Fatalf("unexpected hak path: %s", result.OutputHAKPath) + } + if result.OutputTLKPath != proj.TopDataPackageTLKPath() { + t.Fatalf("unexpected tlk path: %s", result.OutputTLKPath) + } + if _, err := os.Stat(proj.TopDataPackageHAKPath()); err != nil { + t.Fatalf("expected configured packaged hak output: %v", err) + } + if _, err := os.Stat(proj.TopDataPackageTLKPath()); err != nil { + t.Fatalf("expected configured packaged tlk output: %v", err) + } +} + +func TestBuildAndPackageNoOpsWhenOutputsAreCurrent(t *testing.T) { + root := topPackageTestProject(t) + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.Autogen.Consumers = nil + + result, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("initial BuildAndPackage failed: %v", err) + } + + sourceTime := time.Now().Add(-4 * time.Hour) + outputTime := time.Now().Add(-10 * time.Minute) + setTopDataSourceTimes(t, root, sourceTime) + setBuildOutputTimes(t, result, outputTime) + + beforeHAK := mustStat(t, result.OutputHAKPath).ModTime() + beforeTLK := mustStat(t, result.OutputTLKPath).ModTime() + incremental, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("incremental BuildAndPackage failed: %v", err) + } + if incremental.Mode != "incremental-noop" { + t.Fatalf("expected incremental-noop mode, got %q", incremental.Mode) + } + if after := mustStat(t, result.OutputHAKPath).ModTime(); !after.Equal(beforeHAK) { + t.Fatalf("expected hak mtime to be unchanged, got %s want %s", after, beforeHAK) + } + if after := mustStat(t, result.OutputTLKPath).ModTime(); !after.Equal(beforeTLK) { + t.Fatalf("expected tlk mtime to be unchanged, got %s want %s", after, beforeTLK) + } +} + +func TestBuildAndPackageRebuildsWhenDatasetWasDeleted(t *testing.T) { + root := topPackageTestProject(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "ATHLETICS"}] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.Autogen.Consumers = nil + + result, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("initial BuildAndPackage failed: %v", err) + } + stale2DA := filepath.Join(result.Output2DADir, "repadjust.2da") + if _, err := os.Stat(stale2DA); err != nil { + t.Fatalf("expected initial repadjust output: %v", err) + } + + sourceTime := time.Now().Add(-4 * time.Hour) + outputTime := time.Now().Add(-10 * time.Minute) + if err := os.RemoveAll(filepath.Join(root, "topdata", "data", "repadjust")); err != nil { + t.Fatalf("remove deleted dataset: %v", err) + } + setTopDataSourceTimes(t, root, sourceTime) + setBuildOutputTimes(t, result, outputTime) + + incremental, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("incremental BuildAndPackage failed: %v", err) + } + if incremental.Mode == "incremental-noop" { + t.Fatalf("expected deleted dataset to force rebuild, got %q", incremental.Mode) + } + if _, err := os.Stat(stale2DA); !os.IsNotExist(err) { + t.Fatalf("expected stale deleted dataset output to be pruned, got %v", err) + } + if _, err := os.Stat(filepath.Join(result.Output2DADir, "skills.2da")); err != nil { + t.Fatalf("expected remaining dataset output: %v", err) + } +} + +func TestBuildPackageFailsWhenAutogenManifestCacheIsStale(t *testing.T) { + root := topPackageTestProject(t) + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.Autogen.Consumers = nil + + result, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("initial BuildAndPackage failed: %v", err) + } + + sourceTime := time.Now().Add(-6 * time.Hour) + outputTime := time.Now().Add(-4 * time.Hour) + setTopDataSourceTimes(t, root, sourceTime) + setCompiledOutputTimes(t, result, outputTime) + + proj.Config.Autogen.Consumers = []project.AutogenConsumerConfig{ + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "accessory_visualeffects", + Optional: true, + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "model_stem", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "sow-accessory-vfx-manifest.json", + CacheName: "sow-accessory-vfx-manifest.json", + }, + }, + } + cachePath := filepath.Join(root, ".cache", "sow-accessory-vfx-manifest.json") + writeFile(t, cachePath, `{"id":"accessory_visualeffects","entries":[{"group":"head_accessories","model_stem":"hfx_bandana","source":"head_accessories/hfx_bandana.mdl"}]}`+"\n") + setFileTime(t, cachePath, time.Now().Add(-2*autogenManifestCacheMaxAge)) + + if _, err := BuildPackage(proj, nil); err == nil || !strings.Contains(err.Error(), "older than source file") { + t.Fatalf("expected stale autogen manifest cache to invalidate compiled output, got %v", err) + } +} + +func TestBuildAndPackageRefreshesPackageWhenCompiledOutputsAreCurrent(t *testing.T) { + root := topPackageTestProject(t) + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.Autogen.Consumers = nil + + result, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("initial BuildAndPackage failed: %v", err) + } + + sourceTime := time.Now().Add(-6 * time.Hour) + compiledTime := time.Now().Add(-4 * time.Hour) + packageTime := time.Now().Add(-8 * time.Hour) + setTopDataSourceTimes(t, root, sourceTime) + setCompiledOutputTimes(t, result, compiledTime) + setFileTime(t, result.OutputHAKPath, packageTime) + + compiled2DA := filepath.Join(result.Output2DADir, "repadjust.2da") + before2DA := mustStat(t, compiled2DA).ModTime() + refreshed, err := BuildAndPackage(proj, nil) + if err != nil { + t.Fatalf("package refresh BuildAndPackage failed: %v", err) + } + if refreshed.Mode != "native" { + t.Fatalf("expected native mode from packaged compiled output, got %q", refreshed.Mode) + } + if after := mustStat(t, compiled2DA).ModTime(); !after.Equal(before2DA) { + t.Fatalf("expected compiled 2da mtime to be unchanged, got %s want %s", after, before2DA) + } + if after := mustStat(t, result.OutputHAKPath).ModTime(); !after.After(packageTime) { + t.Fatalf("expected hak to be refreshed after %s, got %s", packageTime, after) + } +} + +func TestNativeCompileGroupStatsCountSourceFragmentsForBaseDatasets(t *testing.T) { + root := testProjectRoot(t) + modulesDir := filepath.Join(root, "placeables", "modules") + generatedDir := filepath.Join(root, "feat", "generated") + mkdirAll(t, modulesDir) + mkdirAll(t, generatedDir) + writeFile(t, filepath.Join(modulesDir, "a.json"), "{}\n") + writeFile(t, filepath.Join(modulesDir, "b.json"), "{}\n") + writeFile(t, filepath.Join(generatedDir, "family.json"), "{}\n") + + stats := nativeCompileGroupStats([]nativeCollectedDataset{ + { + Dataset: nativeDataset{ + Kind: nativeDatasetBase, + Name: "placeables", + ModulesDir: modulesDir, + GeneratedDir: filepath.Join(root, "missing-generated"), + }, + }, + { + Dataset: nativeDataset{ + Kind: nativeDatasetBase, + Name: "feat", + ModulesDir: filepath.Join(root, "missing-modules"), + GeneratedDir: generatedDir, + }, + }, + { + Dataset: nativeDataset{ + Kind: nativeDatasetPlain, + Name: "classes/skills/fighter", + }, + }, + }) + + placeables := stats["placeables"] + if placeables.OutputTables != 1 || placeables.SourceFragments != 3 { + t.Fatalf("unexpected placeables stats: %#v", placeables) + } + + feat := stats["feat"] + if feat.OutputTables != 1 || feat.SourceFragments != 2 { + t.Fatalf("unexpected feat stats: %#v", feat) + } + + classes := stats["classes"] + if classes.OutputTables != 1 || classes.SourceFragments != 1 { + t.Fatalf("unexpected classes stats: %#v", classes) + } +} + +func topPackageTestProject(t *testing.T) string { + t.Helper() + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.png"), "icon-data") + return root +} + +func setTopDataSourceTimes(t *testing.T, root string, modTime time.Time) { + t.Helper() + setTreeFileTimes(t, filepath.Join(root, "topdata"), modTime) +} + +func setBuildOutputTimes(t *testing.T, result PackageResult, modTime time.Time) { + t.Helper() + setCompiledOutputTimes(t, result, modTime) + setFileTime(t, result.OutputHAKPath, modTime) + setFileTime(t, result.OutputTLKPath, modTime) +} + +func setCompiledOutputTimes(t *testing.T, result PackageResult, modTime time.Time) { + t.Helper() + setTreeFileTimes(t, result.Output2DADir, modTime) + setTreeFileTimes(t, result.OutputTLKDir, modTime) +} + +func setTreeFileTimes(t *testing.T, root string, modTime time.Time) { + t.Helper() + err := filepath.WalkDir(root, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := os.Chtimes(path, modTime, modTime); err != nil { + return err + } + return nil + }) + if err != nil { + t.Fatalf("set tree times under %s: %v", root, err) + } +} + +func setFileTime(t *testing.T, path string, modTime time.Time) { + t.Helper() + if err := os.Chtimes(path, modTime, modTime); err != nil { + t.Fatalf("set file time %s: %v", path, err) + } +} + +func mustStat(t *testing.T, path string) os.FileInfo { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + return info +} + +func diagnosticsContain(diagnostics []Diagnostic, text string) bool { + for _, diagnostic := range diagnostics { + if strings.Contains(diagnostic.Message, text) { + return true + } + } + return false +} + +func TestValidateProjectAcceptsPNGTopPackageAsset(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.png"), "icon-data") + + report := ValidateProject(testProject(root)) + if report.HasErrors() { + t.Fatalf("expected png topdata asset to validate, got %#v", report.Diagnostics) + } +} + +func TestBuildPackageRequiresExistingCompiledOutput(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + if _, err := BuildPackage(proj, nil); err == nil || !strings.Contains(err.Error(), "run build-topdata first") { + t.Fatalf("expected missing compiled output error, got %v", err) + } +} + +func TestBuildPackageFailsWhenCompiledOutputIsStale(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + basePath := filepath.Join(root, "topdata", "data", "repadjust", "base.json") + writeFile(t, basePath, `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + newTime := time.Now().Add(2 * time.Second) + if err := os.Chtimes(basePath, newTime, newTime); err != nil { + t.Fatalf("touch topdata source: %v", err) + } + + if _, err := BuildPackage(proj, nil); err == nil || !strings.Contains(err.Error(), "older than source file") { + t.Fatalf("expected stale compiled output error, got %v", err) + } +} + +func TestBuildPackageFailsWhenTopDataSourceFileIsDeleted(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + lockPath := filepath.Join(root, "topdata", "data", "repadjust", "lock.json") + writeFile(t, lockPath, "{}\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + if err := os.Remove(lockPath); err != nil { + t.Fatalf("delete topdata source file: %v", err) + } + + if _, err := BuildPackage(proj, nil); err == nil || !strings.Contains(err.Error(), "older than source file") { + t.Fatalf("expected deleted source file to invalidate compiled output, got %v", err) + } +} + +func TestBuildPackageIgnoresTemplateSourcesForFreshness(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.Autogen.Consumers = nil + + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + templatePath := filepath.Join(root, "topdata", "templates", "scratch.2da") + writeFile(t, templatePath, "2DA V2.0\n\nLabel\n0\tScratch\n") + newTime := time.Now().Add(2 * time.Second) + if err := os.Chtimes(templatePath, newTime, newTime); err != nil { + t.Fatalf("touch topdata template: %v", err) + } + + if _, err := BuildPackage(proj, nil); err != nil { + t.Fatalf("expected template sources to be ignored for freshness, got %v", err) + } +} + +func TestValidateProjectWarnsWhenKeyOnlyOverrideMissesCanonicalKeyByCase(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["Label", "MaxRange"], + "rows": [ + {"id": 0, "key": "baseitems:whip", "Label": "Whip", "MaxRange": "3"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:whip":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "ovr_baseitems_whip.json"), `{ + "overrides": [ + {"key": "baseitems:Whip", "MaxRange": "4"} + ] +}`+"\n") + + report := ValidateProject(testProject(root)) + if report.HasErrors() { + t.Fatalf("expected warning-only validation result, got errors:\n%s", diagnosticsText(report.Diagnostics)) + } + if report.WarningCount() == 0 { + t.Fatal("expected authoring warning for key-only override case mismatch") + } + text := diagnosticsText(report.Diagnostics) + if !strings.Contains(text, `override 0 targets key "baseitems:Whip", but the live canonical key at that point is "baseitems:whip"`) { + t.Fatalf("expected near-miss override warning, got:\n%s", text) + } +} + +func TestValidateProjectWarnsWhenLockKeepsNormalizedEquivalentKeysAlive(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [ + {"id": 0, "key": "feat:trackless_step", "LABEL": "TRACKLESS_STEP", "FEAT": "100", "DESCRIPTION": "200"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:trackless_step": 0, + "feat:tracklessstep": 1200 +}`+"\n") + + report := ValidateProject(testProject(root)) + if report.HasErrors() { + t.Fatalf("expected warning-only validation result, got errors:\n%s", diagnosticsText(report.Diagnostics)) + } + if report.WarningCount() == 0 { + t.Fatal("expected normalized-equivalent lock warning") + } + text := diagnosticsText(report.Diagnostics) + if !strings.Contains(text, `lockfile keeps multiple normalized-equivalent keys alive: feat:trackless_step, feat:tracklessstep`) { + t.Fatalf("expected normalized-equivalent key warning, got:\n%s", text) + } +} + +func TestBuildNativeSupportsLoosePlainTableAtTopdataDataRoot(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "ruleset.json"), `{ + "columns": ["Name", "Value"], + "rows": [ + {"id": 0, "Name": "TEST_RULE", "Value": "1"} + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + report := ValidateProject(proj) + if report.HasErrors() { + t.Fatalf("expected loose root table to validate cleanly, got:\n%s", diagnosticsText(report.Diagnostics)) + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("expected loose root table to build: %v", err) + } + if _, err := os.Stat(filepath.Join(result.Output2DADir, "ruleset.2da")); err != nil { + t.Fatalf("expected ruleset.2da output for loose root table: %v", err) + } +} + +func TestBuildNativeDoesNotCreateRootLockfileForExplicitRootPlainTable(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "categories.json"), `{ + "output": "categories.2da", + "columns": ["Category"], + "rows": [ + {"id": 1, "key": "categories:harmful", "Category": "Harmful"}, + {"id": 2, "key": "categories:beneficial", "Category": "Beneficial"} + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("expected explicit loose root table to build: %v", err) + } + if _, err := os.Stat(filepath.Join(result.Output2DADir, "categories.2da")); err != nil { + t.Fatalf("expected categories.2da output for loose root table: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "topdata", "data", "lock.json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expected no root lock.json for fully explicit loose root table, got err %v", err) + } +} + +func TestBuildNativeCreatesRootLockfileForImplicitRootPlainTableKeys(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "categories.json"), `{ + "output": "categories.2da", + "columns": ["Category"], + "rows": [ + {"key": "categories:harmful", "Category": "Harmful"}, + {"id": 5, "key": "categories:beneficial", "Category": "Beneficial"} + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + if _, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil); err != nil { + t.Fatalf("expected implicit loose root table to build: %v", err) + } + + lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "lock.json")) + if err != nil { + t.Fatalf("expected root lock.json for implicit keyed row: %v", err) + } + var lockData map[string]int + if err := json.Unmarshal(lockRaw, &lockData); err != nil { + t.Fatalf("unmarshal lockfile: %v", err) + } + if lockData["categories:harmful"] != 0 || lockData["categories:beneficial"] != 5 { + t.Fatalf("unexpected lock data: %#v", lockData) + } +} + +func TestBuildNativeAppliesGlobalJSONAllOverrideToBaseDataset(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "base.json"), `{ + "output": "spells.2da", + "columns": ["Label", "Metamagic"], + "rows": [ + {"id": 0, "key": "spells:acid_fog", "Label": "AcidFog", "Metamagic": "255"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "lock.json"), `{"spells:acid_fog":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "extra.json"), `{ + "entries": { + "spells:burning_hands": {"Label": "BurningHands", "Metamagic": "127"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "global.json"), `{ + "overrides": [ + {"match": "all", "Metamagic": "0"} + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "spells.2da")) + if err != nil { + t.Fatalf("read spells.2da: %v", err) + } + text := string(got) + if !strings.Contains(text, "0\tAcidFog\t0\n") { + t.Fatalf("expected base spell metamagic to be globally zeroed, got:\n%s", text) + } + if !strings.Contains(text, "1\tBurningHands\t0\n") { + t.Fatalf("expected module-added spell metamagic to be globally zeroed, got:\n%s", text) + } + if strings.Contains(text, "global") { + t.Fatalf("global.json must not be emitted as a separate dataset, got:\n%s", text) + } +} + +func TestBuildNativeAppliesGlobalJSONDefaultsToEntryRows(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "base.json"), `{ + "output": "spells.2da", + "columns": ["Label", "ImpactScript"], + "rows": [ + {"id": 0, "key": "spells:base_missing", "Label": "BaseMissing"}, + {"id": 1, "key": "spells:base_null", "Label": "BaseNull", "ImpactScript": null} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "lock.json"), `{ + "spells:base_missing": 0, + "spells:base_null": 1, + "spells:new_missing": 10, + "spells:new_null": 11, + "spells:new_stars": 12, + "spells:new_empty": 13, + "spells:new_explicit": 14 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "new.json"), `{ + "entries": { + "spells:new_missing": {"Label": "NewMissing"}, + "spells:new_null": {"Label": "NewNull", "ImpactScript": null}, + "spells:new_stars": {"Label": "NewStars", "ImpactScript": "****"}, + "spells:new_empty": {"Label": "NewEmpty", "ImpactScript": ""}, + "spells:new_explicit": {"Label": "NewExplicit", "ImpactScript": "custom_spell_script"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "global.json"), `{ + "defaults": [ + { + "match": {"source": "entries"}, + "values": { + "ImpactScript": {"format": "ss_{id}"} + } + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "spells.2da")) + if err != nil { + t.Fatalf("read spells.2da: %v", err) + } + text := string(got) + for _, want := range []string{ + "0\tBaseMissing\t****\n", + "1\tBaseNull\t****\n", + "10\tNewMissing\tss_10\n", + "11\tNewNull\tss_11\n", + "12\tNewStars\tss_12\n", + "13\tNewEmpty\tss_13\n", + "14\tNewExplicit\tcustom_spell_script\n", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected spells.2da to contain %q, got:\n%s", want, text) + } + } +} + +func TestBuildNativeGlobalJSONDefaultsUseDistinctProvenanceForDuplicateIDs(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "base.json"), `{ + "output": "spells.2da", + "columns": ["Label", "ImpactScript"], + "rows": [ + {"id": 10, "key": "spells:base", "Label": "BaseDuplicate"}, + {"id": 10, "key": "spells:entry", "Label": "EntryDuplicate"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "entry.json"), `{ + "entries": { + "spells:entry": {"Label": "EntryDuplicate"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "global.json"), `{ + "defaults": [ + { + "match": {"source": "entries"}, + "values": { + "ImpactScript": {"format": "ss_{key}"} + } + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "spells.2da")) + if err != nil { + t.Fatalf("read spells.2da: %v", err) + } + got := string(raw) + for _, want := range []string{ + "10\tBaseDuplicate\t****\n", + "10\tEntryDuplicate\tss_spells:entry\n", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected spells.2da to contain %q, got:\n%s", want, got) + } + } +} + +func TestBuildNativeGlobalJSONDefaultsPreserveProvenanceForDisplacedRows(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "base.json"), `{ + "output": "spells.2da", + "columns": ["Label", "ImpactScript"], + "rows": [ + {"id": 0, "key": "spells:base", "Label": "BaseMoved"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "lock.json"), `{ + "spells:base": 0, + "spells:entry": 10 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "entry.json"), `{ + "entries": { + "spells:entry": {"Label": "EntryDisplaced"} + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "global.json"), `{ + "overrides": [ + {"id": 0, "key": "spells:entry"} + ], + "defaults": [ + { + "match": {"source": "entries"}, + "values": { + "ImpactScript": "entry_default" + } + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "spells.2da")) + if err != nil { + t.Fatalf("read spells.2da: %v", err) + } + got := string(raw) + for _, want := range []string{ + "0\tBaseMoved\t****\n", + "10\tEntryDisplaced\tentry_default\n", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected spells.2da to contain %q, got:\n%s", want, got) + } + } +} + +func TestBuildNativeGlobalJSONDefaultsSupportLiteralAllRows(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "ECL", "DefaultScale"], + "rows": [ + {"id": 6, "key": "racialtypes:human", "Label": "Human"}, + {"id": 7, "key": "racialtypes:elf", "Label": "Elf", "ECL": null, "DefaultScale": null}, + {"id": 8, "key": "racialtypes:drow", "Label": "Drow", "ECL": 2, "DefaultScale": "0.95"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{ + "racialtypes:human": 6, + "racialtypes:elf": 7, + "racialtypes:drow": 8 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "global.json"), `{ + "defaults": [ + { + "match": "all", + "values": { + "ECL": 0, + "DefaultScale": 1 + } + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "racialtypes.2da")) + if err != nil { + t.Fatalf("read racialtypes.2da: %v", err) + } + got := string(raw) + for _, want := range []string{ + "6\tHuman\t0\t1\n", + "7\tElf\t0\t1\n", + "8\tDrow\t2\t0.95\n", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected racialtypes.2da to contain %q, got:\n%s", want, got) + } + } +} + +func TestBuildNativeAppliesRecursiveGlobalJSONPlainDatasetInjections(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "skills", "baseclasses")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "skills", "prestigeclasses")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "global.json"), `{ + "injections": [ + { + "row": { + "SkillLabel": "ProfessionFarmer", + "SkillIndex": {"id": "skills:profession_farmer"}, + "ClassSkill": "1" + } + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "baseclasses", "fighter.json"), `{ + "key": "classes/skills:fighter", + "output": "cls_skill_fighter.2da", + "columns": ["SkillLabel", "SkillIndex", "ClassSkill"], + "rows": [ + {"SkillLabel": "Athletics", "SkillIndex": {"id": "skills:athletics"}, "ClassSkill": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "prestigeclasses", "assassin.json"), `{ + "key": "classes/skills:assassin", + "output": "cls_skill_assassin.2da", + "columns": ["SkillLabel", "SkillIndex", "ClassSkill"], + "rows": [ + {"SkillLabel": "ProfessionFarmer", "SkillIndex": {"id": "skills:profession_farmer"}, "ClassSkill": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label"], + "rows": [ + {"id": 1, "key": "skills:athletics", "Label": "Athletics"}, + {"id": 2, "key": "skills:profession_farmer", "Label": "ProfessionFarmer"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":1,"skills:profession_farmer":2}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + fighterBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "cls_skill_fighter.2da")) + if err != nil { + t.Fatalf("read cls_skill_fighter.2da: %v", err) + } + fighterText := string(fighterBytes) + if !strings.Contains(fighterText, "ProfessionFarmer\t2\t1\n") { + t.Fatalf("expected recursive global skill injection, got:\n%s", fighterText) + } + assassinBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "cls_skill_assassin.2da")) + if err != nil { + t.Fatalf("read cls_skill_assassin.2da: %v", err) + } + if got := strings.Count(string(assassinBytes), "ProfessionFarmer"); got != 1 { + t.Fatalf("expected authored profession row to dedupe global injection, got %d occurrences:\n%s", got, string(assassinBytes)) + } + if _, err := os.Stat(filepath.Join(result.Output2DADir, "global.2da")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("global.json must not emit global.2da, got err %v", err) + } +} + +func TestBuildNativeRejectsGlobalDefaultsOnPlainDataset(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "global.json"), `{ + "defaults": [ + {"match": "all", "values": {"ClassSkill": 1}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "fighter.json"), `{ + "columns": ["SkillLabel", "ClassSkill"], + "rows": [ + {"id": 0, "SkillLabel": "Concentration", "ClassSkill": 1} + ] +}`+"\n") + + _, err := BuildNativeWithOptions(testProject(root), NativeBuildOptions{BuildWiki: false}, nil) + if err == nil || !strings.Contains(err.Error(), "defaults") || !strings.Contains(err.Error(), "base.json dataset") { + t.Fatalf("expected plain dataset defaults build error, got %v", err) + } +} + +func TestBuildNativeExtendsConfiguredPlainRowsByRepeatingLastLevel(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "spellsgained")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "spellsknown")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "ruleset")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "spellsgained", "acolyte.json"), `{ + "output": "cls_spgn_acolyte.2da", + "columns": ["Level", "SpellLevel0", "SpellLevel1"], + "rows": [ + {"id": 0, "Level": 1, "SpellLevel0": 3, "SpellLevel1": 1}, + {"id": 1, "Level": 2, "SpellLevel0": 4, "SpellLevel1": 2}, + {"id": 2, "Level": 3, "SpellLevel0": 5, "SpellLevel1": 3} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "spellsknown", "bard_4th.json"), `{ + "output": "cls_spkn_bard4.2da", + "columns": ["Level", "SpellLevel0", "SpellLevel1"], + "rows": [ + {"id": 0, "Level": 1, "SpellLevel0": 2, "SpellLevel1": null}, + {"id": 1, "Level": 2, "SpellLevel0": 3, "SpellLevel1": 1} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "ruleset", "base.json"), `{ + "output": "ruleset.2da", + "columns": ["Name", "Value"], + "rows": [ + {"id": 0, "Name": "TEST_RULE", "Value": "1"} + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.TopData.RowExtensions = []project.TopDataRowExtensionConfig{ + {Dataset: "classes/spellsgained/*", Mode: "repeat_last", LevelColumn: "Level", TargetLevel: 5}, + {Dataset: "classes/spellsknown/*", Mode: "repeat_last", LevelColumn: "Level", TargetLevel: 5}, + } + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + + gained := read2DATable(t, filepath.Join(result.Output2DADir, "cls_spgn_acolyte.2da")) + for rowID, values := range map[string]map[string]string{ + "2": {"Level": "3", "SpellLevel0": "5", "SpellLevel1": "3"}, + "3": {"Level": "4", "SpellLevel0": "5", "SpellLevel1": "3"}, + "4": {"Level": "5", "SpellLevel0": "5", "SpellLevel1": "3"}, + } { + for column, want := range values { + if got := gained.cell(rowID, column); got != want { + t.Fatalf("spellsgained row %s column %s: got %q, want %q", rowID, column, got, want) + } + } + } + + known := read2DATable(t, filepath.Join(result.Output2DADir, "cls_spkn_bard4.2da")) + if got := known.cell("4", "Level"); got != "5" { + t.Fatalf("expected spellsknown glob rule to extend to level 5, got row 4 Level %q", got) + } + if got := known.cell("4", "SpellLevel1"); got != "1" { + t.Fatalf("expected spellsknown generated row to copy last authored values, got SpellLevel1 %q", got) + } + + ruleset := read2DATable(t, filepath.Join(result.Output2DADir, "ruleset.2da")) + if got := ruleset.cell("1", "Value"); got != "" { + t.Fatalf("expected unmatched ruleset table to remain unextended, got row 1 Value %q", got) + } +} + +func TestBuildNativeExtendsConfiguredRowsAfterHighestAuthoredLevel(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "spellsgained")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "spellsgained", "acolyte.json"), `{ + "output": "cls_spgn_acolyte.2da", + "columns": ["Level", "SpellLevel0", "SpellLevel1"], + "rows": [ + {"id": 0, "Level": 1, "SpellLevel0": 3, "SpellLevel1": 1}, + {"id": 1, "Level": 2, "SpellLevel0": 4, "SpellLevel1": 2}, + {"id": 2, "Level": 3, "SpellLevel0": 5, "SpellLevel1": 3}, + {"id": 3, "Level": 4, "SpellLevel0": 6, "SpellLevel1": 4} + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.TopData.RowExtensions = []project.TopDataRowExtensionConfig{ + {Dataset: "classes/spellsgained/*", Mode: "repeat_last", LevelColumn: "Level", TargetLevel: 5}, + } + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + + table := read2DATable(t, filepath.Join(result.Output2DADir, "cls_spgn_acolyte.2da")) + if got := table.cell("3", "SpellLevel0"); got != "6" { + t.Fatalf("expected authored row 3 to be preserved, got SpellLevel0 %q", got) + } + if got := table.cell("4", "Level"); got != "5" { + t.Fatalf("expected generation to start after highest authored level, got row 4 Level %q", got) + } + if got := table.cell("4", "SpellLevel0"); got != "6" { + t.Fatalf("expected generated row to copy level 4 values, got SpellLevel0 %q", got) + } +} + +func TestBuildNativeRejectsInvalidConfiguredRowExtensions(t *testing.T) { + for _, tc := range []struct { + name string + rows string + levelColumn string + targetLevel int + want string + }{ + { + name: "empty rows", + rows: `[]`, + levelColumn: "Level", + targetLevel: 5, + want: "requires at least one authored row", + }, + { + name: "missing level column", + rows: `[{"id": 0, "Level": 1, "SpellLevel0": 1}]`, + levelColumn: "ClassLevel", + targetLevel: 5, + want: `level_column "ClassLevel" is not present`, + }, + { + name: "nonnumeric level", + rows: `[{"id": 0, "Level": "one", "SpellLevel0": 1}]`, + levelColumn: "Level", + targetLevel: 5, + want: `level_column "Level" must be numeric`, + }, + { + name: "target below authored", + rows: `[{"id": 0, "Level": 6, "SpellLevel0": 1}]`, + levelColumn: "Level", + targetLevel: 5, + want: "target_level 5 is below highest authored Level 6", + }, + } { + t.Run(tc.name, func(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "spellsgained")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "spellsgained", "acolyte.json"), `{ + "output": "cls_spgn_acolyte.2da", + "columns": ["Level", "SpellLevel0"], + "rows": `+tc.rows+` +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.TopData.RowExtensions = []project.TopDataRowExtensionConfig{ + {Dataset: "classes/spellsgained/*", Mode: "repeat_last", LevelColumn: tc.levelColumn, TargetLevel: tc.targetLevel}, + } + _, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err == nil { + t.Fatal("expected BuildNativeWithOptions to fail") + } + if !strings.Contains(err.Error(), tc.want) { + t.Fatalf("expected error containing %q, got %v", tc.want, err) + } + }) + } +} + +func TestBuildNativeGeneratesSpellColumnsFromClassSpellbooks(t *testing.T) { + root := testProjectRoot(t) + writeMinimalSpellbookProject(t, root) + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "spellbooks", "bard.json"), `{ + "key": "classes/spellbooks:bard", + "class": "classes:bard", + "levels": { + "0": ["spells:flare"], + "1": ["spells:aid"] + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "spellbooks", "wizard.json"), `{ + "key": "classes/spellbooks:wizard", + "class": "classes:wizard", + "levels": { + "1": ["spells:shield"] + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "spellbooks", "sorcerer.json"), `{ + "key": "classes/spellbooks:sorcerer", + "class": "classes:sorcerer", + "levels": { + "2": ["spells:shield"] + } +}`+"\n") + + proj := testProject(root) + report := ValidateProject(proj) + if report.HasErrors() { + t.Fatalf("expected spellbook project to validate cleanly, got:\n%s", diagnosticsText(report.Diagnostics)) + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + table := read2DATable(t, filepath.Join(result.Output2DADir, "spells.2da")) + if !table.hasColumn("Bard") || !table.hasColumn("Wizard") || !table.hasColumn("Sorcerer") { + t.Fatalf("expected generated spell columns, got %v", table.columns) + } + for row, values := range map[string]map[string]string{ + "0": {"Bard": "0", "Wizard": "****", "Sorcerer": "****"}, + "1": {"Bard": "1", "Wizard": "****", "Sorcerer": "****"}, + "2": {"Bard": "****", "Wizard": "1", "Sorcerer": "2"}, + } { + for column, want := range values { + if got := table.cell(row, column); got != want { + t.Fatalf("row %s column %s: got %q, want %q", row, column, got, want) + } + } + } +} + +func TestValidateProjectRejectsInvalidClassSpellbooks(t *testing.T) { + root := testProjectRoot(t) + writeMinimalSpellbookProject(t, root) + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "spellbooks", "broken.json"), `{ + "key": "classes/spellbooks:broken", + "class": "classes:missing", + "levels": { + "x": ["spells:flare"], + "1": ["spells:missing"] + } +}`+"\n") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatal("expected invalid spellbook validation errors") + } + text := diagnosticsText(report.Diagnostics) + for _, want := range []string{ + `unknown class "classes:missing"`, + `level key "x" must be a nonnegative integer`, + `unknown spell "spells:missing"`, + } { + if !strings.Contains(text, want) { + t.Fatalf("expected validation message %q, got:\n%s", want, text) + } + } +} + +func TestValidateProjectWarnsWhenSpellModuleAuthorsManagedSpellbookColumn(t *testing.T) { + root := testProjectRoot(t) + writeMinimalSpellbookProject(t, root) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells", "modules")) + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "spellbooks", "bard.json"), `{ + "key": "classes/spellbooks:bard", + "class": "classes:bard", + "levels": { + "0": ["spells:flare"] + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "modules", "direct_class_column.json"), `{ + "columns": ["Sorcerer"], + "entries": { + "spells:module_spell": { + "Label": "ModuleSpell", + "Bard": 1, + "Sorcerer": 1 + } + } +}`+"\n") + + report := ValidateProject(testProject(root)) + if report.HasErrors() { + t.Fatalf("expected warning-only validation result, got errors:\n%s", diagnosticsText(report.Diagnostics)) + } + text := diagnosticsText(report.Diagnostics) + if !strings.Contains(text, `spell module authors class spell column "Bard" managed by class spellbook`) { + t.Fatalf("expected managed Bard column warning, got:\n%s", text) + } + if !strings.Contains(text, `use topdata/data/classes/spellbooks/bard.json instead`) { + t.Fatalf("expected warning to point at spellbook file, got:\n%s", text) + } +} + +func TestValidateProjectDoesNotRewritePlainDatasetLockfile(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "ruleset")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + lockPath := filepath.Join(root, "topdata", "data", "ruleset", "lock.json") + writeFile(t, filepath.Join(root, "topdata", "data", "ruleset", "base.json"), `{ + "output": "ruleset.2da", + "columns": ["Name", "Value"], + "rows": [ + {"id": 0, "key": "ruleset:test_rule", "Name": "TEST_RULE", "Value": "1"} + ] +}`+"\n") + writeFile(t, lockPath, "{}\n") + + before, err := os.ReadFile(lockPath) + if err != nil { + t.Fatalf("read initial lockfile: %v", err) + } + + report := ValidateProject(testProject(root)) + if report.HasErrors() { + t.Fatalf("expected validation to succeed, got:\n%s", diagnosticsText(report.Diagnostics)) + } + + after, err := os.ReadFile(lockPath) + if err != nil { + t.Fatalf("read lockfile after validation: %v", err) + } + if string(after) != string(before) { + t.Fatalf("expected validation to leave plain dataset lockfile unchanged, got:\n%s", string(after)) + } +} + +func TestValidateProjectErrorsOnTopPackageAssetCollision(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "repadjust.2da"), "collision") + + report := ValidateProject(testProject(root)) + if !report.HasErrors() { + t.Fatalf("expected topdata asset collision to be an error") + } + if !strings.Contains(diagnosticsText(report.Diagnostics), "collides with compiled topdata output") { + t.Fatalf("expected collision diagnostic, got %#v", report.Diagnostics) + } +} + +func diagnosticsText(diags []Diagnostic) string { + lines := make([]string, 0, len(diags)) + for _, diag := range diags { + lines = append(lines, diag.Message) + } + return strings.Join(lines, "\n") +} + +func writeMinimalSpellbookProject(t *testing.T, root string) { + t.Helper() + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "core")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "classes", "spellbooks")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "spells")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "base.json"), `{ + "output": "classes.2da", + "columns": ["Label", "SpellTableColumn"], + "rows": [ + {"id": 0, "key": "classes:bard", "Label": "Bard", "SpellTableColumn": "Bard"}, + {"id": 1, "key": "classes:wizard", "Label": "Wizard", "SpellTableColumn": "Wizard"}, + {"id": 2, "key": "classes:sorcerer", "Label": "Sorcerer", "SpellTableColumn": "Sorcerer"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "lock.json"), `{ + "classes:bard": 0, + "classes:wizard": 1, + "classes:sorcerer": 2 +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "base.json"), `{ + "output": "spells.2da", + "columns": ["Label", "Bard", "Wiz_Sorc"], + "rows": [ + {"id": 0, "key": "spells:flare", "Label": "Flare", "Bard": "9", "Wiz_Sorc": "9"}, + {"id": 1, "key": "spells:aid", "Label": "Aid", "Bard": "9", "Wiz_Sorc": "9"}, + {"id": 2, "key": "spells:shield", "Label": "Shield", "Bard": "9", "Wiz_Sorc": "9"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "spells", "lock.json"), `{ + "spells:flare": 0, + "spells:aid": 1, + "spells:shield": 2 +}`+"\n") +} + +type twoDATable struct { + columns []string + rows map[string]map[string]string +} + +func read2DATable(t *testing.T, path string) twoDATable { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read 2da %s: %v", path, err) + } + lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + if len(lines) < 3 { + t.Fatalf("expected 2da header and columns in %s, got:\n%s", path, string(raw)) + } + columns := strings.Fields(lines[2]) + rows := map[string]map[string]string{} + for _, line := range lines[3:] { + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + row := map[string]string{} + for index, column := range columns { + value := "****" + if index+1 < len(fields) { + value = fields[index+1] + } + row[column] = value + } + rows[fields[0]] = row + } + return twoDATable{columns: columns, rows: rows} +} + +func (t twoDATable) hasColumn(column string) bool { + for _, candidate := range t.columns { + if candidate == column { + return true + } + } + return false +} + +func (t twoDATable) cell(rowID, column string) string { + row := t.rows[rowID] + if row == nil { + return "" + } + return row[column] +} + +func runGitTest(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + if dir != "" { + cmd.Dir = dir + } + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v failed: %v\n%s", args, err, string(output)) + } +} + +func testProjectRoot(t *testing.T) string { + t.Helper() + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + return root +} + +func testProject(root string) *project.Project { + return &project.Project{ + Root: root, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: project.TopDataConfig{ + Source: "topdata", + Build: ".cache", + ReferenceBuilder: "reference", + Assets: ".", + }, + Autogen: project.AutogenConfig{ + Consumers: []project.AutogenConsumerConfig{ + { + ID: "parts", + Producer: "parts", + Dataset: "parts", + Mode: "parts_rows", + Root: "part", + Include: []string{"**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "trailing_numeric_suffix", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "parts-manifest-current", + AssetName: "sow-parts-manifest.json", + CacheName: "sow-parts-manifest.json", + }, + }, + { + ID: "accessory_visualeffects", + Producer: "accessory_visualeffects", + Dataset: "visualeffects", + Mode: "accessory_visualeffects", + Optional: true, + Root: "vfxs", + Include: []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"}, + Derive: project.AutogenDeriveConfig{ + Kind: "model_stem", + GroupFrom: "first_path_segment", + }, + Manifest: project.AutogenManifestConfig{ + ReleaseTag: "head-vfx-manifest-current", + AssetName: "sow-accessory-vfx-manifest.json", + CacheName: "sow-accessory-vfx-manifest.json", + }, + }, + }, + }, + }, + } +} + +func mkdirAll(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func writeBytes(t *testing.T, path string, content []byte) { + t.Helper() + if err := os.WriteFile(path, content, 0o755); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} diff --git a/internal/topdata/vfx_persistent_migrate.go b/internal/topdata/vfx_persistent_migrate.go new file mode 100644 index 0000000..3057fe7 --- /dev/null +++ b/internal/topdata/vfx_persistent_migrate.go @@ -0,0 +1,198 @@ +package topdata + +import ( + "os" + "path/filepath" + "strings" +) + +func importLegacyVFXPersistent(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + if strings.TrimSpace(referenceBuilderDir) == "" { + return 0, nil + } + + legacyDir := filepath.Join(referenceBuilderDir, "data", "vfx_persistent") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "vfx_persistent") + targetBasePath := filepath.Join(targetDir, "base.json") + targetLockPath := filepath.Join(targetDir, "lock.json") + targetModulesDir := filepath.Join(targetDir, "modules") + targetModulePath := filepath.Join(targetModulesDir, "freeformaoes.json") + if fileExists(targetBasePath) && fileExists(targetLockPath) && fileExists(targetModulePath) { + baseObj, baseErr := loadJSONObject(targetBasePath) + lockObj, lockErr := loadJSONObject(targetLockPath) + if baseErr == nil && lockErr == nil && vfxPersistentImportComplete(baseObj, lockObj) { + legacyModulePaths, err := collectJSONRelativePaths(filepath.Join(legacyDir, "modules")) + if err == nil { + targetModulePaths, err := collectJSONRelativePaths(targetModulesDir) + if err == nil && equalStringSlices(legacyModulePaths, targetModulePaths) { + return 0, nil + } + } + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + lockObj, err := synthesizeVFXPersistentKeys(baseObj) + if err != nil { + return 0, err + } + legacyLockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) + if err != nil { + return 0, err + } + for key, value := range legacyLockObj { + lockObj[key] = value + } + if err := os.MkdirAll(targetModulesDir, 0o755); err != nil { + return 0, err + } + + updated := 0 + for _, write := range []struct { + path string + obj map[string]any + }{ + {path: targetBasePath, obj: baseObj}, + {path: targetLockPath, obj: lockObj}, + } { + changed, err := writeJSONObjectIfChanged(write.path, write.obj) + if err != nil { + return 0, err + } + if changed { + updated++ + } + } + + moduleRelPaths, err := collectJSONRelativePaths(filepath.Join(legacyDir, "modules")) + if err != nil { + return 0, err + } + for _, relPath := range moduleRelPaths { + moduleObj, err := loadJSONObject(filepath.Join(legacyDir, "modules", filepath.FromSlash(relPath))) + if err != nil { + return 0, err + } + targetPath := filepath.Join(targetModulesDir, filepath.FromSlash(relPath)) + if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { + return 0, err + } + changed, err := writeJSONObjectIfChanged(targetPath, moduleObj) + if err != nil { + return 0, err + } + if changed { + updated++ + } + } + + targetModulePaths, err := collectJSONRelativePaths(targetModulesDir) + if err != nil { + return 0, err + } + legacyModules := make(map[string]struct{}, len(moduleRelPaths)) + for _, relPath := range moduleRelPaths { + legacyModules[relPath] = struct{}{} + } + for _, relPath := range targetModulePaths { + if _, ok := legacyModules[relPath]; ok { + continue + } + if err := os.Remove(filepath.Join(targetModulesDir, filepath.FromSlash(relPath))); err != nil && !os.IsNotExist(err) { + return 0, err + } + updated++ + } + + return updated, nil +} + +func vfxPersistentImportComplete(baseObj, lockObj map[string]any) bool { + if !vfxPersistentRowsHaveKeys(baseObj) { + return false + } + expected := map[string]int{ + "vfx_persistent:blankradius5": 47, + "vfx_persistent:blankradius10": 48, + "vfx_persistent:blankradius15": 49, + "vfx_persistent:blankradius20": 50, + "vfx_persistent:blankradius25": 51, + "vfx_persistent:blankradius30": 52, + } + for key, want := range expected { + raw, ok := lockObj[key] + if !ok { + return false + } + got, err := asInt(raw) + if err != nil || got != want { + return false + } + } + return true +} + +func synthesizeVFXPersistentKeys(baseObj map[string]any) (map[string]any, error) { + rawRows, ok := baseObj["rows"].([]any) + if !ok { + return map[string]any{}, nil + } + lockObj := map[string]any{} + for _, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + label, _ := row["LABEL"].(string) + if label == "" { + continue + } + key := "vfx_persistent:" + strings.ToLower(label) + row["key"] = key + if rawID, ok := row["id"]; ok { + lockObj[key] = rawID + } + } + return lockObj, nil +} + +func vfxPersistentRowsHaveKeys(baseObj map[string]any) bool { + rawRows, ok := baseObj["rows"].([]any) + if !ok { + return false + } + for _, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + return false + } + key, _ := row["key"].(string) + if key == "" { + return false + } + } + return true +} + +func equalStringSlices(left, right []string) bool { + if len(left) != len(right) { + return false + } + for i := range left { + if left[i] != right[i] { + return false + } + } + return true +} diff --git a/internal/topdata/visualeffects_migrate.go b/internal/topdata/visualeffects_migrate.go new file mode 100644 index 0000000..36a29aa --- /dev/null +++ b/internal/topdata/visualeffects_migrate.go @@ -0,0 +1,6 @@ +package topdata + +func importLegacyVisualeffects(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "visualeffects", "visualeffects.2da", nil) +} diff --git a/internal/topdata/wiki_data_providers.go b/internal/topdata/wiki_data_providers.go new file mode 100644 index 0000000..0ed671f --- /dev/null +++ b/internal/topdata/wiki_data_providers.go @@ -0,0 +1,518 @@ +package topdata + +import ( + "fmt" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +type wikiDataProvidersDocument struct { + Providers map[string]wikiDataProviderDefinition `json:"providers" yaml:"providers"` +} + +type wikiDataProviderDefinition struct { + Source string `json:"source" yaml:"source"` + Levels string `json:"levels" yaml:"levels"` + ClassFeats wikiDataProviderClassFeatConfig `json:"class_feats" yaml:"class_feats"` + Attacks wikiDataProviderAttacksConfig `json:"attacks" yaml:"attacks"` + Facts wikiDataProviderFactsConfig `json:"facts" yaml:"facts"` + Provider string `json:"provider" yaml:"provider"` + Columns int `json:"columns" yaml:"columns"` + ItemsPerColumn int `json:"items_per_column" yaml:"items_per_column"` + ForceColumns bool `json:"force_columns" yaml:"force_columns"` +} + +type wikiDataProviderClassFeatConfig struct { + Fields map[string]wikiDataProviderProjectionField `json:"fields" yaml:"fields"` +} + +type wikiDataProviderProjectionField struct { + When string `json:"when" yaml:"when"` +} + +type wikiDataProviderFactsConfig struct { + Rows []wikiDataProviderFactRow `json:"rows" yaml:"rows"` +} + +type wikiDataProviderFactRow struct { + Key string `json:"key" yaml:"key"` + Label string `json:"label" yaml:"label"` + Value string `json:"value" yaml:"value"` + When string `json:"when" yaml:"when"` +} + +type wikiDataProviderAttacksConfig struct { + Field string `json:"field" yaml:"field"` + CountField string `json:"count_field" yaml:"count_field"` + Mode string `json:"mode" yaml:"mode"` + Step int `json:"step" yaml:"step"` + Attacks int `json:"attacks" yaml:"attacks"` + Thresholds []wikiDataProviderAttackThreshold `json:"thresholds" yaml:"thresholds"` + Overrides []wikiDataProviderAttackOverrideRule `json:"overrides" yaml:"overrides"` +} + +type wikiDataProviderAttackThreshold struct { + MinBAB int `json:"min_bab" yaml:"min_bab"` + Attacks int `json:"attacks" yaml:"attacks"` +} + +type wikiDataProviderAttackOverrideRule struct { + Classes []string `json:"classes" yaml:"classes"` + Levels any `json:"levels" yaml:"levels"` + Attacks int `json:"attacks" yaml:"attacks"` + Add int `json:"add" yaml:"add"` +} + +func loadWikiDataProviders(path string) (map[string]wikiDataProviderDefinition, error) { + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return map[string]wikiDataProviderDefinition{}, nil + } + return nil, fmt.Errorf("read wiki data providers %s: %w", path, err) + } + var doc wikiDataProvidersDocument + if err := yaml.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("parse wiki data providers %s: %w", path, err) + } + for name, def := range doc.Providers { + if strings.TrimSpace(name) == "" { + return nil, fmt.Errorf("wiki data providers %s: provider name must not be empty", path) + } + if strings.TrimSpace(def.Source) == "" { + return nil, fmt.Errorf("wiki data providers %s: provider %s missing source", path, name) + } + source := strings.TrimSpace(def.Source) + switch source { + case "class_progression": + for field, projection := range def.ClassFeats.Fields { + if strings.TrimSpace(field) == "" { + return nil, fmt.Errorf("wiki data providers %s: provider %s has class feat projection without field name", path, name) + } + if strings.TrimSpace(projection.When) == "" { + return nil, fmt.Errorf("wiki data providers %s: provider %s class feat projection %s missing when", path, name, field) + } + } + if err := validateWikiDataProviderAttacks(path, name, def.Attacks); err != nil { + return nil, err + } + case "columns": + if len(def.ClassFeats.Fields) > 0 { + return nil, fmt.Errorf("wiki data providers %s: provider %s class feat projections require source class_progression", path, name) + } + if strings.TrimSpace(def.Attacks.Mode) != "" { + return nil, fmt.Errorf("wiki data providers %s: provider %s attacks config requires source class_progression", path, name) + } + if strings.TrimSpace(def.Provider) == "" { + return nil, fmt.Errorf("wiki data providers %s: provider %s columns provider missing provider", path, name) + } + if strings.TrimSpace(def.Provider) == name { + return nil, fmt.Errorf("wiki data providers %s: provider %s columns provider must not reference itself", path, name) + } + if def.Columns <= 0 { + return nil, fmt.Errorf("wiki data providers %s: provider %s columns must be a positive integer", path, name) + } + if def.ItemsPerColumn <= 0 { + return nil, fmt.Errorf("wiki data providers %s: provider %s items_per_column must be a positive integer", path, name) + } + case "facts": + if len(def.Facts.Rows) == 0 { + return nil, fmt.Errorf("wiki data providers %s: provider %s facts source requires rows", path, name) + } + for index, row := range def.Facts.Rows { + if strings.TrimSpace(row.Label) == "" { + return nil, fmt.Errorf("wiki data providers %s: provider %s facts row %d missing label", path, name, index+1) + } + if strings.TrimSpace(row.Value) == "" { + return nil, fmt.Errorf("wiki data providers %s: provider %s facts row %d missing value", path, name, index+1) + } + } + case "class_feats", "class_skills", "class_summary", "class_spellbook", "race_feats", "race_name_forms": + if len(def.ClassFeats.Fields) > 0 { + return nil, fmt.Errorf("wiki data providers %s: provider %s class feat projections require source class_progression", path, name) + } + if strings.TrimSpace(def.Attacks.Mode) != "" { + return nil, fmt.Errorf("wiki data providers %s: provider %s attacks config requires source class_progression", path, name) + } + default: + return nil, fmt.Errorf("wiki data providers %s: provider %s has unknown source %q", path, name, def.Source) + } + } + for name, def := range doc.Providers { + if strings.TrimSpace(def.Source) != "columns" { + continue + } + if _, ok := doc.Providers[strings.TrimSpace(def.Provider)]; !ok { + return nil, fmt.Errorf("wiki data providers %s: provider %s columns provider references missing provider %q", path, name, def.Provider) + } + } + if doc.Providers == nil { + return map[string]wikiDataProviderDefinition{}, nil + } + return doc.Providers, nil +} + +func (ctx *wikiContext) wikiDataProviderRows(name string, page wikiTemplatePage) ([]map[string]any, error) { + def, ok := ctx.wikiDataProviders[name] + if !ok { + return nil, fmt.Errorf("missing wiki data provider %q in %s", name, ctx.wikiDataPath) + } + switch strings.TrimSpace(def.Source) { + case "columns": + sourceRows, err := ctx.wikiDataProviderRows(strings.TrimSpace(def.Provider), page) + if err != nil { + return nil, err + } + return wikiColumnProviderRows(sourceRows, def.Columns, def.ItemsPerColumn, def.ForceColumns), nil + case "class_progression": + if page.Category != "classes" { + return nil, nil + } + return ctx.classProgressionRows(def.Levels, def.ClassFeats.Fields, def.Attacks, page.Key, page.Row) + case "facts": + return ctx.wikiFactRows(name, def.Facts, page) + case "class_feats": + if page.Category != "classes" { + return nil, nil + } + return ctx.classFeatRows(page.Row), nil + case "class_skills": + if page.Category != "classes" { + return nil, nil + } + return ctx.classSkillRows(page.Row), nil + case "class_summary": + if page.Category != "classes" { + return nil, nil + } + return ctx.classSummaryRows(page.Row), nil + case "class_spellbook": + if page.Category != "classes" { + return nil, nil + } + return ctx.classSpellbookRows(page.Key, page.Row), nil + case "race_feats": + if page.Category != "racialtypes" { + return nil, nil + } + return ctx.raceFeatRows(page.Row), nil + case "race_name_forms": + if page.Category != "racialtypes" { + return nil, nil + } + return ctx.raceNameFormRows(page.Row), nil + default: + return nil, fmt.Errorf("unknown wiki data provider source %q", def.Source) + } +} + +func (ctx *wikiContext) wikiFactRows(providerName string, cfg wikiDataProviderFactsConfig, page wikiTemplatePage) ([]map[string]any, error) { + out := []map[string]any{} + renderCtx := wikiTableRenderContext{Page: page, TableName: providerName, Path: ctx.wikiDataPath} + for index, spec := range cfg.Rows { + if strings.TrimSpace(spec.When) != "" { + value, err := ctx.evalWikiDataProviderTemplate(spec.When, page.Row, renderCtx) + if err != nil { + return nil, fmt.Errorf("facts provider %s row %d condition %q: %w", providerName, index+1, spec.When, err) + } + if !truthyWikiTableValue(value) { + continue + } + } + value, err := ctx.evalWikiDataProviderTemplate(spec.Value, page.Row, renderCtx) + if err != nil { + return nil, fmt.Errorf("facts provider %s row %d value %q: %w", providerName, index+1, spec.Value, err) + } + text := stringifyWikiTableValue(value) + if strings.TrimSpace(text) == "" || strings.TrimSpace(text) == nullValue { + continue + } + key := strings.TrimSpace(spec.Key) + if key == "" { + key = wikiFactKeyFromLabel(spec.Label) + } + out = append(out, map[string]any{ + "Key": key, + "Label": spec.Label, + "Value": text, + }) + } + return out, nil +} + +func (ctx *wikiContext) evalWikiDataProviderTemplate(template string, row map[string]any, renderCtx wikiTableRenderContext) (any, error) { + template = strings.TrimSpace(template) + if strings.HasPrefix(template, "{{") && strings.HasSuffix(template, "}}") { + expr := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(template, "{{"), "}}")) + parser := wikiExprParser{tokens: tokenizeWikiExpr(expr), row: row, ctx: ctx, renderCtx: renderCtx, allowMissingFields: true} + value, err := parser.parseComparison() + if err != nil { + return nil, err + } + if parser.peek().kind != wikiExprEOF { + return nil, fmt.Errorf("unexpected token %q", parser.peek().text) + } + return value, nil + } + if strings.Contains(template, "{{") { + return nil, fmt.Errorf("mixed literal/expression templates are not supported") + } + return template, nil +} + +func wikiFactKeyFromLabel(label string) string { + key := strings.ToLower(strings.TrimSpace(label)) + key = strings.Map(func(r rune) rune { + switch { + case r >= 'a' && r <= 'z': + return r + case r >= '0' && r <= '9': + return r + case r == ' ' || r == '-' || r == '_': + return '_' + default: + return -1 + } + }, key) + for strings.Contains(key, "__") { + key = strings.ReplaceAll(key, "__", "_") + } + return strings.Trim(key, "_") +} + +func validateWikiDataProviderAttacks(path, provider string, cfg wikiDataProviderAttacksConfig) error { + if strings.TrimSpace(cfg.Mode) == "" { + return nil + } + switch strings.TrimSpace(cfg.Mode) { + case "fixed": + if cfg.Attacks < 1 { + return fmt.Errorf("wiki data providers %s: provider %s attacks fixed mode requires positive attacks", path, provider) + } + case "bab_thresholds": + if len(cfg.Thresholds) == 0 { + return fmt.Errorf("wiki data providers %s: provider %s attacks bab_thresholds mode requires thresholds", path, provider) + } + for i, threshold := range cfg.Thresholds { + if threshold.MinBAB < 0 { + return fmt.Errorf("wiki data providers %s: provider %s attacks threshold %d min_bab must be non-negative", path, provider, i+1) + } + if threshold.Attacks < 1 { + return fmt.Errorf("wiki data providers %s: provider %s attacks threshold %d attacks must be positive", path, provider, i+1) + } + } + default: + return fmt.Errorf("wiki data providers %s: provider %s attacks has unknown mode %q", path, provider, cfg.Mode) + } + for i, override := range cfg.Overrides { + if len(override.Classes) == 0 { + return fmt.Errorf("wiki data providers %s: provider %s attacks override %d missing classes", path, provider, i+1) + } + for _, class := range override.Classes { + if strings.TrimSpace(class) == "" { + return fmt.Errorf("wiki data providers %s: provider %s attacks override %d has empty class", path, provider, i+1) + } + } + if _, err := parseWikiAttackLevels(override.Levels); err != nil { + return fmt.Errorf("wiki data providers %s: provider %s attacks override %d bad levels: %w", path, provider, i+1, err) + } + if override.Attacks != 0 && override.Add != 0 { + return fmt.Errorf("wiki data providers %s: provider %s attacks override %d must not set both attacks and add", path, provider, i+1) + } + if override.Attacks == 0 && override.Add == 0 { + return fmt.Errorf("wiki data providers %s: provider %s attacks override %d must set attacks or add", path, provider, i+1) + } + if override.Attacks < 0 { + return fmt.Errorf("wiki data providers %s: provider %s attacks override %d attacks must be positive", path, provider, i+1) + } + } + return nil +} + +func parseWikiAttackLevels(raw any) (map[int]struct{}, error) { + out := map[int]struct{}{} + addLevel := func(value any) error { + switch typed := value.(type) { + case int: + if typed < 1 { + return fmt.Errorf("level must be positive") + } + out[typed] = struct{}{} + return nil + case int64: + if typed < 1 { + return fmt.Errorf("level must be positive") + } + out[int(typed)] = struct{}{} + return nil + case float64: + level := int(typed) + if typed != float64(level) || level < 1 { + return fmt.Errorf("level must be a positive integer") + } + out[level] = struct{}{} + return nil + case string: + text := strings.TrimSpace(typed) + if text == "" { + return fmt.Errorf("level must not be empty") + } + if strings.Contains(text, "-") { + start, end, err := parseLevelRange(text) + if err != nil { + return err + } + for level := start; level <= end; level++ { + out[level] = struct{}{} + } + return nil + } + level, err := strconv.Atoi(text) + if err != nil || level < 1 { + return fmt.Errorf("level must be a positive integer") + } + out[level] = struct{}{} + return nil + default: + return fmt.Errorf("unsupported level value") + } + } + switch typed := raw.(type) { + case nil: + return nil, fmt.Errorf("levels is required") + case []any: + if len(typed) == 0 { + return nil, fmt.Errorf("levels must not be empty") + } + for _, value := range typed { + if err := addLevel(value); err != nil { + return nil, err + } + } + default: + if err := addLevel(raw); err != nil { + return nil, err + } + } + return out, nil +} + +func wikiColumnProviderRows(sourceRows []map[string]any, configuredColumns, itemsPerColumn int, forceColumns bool) []map[string]any { + if configuredColumns <= 0 || itemsPerColumn <= 0 { + return nil + } + needed := 0 + if len(sourceRows) > 0 { + needed = (len(sourceRows) + itemsPerColumn - 1) / itemsPerColumn + } + total := needed + if forceColumns && configuredColumns > total { + total = configuredColumns + } + if total == 0 { + return nil + } + rows := make([]map[string]any, 0, total) + for i := 0; i < total; i++ { + start := i * itemsPerColumn + end := start + itemsPerColumn + if start > len(sourceRows) { + start = len(sourceRows) + } + if end > len(sourceRows) { + end = len(sourceRows) + } + items := make([]map[string]any, 0, end-start) + for _, source := range sourceRows[start:end] { + items = append(items, cloneRowMap(source)) + } + rows = append(rows, map[string]any{ + "Index": i + 1, + "Count": len(items), + "Items": items, + }) + } + return rows +} + +func (ctx *wikiContext) classSkillRows(classRow map[string]any) []map[string]any { + table := ctx.tableForValue(fieldValue(classRow, "SkillsTable"), ctx.classSkillTables) + if table == nil { + return nil + } + rows := []map[string]any{} + for _, source := range table.Rows { + if stringValue(source, "ClassSkill") != "1" { + continue + } + key := resolveReferenceKey(fieldValue(source, "SkillIndex"), ctx.skillIDToKey) + if key == "" { + key = resolveReferenceKey(fieldValue(source, "SkillIndex"), nil) + } + if key == "" || !ctx.canReferenceWikiKey(key) { + continue + } + row := cloneRowMap(source) + row["SkillKey"] = key + row["SkillName"] = ctx.resolveSkillName(key) + if skillRow := ctx.skillRows[key]; skillRow != nil { + row["SkillAbility"] = stringValue(skillRow, "KeyAbility") + } + rows = append(rows, row) + } + return rows +} + +func (ctx *wikiContext) raceFeatRows(raceRow map[string]any) []map[string]any { + values := []any{} + switch typed := fieldValue(raceRow, "FeatsTable").(type) { + case []any: + values = append(values, typed...) + case []string: + for _, value := range typed { + values = append(values, value) + } + default: + if table := ctx.tableForValue(typed, ctx.raceFeatTables); table != nil { + for _, row := range table.Rows { + values = append(values, fieldValue(row, "FeatIndex")) + } + } + } + rows := []map[string]any{} + for _, value := range values { + key := resolveReferenceKey(value, ctx.featIDToKey) + if key == "" { + key = resolveReferenceKey(value, nil) + } + if key == "" || !ctx.canReferenceWikiKey(key) { + continue + } + rows = append(rows, map[string]any{ + "FeatKey": key, + "FeatName": ctx.resolveFeatName(key), + }) + } + return rows +} + +func (ctx *wikiContext) raceNameFormRows(raceRow map[string]any) []map[string]any { + candidates := []struct { + label string + field string + }{ + {"Plural", "NamePlural"}, + {"Converted Name", "ConverName"}, + {"Lower Name", "ConverNameLower"}, + } + rows := []map[string]any{} + for _, candidate := range candidates { + if value := ctx.resolveTextValue(fieldValue(raceRow, candidate.field), nil); value != "" { + rows = append(rows, map[string]any{"Label": candidate.label, "Value": value}) + } + } + return rows +} diff --git a/internal/topdata/wiki_deploy.go b/internal/topdata/wiki_deploy.go new file mode 100644 index 0000000..2697ced --- /dev/null +++ b/internal/topdata/wiki_deploy.go @@ -0,0 +1,1861 @@ +package topdata + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "html" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "gopkg.in/yaml.v3" +) + +const ( + managedStartMarker = "" + legacyMarkdownStartMarker = "[//]: # (sow-topdata-wiki:managed:start)" + legacyMarkdownEndMarker = "[//]: # (sow-topdata-wiki:managed:end)" + legacyManagedStartMarker = "" + legacyManagedEndMarker = "" + manualStartMarkerPrefix = "") + end := strings.Index(content, managedEndMarker) + if startEnd >= 0 && end >= 0 { + startMarker := content[start : start+startEnd+len("-->")] + return startMarker, managedEndMarker, start, end + } + } + for _, markers := range [][2]string{ + {legacyMarkdownStartMarker, legacyMarkdownEndMarker}, + {legacyManagedStartMarker, legacyManagedEndMarker}, + } { + start := strings.Index(content, markers[0]) + end := strings.Index(content, markers[1]) + if start >= 0 && end >= 0 { + return markers[0], markers[1], start, end + } + } + return "", "", -1, -1 +} + +func extractManagedMarkers(content string) (string, string, bool) { + startMarker, endMarker, start, end := managedRegionBounds(content) + return startMarker, endMarker, start >= 0 && end >= 0 +} + +func mergeManualRegions(merged, existing string) string { + existingManual := extractManualRegions(existing) + if len(existingManual) == 0 { + return merged + } + return replaceManualRegions(merged, existingManual) +} + +func extractManualRegions(content string) map[string]string { + regions := map[string]string{} + offset := 0 + for { + startRel := strings.Index(content[offset:], manualStartMarkerPrefix) + if startRel < 0 { + break + } + start := offset + startRel + startEndRel := strings.Index(content[start:], "-->") + if startEndRel < 0 { + break + } + startMarker := content[start : start+startEndRel+len("-->")] + id := markerID(startMarker) + if id == "" { + offset = start + len(startMarker) + continue + } + endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\"" + end := strings.Index(content[start+len(startMarker):], endPrefix) + if end < 0 { + offset = start + len(startMarker) + continue + } + end += start + len(startMarker) + endEndRel := strings.Index(content[end:], "-->") + if endEndRel < 0 { + break + } + endMarker := content[end : end+endEndRel+len("-->")] + regions[id] = startMarker + strings.TrimRight(content[start+len(startMarker):end], "\n") + "\n" + endMarker + offset = end + len(endMarker) + } + return regions +} + +func replaceManualRegions(content string, replacements map[string]string) string { + for id, replacement := range replacements { + start := strings.Index(content, manualStartMarkerPrefix+" id=\""+id+"\"") + if start < 0 { + continue + } + endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\"" + end := strings.Index(content[start:], endPrefix) + if end < 0 { + continue + } + end += start + endEndRel := strings.Index(content[end:], "-->") + if endEndRel < 0 { + continue + } + endEnd := end + endEndRel + len("-->") + content = content[:start] + replacement + content[endEnd:] + } + return content +} + +func markerID(marker string) string { + const prefix = `id="` + start := strings.Index(marker, prefix) + if start < 0 { + return "" + } + start += len(prefix) + end := strings.Index(marker[start:], `"`) + if end < 0 { + return "" + } + return marker[start : start+end] +} + +func renderArchivedWikiPage(pageID, title string) string { + if title == "" { + title = extractHTMLTitle("", pageID) + } + body := strings.Join([]string{ + "

" + html.EscapeString(title) + "

", + "

This generated page is no longer present in the current Shadows Over Westgate topdata wiki source.

", + }, "\n") + hash := computeManagedHash(body) + return ensureTrailingNewline(strings.Join([]string{ + "", + "", + body, + "", + }, "\n")) +} + +func extractWikiPageID(content string) string { + const pageMarker = "sow-topdata-wiki:page=" + start := strings.Index(content, pageMarker) + if start < 0 { + return "" + } + value := content[start+len(pageMarker):] + if end := strings.Index(value, "-->"); end >= 0 { + value = value[:end] + } + if fields := strings.Fields(value); len(fields) > 0 { + return strings.Trim(strings.TrimSpace(fields[0]), `"`) + } + return "" +} + +func extractWikiPagePublicPath(content string) string { + const pageMarker = "sow-topdata-wiki:page=" + const pathField = "public_path=" + start := strings.Index(content, pageMarker) + if start < 0 { + return "" + } + end := strings.Index(content[start:], "-->") + if end < 0 { + return "" + } + marker := content[start : start+end] + fieldStart := strings.Index(marker, pathField) + if fieldStart < 0 { + return "" + } + value := marker[fieldStart+len(pathField):] + if fields := strings.Fields(value); len(fields) > 0 { + path := strings.Trim(strings.TrimSpace(fields[0]), `"`) + if path != "" && canonicalWikiSlashPath(path) == path { + return path + } + } + return "" +} + +func extractPageTitle(content, fallback string) string { + if title := extractMarkdownTitle(content); title != "" { + return title + } + return extractHTMLTitle(content, fallback) +} + +func extractMarkdownTitle(content string) string { + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "# ") { + continue + } + title := strings.TrimSpace(strings.TrimPrefix(trimmed, "# ")) + if title != "" { + return title + } + } + return "" +} + +func extractHTMLTitle(content, fallback string) string { + lower := strings.ToLower(content) + for _, tag := range []string{"h1", "h2"} { + open := "<" + tag + ">" + close := "" + start := strings.Index(lower, open) + end := strings.Index(lower, close) + if start >= 0 && end > start { + title := strings.TrimSpace(content[start+len(open) : end]) + title = strings.ReplaceAll(title, "\n", " ") + if title != "" { + return html.UnescapeString(stripSimpleTags(title)) + } + } + } + parts := strings.Split(fallback, ":") + return namespaceTitle(parts[len(parts)-1]) +} + +func nodeBBTopicTitle(page wikiDeployPage, minLength int) string { + if minLength <= 0 { + minLength = project.DefaultTopDataWikiTitlePrefixMinLength + } + title := strings.TrimSpace(page.Title) + if len([]rune(title)) >= minLength { + return title + } + namespace := strings.TrimSpace(namespaceTitle(page.Namespace)) + if namespace == "" { + namespace = "Wiki" + } + if title == "" { + parts := strings.Split(page.PageID, ":") + title = namespaceTitle(parts[len(parts)-1]) + } + return namespace + ": " + title +} + +func stripSimpleTags(text string) string { + var out strings.Builder + inTag := false + for _, r := range text { + switch r { + case '<': + inTag = true + case '>': + inTag = false + default: + if !inTag { + out.WriteRune(r) + } + } + } + return out.String() +} + +func sortedKeysFromMap[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +type nodeBBClient struct { + endpoint string + token string + timeout time.Duration +} + +type nodeBBHTTPError struct { + Method string + Path string + Status int + Body string +} + +func (e nodeBBHTTPError) Error() string { + return fmt.Sprintf("NodeBB %s %s failed: HTTP %d: %s", e.Method, e.Path, e.Status, e.Body) +} + +func isNodeBBMissingResource(err error) bool { + var httpErr nodeBBHTTPError + return errors.As(err, &httpErr) && (httpErr.Status == http.StatusNotFound || httpErr.Status == http.StatusGone) +} + +type nodeBBPost struct { + PID int + TID int + Title string + Content string + SourceContent string +} + +type nodeBBEditLock struct { + Token string +} + +type nodeBBWikiPage struct { + TID int + Title string + TitleLeaf string + Slug string + WikiPath string + CanonicalPath string +} + +func newNodeBBClient(endpoint, token string) *nodeBBClient { + return &nodeBBClient{ + endpoint: strings.TrimRight(endpoint, "/"), + token: token, + timeout: 30 * time.Second, + } +} + +func (c *nodeBBClient) getPost(pid int) (nodeBBPost, error) { + var doc any + if err := c.request("GET", fmt.Sprintf("/api/v3/posts/%d", pid), nil, &doc); err != nil { + return nodeBBPost{}, err + } + return parseNodeBBPost(doc), nil +} + +func (c *nodeBBClient) getTopic(tid int) (nodeBBPost, error) { + var doc any + if err := c.request("GET", fmt.Sprintf("/api/v3/topics/%d", tid), nil, &doc); err != nil { + return nodeBBPost{}, err + } + post := parseNodeBBPost(doc) + if post.TID == 0 { + post.TID = tid + } + return post, nil +} + +func (c *nodeBBClient) listNamespacePages(cid int) ([]nodeBBWikiPage, error) { + return c.listNamespacePagesWithQuery(cid, "") +} + +func (c *nodeBBClient) searchNamespacePages(cid int, query string) ([]nodeBBWikiPage, error) { + return c.listNamespacePagesWithQuery(cid, query) +} + +func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nodeBBWikiPage, error) { + var out []nodeBBWikiPage + after := "" + seenCursors := map[string]struct{}{} + seenPages := map[string]struct{}{} + for { + path := fmt.Sprintf("/api/v3/plugins/westgate-wiki/namespace/%d/pages?limit=80", cid) + if query != "" { + path += "&q=" + url.QueryEscape(query) + } + if after != "" { + path += "&after=" + url.QueryEscape(after) + } + var doc any + if err := c.request("GET", path, nil, &doc); err != nil { + return nil, err + } + pages, next, hasMore := parseNodeBBWikiPageList(doc) + newPages := 0 + for _, page := range pages { + identity := nodeBBWikiPageIdentity(page) + if identity == "" { + out = append(out, page) + newPages++ + continue + } + if _, ok := seenPages[identity]; ok { + continue + } + seenPages[identity] = struct{}{} + out = append(out, page) + newPages++ + } + if !hasMore || next == "" { + return out, nil + } + if after != "" && next == after { + if newPages == 0 { + return out, nil + } + return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next) + } + if _, ok := seenCursors[next]; ok { + if newPages == 0 { + return out, nil + } + return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next) + } + seenCursors[next] = struct{}{} + after = next + } +} + +func nodeBBWikiPageIdentity(page nodeBBWikiPage) string { + if page.TID != 0 { + return fmt.Sprintf("tid:%d", page.TID) + } + for _, value := range []string{page.WikiPath, page.CanonicalPath, page.Slug, page.Title} { + if trimmed := strings.TrimSpace(value); trimmed != "" { + return strings.ToLower(trimmed) + } + } + return "" +} + +func (c *nodeBBClient) createTopic(cid int, title, content, summary string) (nodeBBPost, error) { + body := map[string]any{"cid": cid, "title": title, "content": content, "sourceContent": content} + if summary != "" { + body["_uid_note"] = summary + } + var doc any + if err := c.request("POST", "/api/v3/topics", body, &doc); err != nil { + return nodeBBPost{}, err + } + post := parseNodeBBPost(doc) + if post.TID == 0 || post.PID == 0 { + return nodeBBPost{}, fmt.Errorf("NodeBB topic create response did not include tid and pid") + } + return post, nil +} + +func (c *nodeBBClient) updatePost(tid, pid int, content, summary string) error { + if tid == 0 { + post, err := c.getPost(pid) + if err != nil { + return err + } + tid = post.TID + } + lock, err := c.acquireEditLock(tid) + if err != nil { + return err + } + body := map[string]any{"content": content, "sourceContent": content} + if lock.Token != "" { + body["wikiEditLockToken"] = lock.Token + } + if summary != "" { + body["_uid_note"] = summary + } + return c.request("PUT", fmt.Sprintf("/api/v3/posts/%d", pid), body, nil) +} + +func (c *nodeBBClient) renameWikiPage(tid, cid int, title string) error { + if tid == 0 || cid == 0 { + return fmt.Errorf("NodeBB wiki page rename requires topic id and category id") + } + body := map[string]any{"tid": tid, "cid": cid, "title": title} + return c.request("PUT", "/api/v3/plugins/westgate-wiki/page/move", body, nil) +} + +func (c *nodeBBClient) purgeTopic(tid int) error { + if tid == 0 { + return fmt.Errorf("NodeBB topic purge requires topic id") + } + err := c.request(http.MethodDelete, fmt.Sprintf("/api/v3/topics/%d", tid), nil, nil) + var httpErr nodeBBHTTPError + if errors.As(err, &httpErr) && (httpErr.Status == http.StatusNotFound || httpErr.Status == http.StatusGone) { + return nil + } + return err +} + +func (c *nodeBBClient) acquireEditLock(tid int) (nodeBBEditLock, error) { + if tid == 0 { + return nodeBBEditLock{}, fmt.Errorf("NodeBB wiki post update requires a topic id for edit locking") + } + body := map[string]any{"tid": tid} + var doc any + if err := c.request("PUT", "/api/v3/plugins/westgate-wiki/edit-lock", body, &doc); err != nil { + return nodeBBEditLock{}, err + } + lock := parseNodeBBEditLock(doc) + if lock.Token == "" { + return nodeBBEditLock{}, fmt.Errorf("NodeBB wiki edit lock response for topic %d did not include token", tid) + } + return lock, nil +} + +func (c *nodeBBClient) request(method, path string, payload any, out any) error { + var rawPayload []byte + if payload != nil { + var err error + rawPayload, err = json.Marshal(payload) + if err != nil { + return err + } + } + const maxAttempts = 3 + backoff := []time.Duration{5 * time.Second, 15 * time.Second} + for attempt := 0; attempt < maxAttempts; attempt++ { + err := c.doRequest(method, path, rawPayload, out) + if err == nil { + return nil + } + if attempt == maxAttempts-1 || !isNodeBBRetryable(err) { + return err + } + time.Sleep(backoff[attempt]) + } + return nil +} + +func isNodeBBRetryable(err error) bool { + if err == nil { + return false + } + var httpErr nodeBBHTTPError + if errors.As(err, &httpErr) { + return httpErr.Status == http.StatusTooManyRequests || + httpErr.Status == http.StatusBadGateway || + httpErr.Status == http.StatusServiceUnavailable || + httpErr.Status == http.StatusGatewayTimeout + } + // Retry on network/timeout errors (e.g. context deadline exceeded) + return true +} + +func (c *nodeBBClient) doRequest(method, path string, rawPayload []byte, out any) error { + var body io.Reader + if rawPayload != nil { + body = bytes.NewReader(rawPayload) + } + req, err := http.NewRequest(method, c.endpoint+path, body) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+c.token) + req.Header.Set("Accept", "application/json") + if rawPayload != nil { + req.Header.Set("Content-Type", "application/json") + } + client := http.Client{Timeout: c.timeout} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode >= 400 { + return nodeBBHTTPError{Method: method, Path: path, Status: resp.StatusCode, Body: strings.TrimSpace(string(raw))} + } + if out == nil || len(bytes.TrimSpace(raw)) == 0 { + return nil + } + if err := json.Unmarshal(raw, out); err != nil { + return fmt.Errorf("parse NodeBB response: %w", err) + } + return nil +} + +func parseNodeBBPost(doc any) nodeBBPost { + for { + m, ok := doc.(map[string]any) + if !ok { + return nodeBBPost{} + } + if response, ok := m["response"]; ok { + doc = response + continue + } + if postData, ok := m["postData"]; ok { + doc = postData + continue + } + if mainPost, ok := m["mainPost"]; ok { + post := parseNodeBBPost(mainPost) + if post.TID == 0 { + post.TID = intFromAny(m["tid"]) + } + if post.PID == 0 { + post.PID = firstInt(m, "mainPid") + } + if post.Title == "" { + post.Title = firstString(m, "title", "titleRaw", "title_raw") + } + return post + } + sourceContent := firstString(m, "sourceContent", "source_content") + content := firstString(m, "content") + if sourceContent != "" { + content = sourceContent + } + return nodeBBPost{ + PID: firstInt(m, "pid", "mainPid"), + TID: firstInt(m, "tid"), + Title: firstString(m, "title", "titleRaw", "title_raw"), + Content: content, + SourceContent: sourceContent, + } + } +} + +func parseNodeBBEditLock(doc any) nodeBBEditLock { + for { + m, ok := doc.(map[string]any) + if !ok { + return nodeBBEditLock{} + } + if response, ok := m["response"]; ok { + doc = response + continue + } + return nodeBBEditLock{Token: firstString(m, "token")} + } +} + +func parseNodeBBWikiPageList(doc any) ([]nodeBBWikiPage, string, bool) { + for { + m, ok := doc.(map[string]any) + if !ok { + return nil, "", false + } + if response, ok := m["response"]; ok { + doc = response + continue + } + pagesRaw, _ := m["pages"].([]any) + pages := make([]nodeBBWikiPage, 0, len(pagesRaw)) + for _, raw := range pagesRaw { + row, ok := raw.(map[string]any) + if !ok { + continue + } + pages = append(pages, nodeBBWikiPage{ + TID: firstInt(row, "tid"), + Title: firstString(row, "title"), + TitleLeaf: firstString(row, "titleLeaf", "title_leaf"), + Slug: firstString(row, "slug"), + WikiPath: firstString(row, "wikiPath", "wiki_path"), + CanonicalPath: firstString(row, "canonicalPath", "canonical_path"), + }) + } + return pages, firstString(m, "nextCursor", "next_cursor"), boolFromAny(m["hasMore"]) + } +} + +func isNodeBBWikiPageCollision(err error) bool { + var httpErr nodeBBHTTPError + if !errors.As(err, &httpErr) { + return false + } + if httpErr.Status != http.StatusBadRequest { + return false + } + body := strings.ToLower(httpErr.Body) + return strings.Contains(body, "wiki page with this url already exists") || + strings.Contains(body, "page-collision") || + strings.Contains(body, "namespace-page-collision") +} + +func boolFromAny(value any) bool { + switch typed := value.(type) { + case bool: + return typed + case string: + return typed == "true" || typed == "1" + default: + return false + } +} + +func firstInt(m map[string]any, keys ...string) int { + for _, key := range keys { + if value := intFromAny(m[key]); value != 0 { + return value + } + } + return 0 +} + +func intFromAny(value any) int { + switch typed := value.(type) { + case int: + return typed + case float64: + return int(typed) + case json.Number: + i, _ := typed.Int64() + return int(i) + case string: + i, _ := strconv.Atoi(typed) + return i + default: + return 0 + } +} + +func firstString(m map[string]any, keys ...string) string { + for _, key := range keys { + if value, ok := m[key].(string); ok { + return value + } + } + return "" +} diff --git a/internal/topdata/wiki_deploy_test.go b/internal/topdata/wiki_deploy_test.go new file mode 100644 index 0000000..edf2a54 --- /dev/null +++ b/internal/topdata/wiki_deploy_test.go @@ -0,0 +1,2387 @@ +package topdata + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func TestDeployWikiDryRunDoesNotWriteRemoteOrManifest(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Athletics\n\nGenerated athletics page\n[//]: # (sow-topdata-wiki:managed:end)\n" + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.md"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + old := "\n

Athletics

\n

Existing athletics page

\n\n" + oldHash := computeManagedHash(old) + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:athletics": {Hash: oldHash, TID: 7, PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + updateCalls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer nodebb-token" { + t.Fatalf("unexpected authorization header %q", got) + } + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7, "content": old}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42": + updateCalls++ + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + DryRun: true, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions dry run failed: %v", err) + } + + if result.Updated != 1 { + t.Fatalf("expected one planned update, got %d", result.Updated) + } + if updateCalls != 0 { + t.Fatalf("expected dry run not to update NodeBB posts, got %d calls", updateCalls) + } + after := loadDeployManifest(manifestPath) + if after.Pages["skills:athletics"].Hash != oldHash { + t.Fatalf("expected dry run not to rewrite manifest") + } +} + +func TestDeployWikiReportsPlanningProgressBeforeRemoteWork(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Athletics

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + + progress := []string{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/3/pages": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []map[string]any{}, "hasMore": false}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + _, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: filepath.Join(root, "deploy-manifest.json"), + CategoryIDs: map[string]int{"skills": 3}, + DryRun: true, + AllowCreates: true, + }, func(message string) { + progress = append(progress, message) + }) + if err != nil { + t.Fatalf("DeployWikiWithOptions dry run failed: %v", err) + } + + joined := strings.Join(progress, "\n") + for _, expected := range []string{ + "Collecting local wiki pages from", + "Loaded 1 local wiki page(s)", + "Loaded wiki deploy manifest", + "Planning NodeBB wiki deploy for 1 local page(s)", + "Listing NodeBB wiki namespace category 3", + } { + if !strings.Contains(joined, expected) { + t.Fatalf("expected progress %q, got:\n%s", expected, joined) + } + } +} + +func TestDeployWikiReportsLiveExecutionProgress(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "feat"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + updatePage := ` + +

Updated

+

New generated content

+ +` + renamePage := ` + +

Renamed

+

Unchanged generated content

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "feat", "updated.html"), []byte(updatePage), 0644); err != nil { + t.Fatalf("write update page: %v", err) + } + if err := os.WriteFile(filepath.Join(sourceDir, "feat", "renamed.html"), []byte(renamePage), 0644); err != nil { + t.Fatalf("write rename page: %v", err) + } + oldUpdate := ` + +

Updated

+

Old generated content

+ +` + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "feat:renamed": {Hash: computeManagedHash(renamePage), Title: "Old Renamed", TopicTitle: "Old Renamed", Namespace: "feat", SourceContentSynced: true, TID: 7, PID: 70, CID: 5}, + "feat:updated": {Hash: computeManagedHash(oldUpdate), Title: "Updated", TopicTitle: "Updated", Namespace: "feat", TID: 8, PID: 80, CID: 5}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + progress := []string{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/7": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 7, "title": "Old Renamed", "mainPid": 70}}) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/80": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 80, "tid": 8, "content": oldUpdate}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/edit-lock": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"token": "lock-token"}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/80": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 80, "tid": 8}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/page/move": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 7}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + }, func(message string) { + progress = append(progress, message) + }) + if err != nil { + t.Fatalf("DeployWikiWithOptions live deploy failed: %v", err) + } + if result.Updated != 1 || result.Renamed != 1 { + t.Fatalf("expected one update and one rename, got %#v", result) + } + + joined := strings.Join(progress, "\n") + for _, expected := range []string{ + "Executing NodeBB wiki actions: total 2, create 0, update 1, rename 1, archive 0, purge 0", + "Executing NodeBB wiki action 1/2:", + "Executing NodeBB wiki action 2/2:", + "rename feat:renamed", + "update feat:updated", + } { + if !strings.Contains(joined, expected) { + t.Fatalf("expected execution progress %q, got:\n%s", expected, joined) + } + } +} + +func TestNodeBBNamespacePaginationStopsWhenRepeatedCursorReturnsNoNewPages(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "pages": []map[string]any{ + {"tid": 2461, "title": "Favored Enemy: Elves", "titleLeaf": "Favored Enemy: Elves"}, + }, + "hasMore": true, + "nextCursor": "same-cursor", + }, + }) + })) + defer server.Close() + + client := newNodeBBClient(server.URL, "nodebb-token") + pages, err := client.listNamespacePages(9) + if err != nil { + t.Fatalf("expected repeated cursor with duplicate pages to stop cleanly, got %v", err) + } + if len(pages) != 1 || pages[0].TID != 2461 { + t.Fatalf("expected one unique page from repeated cursor response, got %#v", pages) + } + if requests != 2 { + t.Fatalf("expected pagination to stop after repeated cursor, got %d requests", requests) + } +} + +func TestNodeBBNamespacePaginationRejectsRepeatedCursorWithNewPages(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + w.Header().Set("Content-Type", "application/json") + pages := []map[string]any{ + {"tid": 1, "title": "First", "titleLeaf": "First"}, + } + if requests == 2 { + pages = []map[string]any{ + {"tid": 2, "title": "Second", "titleLeaf": "Second"}, + } + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "pages": pages, + "hasMore": true, + "nextCursor": "same-cursor", + }, + }) + })) + defer server.Close() + + client := newNodeBBClient(server.URL, "nodebb-token") + _, err := client.listNamespacePages(9) + if err == nil { + t.Fatal("expected repeated cursor with new pages to fail") + } + if !strings.Contains(err.Error(), "repeated pagination cursor") { + t.Fatalf("expected repeated cursor error, got %v", err) + } +} + +func TestCollectLocalPagesReadsGeneratedPageIDFromMarkerWhenPathIsTitleBased(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "feat"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Inspire Competence

+

Generated page.

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "feat", "Inspire_Competence.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + + pages, err := collectLocalPages(sourceDir, "", []string{"feat"}) + if err != nil { + t.Fatalf("collectLocalPages failed: %v", err) + } + page, ok := pages["feat:bardic_music_inspire_competence"] + if !ok { + t.Fatalf("expected marker page ID to be used as deploy identity, got keys %#v", sortedKeysFromMap(pages)) + } + if page.PageID != "feat:bardic_music_inspire_competence" || page.Namespace != "feat" || page.Title != "Inspire Competence" { + t.Fatalf("unexpected collected page metadata: %#v", page) + } + if _, ok := pages["feat:Inspire_Competence"]; ok { + t.Fatalf("expected title-based path not to become generated page identity") + } +} + +func TestCollectLocalPagesReadsCanonicalPublicPathFromPageIndex(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "spells"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Fear

+

Generated page.

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "spells", "Fear.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + if err := saveWikiPageIndex(filepath.Join(root, "page-index.json"), wikiPageIndex{ + Version: wikiGeneratorVersion, + Pages: []wikiPageIndexEntry{ + { + PageID: "spells:pdk:fear", + PublicPath: "Magic/Fear", + OutputPath: "pages/spells/Fear.html", + }, + }, + }); err != nil { + t.Fatalf("write page index: %v", err) + } + + pages, err := collectLocalPages(sourceDir, "", []string{"spells"}) + if err != nil { + t.Fatalf("collectLocalPages failed: %v", err) + } + if got := pages["spells:pdk:fear"].PublicPath; got != "Magic/Fear" { + t.Fatalf("expected canonical public path from page-index, got %q", got) + } +} + +func TestCollectLocalPagesUsesConfiguredPageIndexPath(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "spells"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Acid Fog

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "spells", "Acid_Fog.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + indexPath := filepath.Join(root, "manifest", "pages.json") + if err := saveWikiPageIndex(indexPath, wikiPageIndex{ + Version: wikiGeneratorVersion, + Pages: []wikiPageIndexEntry{ + { + PageID: "spells:acid_fog", + Title: "Acid Fog", + PublicPath: "Magic/Acid_Fog", + OutputPath: "pages/spells/Acid_Fog.html", + }, + }, + }); err != nil { + t.Fatalf("write page index: %v", err) + } + + pages, err := collectLocalPages(sourceDir, indexPath, []string{"spells"}) + if err != nil { + t.Fatalf("collectLocalPages failed: %v", err) + } + if got := pages["spells:acid_fog"].PublicPath; got != "Magic/Acid_Fog" { + t.Fatalf("expected configured page-index public path, got %q", got) + } +} + +func TestCollectLocalPagesReadsTitleFromPageIndexForHeadinglessHTML(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "spells"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Generated acid fog page.

+
SchoolConjuration
+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "spells", "Acid_Fog.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + if err := saveWikiPageIndex(filepath.Join(root, "page-index.json"), wikiPageIndex{ + Version: wikiGeneratorVersion, + Pages: []wikiPageIndexEntry{ + { + PageID: "spells:acid_fog", + Title: "Acid Fog", + PublicPath: "Spells/Acid_Fog", + OutputPath: "pages/spells/Acid_Fog.html", + }, + }, + }); err != nil { + t.Fatalf("write page index: %v", err) + } + + pages, err := collectLocalPages(sourceDir, "", []string{"spells"}) + if err != nil { + t.Fatalf("collectLocalPages failed: %v", err) + } + page := pages["spells:acid_fog"] + if page.Title != "Acid Fog" { + t.Fatalf("expected page-index title for headingless HTML, got %#v", page) + } + if page.PublicPath != "Spells/Acid_Fog" { + t.Fatalf("expected page-index public path for headingless HTML, got %#v", page) + } +} + +func TestExtractWikiPagePublicPathAcceptsCanonicalSlashPath(t *testing.T) { + content := ` + +

Fear

+ +` + if got := extractWikiPagePublicPath(content); got != "Magic/Fear" { + t.Fatalf("expected slash-separated public path, got %q", got) + } +} + +func TestDeployWikiDryRunReadoptsMissingMappedPost(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "classes"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Acolyte

+

Generated acolyte page

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "classes", "acolyte.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + old := ` + +

Acolyte

+

Old acolyte page

+ +` + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "classes:acolyte": {Hash: computeManagedHash(old), TID: 7, PID: 42, CID: 9}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(map[string]any{ + "status": map[string]any{"code": "not-found", "message": "Post does not exist"}, + "response": map[string]any{}, + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/9/pages": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "pages": []map[string]any{ + {"tid": 77, "title": "Acolyte", "titleLeaf": "Acolyte", "slug": "77/acolyte", "wikiPath": "/wiki/classes/acolyte"}, + }, + "hasMore": false, + }, + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/77": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 77, "mainPid": 88}}) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/88": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77, "content": old}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + DryRun: true, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions dry-run re-adoption failed: %v", err) + } + if result.Updated != 1 { + t.Fatalf("expected one planned update after re-adoption, got %#v", result) + } +} + +func TestMatchExistingNodeBBPageIgnoresRetiredGeneratedPublicSlug(t *testing.T) { + page := wikiDeployPage{ + PageID: "spells:pdk:fear", + Title: "Fear", + PublicPath: "Spells/Fear", + } + matched, ok := matchExistingNodeBBPage(page, []nodeBBWikiPage{ + {TID: 31, Title: "Fear", TitleLeaf: "Fear", WikiPath: "/wiki/spells/fear"}, + {TID: 32, Title: "Fear", TitleLeaf: "Fear", WikiPath: "/wiki/spells/pdk-fear"}, + }) + if !ok || matched.TID != 31 { + t.Fatalf("expected canonical title match to select tid 31, got %#v ok=%t", matched, ok) + } +} + +func TestMatchExistingNodeBBPageDoesNotPreferRetiredGeneratedSlugWhenRemoteOrderChanges(t *testing.T) { + page := wikiDeployPage{ + PageID: "spells:pdk:fear", + Title: "Fear", + PublicPath: "Spells/Fear", + } + matched, ok := matchExistingNodeBBPage(page, []nodeBBWikiPage{ + {TID: 32, Title: "Fear", TitleLeaf: "Fear", Slug: "32/pdk-fear", WikiPath: "/wiki/spells/pdk-fear"}, + {TID: 31, Title: "Fear", TitleLeaf: "Fear", Slug: "31/fear"}, + }) + if !ok || matched.TID != 31 { + t.Fatalf("expected canonical leaf match to select tid 31 over retired generated slug, got %#v ok=%t", matched, ok) + } +} + +func TestMatchExistingNodeBBPageAdoptsCanonicalWikiPathWhenSlugIsRetired(t *testing.T) { + page := wikiDeployPage{ + PageID: "spells:pdk:fear", + Title: "Fear", + } + matched, ok := matchExistingNodeBBPage(page, []nodeBBWikiPage{ + {TID: 32, Title: "Fear", TitleLeaf: "Fear", Slug: "32/pdk-fear", WikiPath: "/wiki/Spells/Fear"}, + }) + if !ok || matched.TID != 32 { + t.Fatalf("expected title fallback to adopt canonical wiki path despite retired NodeBB topic slug, got %#v ok=%t", matched, ok) + } +} + +func TestMatchExistingNodeBBPageRequiresFullPathBeforeLeafFallback(t *testing.T) { + page := wikiDeployPage{ + PageID: "spells:pdk:fear", + Title: "Fear", + PublicPath: "Spells/Necromancy/Fear", + } + matched, ok := matchExistingNodeBBPage(page, []nodeBBWikiPage{ + {TID: 31, Title: "Fear", TitleLeaf: "Fear", WikiPath: "/wiki/Spells/Illusion/Fear"}, + }) + if ok { + t.Fatalf("expected no cross-namespace leaf fallback match, got %#v", matched) + } +} + +func TestMatchExistingNodeBBPageRequiresCanonicalPathForNestedPublicPath(t *testing.T) { + page := wikiDeployPage{ + PageID: "spells:pdk:fear", + Title: "Fear", + PublicPath: "Spells/Necromancy/Fear", + } + matched, ok := matchExistingNodeBBPage(page, []nodeBBWikiPage{ + {TID: 31, Title: "Fear", TitleLeaf: "Fear", Slug: "31/fear"}, + }) + if ok { + t.Fatalf("expected no nested-path title or leaf fallback match without canonical path, got %#v", matched) + } + + matched, ok = matchExistingNodeBBPage(page, []nodeBBWikiPage{ + {TID: 31, Title: "Fear", TitleLeaf: "Fear", CanonicalPath: "Spells/Necromancy/Fear"}, + }) + if !ok || matched.TID != 31 { + t.Fatalf("expected canonical path match for nested public path, got %#v ok=%t", matched, ok) + } +} + +func TestNestedCanonicalAdoptionFallbackError(t *testing.T) { + page := wikiDeployPage{ + PageID: "spells:pdk:fear", + Title: "Fear", + PublicPath: "Spells/Necromancy/Fear", + } + err := nestedCanonicalAdoptionFallbackError(page, []nodeBBWikiPage{ + {TID: 31, Title: "Fear", TitleLeaf: "Fear", Slug: "31/fear"}, + }) + if err == nil || !strings.Contains(err.Error(), "requires exact canonical wikiPath/canonicalPath") { + t.Fatalf("expected exact canonical remapping error, got %v", err) + } +} + +func TestDeployCanonicalWikiSegmentMatchesGeneratedPathPolicy(t *testing.T) { + tests := map[string]string{ + `Grandmaster's Battle Momentum`: "Grandmasters_Battle_Momentum", + `Bigby’s Clenched Fist`: "Bigbys_Clenched_Fist", + `Hardiness vs. Enchantments`: "Hardiness_vs_Enchantments", + `Clairaudience/Clairvoyance`: "Clairaudience_Clairvoyance", + `Élite & Noble Houses`: "Elite_Noble_Houses", + `Alpha!@#$%^&*()[]-=_+/?.,<>` + "`" + `~|\Omega`: "Alpha_Omega", + `Æther Œuvre Øresund Straße Þorn Łódź Đelta`: "Aether_Oeuvre_Oresund_Strasse_Thorn_Lodz_Delta", + } + for input, want := range tests { + if got := canonicalWikiSegment(input); got != want { + t.Fatalf("canonicalWikiSegment(%q) = %q, want %q", input, got, want) + } + } +} + +func TestLoadWikiNamespaceDeclarationsValidatesCanonicalPaths(t *testing.T) { + root := testProjectRoot(t) + wikiDir := filepath.Join(root, "topdata", "wiki") + if err := os.MkdirAll(wikiDir, 0755); err != nil { + t.Fatalf("create wiki dir: %v", err) + } + write := func(text string) { + t.Helper() + if err := os.WriteFile(filepath.Join(wikiDir, "namespaces.yaml"), []byte(text), 0644); err != nil { + t.Fatalf("write namespaces: %v", err) + } + } + + write(` +namespaces: + - id: feat + title: Feats + canonical_path: Wiki/Feats + category_env: SOW_WIKI_FEATS_CID + edit_policy: preserve_manual_sections +`) + declarations, err := loadWikiNamespaceDeclarations(testProject(root)) + if err != nil { + t.Fatalf("expected canonical namespace path declaration to load: %v", err) + } + if got := declarations[0].CanonicalPath; got != "Wiki/Feats" { + t.Fatalf("expected canonical namespace path to be preserved, got %q", got) + } + + write(` +namespaces: + - id: search + title: Search + canonical_path: Wiki/Search + category_env: SOW_WIKI_SEARCH_CID + edit_policy: preserve_manual_sections +`) + declarations, err = loadWikiNamespaceDeclarations(testProject(root)) + if err != nil { + t.Fatalf("expected reserved display title with safe canonical path to load: %v", err) + } + if got := declarations[0].CanonicalPath; got != "Wiki/Search" { + t.Fatalf("expected safe canonical namespace path to be preserved, got %q", got) + } + + write(` +namespaces: + - id: bad + title: "!!!" + category_env: SOW_WIKI_BAD_CID + edit_policy: preserve_manual_sections +`) + _, err = loadWikiNamespaceDeclarations(testProject(root)) + if err == nil || !strings.Contains(err.Error(), `topdata wiki namespace "bad" title`) { + t.Fatalf("expected invalid namespace title error, got %v", err) + } + + write(` +namespaces: + - id: search + title: Search + canonical_path: search + category_env: SOW_WIKI_SEARCH_CID + edit_policy: preserve_manual_sections +`) + _, err = loadWikiNamespaceDeclarations(testProject(root)) + if err == nil || !strings.Contains(err.Error(), `reserved first segment`) { + t.Fatalf("expected reserved namespace path error, got %v", err) + } + + write(` +namespaces: + - id: numeric + title: Feats + canonical_path: 123/Feats + category_env: SOW_WIKI_NUMERIC_CID + edit_policy: preserve_manual_sections +`) + _, err = loadWikiNamespaceDeclarations(testProject(root)) + if err == nil || !strings.Contains(err.Error(), `reserved first segment`) { + t.Fatalf("expected numeric namespace path error, got %v", err) + } +} + +func TestDeployWikiCreatesNodeBBTopicAndWritesManifest(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Athletics\n\nGenerated athletics page\n[//]: # (sow-topdata-wiki:managed:end)\n" + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.md"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + + createCalls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer nodebb-token" { + t.Fatalf("unexpected authorization header %q", got) + } + if r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/5/pages" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}}) + return + } + if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + createCalls++ + var req struct { + CID int `json:"cid"` + Title string `json:"title"` + Content string `json:"content"` + SourceContent string `json:"sourceContent"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode create request: %v", err) + } + if req.CID != 5 || req.Title != "Athletics" || req.Content != generated { + t.Fatalf("unexpected create request: %#v", req) + } + if req.SourceContent != generated { + t.Fatalf("expected create request sourceContent to match generated HTML, got %q", req.SourceContent) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "tid": 11, + "mainPost": map[string]any{ + "pid": 42, + "tid": 11, + }, + }, + }) + })) + defer server.Close() + + manifestPath := filepath.Join(root, "deploy-manifest.json") + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + CategoryIDs: map[string]int{"skills": 5}, + AllowCreates: true, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions create failed: %v", err) + } + if result.Created != 1 || createCalls != 1 { + t.Fatalf("expected one created topic, result=%d calls=%d", result.Created, createCalls) + } + manifest := loadDeployManifest(manifestPath) + entry := manifest.Pages["skills:athletics"] + if entry.TID != 11 || entry.PID != 42 || entry.CID != 5 || entry.Hash != computeManagedHash(generated) { + t.Fatalf("unexpected manifest entry: %#v", entry) + } + if !entry.SourceContentSynced { + t.Fatalf("expected created manifest entry to record sourceContent sync") + } +} + +func TestParseNodeBBPostPrefersSourceContentForWikiHTML(t *testing.T) { + post := parseNodeBBPost(map[string]any{ + "response": map[string]any{ + "pid": 42, + "tid": 7, + "content": "<p>Rendered as text</p>", + "sourceContent": "

Stored source HTML

", + }, + }) + if post.Content != "

Stored source HTML

" { + t.Fatalf("expected sourceContent to be canonical deploy content, got %#v", post) + } + if post.SourceContent != "

Stored source HTML

" { + t.Fatalf("expected sourceContent field to be preserved, got %#v", post) + } +} + +func TestDeployWikiRepairsManifestedPageMissingSourceContentSync(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Ability: Strength.

+
Key AbilitySTR
+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "Athletics.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:athletics": { + Hash: computeManagedHash(generated), + Title: "Athletics", + TopicTitle: "Athletics", + Namespace: "skills", + TID: 7, + PID: 42, + CID: 3, + }, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + updateCalls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7, "content": generated}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/edit-lock": + respondWikiEditLock(t, w, r, 7, "source-sync-lock") + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42": + updateCalls++ + var req struct { + Content string `json:"content"` + SourceContent string `json:"sourceContent"` + WikiEditLockToken string `json:"wikiEditLockToken"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode update request: %v", err) + } + if req.WikiEditLockToken != "source-sync-lock" { + t.Fatalf("expected update to include wiki edit lock token, got %q", req.WikiEditLockToken) + } + if req.Content != generated || req.SourceContent != generated { + t.Fatalf("expected repair update to write content and sourceContent, got %#v", req) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions sourceContent repair failed: %v", err) + } + if result.Updated != 1 || result.Skipped != 0 || updateCalls != 1 { + t.Fatalf("expected one sourceContent repair update, result=%#v updateCalls=%d", result, updateCalls) + } + entry := loadDeployManifest(manifestPath).Pages["skills:athletics"] + if !entry.SourceContentSynced { + t.Fatalf("expected repair deploy to record sourceContent sync, got %#v", entry) + } +} + +func TestMergeManagedContentPreservesUserTopAndBottomSections(t *testing.T) { + existing := strings.Join([]string{ + ``, + ``, + `

Acolyte

`, + ``, + `

User-written overview.

`, + ``, + `

Old generated body.

`, + ``, + `

User-written bottom content.

`, + ``, + ``, + }, "\n") + generated := strings.Join([]string{ + ``, + ``, + `

Acolyte

`, + ``, + `

`, + ``, + `

New generated body.

`, + ``, + `

`, + ``, + ``, + }, "\n") + + merged := mergeManagedContent(existing, generated) + for _, expected := range []string{ + "

User-written overview.

", + "

User-written bottom content.

", + "

New generated body.

", + } { + if !strings.Contains(merged, expected) { + t.Fatalf("expected merged content to contain %q:\n%s", expected, merged) + } + } + if strings.Contains(merged, "Old generated body") { + t.Fatalf("expected old generated body to be replaced:\n%s", merged) + } +} + +func TestDeployWikiCreatesNodeBBTopicWithoutFallbackForDefaultThreeCharacterTitle(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "spells"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Aid\n\nGenerated spell page\n[//]: # (sow-topdata-wiki:managed:end)\n" + if err := os.WriteFile(filepath.Join(sourceDir, "spells", "aid.md"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/5/pages" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}}) + return + } + if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Title string `json:"title"` + Content string `json:"content"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode create request: %v", err) + } + if req.Title != "Aid" { + t.Fatalf("expected unprefixed title, got %q", req.Title) + } + if req.Content != generated { + t.Fatalf("expected page body to keep short h1 unchanged:\n%s", req.Content) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "tid": 11, + "mainPost": map[string]any{ + "pid": 42, + "tid": 11, + }, + }, + }) + })) + defer server.Close() + + _, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: filepath.Join(root, "deploy-manifest.json"), + CategoryIDs: map[string]int{"spells": 5}, + AllowCreates: true, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions create failed: %v", err) + } +} + +func TestDeployWikiCreatesNodeBBTopicWithFallbackForTitleShorterThanConfiguredMinimum(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "spells"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Ox\n\nGenerated spell page\n[//]: # (sow-topdata-wiki:managed:end)\n" + if err := os.WriteFile(filepath.Join(sourceDir, "spells", "ox.md"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/5/pages" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}}) + return + } + if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Title string `json:"title"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode create request: %v", err) + } + if req.Title != "Spells: Ox" { + t.Fatalf("expected fallback title, got %q", req.Title) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "tid": 11, + "mainPost": map[string]any{ + "pid": 42, + "tid": 11, + }, + }, + }) + })) + defer server.Close() + + _, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: filepath.Join(root, "deploy-manifest.json"), + CategoryIDs: map[string]int{"spells": 5}, + AllowCreates: true, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions create failed: %v", err) + } +} + +func TestDeployWikiRenamesExistingPrefixedTopicWhenTitleIsLongEnough(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "itemtypes"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Belt

+

Generated belt page

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "itemtypes", "belt.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "itemtypes:belt": {Hash: computeManagedHash(generated), Title: "Belt", Namespace: "itemtypes", SourceContentSynced: true, TID: 7, PID: 42, CID: 5}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + renameCalls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/7": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "tid": 7, + "title": "Itemtypes: Belt", + "mainPid": 42, + }, + }) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/page/move": + renameCalls++ + var req struct { + TID int `json:"tid"` + CID int `json:"cid"` + Title string `json:"title"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode rename request: %v", err) + } + if req.TID != 7 || req.CID != 5 || req.Title != "Belt" { + t.Fatalf("unexpected rename request: %#v", req) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 7, "cid": 5, "title": "Belt"}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions rename cleanup failed: %v", err) + } + if result.Renamed != 1 || result.Updated != 0 || result.Skipped != 1 || renameCalls != 1 { + t.Fatalf("expected one rename cleanup and skipped body update, result=%#v renameCalls=%d", result, renameCalls) + } + manifest := loadDeployManifest(manifestPath) + entry := manifest.Pages["itemtypes:belt"] + if entry.TopicTitle != "Belt" || entry.Title != "Belt" { + t.Fatalf("expected manifest to record cleaned topic title, got %#v", entry) + } +} + +func TestDeployWikiRenamesManagedTopicWhenGeneratedTitleChanges(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "feat"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Weapon Focus (dwarven waraxe)

+

Generated weapon focus page

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "feat", "weapon:focus:dwaxe.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "feat:weapon:focus:dwaxe": { + Hash: computeManagedHash(generated), + Title: "Weapon Focus (Dwaxe)", + TopicTitle: "Weapon Focus (Dwaxe)", + Namespace: "feat", + SourceContentSynced: true, + TID: 7, + PID: 42, + CID: 5, + }, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + renameCalls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/7": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "tid": 7, + "title": "Weapon Focus (Dwaxe)", + "mainPid": 42, + }, + }) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/page/move": + renameCalls++ + var req struct { + TID int `json:"tid"` + CID int `json:"cid"` + Title string `json:"title"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode rename request: %v", err) + } + if req.TID != 7 || req.CID != 5 || req.Title != "Weapon Focus (dwarven waraxe)" { + t.Fatalf("unexpected rename request: %#v", req) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 7}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions title rename failed: %v", err) + } + if result.Renamed != 1 || result.Updated != 0 || result.Created != 0 || result.Archived != 0 || result.Purged != 0 || renameCalls != 1 { + t.Fatalf("expected generated title rename only, result=%#v renameCalls=%d", result, renameCalls) + } + entry := loadDeployManifest(manifestPath).Pages["feat:weapon:focus:dwaxe"] + if entry.Title != "Weapon Focus (dwarven waraxe)" || entry.TopicTitle != "Weapon Focus (dwarven waraxe)" { + t.Fatalf("expected renamed title state in manifest, got %#v", entry) + } +} + +func TestDeployWikiDoesNotRenameHeadinglessPageToPageIDFallback(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "spells"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Generated acid fog page.

+
SchoolConjuration
+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "spells", "Acid_Fog.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + if err := saveWikiPageIndex(filepath.Join(root, "page-index.json"), wikiPageIndex{ + Version: wikiGeneratorVersion, + Pages: []wikiPageIndexEntry{ + {PageID: "spells:acid_fog", Title: "Acid Fog", PublicPath: "Spells/Acid_Fog", OutputPath: "pages/spells/Acid_Fog.html"}, + }, + }); err != nil { + t.Fatalf("write page index: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "spells:acid_fog": { + Hash: computeManagedHash(generated), + Title: "Acid Fog", + TopicTitle: "Acid Fog", + Namespace: "spells", + SourceContentSynced: true, + TID: 7, + PID: 42, + CID: 5, + }, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("headingless unchanged page must not call NodeBB, got %s %s", r.Method, r.URL.String()) + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions headingless skip failed: %v", err) + } + if result.Renamed != 0 || result.Updated != 0 || result.Skipped != 1 { + t.Fatalf("expected headingless page to skip without page-id fallback rename, got %#v", result) + } +} + +func TestDeployWikiRenamesBrokenHeadinglessFallbackTitleBackToPageIndexTitle(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "spells"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Generated acid fog page.

+
SchoolConjuration
+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "spells", "Acid_Fog.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + if err := saveWikiPageIndex(filepath.Join(root, "page-index.json"), wikiPageIndex{ + Version: wikiGeneratorVersion, + Pages: []wikiPageIndexEntry{ + {PageID: "spells:acid_fog", Title: "Acid Fog", PublicPath: "Spells/Acid_Fog", OutputPath: "pages/spells/Acid_Fog.html"}, + }, + }); err != nil { + t.Fatalf("write page index: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "spells:acid_fog": { + Hash: computeManagedHash(generated), + Title: "Acid_fog", + TopicTitle: "Acid_fog", + Namespace: "spells", + SourceContentSynced: true, + TID: 7, + PID: 42, + CID: 5, + }, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + renameCalls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/7": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{"tid": 7, "title": "Acid_fog", "mainPid": 42}, + }) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/page/move": + renameCalls++ + var req struct { + TID int `json:"tid"` + CID int `json:"cid"` + Title string `json:"title"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode rename request: %v", err) + } + if req.TID != 7 || req.CID != 5 || req.Title != "Acid Fog" { + t.Fatalf("unexpected rename request: %#v", req) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 7}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions headingless title repair failed: %v", err) + } + if result.Renamed != 1 || result.Updated != 0 || result.Skipped != 1 || renameCalls != 1 { + t.Fatalf("expected one title repair rename and skipped body update, result=%#v renameCalls=%d", result, renameCalls) + } + entry := loadDeployManifest(manifestPath).Pages["spells:acid_fog"] + if entry.Title != "Acid Fog" || entry.TopicTitle != "Acid Fog" { + t.Fatalf("expected repaired manifest title from page-index, got %#v", entry) + } +} + +func TestDeployWikiAdoptsExistingNodeBBPageWhenManifestIsMissingWithoutCreate(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "classes"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Acolyte

+

Generated acolyte page

+ + +

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "classes", "acolyte.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + old := ` + +

Acolyte

+

Old acolyte page

+ + +

Keep this remote note.

+ +` + + createCalls := 0 + var updated string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/9/pages": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "pages": []map[string]any{ + { + "tid": 77, + "title": "Acolyte", + "titleLeaf": "Acolyte", + "slug": "77/acolyte", + "wikiPath": "/wiki/classes/acolyte", + }, + }, + "hasMore": false, + }, + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/77": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 77, "mainPid": 88}}) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/88": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77, "content": old}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/edit-lock": + respondWikiEditLock(t, w, r, 77, "adopt-lock") + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/88": + var req struct { + Content string `json:"content"` + WikiEditLockToken string `json:"wikiEditLockToken"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode update request: %v", err) + } + if req.WikiEditLockToken != "adopt-lock" { + t.Fatalf("expected adopted update to include wiki edit lock token, got %q", req.WikiEditLockToken) + } + updated = req.Content + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77}}) + case r.Method == http.MethodPost && r.URL.Path == "/api/v3/topics": + createCalls++ + t.Fatalf("existing wiki page should be adopted and updated, not recreated") + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + manifestPath := filepath.Join(root, "deploy-manifest.json") + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + CategoryIDs: map[string]int{"classes": 9}, + StalePolicy: "report", + NotesDelimiter: "", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions adopt failed: %v", err) + } + if result.Created != 0 || result.Updated != 1 || createCalls != 0 { + t.Fatalf("expected one adopted update and no creates, result=%#v createCalls=%d", result, createCalls) + } + if !containsAll(updated, "

Generated acolyte page

", "

Keep this remote note.

") { + t.Fatalf("expected adopted update to merge generated and remote manual content:\n%s", updated) + } + manifest := loadDeployManifest(manifestPath) + entry := manifest.Pages["classes:acolyte"] + if entry.TID != 77 || entry.PID != 88 || entry.CID != 9 || entry.Hash != computeManagedHash(generated) { + t.Fatalf("unexpected adopted manifest entry: %#v", entry) + } +} + +func TestDeployWikiMergesHTMLManagedAndManualRegions(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Athletics

+

Generated athletics page

+ + +

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + old := ` + +

Athletics

+

Old generated text

+ + +

Keep this human note.

+ +` + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:athletics": {Hash: computeManagedHash(old), TID: 7, PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + var updated string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7, "content": old}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/edit-lock": + respondWikiEditLock(t, w, r, 7, "merge-lock") + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42": + var req struct { + Content string `json:"content"` + WikiEditLockToken string `json:"wikiEditLockToken"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode update request: %v", err) + } + if req.WikiEditLockToken != "merge-lock" { + t.Fatalf("expected update to include wiki edit lock token, got %q", req.WikiEditLockToken) + } + updated = req.Content + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions update failed: %v", err) + } + if result.Updated != 1 { + t.Fatalf("expected one update, got %#v", result) + } + if !containsAll(updated, "

Generated athletics page

", "

Keep this human note.

", `manual:start id="user_bottom"`) { + t.Fatalf("expected update to replace managed HTML and preserve/bootstrap manual regions:\n%s", updated) + } + if strings.Contains(updated, `manual:start id="notes"`) || strings.Contains(updated, `manual:start id="see_also"`) { + t.Fatalf("expected update to omit legacy split bottom manual regions:\n%s", updated) + } +} + +func TestDeployWikiUpdateAcquiresWestgateWikiEditLock(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Athletics

+

Generated athletics page

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + old := ` + +

Athletics

+

Old athletics page

+ +` + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:athletics": {Hash: computeManagedHash(old), TID: 7, PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + lockAcquired := false + var updateToken string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7, "content": old}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/edit-lock": + var req struct { + TID int `json:"tid"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode edit lock request: %v", err) + } + if req.TID != 7 { + t.Fatalf("expected edit lock for tid 7, got %d", req.TID) + } + lockAcquired = true + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"status": "ok", "tid": 7, "token": "lock-token"}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42": + var req struct { + Content string `json:"content"` + WikiEditLockToken string `json:"wikiEditLockToken"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode update request: %v", err) + } + updateToken = req.WikiEditLockToken + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions update failed: %v", err) + } + if result.Updated != 1 { + t.Fatalf("expected one update, got %#v", result) + } + if !lockAcquired { + t.Fatalf("expected deployer to acquire a wiki edit lock before update") + } + if updateToken != "lock-token" { + t.Fatalf("expected update to include wiki edit lock token, got %q", updateToken) + } +} + +func TestDeployWikiCreateCollisionAdoptsExistingNodeBBPage(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "classes"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Bard

+

Generated bard page

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "classes", "bard.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + old := ` + +

Bard

+

Old bard page

+ +` + + createCalls := 0 + var updated string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/9/pages" && r.URL.Query().Get("q") == "": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}}) + case r.Method == http.MethodPost && r.URL.Path == "/api/v3/topics": + createCalls++ + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + _ = json.NewEncoder(w).Encode(map[string]any{ + "status": map[string]any{ + "code": "bad-request", + "message": "A wiki page with this URL already exists in this namespace. Rename the page before publishing.", + }, + "response": map[string]any{}, + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/9/pages" && r.URL.Query().Get("q") == "bard": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "pages": []map[string]any{ + {"tid": 77, "title": "Bard", "titleLeaf": "Bard", "slug": "77/bard", "wikiPath": "/wiki/classes/bard"}, + }, + "hasMore": false, + }, + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/77": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 77, "mainPid": 88}}) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/88": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77, "content": old}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/edit-lock": + respondWikiEditLock(t, w, r, 77, "collision-lock") + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/88": + var req struct { + Content string `json:"content"` + WikiEditLockToken string `json:"wikiEditLockToken"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode update request: %v", err) + } + if req.WikiEditLockToken != "collision-lock" { + t.Fatalf("expected collision recovery update to include edit lock token, got %q", req.WikiEditLockToken) + } + updated = req.Content + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + manifestPath := filepath.Join(root, "deploy-manifest.json") + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + CategoryIDs: map[string]int{"classes": 9}, + AllowCreates: true, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions collision recovery failed: %v", err) + } + if result.Created != 0 || result.Updated != 1 || createCalls != 1 { + t.Fatalf("expected collision recovery to update existing page, result=%#v createCalls=%d", result, createCalls) + } + if !containsAll(updated, "

Generated bard page

") { + t.Fatalf("expected collision recovery to update adopted page:\n%s", updated) + } + manifest := loadDeployManifest(manifestPath) + entry := manifest.Pages["classes:bard"] + if entry.TID != 77 || entry.PID != 88 || entry.CID != 9 || entry.Hash != computeManagedHash(generated) { + t.Fatalf("unexpected adopted manifest entry: %#v", entry) + } +} + +func TestDeployWikiCreateCollisionSearchesCanonicalTitleSegment(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "itemtypes"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Wand (not enchanted)

+

Generated wand page

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "itemtypes", "Wand_not_enchanted.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + old := ` + +

Wand (not enchanted)

+

Old wand page

+ +` + + createCalls := 0 + var updated string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/58/pages" && r.URL.Query().Get("q") == "": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}}) + case r.Method == http.MethodPost && r.URL.Path == "/api/v3/topics": + createCalls++ + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + _ = json.NewEncoder(w).Encode(map[string]any{ + "status": map[string]any{ + "code": "bad-request", + "message": "A wiki page with this URL already exists in this namespace. Rename the page before publishing.", + }, + "response": map[string]any{}, + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/58/pages" && r.URL.Query().Get("q") == "blank_magicwand": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}}) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/58/pages" && r.URL.Query().Get("q") == "Wand (not enchanted)": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}}) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/58/pages" && r.URL.Query().Get("q") == "Wand_not_enchanted": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "pages": []map[string]any{ + {"tid": 77, "title": "Wand (not enchanted)", "titleLeaf": "Wand (not enchanted)", "slug": "77/wand-not-enchanted", "wikiPath": "/wiki/Itemtypes/Wand_not_enchanted"}, + }, + "hasMore": false, + }, + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/77": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 77, "mainPid": 88}}) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/88": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77, "content": old}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/edit-lock": + respondWikiEditLock(t, w, r, 77, "wand-collision-lock") + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/88": + var req struct { + Content string `json:"content"` + WikiEditLockToken string `json:"wikiEditLockToken"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode update request: %v", err) + } + if req.WikiEditLockToken != "wand-collision-lock" { + t.Fatalf("expected collision recovery update to include edit lock token, got %q", req.WikiEditLockToken) + } + updated = req.Content + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + manifestPath := filepath.Join(root, "deploy-manifest.json") + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + CategoryIDs: map[string]int{"itemtypes": 58}, + AllowCreates: true, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions collision recovery failed: %v", err) + } + if result.Created != 0 || result.Updated != 1 || createCalls != 1 { + t.Fatalf("expected collision recovery to update existing page, result=%#v createCalls=%d", result, createCalls) + } + if !containsAll(updated, "

Generated wand page

") { + t.Fatalf("expected collision recovery to update adopted page:\n%s", updated) + } +} + +func TestDeployWikiReportsAndArchivesStalePages(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + var archived string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut && r.URL.Path == "/api/v3/plugins/westgate-wiki/edit-lock" { + respondWikiEditLock(t, w, r, 7, "archive-lock") + return + } + if r.Method != http.MethodPut || r.URL.Path != "/api/v3/posts/42" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Content string `json:"content"` + WikiEditLockToken string `json:"wikiEditLockToken"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode archive request: %v", err) + } + if req.WikiEditLockToken != "archive-lock" { + t.Fatalf("expected archive to include wiki edit lock token, got %q", req.WikiEditLockToken) + } + archived = req.Content + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}}) + })) + defer server.Close() + + report, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + DryRun: true, + StalePolicy: "report", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions stale report failed: %v", err) + } + if report.Stale != 1 { + t.Fatalf("expected one stale report, got %#v", report) + } + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + StalePolicy: "archive", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions stale archive failed: %v", err) + } + if result.Stale != 1 || result.Archived != 1 { + t.Fatalf("expected one stale archive, got %#v", result) + } + if !containsAll(archived, "

Retired Skill

", "no longer present", ` + +

Replacement

+

Generated replacement page

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "replacement.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:retired": {Hash: "old-hash", Title: "Replacement", Namespace: "skills", TID: 7, PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/3/pages": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}}) + case r.Method == http.MethodDelete && r.URL.Path == "/api/v3/topics/7": + calls = append(calls, "purge") + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]any{"code": "ok"}}) + case r.Method == http.MethodPost && r.URL.Path == "/api/v3/topics": + calls = append(calls, "create") + if len(calls) != 2 || calls[0] != "purge" { + t.Fatalf("expected stale purge before create, got calls %#v", calls) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "tid": 11, + "mainPost": map[string]any{ + "pid": 99, + "tid": 11, + }, + }, + }) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + CategoryIDs: map[string]int{"skills": 3}, + AllowCreates: true, + StalePolicy: "purge", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions purge-and-create failed: %v", err) + } + if result.Created != 1 || result.Purged != 1 { + t.Fatalf("expected one create and one purge, got %#v", result) + } + if strings.Join(calls, ",") != "purge,create" { + t.Fatalf("expected purge before create, got %#v", calls) + } +} + +func TestDeployWikiResetManagedNamespacesPurgesRemotePagesBeforeCreatingFreshManifest(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Replacement

+

Generated replacement page

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "replacement.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:stale": {Hash: "old-hash", Title: "Replacement", Namespace: "skills", TID: 7, PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/3/pages": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "pages": []map[string]any{ + {"tid": 7, "title": "Old Replacement", "titleLeaf": "Old Replacement", "slug": "7/replacement", "wikiPath": "/wiki/skills/replacement"}, + {"tid": 8, "title": "Manual Note", "titleLeaf": "Manual Note", "slug": "8/manual-note", "wikiPath": "/wiki/skills/manual-note"}, + }, + "hasMore": false, + }, + }) + case r.Method == http.MethodDelete && r.URL.Path == "/api/v3/topics/7": + calls = append(calls, "purge:7") + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]any{"code": "ok"}}) + case r.Method == http.MethodDelete && r.URL.Path == "/api/v3/topics/8": + calls = append(calls, "purge:8") + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]any{"code": "ok"}}) + case r.Method == http.MethodPost && r.URL.Path == "/api/v3/topics": + calls = append(calls, "create") + if strings.Join(calls, ",") != "purge:7,purge:8,create" { + t.Fatalf("expected namespace reset purges before create, got %#v", calls) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "tid": 11, + "mainPost": map[string]any{ + "pid": 99, + "tid": 11, + }, + }, + }) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + CategoryIDs: map[string]int{"skills": 3}, + AllowCreates: true, + ResetManagedNamespaces: true, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions reset-managed-namespaces failed: %v", err) + } + if result.Created != 1 || result.Purged != 2 || result.Stale != 0 { + t.Fatalf("expected one create and two namespace purges, got %#v", result) + } + if strings.Join(calls, ",") != "purge:7,purge:8,create" { + t.Fatalf("expected reset purges before create, got %#v", calls) + } + manifest := loadDeployManifest(manifestPath) + if _, ok := manifest.Pages["skills:stale"]; ok { + t.Fatalf("expected reset to remove previous manifest mapping") + } + entry := manifest.Pages["skills:replacement"] + if entry.TID != 11 || entry.PID != 99 || entry.CID != 3 { + t.Fatalf("expected fresh created mapping in manifest, got %#v", entry) + } +} + +func TestDeployWikiResetManagedNamespacesRequiresCreateForLocalPages(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Replacement

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "replacement.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("reset without --create must fail before remote calls, got %s %s", r.Method, r.URL.String()) + })) + defer server.Close() + + _, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + CategoryIDs: map[string]int{"skills": 3}, + ResetManagedNamespaces: true, + }, nil) + if err == nil || !strings.Contains(err.Error(), "requires --create") { + t.Fatalf("expected reset without create to fail clearly, got %v", err) + } +} + +func TestDeployWikiPurgeTreatsAlreadyMissingTrackedTopicAsSuccess(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3}, + "skills:gone": {Hash: "old-hash", Title: "Gone Skill", Namespace: "skills", TID: 8, PID: 43, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + seen := map[string]int{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + seen[r.URL.Path]++ + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v3/topics/7": + http.Error(w, `{"status":{"message":"topic not found"}}`, http.StatusNotFound) + case "/api/v3/topics/8": + _ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]any{"code": "ok"}}) + default: + t.Fatalf("unexpected purge path %s", r.URL.Path) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + StalePolicy: "purge", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions stale purge with missing topic failed: %v", err) + } + if result.Stale != 2 || result.Purged != 2 { + t.Fatalf("expected two stale purges, got %#v", result) + } + if seen["/api/v3/topics/7"] != 1 || seen["/api/v3/topics/8"] != 1 { + t.Fatalf("expected one purge call for each tracked topic, got %#v", seen) + } + manifest := loadDeployManifest(manifestPath) + if _, ok := manifest.Pages["skills:retired"]; ok { + t.Fatalf("expected already-missing stale manifest entry to be removed") + } + if _, ok := manifest.Pages["skills:gone"]; ok { + t.Fatalf("expected purged stale manifest entry to be removed") + } +} + +func TestDeployWikiPurgeDoesNotTargetCurrentGeneratedPages(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Athletics\n\nGenerated athletics page\n[//]: # (sow-topdata-wiki:managed:end)\n" + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.md"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:athletics": { + Hash: computeManagedHash(generated), + Title: "Athletics", + Namespace: "skills", + SourceContentSynced: true, + TID: 7, + PID: 42, + CID: 3, + }, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("current generated page must not call NodeBB during matching-hash deploy, got %s %s", r.Method, r.URL.Path) + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + StalePolicy: "purge", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions current page purge deploy failed: %v", err) + } + if result.Stale != 0 || result.Purged != 0 || result.Skipped != 1 { + t.Fatalf("expected current generated page to skip purge, got %#v", result) + } + if _, ok := loadDeployManifest(manifestPath).Pages["skills:athletics"]; !ok { + t.Fatalf("expected current generated page to remain in deploy manifest") + } +} + +func containsAll(haystack string, needles ...string) bool { + for _, needle := range needles { + if !strings.Contains(haystack, needle) { + return false + } + } + return true +} + +func respondWikiEditLock(t *testing.T, w http.ResponseWriter, r *http.Request, expectedTID int, token string) { + t.Helper() + var req struct { + TID int `json:"tid"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode edit lock request: %v", err) + } + if req.TID != expectedTID { + t.Fatalf("expected edit lock for tid %d, got %d", expectedTID, req.TID) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"status": "ok", "tid": expectedTID, "token": token}}) +} diff --git a/internal/topdata/wiki_formatters.go b/internal/topdata/wiki_formatters.go new file mode 100644 index 0000000..0a9dfed --- /dev/null +++ b/internal/topdata/wiki_formatters.go @@ -0,0 +1,286 @@ +package topdata + +import ( + "fmt" + "html" + "strings" +) + +func (ctx *wikiContext) renderWikiFormatter(spec string, page wikiTemplatePage) (string, error) { + name := strings.Fields(spec) + if len(name) == 0 { + return "", fmt.Errorf("render wiki template for %s: empty formatter", page.PageID) + } + switch name[0] { + case "Description": + return renderWikiTemplatePlainHTML(ctx.resolveRowDescription(page.Category, page.Row)), nil + case "FactsTable": + return ctx.renderFactsHTML(page.Category, page.Key, page.Row), nil + case "Prerequisites": + return renderWikiFactTableHTML([]wikiHTMLFact{{Label: "Prerequisites", Value: ctx.renderFeatPrerequisites(page.Row)}}), nil + case "SkillList": + if page.Category != "classes" { + return "", nil + } + return ctx.renderClassSkillListHTML(page.Row), nil + case "ClassFeatureTable": + if page.Category != "classes" { + return "", nil + } + return ctx.renderClassFeatureTableHTML(page.Row), nil + case "FeatProgression": + if page.Category != "classes" { + return "", nil + } + return ctx.renderClassFeatProgressionHTML(page.Row), nil + case "SpellTable", "SavesProgression", "BABProgression": + return "", nil + case "RaceFeatList": + return renderWikiFactTableHTML([]wikiHTMLFact{{Label: "Racial Feats", Value: ctx.renderRaceFeatList(page.Row)}}), nil + case "StatusCallout": + return buildWikiStatusBlock(page.Status, page.Category), nil + case "RaceNameForms": + return ctx.renderRaceNameFormsHTML(page.Row), nil + case "UserBottomFallback": + return "", nil + default: + return "", fmt.Errorf("render wiki template for %s: unknown formatter %q", page.PageID, name[0]) + } +} + +type wikiHTMLFact struct { + Label string + Value string +} + +func (ctx *wikiContext) renderFactsHTML(category, key string, row map[string]any) string { + facts := []wikiHTMLFact{} + add := func(label, value string) { + if strings.TrimSpace(value) != "" { + facts = append(facts, wikiHTMLFact{Label: label, Value: value}) + } + } + switch category { + case "feat": + for _, fact := range ctx.renderFeatFactsHTML(row) { + add(fact.Label, fact.Value) + } + case "skills": + add("Key Ability", stringValue(row, "KeyAbility")) + add("Untrained", yesNoValue(stringValue(row, "Untrained"))) + add("Armor Check Penalty", yesNoValue(stringValue(row, "ArmorCheckPenalty"))) + case "spells": + add("Constant", stringValue(row, "Constant")) + case "baseitems": + add("Item Class", stringValue(row, "ItemClass")) + add("Weapon Type", stringValue(row, "WeaponType")) + add("Required Feats", ctx.renderReferenceList(ctx.wikiReqFeats(row))) + add("Base Item Stats", ctx.resolveTextValue(fieldValue(row, "BaseItemStatRef"), nil)) + case "classes": + add("Hit Die", "d"+stringValue(row, "HitDie")) + add("Skill Points", stringValue(row, "SkillPointBase")) + add("Primary Ability", stringValue(row, "PrimaryAbil")) + case "racialtypes": + add("Ability Adjustments", formatAbilityAdjustments(row)) + add("Favored Class", ctx.renderReference(fieldValue(row, "Favored"), ctx.classIDToKey, ctx.resolveClassName)) + add("Favored Enemy", ctx.renderReference(fieldValue(row, "FavoredEnemyFeat"), ctx.featIDToKey, ctx.resolveFeatName)) + add("Racial Feats", ctx.renderRaceFeatList(row)) + } + _ = key + return renderWikiFactTableHTML(facts) +} + +func (ctx *wikiContext) renderFeatFactsHTML(row map[string]any) []wikiHTMLFact { + return []wikiHTMLFact{ + {Label: "Prerequisites", Value: ctx.renderFeatPrerequisites(row)}, + {Label: "Minimum Attack Bonus", Value: stringValue(row, "MINATTACKBONUS")}, + {Label: "Minimum Spell Level", Value: stringValue(row, "MINSPELLLVL")}, + {Label: "Minimum Fortitude Save", Value: stringValue(row, "MinFortSave")}, + } +} + +func renderWikiFactTableHTML(facts []wikiHTMLFact) string { + rows := []string{} + for _, fact := range facts { + if strings.TrimSpace(fact.Value) == "" { + continue + } + rows = append(rows, ``+html.EscapeString(fact.Label)+``+renderMultilineInlineHTML(fact.Value)+``) + } + if len(rows) == 0 { + return "" + } + return `` + strings.Join(rows, "\n") + `
` +} + +func renderMultilineInlineHTML(text string) string { + lines := []string{} + for _, line := range strings.Split(strings.ReplaceAll(strings.TrimSpace(text), "\r\n", "\n"), "\n") { + line = strings.TrimSpace(line) + if line != "" { + lines = append(lines, renderInlineHTML(line)) + } + } + return strings.Join(lines, "
") +} + +func renderWikiListHTML(class string, items []string) string { + rendered := []string{} + for _, item := range items { + if strings.TrimSpace(item) != "" { + rendered = append(rendered, "
  • "+renderInlineHTML(item)+"
    • ") + } + } + if len(rendered) == 0 { + return "" + } + classAttr := "" + if class != "" { + classAttr = ` class="` + html.EscapeString(class) + `"` + } + return "" + strings.Join(rendered, "\n") + "" +} + +func (ctx *wikiContext) renderClassSkillList(row map[string]any) string { + table := ctx.tableForValue(fieldValue(row, "SkillsTable"), ctx.classSkillTables) + if table == nil { + return "" + } + skills := []string{} + for _, skillRow := range table.Rows { + if stringValue(skillRow, "ClassSkill") != "1" { + continue + } + name := ctx.renderReference(fieldValue(skillRow, "SkillIndex"), ctx.skillIDToKey, ctx.resolveSkillName) + if name != "" { + skills = append(skills, name) + } + } + if len(skills) == 0 { + return "" + } + return "==== Class Skills ====\n\n * " + strings.Join(skills, ", ") +} + +func (ctx *wikiContext) renderClassSkillListHTML(row map[string]any) string { + table := ctx.tableForValue(fieldValue(row, "SkillsTable"), ctx.classSkillTables) + if table == nil { + return "" + } + skills := []string{} + for _, skillRow := range table.Rows { + if stringValue(skillRow, "ClassSkill") != "1" { + continue + } + name := ctx.renderReference(fieldValue(skillRow, "SkillIndex"), ctx.skillIDToKey, ctx.resolveSkillName) + if name != "" { + skills = append(skills, name) + } + } + if len(skills) == 0 { + return "" + } + return `

    Class Skills

    ` + renderWikiListHTML("wiki-list wiki-class-skills", skills) +} + +func (ctx *wikiContext) renderClassFeatProgression(row map[string]any) string { + table := ctx.tableForValue(fieldValue(row, "FeatsTable"), ctx.classFeatTables) + if table == nil { + return "" + } + lines := []string{} + byLevel := map[string][]string{} + selectable := []string{} + for _, featRow := range table.Rows { + name := ctx.renderReference(fieldValue(featRow, "FeatIndex"), ctx.featIDToKey, ctx.resolveFeatName) + if name == "" { + name = ctx.renderReference(fieldValue(featRow, "FeatIndex"), nil, func(string) string { return "" }) + } + if name == "" { + continue + } + level := stringValue(featRow, "GrantedOnLevel") + if level == "" || strings.HasPrefix(level, "-") { + selectable = append(selectable, name) + continue + } + byLevel[level] = append(byLevel[level], name) + } + if len(byLevel) > 0 { + lines = append(lines, "==== Granted Feats ====", "") + for _, level := range sortedKeys(byLevel) { + lines = append(lines, " * Level "+level+": "+strings.Join(byLevel[level], ", ")) + } + lines = append(lines, "") + } + if len(selectable) > 0 { + lines = append(lines, "==== Selectable Feats ====", "", " * "+strings.Join(selectable, ", "), "") + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} + +func (ctx *wikiContext) renderClassFeatProgressionHTML(row map[string]any) string { + table := ctx.tableForValue(fieldValue(row, "FeatsTable"), ctx.classFeatTables) + if table == nil { + return "" + } + byLevel := map[string][]string{} + selectable := []string{} + for _, featRow := range table.Rows { + name := ctx.renderReference(fieldValue(featRow, "FeatIndex"), ctx.featIDToKey, ctx.resolveFeatName) + if name == "" { + name = ctx.renderReference(fieldValue(featRow, "FeatIndex"), nil, func(string) string { return "" }) + } + if name == "" { + continue + } + level := stringValue(featRow, "GrantedOnLevel") + if level == "" || strings.HasPrefix(level, "-") { + selectable = append(selectable, name) + continue + } + byLevel[level] = append(byLevel[level], name) + } + parts := []string{} + if len(byLevel) > 0 { + items := []string{} + for _, level := range sortedKeys(byLevel) { + items = append(items, "Level "+level+": "+strings.Join(byLevel[level], ", ")) + } + parts = append(parts, `

    Granted Feats

    `+renderWikiListHTML("wiki-list wiki-class-granted-feats", items)) + } + if len(selectable) > 0 { + parts = append(parts, `

    Selectable Feats

    `+renderWikiListHTML("wiki-list wiki-class-selectable-feats", []string{strings.Join(selectable, ", ")})) + } + return strings.Join(parts, "\n") +} + +func (ctx *wikiContext) renderClassFeatureTable(row map[string]any) string { + table := ctx.tableForValue(fieldValue(row, "BonusFeatsTable"), ctx.classBonusTables) + if table == nil || len(table.Rows) == 0 { + return "" + } + return "==== Bonus Feats ====\n\n * Bonus feat table: " + table.OutputStem +} + +func (ctx *wikiContext) renderClassFeatureTableHTML(row map[string]any) string { + table := ctx.tableForValue(fieldValue(row, "BonusFeatsTable"), ctx.classBonusTables) + if table == nil || len(table.Rows) == 0 { + return "" + } + return `

    Bonus Feats

    ` + renderWikiListHTML("wiki-list wiki-class-bonus-feats", []string{"Bonus feat table: " + table.OutputStem}) +} + +func (ctx *wikiContext) renderRaceNameFormsHTML(row map[string]any) string { + facts := []wikiHTMLFact{} + if plural := ctx.resolveTextValue(fieldValue(row, "NamePlural"), nil); plural != "" { + facts = append(facts, wikiHTMLFact{Label: "Plural", Value: plural}) + } + if converted := ctx.resolveTextValue(fieldValue(row, "ConverName"), nil); converted != "" { + facts = append(facts, wikiHTMLFact{Label: "Converted Name", Value: converted}) + } + if lower := ctx.resolveTextValue(fieldValue(row, "ConverNameLower"), nil); lower != "" { + facts = append(facts, wikiHTMLFact{Label: "Lower Name", Value: lower}) + } + return renderWikiFactTableHTML(facts) +} diff --git a/internal/topdata/wiki_native.go b/internal/topdata/wiki_native.go new file mode 100644 index 0000000..6a4a1c8 --- /dev/null +++ b/internal/topdata/wiki_native.go @@ -0,0 +1,2159 @@ +package topdata + +import ( + "crypto/sha256" + "embed" + "encoding/json" + "fmt" + "html" + "io/fs" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "gopkg.in/yaml.v3" +) + +const ( + wikiStatusNew = "new" + wikiStatusModified = "modified" + wikiStatusVanilla = "vanilla" + wikiRootDirName = "wiki" + legacyWikiRootDirName = ".wiki" + wikiPagesDirName = "pages" + wikiStateFileName = "state.json" + wikiGeneratorVersion = "nodebb-tiptap-html-v3" + wikiGeneratedStatus = "generated" + wikiSkippedStatus = "skipped" +) + +var ( + wikiStatusPriority = map[string]int{wikiStatusVanilla: 0, wikiStatusModified: 1, wikiStatusNew: 2} + wikiStatusLabels = map[string]string{wikiStatusNew: "New", wikiStatusModified: "Modified", wikiStatusVanilla: "Vanilla"} +) + +//go:embed wiki_templates/*.txt +var wikiTemplateFS embed.FS + +type wikiStateDocument struct { + Version string `json:"version"` + Digest string `json:"digest"` + Pages int `json:"pages"` +} + +type wikiPageIndex struct { + Version string `json:"version"` + Pages []wikiPageIndexEntry `json:"pages"` +} + +type wikiPageIndexEntry struct { + PageID string `json:"page_id"` + Namespace string `json:"namespace"` + SourceDataset string `json:"source_dataset"` + SourceKey string `json:"source_key"` + Title string `json:"title"` + PublicPath string `json:"public_path"` + Hash string `json:"hash"` + OutputPath string `json:"output_path"` + EditPolicy string `json:"edit_policy"` + StalePolicy string `json:"stale_policy"` +} + +type wikiManualSectionsDocument struct { + ManualSections []wikiManualSection `json:"manual_sections" yaml:"manual_sections"` +} + +type wikiManualSection struct { + ID string `json:"id" yaml:"id"` + Alias string `json:"alias" yaml:"alias"` + Heading string `json:"heading" yaml:"heading"` + Placement string `json:"placement" yaml:"placement"` + InitialHTML string `json:"initial_html" yaml:"initial_html"` +} + +type wikiResult struct { + OutputDir string + PageCount int + Status string + Digest string +} + +type wikiTable struct { + Key string + Output string + OutputStem string + Rows []map[string]any +} + +type wikiContext struct { + dialog map[string]string + rowsByKey map[string]map[string]any + featRows map[string]map[string]any + featIDToKey map[int]string + skillRows map[string]map[string]any + skillIDToKey map[int]string + spellRows map[string]map[string]any + baseitemRows map[string]map[string]any + classRows map[string]map[string]any + classIDToKey map[int]string + raceRows map[string]map[string]any + raceIDToKey map[int]string + masterfeatRows map[string]map[string]any + classFeatTables map[string]wikiTable + classSkillTables map[string]wikiTable + classBonusTables map[string]wikiTable + classSaveTables map[string]wikiTable + classSpellGainTables map[string]wikiTable + classSpellKnownTables map[string]wikiTable + classSpellbooks map[string]classSpellbookSpec + raceFeatTables map[string]wikiTable + statuses map[string]map[string]string + implementedFeats map[string]struct{} + manualSections []wikiManualSection + templateDir string + wikiTablesPath string + wikiTableDefinitions map[string]wikiTableDefinition + wikiDataPath string + wikiDataProviders map[string]wikiDataProviderDefinition + visibilityPolicy *wikiVisibilityDefinitions + visibleWikiKeys map[string]bool + namespaceTitles map[string]string + namespacePaths map[string]string + alignmentLinkPattern string + alignmentLinkFormat string +} + +func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progress func(string)) (wikiResult, error) { + if progress == nil { + progress = func(string) {} + } + rootDir := wikiOutputRootDir(p) + outputDir := wikiOutputPagesDir(p) + statePath := wikiOutputStatePath(p) + + digest, err := computeWikiSourceDigest(p) + if err != nil { + return wikiResult{}, err + } + + state, _ := loadWikiState(statePath) + if !force && digest != "" && state.Digest == digest && wikiCachedPagesAreCurrent(outputDir, state) { + return wikiResult{ + OutputDir: outputDir, + PageCount: state.Pages, + Status: wikiSkippedStatus, + Digest: digest, + }, nil + } + + progress("Building native wiki pages...") + ctx, err := loadWikiContext(filepath.Join(p.TopDataSourceDir(), "data"), p.TopDataSourceDir()) + if err != nil { + return wikiResult{}, err + } + manualSections := loadWikiManualSections(p) + ctx.manualSections = manualSections + ctx.templateDir = p.TopDataWikiPageTemplatesDir() + ctx.alignmentLinkPattern = p.EffectiveConfig().TopData.Wiki.AlignmentLinks.TargetPattern + ctx.alignmentLinkFormat = p.EffectiveConfig().TopData.Wiki.AlignmentLinks.LinkFormat + ctx.wikiTablesPath = filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.TablesFile)) + tableDefinitions, err := loadWikiTableDefinitions(ctx.wikiTablesPath) + if err != nil { + return wikiResult{}, err + } + ctx.wikiTableDefinitions = tableDefinitions + ctx.wikiDataPath = p.TopDataWikiDataPath() + dataProviders, err := loadWikiDataProviders(ctx.wikiDataPath) + if err != nil { + return wikiResult{}, err + } + ctx.wikiDataProviders = dataProviders + visibilityPath := filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.VisibilityFile)) + if err := ctx.loadWikiVisibility(visibilityPath); err != nil { + return wikiResult{}, err + } + pageSlugOverrides, err := loadWikiPagePathDeclarations(p.TopDataWikiPagePathsPath()) + if err != nil { + return wikiResult{}, err + } + _ = pageSlugOverrides + namespaceDeclarations, err := loadWikiNamespaceDeclarations(p) + if err != nil { + return wikiResult{}, err + } + ctx.namespaceTitles = wikiNamespaceTitles(namespaceDeclarations) + ctx.namespacePaths = wikiNamespacePaths(namespaceDeclarations) + + if err := os.RemoveAll(outputDir); err != nil { + return wikiResult{}, fmt.Errorf("clean wiki output: %w", err) + } + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return wikiResult{}, fmt.Errorf("create wiki output dir: %w", err) + } + + pageTitles := map[string]string{} + pageStatuses := map[string]string{} + pageIndex := wikiPageIndex{Version: wikiGeneratorVersion} + outputPaths := map[string]string{} + + writePage := func(pageID, content, title, status string) error { + relPath := wikiPageOutputRelPath(pageID, title) + if previous := outputPaths[relPath]; previous != "" { + return fmt.Errorf("duplicate wiki page output path %q for %s and %s", filepath.ToSlash(relPath), previous, pageID) + } + outputPaths[relPath] = pageID + path := filepath.Join(outputDir, relPath) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + namespace := strings.SplitN(pageID, ":", 2)[0] + publicPath := ctx.publicWikiPath(namespace, title) + content = renderNodeBBManagedHTML(pageID, content, manualSections) + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + return err + } + pageTitles[pageID] = title + pageStatuses[pageID] = status + rel, err := filepath.Rel(rootDir, path) + if err != nil { + rel = filepath.Base(path) + } + pageIndex.Pages = append(pageIndex.Pages, wikiPageIndexEntry{ + PageID: pageID, + Namespace: namespace, + SourceDataset: categoryFromWikiNamespace(namespace), + SourceKey: pageID, + Title: title, + PublicPath: publicPath, + Hash: computeManagedHash(content), + OutputPath: filepath.ToSlash(rel), + EditPolicy: wikiEditPolicy(namespace), + StalePolicy: p.EffectiveConfig().TopData.Wiki.StalePages.Default, + }) + return nil + } + + if err := generateEntityPages(outputDir, ctx, writePage); err != nil { + return wikiResult{}, err + } + if wikiStatusPagesEnabled(p.EffectiveConfig().TopData.Wiki) { + if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces, p.EffectiveConfig().TopData.Wiki.StatusListingScope, manualSections, ctx); err != nil { + return wikiResult{}, err + } + } + if err := indexUnregisteredWikiPages(rootDir, outputDir, p.EffectiveConfig().TopData.Wiki.StalePages.Default, ctx, &pageIndex); err != nil { + return wikiResult{}, err + } + if err := saveWikiPageIndex(p.TopDataWikiPageIndexPath(), pageIndex); err != nil { + return wikiResult{}, err + } + + if err := os.MkdirAll(rootDir, 0o755); err != nil { + return wikiResult{}, fmt.Errorf("create wiki root dir: %w", err) + } + pageCount := len(pageIndex.Pages) + state = wikiStateDocument{Version: wikiGeneratorVersion, Digest: digest, Pages: pageCount} + if err := saveWikiState(statePath, state); err != nil { + return wikiResult{}, err + } + return wikiResult{ + OutputDir: outputDir, + PageCount: pageCount, + Status: wikiGeneratedStatus, + Digest: digest, + }, nil +} + +func wikiStatusPagesEnabled(cfg project.TopDataWikiConfig) bool { + return cfg.StatusPages == nil || *cfg.StatusPages +} + +func wikiCachedPagesAreCurrent(outputDir string, state wikiStateDocument) bool { + if state.Version != wikiGeneratorVersion { + return false + } + if state.Pages <= 0 { + return false + } + count := 0 + err := filepath.WalkDir(outputDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.EqualFold(filepath.Ext(path), ".html") { + count++ + } + return nil + }) + return err == nil && count == state.Pages +} + +func loadWikiManualSections(p *project.Project) []wikiManualSection { + defaults := defaultWikiManualSections() + path := p.TopDataWikiManualSectionsPath() + raw, err := os.ReadFile(path) + if err != nil { + return defaults + } + var doc wikiManualSectionsDocument + if err := yaml.Unmarshal(raw, &doc); err != nil || len(doc.ManualSections) == 0 { + return defaults + } + out := []wikiManualSection{} + for _, section := range doc.ManualSections { + section.ID = strings.TrimSpace(section.ID) + if section.ID == "" { + continue + } + section.Alias = strings.TrimSpace(section.Alias) + if section.Alias == "" { + switch section.ID { + case "user_top": + section.Alias = "UserTopSection" + case "user_bottom": + section.Alias = "UserBottomSection" + } + } + if strings.TrimSpace(section.InitialHTML) == "" { + heading := strings.TrimSpace(section.Heading) + if heading == "" { + heading = section.ID + } + section.InitialHTML = "

    " + html.EscapeString(heading) + "

    " + } + out = append(out, section) + } + if len(out) == 0 { + return defaults + } + return out +} + +func saveWikiPageIndex(path string, index wikiPageIndex) error { + if err := validateWikiPageIndex(index); err != nil { + return err + } + slices.SortFunc(index.Pages, func(a, b wikiPageIndexEntry) int { + return strings.Compare(a.PageID, b.PageID) + }) + raw, err := json.MarshalIndent(index, "", " ") + if err != nil { + return fmt.Errorf("marshal wiki page index: %w", err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, append(raw, '\n'), 0o644) +} + +func validateWikiPageIndex(index wikiPageIndex) error { + pageIDs := map[string]string{} + outputPaths := map[string]string{} + publicPaths := map[string]string{} + for _, entry := range index.Pages { + if previous, ok := pageIDs[entry.PageID]; ok { + return fmt.Errorf("duplicate wiki page-index page_id %q for %s and %s", entry.PageID, previous, entry.OutputPath) + } + if previous, ok := outputPaths[entry.OutputPath]; ok { + return fmt.Errorf("duplicate wiki page-index output_path %q for %s and %s", entry.OutputPath, previous, entry.PageID) + } + if entry.PublicPath != "" { + publicPathKey := canonicalWikiSlashPathFoldedKey(entry.PublicPath) + if publicPathKey == "" { + publicPathKey = entry.PublicPath + } + if previous, ok := publicPaths[publicPathKey]; ok { + return fmt.Errorf("duplicate wiki page-index public_path %q for %s and %s", entry.PublicPath, previous, entry.PageID) + } + publicPaths[publicPathKey] = entry.PageID + } + pageIDs[entry.PageID] = entry.OutputPath + outputPaths[entry.OutputPath] = entry.PageID + } + return nil +} + +func indexUnregisteredWikiPages(rootDir, outputDir, stalePolicy string, ctx *wikiContext, index *wikiPageIndex) error { + seen := map[string]struct{}{} + for _, entry := range index.Pages { + seen[entry.PageID] = struct{}{} + } + if ctx == nil { + ctx = &wikiContext{} + } + return filepath.WalkDir(outputDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".html") { + return nil + } + relToPages, err := filepath.Rel(outputDir, path) + if err != nil { + return err + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + pageID := extractWikiPageID(string(raw)) + if pageID == "" { + pageID = strings.TrimSuffix(filepath.ToSlash(relToPages), filepath.Ext(relToPages)) + pageID = strings.ReplaceAll(pageID, "/", ":") + } + if _, ok := seen[pageID]; ok { + return nil + } + namespace := strings.SplitN(pageID, ":", 2)[0] + relToRoot, err := filepath.Rel(rootDir, path) + if err != nil { + relToRoot = filepath.Base(path) + } + title := extractHTMLTitle(string(raw), pageID) + index.Pages = append(index.Pages, wikiPageIndexEntry{ + PageID: pageID, + Namespace: namespace, + SourceDataset: categoryFromWikiNamespace(namespace), + SourceKey: pageID, + Title: title, + PublicPath: ctx.publicWikiPath(namespace, title), + Hash: computeManagedHash(string(raw)), + OutputPath: filepath.ToSlash(relToRoot), + EditPolicy: wikiEditPolicy(namespace), + StalePolicy: stalePolicy, + }) + seen[pageID] = struct{}{} + return nil + }) +} + +func loadWikiState(path string) (wikiStateDocument, error) { + raw, err := os.ReadFile(path) + if err != nil { + return wikiStateDocument{}, err + } + var doc wikiStateDocument + if err := json.Unmarshal(raw, &doc); err != nil { + return wikiStateDocument{}, fmt.Errorf("parse wiki state: %w", err) + } + return doc, nil +} + +func saveWikiState(path string, state wikiStateDocument) error { + raw, err := json.MarshalIndent(state, "", " ") + if err != nil { + return fmt.Errorf("marshal wiki state: %w", err) + } + raw = append(raw, '\n') + if err := os.WriteFile(path, raw, 0o644); err != nil { + return fmt.Errorf("write wiki state: %w", err) + } + return nil +} + +func computeWikiSourceDigest(p *project.Project) (string, error) { + hasher := sha256.New() + _, _ = hasher.Write([]byte(wikiGeneratorVersion)) + sourceDir := p.TopDataSourceDir() + wikiSourceDir := p.TopDataWikiSourceDir() + files := []string{} + relevantSourceDirs := []string{ + filepath.Join(sourceDir, "data", "feat"), + filepath.Join(sourceDir, "data", "skills"), + filepath.Join(sourceDir, "data", "spells"), + filepath.Join(sourceDir, "data", "baseitems"), + filepath.Join(sourceDir, "data", "classes"), + filepath.Join(sourceDir, "data", "racialtypes"), + wikiSourceDir, + } + for _, dir := range relevantSourceDirs { + _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + switch strings.ToLower(filepath.Ext(path)) { + case ".json", ".yaml", ".yml", ".html": + files = append(files, path) + } + return nil + }) + } + if len(files) == 0 { + return "", nil + } + slices.Sort(files) + for _, path := range files { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + continue + } + return "", err + } + rel, err := filepath.Rel(sourceDir, path) + if err == nil && !strings.HasPrefix(rel, "..") { + rel = filepath.ToSlash(rel) + } else { + rel = filepath.Base(path) + } + _, _ = hasher.Write([]byte(rel)) + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read wiki digest input %s: %w", path, err) + } + _, _ = hasher.Write(raw) + } + return fmt.Sprintf("%x", hasher.Sum(nil)), nil +} + +func newestRelevantWikiSource(p *project.Project) (time.Time, string, error) { + newest := time.Time{} + newestPath := "" + sourceDir := p.TopDataSourceDir() + relevantSourceDirs := []string{ + filepath.Join(sourceDir, "data", "feat"), + filepath.Join(sourceDir, "data", "skills"), + filepath.Join(sourceDir, "data", "spells"), + filepath.Join(sourceDir, "data", "baseitems"), + filepath.Join(sourceDir, "data", "classes"), + filepath.Join(sourceDir, "data", "racialtypes"), + p.TopDataWikiSourceDir(), + } + for _, dir := range relevantSourceDirs { + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if d.IsDir() { + return nil + } + if !strings.EqualFold(filepath.Ext(path), ".json") { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if newest.IsZero() || info.ModTime().After(newest) { + newest = info.ModTime() + newestPath = path + } + return nil + }) + if err != nil { + return time.Time{}, "", err + } + } + return newest, newestPath, nil +} + +func loadWikiContext(dataDir, sourceDir string) (*wikiContext, error) { + dialogData, err := loadBaseDialogData(filepath.Join(sourceDir, "base_dialog.json")) + if err != nil { + return nil, err + } + dialog := map[string]string{} + if dialogData != nil { + for key, entry := range dialogData.Entries { + dialog[key] = entry.Text + } + } + + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return nil, err + } + loadBase := func(name string) (nativeCollectedDataset, bool, error) { + for _, dataset := range datasets { + if dataset.Name == name { + collected, err := collectNativeDataset(dataset) + return collected, true, err + } + } + return nativeCollectedDataset{}, false, nil + } + + featDataset, ok, err := loadBase("feat") + if err != nil { + return nil, err + } + skillDataset, skillOK, err := loadBase("skills") + if err != nil { + return nil, err + } + spellDataset, spellOK, err := loadBase("spells") + if err != nil { + return nil, err + } + baseitemDataset, baseitemOK, err := loadBase("baseitems") + if err != nil { + return nil, err + } + classDataset, classOK, err := loadBase("classes/core") + if err != nil { + return nil, err + } + masterfeatDataset, masterfeatOK, err := loadBase("masterfeats") + if err != nil { + return nil, err + } + raceDatasets, err := collectWikiRacialtypesDatasets(dataDir, datasets) + if err != nil { + return nil, err + } + raceRows := map[string]map[string]any{} + raceIDToKey := map[int]string{} + raceFeatTables := map[string]wikiTable{} + for _, dataset := range raceDatasets { + if dataset.Dataset.OutputName == "racialtypes.2da" { + raceRows, raceIDToKey = rowsByKeyAndID(dataset.Rows) + continue + } + table := wikiTable{ + Key: dataset.TableKey, + Output: dataset.Dataset.OutputName, + OutputStem: outputStem(dataset.Dataset.OutputName), + Rows: cloneRows(dataset.Rows), + } + raceFeatTables[table.OutputStem] = table + if table.Key != "" { + raceFeatTables[table.Key] = table + } + } + + classFeatTables, err := loadWikiTables(filepath.Join(dataDir, "classes", "feats")) + if err != nil { + return nil, err + } + classSkillTables, err := loadWikiTables(filepath.Join(dataDir, "classes", "skills")) + if err != nil { + return nil, err + } + classBonusTables, err := loadWikiTablesFromDirs(filepath.Join(dataDir, "classes", "bonusfeats"), filepath.Join(dataDir, "classes", "bfeat")) + if err != nil { + return nil, err + } + classSaveTables, err := loadWikiTablesFromDirs(filepath.Join(dataDir, "classes", "savingthrows"), filepath.Join(dataDir, "classes", "savthr")) + if err != nil { + return nil, err + } + classSpellGainTables, err := loadWikiTables(filepath.Join(dataDir, "classes", "spellsgained")) + if err != nil { + return nil, err + } + classSpellKnownTables, err := loadWikiTables(filepath.Join(dataDir, "classes", "spellsknown")) + if err != nil { + return nil, err + } + classSpellbookList, diagnostics := loadClassSpellbookSpecs(filepath.Join(dataDir, "classes", "spellbooks")) + if len(diagnostics) > 0 { + return nil, fmt.Errorf("load class spellbooks for wiki: %s", diagnosticsTextForError(diagnostics)) + } + classSpellbooks := map[string]classSpellbookSpec{} + for _, spec := range classSpellbookList { + classSpellbooks[spec.Class] = spec + } + + ctx := &wikiContext{ + dialog: dialog, + rowsByKey: map[string]map[string]any{}, + statuses: map[string]map[string]string{}, + raceFeatTables: raceFeatTables, + classFeatTables: classFeatTables, + classSkillTables: classSkillTables, + classBonusTables: classBonusTables, + classSaveTables: classSaveTables, + classSpellGainTables: classSpellGainTables, + classSpellKnownTables: classSpellKnownTables, + classSpellbooks: classSpellbooks, + } + if ok { + ctx.featRows, ctx.featIDToKey = rowsByKeyAndID(featDataset.Rows) + } else { + ctx.featRows, ctx.featIDToKey = map[string]map[string]any{}, map[int]string{} + } + if skillOK { + ctx.skillRows, ctx.skillIDToKey = rowsByKeyAndID(skillDataset.Rows) + } else { + ctx.skillRows, ctx.skillIDToKey = map[string]map[string]any{}, map[int]string{} + } + if spellOK { + ctx.spellRows, _ = rowsByKeyAndID(spellDataset.Rows) + } else { + ctx.spellRows = map[string]map[string]any{} + } + if baseitemOK { + ctx.baseitemRows, _ = rowsByKeyAndID(baseitemDataset.Rows) + } else { + ctx.baseitemRows = map[string]map[string]any{} + } + if classOK { + ctx.classRows, ctx.classIDToKey = rowsByKeyAndID(classDataset.Rows) + } else { + ctx.classRows, ctx.classIDToKey = map[string]map[string]any{}, map[int]string{} + } + if masterfeatOK { + ctx.masterfeatRows, _ = rowsByKeyAndID(masterfeatDataset.Rows) + } else { + ctx.masterfeatRows = map[string]map[string]any{} + } + ctx.raceRows = raceRows + ctx.raceIDToKey = raceIDToKey + for _, group := range []map[string]map[string]any{ctx.featRows, ctx.skillRows, ctx.spellRows, ctx.baseitemRows, ctx.classRows, ctx.raceRows, ctx.masterfeatRows} { + for key, row := range group { + ctx.rowsByKey[key] = row + } + } + if ok { + ctx.statuses["feat"] = collectDatasetStatuses(filepath.Join(dataDir, "feat"), "feat") + } + if skillOK { + ctx.statuses["skills"] = collectDatasetStatuses(filepath.Join(dataDir, "skills"), "skills") + } + if spellOK { + ctx.statuses["spells"] = collectDatasetStatuses(filepath.Join(dataDir, "spells"), "spells") + } + if baseitemOK { + ctx.statuses["baseitems"] = collectDatasetStatuses(filepath.Join(dataDir, "baseitems"), "baseitems") + } + if classOK { + ctx.statuses["classes"] = collectDatasetStatuses(filepath.Join(dataDir, "classes", "core"), "classes/core") + } + ctx.statuses["racialtypes"] = collectWikiRacialtypeStatuses(dataDir) + ctx.implementedFeats = ctx.collectImplementedFeatKeys() + return ctx, nil +} + +func collectWikiRacialtypesDatasets(dataDir string, datasets []nativeDataset) ([]nativeCollectedDataset, error) { + out := []nativeCollectedDataset{} + for _, dataset := range datasets { + name := filepath.ToSlash(dataset.Name) + if !strings.HasPrefix(name, "racialtypes/") { + continue + } + collected, err := collectNativeDataset(dataset) + if err != nil { + return nil, err + } + if collected.Dataset.OutputName == "racialtypes.2da" || collected.TableKey != "" { + out = append(out, collected) + } + } + if len(out) > 0 { + return out, nil + } + return collectRacialtypesRegistryDatasets(dataDir) +} + +func collectWikiRacialtypeStatuses(dataDir string) map[string]string { + coreRoot := filepath.Join(dataDir, "racialtypes", "core") + if _, err := os.Stat(coreRoot); err == nil { + return collectDatasetStatuses(coreRoot, "racialtypes/core") + } + return collectRaceStatuses(filepath.Join(dataDir, "racialtypes", "registry")) +} + +func rowsByKeyAndID(rows []map[string]any) (map[string]map[string]any, map[int]string) { + byKey := map[string]map[string]any{} + byID := map[int]string{} + for _, row := range rows { + key, _ := row["key"].(string) + if key == "" { + continue + } + byKey[key] = row + if id, ok := row["id"].(int); ok { + byID[id] = key + } + } + return byKey, byID +} + +func cloneRows(rows []map[string]any) []map[string]any { + out := make([]map[string]any, 0, len(rows)) + for _, row := range rows { + out = append(out, cloneRowMap(row)) + } + return out +} + +func loadWikiTables(root string) (map[string]wikiTable, error) { + info, err := os.Stat(root) + if err != nil { + if os.IsNotExist(err) { + return map[string]wikiTable{}, nil + } + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("%s must be a directory", root) + } + tables := map[string]wikiTable{} + err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.EqualFold(filepath.Ext(path), ".json") || strings.EqualFold(filepath.Base(path), "lock.json") || strings.EqualFold(filepath.Base(path), "global.json") { + return nil + } + obj, err := loadJSONObject(path) + if err != nil { + return err + } + rawRows, ok := obj["rows"].([]any) + if !ok { + return fmt.Errorf("%s: rows must be an array", path) + } + rows := make([]map[string]any, 0, len(rawRows)) + for _, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + return fmt.Errorf("%s: row must be an object", path) + } + rows = append(rows, row) + } + output, _ := obj["output"].(string) + if output == "" { + output = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + ".2da" + } + table := wikiTable{ + Key: stringField(obj, "key"), + Output: output, + OutputStem: outputStem(output), + Rows: rows, + } + tables[table.OutputStem] = table + if table.Key != "" { + tables[table.Key] = table + } + return nil + }) + if err != nil { + return nil, err + } + return tables, nil +} + +func loadWikiTablesFromDirs(roots ...string) (map[string]wikiTable, error) { + merged := map[string]wikiTable{} + for _, root := range roots { + tables, err := loadWikiTables(root) + if err != nil { + return nil, err + } + for key, table := range tables { + merged[key] = table + } + } + return merged, nil +} + +func generateEntityPages(outputDir string, ctx *wikiContext, writePage func(pageID, content, title, status string) error) error { + if err := generateCategoryPages("classes", ctx.classRows, ctx, writePage); err != nil { + return err + } + if err := generateCategoryPages("feat", ctx.featRows, ctx, writePage); err != nil { + return err + } + if err := generateCategoryPages("skills", ctx.skillRows, ctx, writePage); err != nil { + return err + } + if err := generateCategoryPages("spells", ctx.spellRows, ctx, writePage); err != nil { + return err + } + if err := generateCategoryPages("baseitems", ctx.baseitemRows, ctx, writePage); err != nil { + return err + } + if err := generateCategoryPages("racialtypes", ctx.raceRows, ctx, writePage); err != nil { + return err + } + _ = outputDir + return nil +} + +func generateCategoryPages(category string, rows map[string]map[string]any, ctx *wikiContext, writePage func(pageID, content, title, status string) error) error { + keys := sortedKeys(rows) + for _, key := range keys { + row := rows[key] + if !ctx.shouldGeneratePage(category, key, row) { + continue + } + title := ctx.resolveRowName(category, row) + if title == "" { + continue + } + content, err := ctx.renderPage(category, key, row, title) + if err != nil { + return err + } + if err := writePage(wikiPageIDForKey(key), content, title, ctx.pageStatus(category, key, row)); err != nil { + return err + } + } + return nil +} + +func (ctx *wikiContext) shouldGeneratePage(category, key string, row map[string]any) bool { + if row == nil { + return false + } + if ctx.visibilityPolicy != nil { + return ctx.canReferenceWikiKey(key) + } + override := ctx.wikiGenerateOverride(row) + if override == "0" { + return false + } + name := ctx.resolveRowName(category, row) + description := ctx.resolveRowDescription(category, row) + switch category { + case "classes": + if isToolLike(name, stringField(row, "Label"), stringField(row, "Constant")) { + return false + } + return override == "1" || stringValue(row, "PlayerClass") == "1" + case "feat": + if name == "" || description == "" || strings.EqualFold(stringValue(row, "FEAT"), nullValue) { + return false + } + if override == "1" { + return true + } + if stringValue(row, "ALLCLASSESCANUSE") == "1" { + return true + } + _, ok := ctx.implementedFeats[key] + return ok + case "skills": + if name == "" || description == "" { + return false + } + return override == "1" || stringValue(row, "HideFromLevelUp") != "1" + case "racialtypes": + if name == "" || description == "" { + return false + } + return override == "1" || stringValue(row, "PlayerRace") == "1" + case "spells": + if isToolLike(name, stringField(row, "Label"), stringField(row, "Constant")) { + return false + } + return name != "" && description != "" + default: + return name != "" && description != "" + } +} + +func (ctx *wikiContext) wikiRowCanRender(category string, row map[string]any) bool { + if row == nil { + return false + } + name := ctx.resolveRowName(category, row) + description := ctx.resolveRowDescription(category, row) + switch category { + case "classes", "spells": + if isToolLike(name, stringField(row, "Label"), stringField(row, "Constant")) { + return false + } + } + return name != "" && description != "" +} + +func (ctx *wikiContext) renderPage(category, key string, row map[string]any, title string) (string, error) { + manualSections := ctx.manualSections + if len(manualSections) == 0 { + manualSections = defaultWikiManualSections() + } + page := wikiTemplatePage{ + PageID: wikiPageIDForKey(key), + Category: category, + Key: key, + Title: title, + Status: ctx.pageStatus(category, key, row), + Row: row, + ManualSections: manualSections, + } + return ctx.renderWikiPageTemplate(category, page) +} + +func (ctx *wikiContext) renderFacts(category, key string, row map[string]any) string { + lines := []string{} + switch category { + case "feat": + lines = append(lines, ctx.renderFeatFacts(row)...) + case "skills": + lines = append(lines, formatFact("Key Ability", stringValue(row, "KeyAbility"))) + lines = append(lines, formatFact("Untrained", yesNoValue(stringValue(row, "Untrained")))) + lines = append(lines, formatFact("Armor Check Penalty", yesNoValue(stringValue(row, "ArmorCheckPenalty")))) + case "spells": + lines = append(lines, formatFact("Constant", stringValue(row, "Constant"))) + case "baseitems": + lines = append(lines, formatFact("Item Class", stringValue(row, "ItemClass"))) + lines = append(lines, formatFact("Weapon Type", stringValue(row, "WeaponType"))) + lines = append(lines, formatFact("Required Feats", ctx.renderReferenceList(ctx.wikiReqFeats(row)))) + lines = append(lines, formatFact("Base Item Stats", ctx.resolveTextValue(fieldValue(row, "BaseItemStatRef"), nil))) + case "classes": + lines = append(lines, formatFact("Hit Die", "d"+stringValue(row, "HitDie"))) + lines = append(lines, formatFact("Skill Points", stringValue(row, "SkillPointBase"))) + lines = append(lines, formatFact("Primary Ability", stringValue(row, "PrimaryAbil"))) + case "racialtypes": + lines = append(lines, formatFact("Ability Adjustments", formatAbilityAdjustments(row))) + lines = append(lines, formatFact("Favored Class", ctx.renderReference(fieldValue(row, "Favored"), ctx.classIDToKey, ctx.resolveClassName))) + lines = append(lines, formatFact("Favored Enemy", ctx.renderReference(fieldValue(row, "FavoredEnemyFeat"), ctx.featIDToKey, ctx.resolveFeatName))) + lines = append(lines, formatFact("Racial Feats", ctx.renderRaceFeatList(row))) + } + out := make([]string, 0, len(lines)) + for _, line := range lines { + if strings.TrimSpace(line) != "" { + out = append(out, line) + } + } + return strings.Join(out, "\n") +} + +func (ctx *wikiContext) renderFeatFacts(row map[string]any) []string { + lines := []string{} + lines = append(lines, formatFact("Prerequisites", ctx.renderFeatPrerequisites(row))) + lines = append(lines, formatFact("Minimum Attack Bonus", stringValue(row, "MINATTACKBONUS"))) + lines = append(lines, formatFact("Minimum Spell Level", stringValue(row, "MINSPELLLVL"))) + lines = append(lines, formatFact("Minimum Fortitude Save", stringValue(row, "MinFortSave"))) + return lines +} + +func (ctx *wikiContext) renderFeatPrerequisites(row map[string]any) string { + parts := []string{} + for _, field := range []struct { + Key string + Label string + }{ + {"MINSTR", "Str"}, {"MINDEX", "Dex"}, {"MINCON", "Con"}, {"MININT", "Int"}, {"MINWIS", "Wis"}, {"MINCHA", "Cha"}, + } { + if value := stringValue(row, field.Key); value != "" { + parts = append(parts, field.Label+" "+value) + } + } + for _, field := range []string{"PREREQFEAT1", "PREREQFEAT2"} { + if rendered := ctx.renderReference(fieldValue(row, field), ctx.featIDToKey, ctx.resolveFeatName); rendered != "" { + parts = append(parts, rendered) + } + } + orReqs := []string{} + for _, field := range []string{"OrReqFeat0", "OrReqFeat1", "OrReqFeat2", "OrReqFeat3", "OrReqFeat4"} { + if rendered := ctx.renderReference(fieldValue(row, field), ctx.featIDToKey, ctx.resolveFeatName); rendered != "" { + orReqs = append(orReqs, rendered) + } + } + if len(orReqs) > 0 { + parts = append(parts, strings.Join(orReqs, " or ")) + } + if rendered := ctx.renderReference(fieldValue(row, "MinLevelClass"), ctx.classIDToKey, ctx.resolveClassName); rendered != "" { + if value := stringValue(row, "MinLevel"); value != "" { + parts = append(parts, "Level "+value+" of "+rendered) + } + } + for _, spec := range []struct { + SkillField string + RankField string + }{ + {"REQSKILL", "ReqSkillMinRanks"}, + {"REQSKILL2", "ReqSkillMinRanks2"}, + } { + if skill := ctx.renderReference(fieldValue(row, spec.SkillField), ctx.skillIDToKey, ctx.resolveSkillName); skill != "" { + ranks := stringValue(row, spec.RankField) + if ranks != "" { + parts = append(parts, skill+" "+ranks+" ranks") + } else { + parts = append(parts, skill) + } + } + } + return strings.Join(parts, ", ") +} + +func (ctx *wikiContext) renderProgression(category string, row map[string]any) string { + if category != "classes" { + return "" + } + lines := []string{} + if table := ctx.tableForValue(fieldValue(row, "SkillsTable"), ctx.classSkillTables); table != nil { + skills := []string{} + for _, skillRow := range table.Rows { + if stringValue(skillRow, "ClassSkill") != "1" { + continue + } + name := ctx.renderReference(fieldValue(skillRow, "SkillIndex"), ctx.skillIDToKey, ctx.resolveSkillName) + if name != "" { + skills = append(skills, name) + } + } + if len(skills) > 0 { + lines = append(lines, "==== Class Skills ====") + lines = append(lines, "") + lines = append(lines, " * "+strings.Join(skills, ", ")) + lines = append(lines, "") + } + } + if table := ctx.tableForValue(fieldValue(row, "FeatsTable"), ctx.classFeatTables); table != nil { + byLevel := map[string][]string{} + selectable := []string{} + for _, featRow := range table.Rows { + name := ctx.renderReference(fieldValue(featRow, "FeatIndex"), ctx.featIDToKey, ctx.resolveFeatName) + if name == "" { + name = ctx.renderReference(fieldValue(featRow, "FeatIndex"), nil, func(string) string { return "" }) + } + if name == "" { + continue + } + level := stringValue(featRow, "GrantedOnLevel") + if level == "" || strings.HasPrefix(level, "-") { + selectable = append(selectable, name) + continue + } + byLevel[level] = append(byLevel[level], name) + } + if len(byLevel) > 0 { + lines = append(lines, "==== Granted Feats ====") + lines = append(lines, "") + levels := sortedKeys(byLevel) + for _, level := range levels { + lines = append(lines, " * Level "+level+": "+strings.Join(byLevel[level], ", ")) + } + lines = append(lines, "") + } + if len(selectable) > 0 { + lines = append(lines, "==== Selectable Feats ====") + lines = append(lines, "") + lines = append(lines, " * "+strings.Join(selectable, ", ")) + lines = append(lines, "") + } + } + if table := ctx.tableForValue(fieldValue(row, "BonusFeatsTable"), ctx.classBonusTables); table != nil && len(table.Rows) > 0 { + lines = append(lines, "==== Bonus Feats ====") + lines = append(lines, "") + lines = append(lines, " * Bonus feat table: "+table.OutputStem) + lines = append(lines, "") + } + if len(lines) == 0 { + return "No class progression data." + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} + +func (ctx *wikiContext) renderRaceNameForms(row map[string]any) string { + parts := []string{} + if plural := ctx.resolveTextValue(fieldValue(row, "NamePlural"), nil); plural != "" { + parts = append(parts, "**Plural:** "+plural+`\\`) + } + if converted := ctx.resolveTextValue(fieldValue(row, "ConverName"), nil); converted != "" { + parts = append(parts, "**Converted Name:** "+converted+`\\`) + } + if lower := ctx.resolveTextValue(fieldValue(row, "ConverNameLower"), nil); lower != "" { + parts = append(parts, "**Lower Name:** "+lower+`\\`) + } + return strings.Join(parts, "\n") +} + +func (ctx *wikiContext) renderRaceFeatList(row map[string]any) string { + table := ctx.tableForValue(fieldValue(row, "FeatsTable"), ctx.raceFeatTables) + if table == nil { + return "" + } + parts := []string{} + for _, featRow := range table.Rows { + if rendered := ctx.renderReference(fieldValue(featRow, "FeatIndex"), ctx.featIDToKey, ctx.resolveFeatName); rendered != "" { + parts = append(parts, rendered) + } + } + return strings.Join(parts, ", ") +} + +func (ctx *wikiContext) renderReference(value any, idToKey map[int]string, nameResolver func(string) string) string { + key := resolveReferenceKey(value, idToKey) + if key == "" || !ctx.canReferenceWikiKey(key) { + return "" + } + name := nameResolver(key) + if name == "" { + name = key + } + if target := ctx.publicWikiTargetForKey(key, name); target != "" { + return "[[" + target + "|" + name + "]]" + } + return name +} + +func (ctx *wikiContext) renderReferenceList(values []any) string { + parts := []string{} + for _, value := range values { + if rendered := ctx.renderReference(value, ctx.featIDToKey, ctx.resolveFeatName); rendered != "" { + parts = append(parts, rendered) + } + } + return strings.Join(parts, ", ") +} + +func (ctx *wikiContext) resolveRowName(category string, row map[string]any) string { + spec := specForDataset(category) + nameFields := spec.NameFields + if category == "classes" { + nameFields = []string{"Name", "Plural", "Lower", "CLASS", "Label", "Short"} + } + for _, field := range nameFields { + if value, ok := lookupField(row, field); ok { + if text := ctx.resolveTextValue(value, nil); text != "" { + return text + } + } + } + return "" +} + +func (ctx *wikiContext) resolveRowDescription(category string, row map[string]any) string { + spec := specForDataset(category) + for _, field := range spec.DescriptionFields { + if value, ok := lookupField(row, field); ok { + if text := ctx.resolveTextValue(value, nil); text != "" { + return text + } + } + } + return "" +} + +func (ctx *wikiContext) resolveTextValue(value any, stack map[string]struct{}) string { + switch typed := value.(type) { + case nil: + return "" + case string: + trimmed := strings.TrimSpace(typed) + if trimmed == "" || trimmed == nullValue { + return "" + } + if entry, ok := ctx.dialog[trimmed]; ok { + return entry + } + return trimmed + case int: + if entry, ok := ctx.dialog[strconv.Itoa(typed)]; ok { + return entry + } + return strconv.Itoa(typed) + case float64: + text := strconv.Itoa(int(typed)) + if entry, ok := ctx.dialog[text]; ok { + return entry + } + return text + case map[string]any: + payload, ok, err := parseTLKPayload(typed, true) + if err == nil && ok { + switch { + case payload.Text != "": + return payload.Text + case payload.Ref != "": + if stack == nil { + stack = map[string]struct{}{} + } + token := payload.Ref + "." + payload.RefField + if _, seen := stack[token]; seen { + return "" + } + stack[token] = struct{}{} + defer delete(stack, token) + target := ctx.rowsByKey[payload.Ref] + if target == nil { + return "" + } + field, ok := lookupField(target, payload.RefField) + if !ok { + return "" + } + return ctx.resolveTextValue(field, stack) + case payload.Key != "": + if entry, ok := ctx.dialog[payload.Key]; ok { + return entry + } + return "" + } + } + } + return "" +} + +func (ctx *wikiContext) resolveFeatName(key string) string { + if row := ctx.featRows[key]; row != nil { + return ctx.resolveRowName("feat", row) + } + if row := ctx.masterfeatRows[key]; row != nil { + return ctx.resolveRowName("masterfeats", row) + } + return "" +} + +func (ctx *wikiContext) resolveSkillName(key string) string { + if row := ctx.skillRows[key]; row != nil { + return ctx.resolveRowName("skills", row) + } + return "" +} + +func (ctx *wikiContext) resolveSpellName(key string) string { + if row := ctx.spellRows[key]; row != nil { + return ctx.resolveRowName("spells", row) + } + return "" +} + +func (ctx *wikiContext) resolveClassName(key string) string { + if row := ctx.classRows[key]; row != nil { + return ctx.resolveRowName("classes", row) + } + return "" +} + +func (ctx *wikiContext) resolveRaceName(key string) string { + if row := ctx.raceRows[key]; row != nil { + return ctx.resolveRowName("racialtypes", row) + } + return "" +} + +func (ctx *wikiContext) resolveBaseItemName(key string) string { + if row := ctx.baseitemRows[key]; row != nil { + return ctx.resolveRowName("baseitems", row) + } + return "" +} + +func (ctx *wikiContext) wikiGenerateOverride(row map[string]any) string { + if meta, err := parseExistingMetadata(row); err == nil && meta != nil { + if wiki, ok := meta["wiki"].(map[string]any); ok { + if value, ok := wiki["generate"]; ok { + return normalizeWikiScalar(value) + } + } + } + if value, ok := lookupField(row, "WIKIGENERATE"); ok { + return normalizeWikiScalar(value) + } + return "" +} + +func (ctx *wikiContext) wikiReqFeats(row map[string]any) []any { + if meta, err := parseExistingMetadata(row); err == nil && meta != nil { + if wiki, ok := meta["wiki"].(map[string]any); ok { + if value, ok := wiki["reqfeats"]; ok { + if typed, ok := value.([]any); ok { + return typed + } + } + } + } + return nil +} + +func (ctx *wikiContext) pageStatus(category, key string, row map[string]any) string { + if meta, err := parseExistingMetadata(row); err == nil && meta != nil { + if wiki, ok := meta["wiki"].(map[string]any); ok { + if value, ok := wiki["status"]; ok { + if normalized := normalizeWikiScalar(value); normalized != "" { + return normalized + } + } + } + } + if categoryStatuses, ok := ctx.statuses[category]; ok { + if status, ok := categoryStatuses[key]; ok && status != "" { + return status + } + } + return wikiStatusVanilla +} + +func (ctx *wikiContext) collectImplementedFeatKeys() map[string]struct{} { + implemented := map[string]struct{}{} + masterfeatGroups := map[string][]string{} + for key, row := range ctx.featRows { + if group := masterfeatGroupKey(fieldValue(row, "MASTERFEAT"), ctx.masterfeatRows); group != "" { + masterfeatGroups[group] = append(masterfeatGroups[group], key) + } + } + classCounts := func(row map[string]any) bool { + override := ctx.wikiGenerateOverride(row) + if override == "1" { + return true + } + if override == "0" { + return false + } + return stringValue(row, "PlayerClass") == "1" + } + for _, row := range ctx.classRows { + if !classCounts(row) { + continue + } + for _, tableMap := range []map[string]wikiTable{ctx.classFeatTables, ctx.classBonusTables} { + if table := ctx.tableForValue(fieldValue(row, "FeatsTable"), tableMap); table != nil { + ctx.collectImplementedFromTable(implemented, masterfeatGroups, table.Rows) + } + if table := ctx.tableForValue(fieldValue(row, "BonusFeatsTable"), tableMap); table != nil { + ctx.collectImplementedFromTable(implemented, masterfeatGroups, table.Rows) + } + } + } + for _, row := range ctx.raceRows { + override := ctx.wikiGenerateOverride(row) + if override == "0" || (override != "1" && stringValue(row, "PlayerRace") != "1") { + continue + } + if table := ctx.tableForValue(fieldValue(row, "FeatsTable"), ctx.raceFeatTables); table != nil { + ctx.collectImplementedFromTable(implemented, masterfeatGroups, table.Rows) + } + } + return implemented +} + +func (ctx *wikiContext) collectImplementedFromTable(target map[string]struct{}, masterfeatGroups map[string][]string, rows []map[string]any) { + for _, row := range rows { + key := resolveReferenceKey(fieldValue(row, "FeatIndex"), ctx.featIDToKey) + if key == "" { + if key = resolveReferenceKey(fieldValue(row, "FeatIndex"), nil); key == "" { + continue + } + } + if strings.HasPrefix(key, "masterfeats:") { + for _, featKey := range masterfeatGroups[key] { + target[featKey] = struct{}{} + } + continue + } + target[key] = struct{}{} + } +} + +func (ctx *wikiContext) tableForValue(value any, tables map[string]wikiTable) *wikiTable { + switch typed := value.(type) { + case string: + if table, ok := tables[typed]; ok { + return &table + } + case map[string]any: + if tableKey, ok := typed["table"].(string); ok { + tableKey = outputStem(tableKey) + if table, ok := tables[tableKey]; ok { + return &table + } + } + } + return nil +} + +func collectDatasetStatuses(root, datasetName string) map[string]string { + statuses := map[string]string{} + basePath := filepath.Join(root, "base.json") + baseObj, err := loadJSONObject(basePath) + if err == nil { + if rows, ok := baseObj["rows"].([]any); ok { + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + if key := stringField(row, "key"); key != "" { + statuses[key] = wikiStatusVanilla + } + } + } + } + mergeExplicitWikiStatuses(basePath, statuses) + for _, dirName := range []string{"modules", "generated"} { + dir := filepath.Join(root, dirName) + paths, err := collectModulePaths(dir) + if err != nil { + continue + } + for _, path := range paths { + obj, err := loadJSONObject(path) + if err != nil { + continue + } + if entries, ok := obj["entries"].(map[string]any); ok { + for key, raw := range entries { + row, ok := raw.(map[string]any) + if !ok { + continue + } + if _, exists := statuses[key]; exists { + statuses[key] = wikiStatusModified + } else { + statuses[key] = wikiStatusNew + } + mergeRowWikiStatus(key, row, statuses) + } + } + if overrides, ok := obj["overrides"].([]any); ok { + for _, raw := range overrides { + row, ok := raw.(map[string]any) + if !ok { + continue + } + key := stringField(row, "key") + if key == "" { + continue + } + if _, exists := statuses[key]; exists { + statuses[key] = wikiStatusModified + } else { + statuses[key] = wikiStatusNew + } + mergeRowWikiStatus(key, row, statuses) + } + } + } + } + _ = datasetName + return statuses +} + +func collectRaceStatuses(root string) map[string]string { + statuses := map[string]string{} + baseObj, err := loadJSONObject(filepath.Join(root, "base.json")) + if err == nil { + if rows, ok := baseObj["rows"].([]any); ok { + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + if key := stringField(row, "key"); key != "" { + statuses[key] = wikiStatusVanilla + } + } + } + } + paths, err := collectModulePaths(filepath.Join(root, "races")) + if err != nil { + return statuses + } + for _, path := range paths { + row, err := loadJSONObject(path) + if err != nil { + continue + } + key := stringField(row, "key") + if key == "" { + continue + } + if _, exists := statuses[key]; exists { + statuses[key] = wikiStatusModified + } else { + statuses[key] = wikiStatusNew + } + mergeRowWikiStatus(key, row, statuses) + if core, ok := row["core"].(map[string]any); ok { + mergeRowWikiStatus(key, core, statuses) + } + } + return statuses +} + +func mergeExplicitWikiStatuses(path string, statuses map[string]string) { + obj, err := loadJSONObject(path) + if err != nil { + return + } + if rows, ok := obj["rows"].([]any); ok { + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + if key := stringField(row, "key"); key != "" { + mergeRowWikiStatus(key, row, statuses) + } + } + } +} + +func mergeRowWikiStatus(key string, row map[string]any, statuses map[string]string) { + if meta, err := parseExistingMetadata(row); err == nil && meta != nil { + if wiki, ok := meta["wiki"].(map[string]any); ok { + if value, ok := wiki["status"]; ok { + if normalized := normalizeWikiScalar(value); normalized != "" { + statuses[key] = normalized + } + } + } + } +} + +func buildWikiStatusBlock(status, category string) string { + message := "This " + wikiStatusNoun(category) + " is unchanged from vanilla." + switch status { + case wikiStatusNew: + message = "This is a new " + wikiStatusNoun(category) + "!" + case wikiStatusModified: + message = "This " + wikiStatusNoun(category) + " has been altered from vanilla." + } + return "

    " + html.EscapeString(message) + "

    " +} + +func wikiStatusNoun(category string) string { + switch category { + case "feat": + return "feat" + case "skills": + return "skill" + case "spells": + return "spell" + case "classes": + return "class" + case "racialtypes": + return "race" + case "baseitems": + return "item type" + default: + return "page" + } +} + +func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string, namespaces []string, listingScope string, manualSections []wikiManualSection, ctx *wikiContext) error { + summaries := buildWikiNamespaceSummaries(pageStatuses, namespaces) + if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.html"), renderWikiStatusReportPage(summaries, namespaces, ctx), manualSections); err != nil { + return err + } + for _, namespace := range namespaces { + summary := summaries[namespace] + if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".html"), renderWikiNamespaceFragment(namespace, summary, ctx), manualSections); err != nil { + return err + } + for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} { + pageIDs := filterWikiPageIDs(pageStatuses, namespace, status) + if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".html"), renderWikiStatusListingPage(wikiNamespacePublicTitle(ctx, namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles, ctx), manualSections); err != nil { + return err + } + } + } + if listingScope == "" { + listingScope = project.DefaultTopDataWikiStatusListingScope + } + if listingScope == "all" { + for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} { + pageIDs := filterWikiPageIDs(pageStatuses, "", status) + if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".html"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles, ctx), manualSections); err != nil { + return err + } + } + } + return nil +} + +func writeWikiFile(path, content string, manualSections []wikiManualSection) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + pageID := wikiRelPathToPageID(path) + content = renderNodeBBManagedHTML(pageID, content, manualSections) + return os.WriteFile(path, []byte(ensureTrailingNewline(content)), 0o644) +} + +func buildWikiNamespaceSummaries(pageStatuses map[string]string, namespaces []string) map[string]map[string]any { + summaries := map[string]map[string]any{} + for _, namespace := range namespaces { + statuses := []string{} + for pageID, status := range pageStatuses { + if strings.SplitN(pageID, ":", 2)[0] == namespace { + statuses = append(statuses, status) + } + } + summaries[namespace] = map[string]any{ + "status": rollupWikiStatuses(statuses), + "new": countWikiStatus(statuses, wikiStatusNew), + "modified": countWikiStatus(statuses, wikiStatusModified), + "vanilla": countWikiStatus(statuses, wikiStatusVanilla), + "total": len(statuses), + } + } + return summaries +} + +func countWikiStatus(statuses []string, target string) int { + count := 0 + for _, status := range statuses { + if status == target { + count++ + } + } + return count +} + +func rollupWikiStatuses(statuses []string) string { + best := wikiStatusVanilla + for _, status := range statuses { + if wikiStatusPriority[status] > wikiStatusPriority[best] { + best = status + } + } + return best +} + +func wikiNamespacePublicTitle(ctx *wikiContext, namespace string) string { + if ctx != nil && ctx.namespaceTitles != nil { + if declared := strings.TrimSpace(ctx.namespaceTitles[namespace]); declared != "" { + return declared + } + } + return namespaceTitle(namespace) +} + +func renderWikiNamespaceFragment(namespace string, summary map[string]any, ctx *wikiContext) string { + return renderStatusTable(wikiNamespacePublicTitle(ctx, namespace)+" Namespace", summary, "") +} + +func renderWikiStatusReportPage(summaries map[string]map[string]any, namespaces []string, ctx *wikiContext) string { + if ctx == nil { + ctx = &wikiContext{} + } + lines := []string{"

    Wiki Status

    ", "

    This page is auto-generated from builder source provenance.

    "} + for _, namespace := range namespaces { + summary := summaries[namespace] + title := wikiNamespacePublicTitle(ctx, namespace) + lines = append(lines, renderStatusTable(title, summary, fmt.Sprintf("[[%s|%s]]", ctx.publicWikiPath(namespace, ""), title))) + } + return strings.Join(lines, "\n") +} + +func renderWikiStatusListingPage(title string, pageIDs []string, pageTitles map[string]string, ctx *wikiContext) string { + if ctx == nil { + ctx = &wikiContext{} + } + lines := []string{"

    " + html.EscapeString(title) + "

    "} + if len(pageIDs) == 0 { + lines = append(lines, "

    No matching generated pages.

    ") + } else { + lines = append(lines, `
      `) + for _, pageID := range pageIDs { + title := pageTitles[pageID] + if title == "" { + title = pageID + } + namespace := strings.SplitN(pageID, ":", 2)[0] + lines = append(lines, "
    • "+renderInlineHTML("[["+ctx.publicWikiPath(namespace, title)+"|"+title+"]]")+"
      • ") + } + lines = append(lines, "
    ") + } + return strings.Join(lines, "\n") +} + +func renderStatusTable(title string, summary map[string]any, namespaceLink string) string { + rows := []string{ + "

    " + html.EscapeString(title) + "

    ", + ``, + "", + fmt.Sprintf("", summary["total"].(int)), + fmt.Sprintf("", summary["new"].(int)), + fmt.Sprintf("", summary["modified"].(int)), + fmt.Sprintf("", summary["vanilla"].(int)), + } + if namespaceLink != "" { + rows = append(rows, "") + } + rows = append(rows, "
    Namespace Status" + html.EscapeString(wikiStatusLabels[summary["status"].(string)]) + "
    Generated Pages%d
    New%d
    Modified%d
    Vanilla%d
    Namespace"+renderInlineHTML(namespaceLink)+"
    ") + return strings.Join(rows, "\n") +} + +func filterWikiPageIDs(pageStatuses map[string]string, namespace, status string) []string { + pageIDs := []string{} + for pageID, candidate := range pageStatuses { + if candidate != status { + continue + } + if namespace != "" && strings.SplitN(pageID, ":", 2)[0] != namespace { + continue + } + pageIDs = append(pageIDs, pageID) + } + slices.Sort(pageIDs) + return pageIDs +} + +func wikiNamespaceTitles(declarations []wikiNamespaceDeclaration) map[string]string { + titles := map[string]string{} + for _, declaration := range declarations { + id := strings.TrimSpace(declaration.ID) + title := strings.TrimSpace(declaration.Title) + if id != "" && title != "" { + titles[id] = title + } + } + return titles +} + +func wikiNamespacePaths(declarations []wikiNamespaceDeclaration) map[string]string { + paths := map[string]string{} + for _, declaration := range declarations { + id := strings.TrimSpace(declaration.ID) + path := strings.TrimSpace(declaration.CanonicalPath) + if id != "" && path != "" { + paths[id] = path + } + } + return paths +} + +func wikiPageIDForKey(key string) string { + if key == "" { + return "" + } + category, value, ok := strings.Cut(key, ":") + if !ok { + return "" + } + namespace := map[string]string{ + "baseitems": "itemtypes", + "racialtypes": "races", + "skills": "skills", + "spells": "spells", + "classes": "classes", + "feat": "feat", + }[category] + if namespace == "" { + return "" + } + value = strings.NewReplacer("/", "_", "\\", "_").Replace(value) + return namespace + ":" + value +} + +func (ctx *wikiContext) publicWikiTargetForKey(key, fallbackTitle string) string { + pageID := wikiPageIDForKey(key) + if pageID == "" { + return "" + } + title := "" + if row, ok := ctx.rowsByKey[key]; ok { + category, _, _ := strings.Cut(key, ":") + title = ctx.resolveRowName(category, row) + } + if strings.TrimSpace(title) == "" { + title = fallbackTitle + } + if strings.TrimSpace(title) == "" { + title = key + } + namespace := strings.SplitN(pageID, ":", 2)[0] + return ctx.publicWikiPath(namespace, title) +} + +func (ctx *wikiContext) publicWikiPath(namespace, title string) string { + namespacePath := canonicalWikiSegment(namespaceTitle(namespace)) + if ctx != nil && ctx.namespacePaths != nil { + if declared := strings.TrimSpace(ctx.namespacePaths[namespace]); declared != "" { + namespacePath = declared + } + } + if ctx != nil && ctx.namespaceTitles != nil && (ctx.namespacePaths == nil || strings.TrimSpace(ctx.namespacePaths[namespace]) == "") { + if declared := strings.TrimSpace(ctx.namespaceTitles[namespace]); declared != "" { + namespacePath = canonicalWikiSegment(declared) + } + } + titlePath := canonicalWikiPath(wikiTitlePathParts(title)...) + switch { + case namespacePath == "": + return titlePath + case titlePath == "": + return namespacePath + default: + return namespacePath + "/" + titlePath + } +} + +func wikiTitlePathParts(title string) []string { + parts := []string{} + for _, part := range strings.Split(title, "::") { + part = strings.TrimSpace(part) + if part != "" { + parts = append(parts, part) + } + } + return parts +} + +func wikiSlugifyTitle(value string) string { + return slugifyWikiText(value, "topic") +} + +func wikiPageOutputRelPath(pageID, title string) string { + namespace := strings.TrimSpace(strings.SplitN(pageID, ":", 2)[0]) + if namespace == "" { + namespace = "page" + } + parts := []string{namespace} + parts = append(parts, wikiTitlePathParts(title)...) + path := canonicalWikiPath(parts...) + if path == "" { + path = wikiPageIDToRelPath(pageID) + return path + } + return filepath.FromSlash(path + ".html") +} + +func wikiPageIDToRelPath(pageID string) string { + return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".html") +} + +func wikiRelPathToPageID(path string) string { + path = filepath.ToSlash(path) + path = strings.TrimSuffix(path, ".md") + path = strings.TrimSuffix(path, ".html") + if index := strings.LastIndex(path, "/pages/"); index >= 0 { + path = path[index+len("/pages/"):] + } + return strings.ReplaceAll(path, "/", ":") +} + +func namespaceTitle(namespace string) string { + switch namespace { + case "itemtypes": + return "Item Types" + case "feat": + return "Feats" + default: + return strings.Title(namespace) + } +} + +func categoryFromWikiNamespace(namespace string) string { + switch namespace { + case "itemtypes": + return "baseitems" + case "races": + return "racialtypes" + default: + return namespace + } +} + +func wikiEditPolicy(namespace string) string { + if namespace == "meta" { + return "generated_only" + } + return "preserve_manual_sections" +} + +func normalizeWikiScalar(value any) string { + switch typed := value.(type) { + case string: + trimmed := strings.TrimSpace(strings.ToLower(typed)) + if _, ok := wikiStatusPriority[trimmed]; ok { + return trimmed + } + if trimmed == "0" || trimmed == "1" { + return trimmed + } + } + return "" +} + +func resolveReferenceKey(value any, idToKey map[int]string) string { + switch typed := value.(type) { + case map[string]any: + if key, ok := typed["key"].(string); ok && strings.Contains(key, ":") { + return key + } + if id, ok := typed["id"].(string); ok && strings.Contains(id, ":") { + return id + } + if idToKey != nil { + if rawID, ok := typed["id"]; ok { + if id, err := asInt(rawID); err == nil { + return idToKey[id] + } + } + } + case string: + if strings.Contains(typed, ":") { + return typed + } + if idToKey != nil { + if id, err := strconv.Atoi(typed); err == nil { + return idToKey[id] + } + } + case int: + if idToKey != nil { + return idToKey[typed] + } + case float64: + if idToKey != nil { + return idToKey[int(typed)] + } + } + return "" +} + +func masterfeatGroupKey(value any, masterfeatRows map[string]map[string]any) string { + key := resolveReferenceKey(value, nil) + if strings.HasPrefix(key, "masterfeats:") { + return key + } + id, err := asInt(value) + if err != nil { + return "" + } + for key, row := range masterfeatRows { + if rowID, ok := row["id"].(int); ok && rowID == id { + return key + } + } + return "" +} + +func fieldValue(row map[string]any, field string) any { + value, _ := lookupField(row, field) + return value +} + +func stringValue(row map[string]any, field string) string { + value, ok := lookupField(row, field) + if !ok || value == nil { + return "" + } + switch typed := value.(type) { + case string: + if typed == nullValue { + return "" + } + return typed + case int: + return strconv.Itoa(typed) + case float64: + return strconv.Itoa(int(typed)) + default: + return "" + } +} + +func formatAbilityAdjustments(row map[string]any) string { + parts := []string{} + for _, spec := range []struct { + Field string + Label string + }{ + {"StrAdjust", "Strength"}, + {"DexAdjust", "Dexterity"}, + {"ConAdjust", "Constitution"}, + {"IntAdjust", "Intelligence"}, + {"WisAdjust", "Wisdom"}, + {"ChaAdjust", "Charisma"}, + } { + value := stringValue(row, spec.Field) + if value == "" || value == "0" { + continue + } + if !strings.HasPrefix(value, "-") { + value = "+" + value + } + parts = append(parts, value+" "+spec.Label) + } + return strings.Join(parts, ", ") +} + +func yesNoValue(value string) string { + switch value { + case "1": + return "Yes" + case "0": + return "No" + default: + return "" + } +} + +func formatFact(label, value string) string { + if strings.TrimSpace(value) == "" { + return "" + } + return "**" + label + ":** " + value + `\\` +} + +func renderNodeBBManagedHTML(pageID, sourceText string, manualSections []wikiManualSection) string { + body := strings.TrimSpace(sourceText) + hash := computeManagedHash(body) + parts := []string{ + "", + "", + body, + "", + } + renderedSections := extractManualRegions(body) + for _, section := range manualSections { + if _, ok := renderedSections[section.ID]; ok { + continue + } + parts = append(parts, + renderWikiManualSection(section), + ) + } + return ensureTrailingNewline(strings.Join(parts, "\n")) +} + +func renderInlineHTML(text string) string { + var out strings.Builder + for { + start := strings.Index(text, "[[") + if start < 0 { + out.WriteString(html.EscapeString(text)) + break + } + end := strings.Index(text[start+2:], "]]") + if end < 0 { + out.WriteString(html.EscapeString(text)) + break + } + end += start + 2 + out.WriteString(html.EscapeString(text[:start])) + out.WriteString(text[start : end+2]) + text = text[end+2:] + } + return out.String() +} + +func renderWikiLinks(text string) string { + var out strings.Builder + for { + start := strings.Index(text, "[[") + if start < 0 { + out.WriteString(text) + break + } + end := strings.Index(text[start+2:], "]]") + if end < 0 { + out.WriteString(text) + break + } + end += start + 2 + out.WriteString(text[:start]) + marker := text[start : end+2] + out.WriteString(marker) + text = text[end+2:] + } + return out.String() +} + +func ensureTrailingNewline(text string) string { + if strings.HasSuffix(text, "\n") { + return text + } + return text + "\n" +} + +func isToolLike(values ...string) bool { + haystack := strings.ToLower(strings.Join(values, " ")) + return strings.Contains(haystack, "dm tool") || strings.Contains(haystack, "player tool") || strings.Contains(haystack, "activate item") || strings.Contains(haystack, "dm_tool") || strings.Contains(haystack, "player_tool") || strings.Contains(haystack, "activate_item") +} diff --git a/internal/topdata/wiki_native_test.go b/internal/topdata/wiki_native_test.go new file mode 100644 index 0000000..98a9b4e --- /dev/null +++ b/internal/topdata/wiki_native_test.go @@ -0,0 +1,3008 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp", "Untrained", "KeyAbility", "ArmorCheckPenalty", "Constant"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, + "Description": {"tlk": {"key": "skills:athletics.description", "text": "Ability: Strength.\n\nUses:\n- Climb\n- Swim"}}, + "HideFromLevelUp": "0", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "Constant": "SKILL_ATHLETICS" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + generatedHTMLCount := countGeneratedWikiHTMLFiles(t, filepath.Join(root, ".cache", "wiki", "pages")) + if result.WikiPages != generatedHTMLCount { + t.Fatalf("expected wiki page count to match generated HTML files, got result=%d files=%d", result.WikiPages, generatedHTMLCount) + } + pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "Athletics.html") + got, err := os.ReadFile(pagePath) + if err != nil { + t.Fatalf("read generated page: %v", err) + } + text := string(got) + if !strings.Contains(text, "

    Athletics

    ") || !strings.Contains(text, `class="wiki-callout wiki-callout--status"`) || !strings.Contains(text, "This skill is unchanged from vanilla.") { + t.Fatalf("unexpected generated page:\n%s", text) + } + if !strings.Contains(text, ``) || strings.Contains(text, `wiki_slug=`) || !strings.Contains(text, ``) { + t.Fatalf("expected generated page to include user_bottom manual section placeholder:\n%s", text) + } + if strings.Contains(text, `manual:start id="notes"`) || strings.Contains(text, `manual:start id="see_also"`) || strings.Contains(text, "

    See Also

    ") { + t.Fatalf("expected generated page to omit legacy split bottom manual sections:\n%s", text) + } + statusPath := filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus.html") + statusRaw, err := os.ReadFile(statusPath) + if err != nil { + t.Fatalf("read status page: %v", err) + } + if !strings.Contains(string(statusRaw), "Vanilla1") { + t.Fatalf("expected wiki status summary to count vanilla page:\n%s", string(statusRaw)) + } + indexRaw, err := os.ReadFile(filepath.Join(root, ".cache", "wiki", "page-index.json")) + if err != nil { + t.Fatalf("read page index: %v", err) + } + var pageIndex wikiPageIndex + if err := json.Unmarshal(indexRaw, &pageIndex); err != nil { + t.Fatalf("parse page index: %v", err) + } + if len(pageIndex.Pages) != generatedHTMLCount { + t.Fatalf("expected page-index entries to match generated HTML files, got index=%d files=%d", len(pageIndex.Pages), generatedHTMLCount) + } + indexText := string(indexRaw) + if !strings.Contains(indexText, `"page_id": "skills:athletics"`) || !strings.Contains(indexText, `"public_path": "Skills/Athletics"`) || strings.Contains(indexText, `"public_slug"`) || !strings.Contains(indexText, `"output_path": "pages/skills/Athletics.html"`) || !strings.Contains(indexText, `"edit_policy": "preserve_manual_sections"`) { + t.Fatalf("expected deterministic page-index metadata, got:\n%s", indexText) + } + if !strings.Contains(indexText, `"page_id": "meta:wikistatus"`) || !strings.Contains(indexText, `"output_path": "pages/meta/wikistatus.html"`) || !strings.Contains(indexText, `"edit_policy": "generated_only"`) { + t.Fatalf("expected status pages in page-index metadata, got:\n%s", indexText) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil { + t.Fatalf("expected state file after first build: %v", err) + } + stateBefore, err := loadWikiState(filepath.Join(root, ".cache", "wiki", "state.json")) + if err != nil { + t.Fatalf("read state before second build: %v", err) + } + digestBefore, err := computeWikiSourceDigest(proj) + if err != nil { + t.Fatalf("compute digest before second build: %v", err) + } + if stateBefore.Digest != digestBefore { + t.Fatalf("expected state digest to match current digest before second build, got state=%s current=%s", stateBefore.Digest, digestBefore) + } + + second, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("second BuildNative failed: %v", err) + } + if second.WikiPages != generatedHTMLCount { + t.Fatalf("expected generated wiki page count after rebuild to remain stable, got %d want %d", second.WikiPages, generatedHTMLCount) + } +} + +func TestBuildWikiRegeneratesMissingCachedPages(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp", "Untrained", "KeyAbility", "ArmorCheckPenalty", "Constant"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, + "Description": {"tlk": {"key": "skills:athletics.description", "text": "Ability: Strength."}}, + "HideFromLevelUp": "0", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "Constant": "SKILL_ATHLETICS" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + pagesDir := filepath.Join(root, ".cache", "wiki", "pages") + if err := os.RemoveAll(pagesDir); err != nil { + t.Fatalf("remove generated wiki pages: %v", err) + } + if err := os.MkdirAll(pagesDir, 0o755); err != nil { + t.Fatalf("recreate empty wiki pages dir: %v", err) + } + + result, err := BuildWikiWithOptions(proj, BuildWikiOptions{}, nil) + if err != nil { + t.Fatalf("BuildWikiWithOptions failed: %v", err) + } + if result.Status != wikiGeneratedStatus { + t.Fatalf("expected missing cached pages to regenerate, got wiki status %q", result.Status) + } + if _, err := os.Stat(filepath.Join(pagesDir, "skills", "Athletics.html")); err != nil { + t.Fatalf("expected regenerated skill page: %v", err) + } + + if err := os.RemoveAll(pagesDir); err != nil { + t.Fatalf("remove regenerated wiki pages: %v", err) + } + if err := os.MkdirAll(pagesDir, 0o755); err != nil { + t.Fatalf("recreate empty wiki pages dir before native build: %v", err) + } + + nativeResult, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative with missing cached pages failed: %v", err) + } + if nativeResult.WikiStatus != wikiGeneratedStatus { + t.Fatalf("expected native build to regenerate missing cached pages, got wiki status %q", nativeResult.WikiStatus) + } +} + +func TestBuildWikiRegeneratesOldGeneratorCache(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp", "Untrained", "KeyAbility", "ArmorCheckPenalty", "Constant"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, + "Description": {"tlk": {"key": "skills:athletics.description", "text": "Ability: Strength."}}, + "HideFromLevelUp": "0", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "Constant": "SKILL_ATHLETICS" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + statePath := filepath.Join(root, ".cache", "wiki", "state.json") + state, err := loadWikiState(statePath) + if err != nil { + t.Fatalf("read state: %v", err) + } + state.Version = "nodebb-tiptap-html-v2" + if err := saveWikiState(statePath, state); err != nil { + t.Fatalf("save old-version state: %v", err) + } + + result, err := BuildWikiWithOptions(proj, BuildWikiOptions{}, nil) + if err != nil { + t.Fatalf("BuildWikiWithOptions failed: %v", err) + } + if result.Status != wikiGeneratedStatus { + t.Fatalf("expected old generator cache to regenerate, got wiki status %q", result.Status) + } + refreshed, err := loadWikiState(statePath) + if err != nil { + t.Fatalf("read refreshed state: %v", err) + } + if refreshed.Version != wikiGeneratorVersion { + t.Fatalf("expected refreshed generator version %q, got %q", wikiGeneratorVersion, refreshed.Version) + } +} + +func TestRenderNodeBBManagedHTMLPreservesHeadinglessTemplateHTML(t *testing.T) { + source := strings.TrimSpace(` +

    This is generated.

    + +

    + +

    Generated description.

    +
    Hit Died6
    +

    Class Skills

    +
    • [[Skills/Heal|Heal]]
    +`) + + got := renderNodeBBManagedHTML("classes:adept", source, []wikiManualSection{ + {ID: "user_top", Alias: "UserTopSection", InitialHTML: "

    "}, + {ID: "user_bottom", Alias: "UserBottomSection", InitialHTML: "

    "}, + }) + + for _, want := range []string{ + ``, + `

    Class Skills

    `, + `
    `, + `
  • [[Skills/Heal|Heal]]
    • `, + } { + if !strings.Contains(got, want) { + t.Fatalf("expected generated HTML to preserve %q:\n%s", want, got) + } + } + if strings.Contains(got, "<table") || strings.Contains(got, "<div") || strings.Contains(got, "

    <") { + t.Fatalf("expected generated HTML not to be escaped into plain text:\n%s", got) + } + if strings.Count(got, `manual:start id="user_top"`) != 1 { + t.Fatalf("expected existing user_top manual section not to be duplicated:\n%s", got) + } + if strings.Count(got, `manual:start id="user_bottom"`) != 1 { + t.Fatalf("expected missing user_bottom manual section to be bootstrapped once:\n%s", got) + } +} + +func TestBuildNativeIndexesUnregisteredStatusPagesWithDeclaredNamespacePath(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + mkdirAll(t, filepath.Join(root, "topdata", "wiki")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "wiki", "namespaces.yaml"), ` +namespaces: + - id: meta + title: Wiki Status + canonical_path: Wiki/Status + category_env: SOW_WIKI_META_CID + edit_policy: generated_only +`) + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp", "Untrained", "KeyAbility", "ArmorCheckPenalty", "Constant"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, + "Description": {"tlk": {"key": "skills:athletics.description", "text": "Ability: Strength."}}, + "HideFromLevelUp": "0", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "Constant": "SKILL_ATHLETICS" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + indexRaw, err := os.ReadFile(filepath.Join(root, ".cache", "wiki", "page-index.json")) + if err != nil { + t.Fatalf("read page index: %v", err) + } + var pageIndex wikiPageIndex + if err := json.Unmarshal(indexRaw, &pageIndex); err != nil { + t.Fatalf("parse page index: %v", err) + } + paths := map[string]string{} + for _, page := range pageIndex.Pages { + paths[page.PageID] = page.PublicPath + } + if got, want := paths["meta:wikistatus"], "Wiki/Status/Wiki_Status"; got != want { + t.Fatalf("expected declared canonical path for status page, got %q want %q", got, want) + } + if got := paths["meta:wikistatus:skills:vanilla"]; !strings.HasPrefix(got, "Wiki/Status/") { + t.Fatalf("expected declared canonical path for status listing, got %q", got) + } +} + +func TestBuildNativeCanOmitAggregateWikiStatusListings(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp", "Untrained", "KeyAbility", "ArmorCheckPenalty", "Constant"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, + "Description": {"tlk": {"key": "skills:athletics.description", "text": "Ability: Strength."}}, + "HideFromLevelUp": "0", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "Constant": "SKILL_ATHLETICS" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.TopData.Wiki.StatusListingScope = "namespace" + + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus", "skills", "vanilla.html")); err != nil { + t.Fatalf("expected namespace status listing to be generated: %v", err) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus", "vanilla.html")); !os.IsNotExist(err) { + t.Fatalf("expected aggregate status listing to be omitted, got err=%v", err) + } +} + +func TestBuildNativeCanDisableWikiStatusPages(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp", "Untrained", "KeyAbility", "ArmorCheckPenalty", "Constant"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, + "Description": {"tlk": {"key": "skills:athletics.description", "text": "Ability: Strength."}}, + "HideFromLevelUp": "0", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "Constant": "SKILL_ATHLETICS" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + + disabled := false + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.TopData.Wiki.StatusPages = &disabled + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "pages", "meta")); !os.IsNotExist(err) { + t.Fatalf("expected wiki status pages to be disabled, got err=%v", err) + } + indexRaw, err := os.ReadFile(filepath.Join(root, ".cache", "wiki", "page-index.json")) + if err != nil { + t.Fatalf("read page index: %v", err) + } + if strings.Contains(string(indexRaw), `"namespace": "meta"`) || strings.Contains(string(indexRaw), `"meta:wikistatus`) { + t.Fatalf("expected disabled status pages to be omitted from page-index:\n%s", indexRaw) + } + if result.WikiPages != 1 { + t.Fatalf("expected only entity page to be generated, got %d pages", result.WikiPages) + } +} + +func TestBuildNativeWikiGeneratesRacePagesFromCoreRacialtypes(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "00_feats")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["FEAT", "DESCRIPTION"], + "rows": [ + {"id": 10, "key": "feat:darkvision", "FEAT": "Darkvision", "DESCRIPTION": "See in the dark."} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:darkvision":10}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "Name", "Description", "Appearance", "PlayerRace", "FeatsTable"], + "rows": [ + {"id": 7, "key": "racialtypes:beast", "Label": "Beast", "Name": "Beast", "Description": "Creature category.", "Appearance": 174, "PlayerRace": 0, "FeatsTable": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:beast":7,"racialtypes:goblin_dynamic":39}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "add_goblin.json"), `{ + "entries": { + "racialtypes:goblin_dynamic": { + "Label": "Goblin_Dynamic", + "Name": {"tlk": {"text": "Goblin"}}, + "Description": {"tlk": {"text": "A playable goblin race."}}, + "Appearance": 15381, + "PlayerRace": 1, + "FeatsTable": {"table": "racialtypes/feats:goblin_dynamic"} + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "00_feats", "goblin.json"), `{ + "key": "racialtypes/feats:goblin_dynamic", + "output": "race_feat_gobpc.2da", + "columns": ["FeatLabel", "FeatIndex"], + "rows": [ + {"id": 0, "FeatIndex": {"id": "feat:darkvision"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "00_feats", "lock.json"), "{}\n") + + disabled := false + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.TopData.Wiki.StatusPages = &disabled + + result, err := BuildNative(proj, nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + if result.WikiPages != 2 { + t.Fatalf("expected one generated race page and its referenced feat page, got %d", result.WikiPages) + } + pagePath := filepath.Join(root, ".cache", "wiki", "pages", "races", "Goblin.html") + got, err := os.ReadFile(pagePath) + if err != nil { + t.Fatalf("read generated race page: %v", err) + } + if !strings.Contains(string(got), "A playable goblin race.") || !strings.Contains(string(got), "[[Feats/Darkvision|Darkvision]]") { + t.Fatalf("expected generated race page content, got:\n%s", got) + } + indexRaw, err := os.ReadFile(filepath.Join(root, ".cache", "wiki", "page-index.json")) + if err != nil { + t.Fatalf("read page index: %v", err) + } + indexText := string(indexRaw) + if !strings.Contains(indexText, `"page_id": "races:goblin_dynamic"`) || !strings.Contains(indexText, `"namespace": "races"`) || !strings.Contains(indexText, `"public_path": "Races/Goblin"`) { + t.Fatalf("expected generated race page-index entry, got:\n%s", indexText) + } +} + +func countGeneratedWikiHTMLFiles(t *testing.T, root string) int { + t.Helper() + count := 0 + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.EqualFold(filepath.Ext(path), ".html") { + count++ + } + return nil + }) + if err != nil { + t.Fatalf("count generated wiki HTML files: %v", err) + } + return count +} + +func TestDefaultWikiManualSectionsUseSingleUserBottomBlock(t *testing.T) { + sections := defaultWikiManualSections() + if len(sections) != 2 { + t.Fatalf("expected top and bottom manual sections only, got %#v", sections) + } + expected := map[string]string{ + "user_top": "UserTopSection", + "user_bottom": "UserBottomSection", + } + for _, section := range sections { + alias, ok := expected[section.ID] + if !ok { + t.Fatalf("unexpected default manual section %#v", section) + } + if section.Alias != alias { + t.Fatalf("expected alias %q for %q, got %q", alias, section.ID, section.Alias) + } + } +} + +func TestWikiClassNamePrefersFullNameInsteadOfShortCode(t *testing.T) { + ctx := &wikiContext{} + got := ctx.resolveRowName("classes", map[string]any{ + "Name": map[string]any{"tlk": map[string]any{"key": "classes:wizard.name", "text": "Wizard"}}, + "Short": map[string]any{"tlk": map[string]any{"key": "classes:wizard.short", "text": "Wiz"}}, + }) + if got != "Wizard" { + t.Fatalf("expected full class name, got %q", got) + } +} + +func TestWikiTemplateRendersFieldsFormattersAndPreservedTopSection(t *testing.T) { + ctx := &wikiContext{} + page := wikiTemplatePage{ + PageID: "classes:acolyte", + Category: "classes", + Key: "classes:acolyte", + Title: "Acolyte", + Status: wikiStatusNew, + Row: map[string]any{ + "HitDie": 6, + "SkillPointBase": 4, + "Description": map[string]any{"tlk": map[string]any{ + "text": "Acolyte description.", + }}, + }, + ManualSections: []wikiManualSection{ + {ID: "user_top", Alias: "UserTopSection", InitialHTML: "

    "}, + {ID: "user_bottom", Alias: "UserBottomSection", InitialHTML: "

    "}, + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `

    {{title}}

    +{{UserTopSection}} +

    Hit Die: {{HitDie}}

    +

    Skill Points: {{SkillPointBase}}

    +{{format:StatusCallout}} +{{format:Description}} +{{UserBottomSection}}`, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + + if !strings.Contains(got, "

    Acolyte

    \n") { + t.Fatalf("expected preserved top section directly after title, got:\n%s", got) + } + for _, expected := range []string{ + "

    Hit Die: 6

    ", + "

    Skill Points: 4

    ", + "This is a new class!", + "

    Acolyte description.

    ", + ``, + } { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in rendered template:\n%s", expected, got) + } + } +} + +func TestWikiTemplateMissingFieldFailsUnlessDefaultProvided(t *testing.T) { + ctx := &wikiContext{} + page := wikiTemplatePage{PageID: "classes:acolyte", Category: "classes", Title: "Acolyte", Row: map[string]any{}} + + if _, err := ctx.renderWikiTemplateString("classes.html", `

    {{HitDie}}

    `, page); err == nil || !strings.Contains(err.Error(), `missing field "HitDie"`) { + t.Fatalf("expected missing field error, got %v", err) + } + + got, err := ctx.renderWikiTemplateString("classes.html", `

    {{field:HitDie|default=unknown}}

    `, page) + if err != nil { + t.Fatalf("expected defaulted field to render, got %v", err) + } + if got != "

    unknown

    " { + t.Fatalf("expected default value, got %q", got) + } +} + +func TestWikiTemplateEachBlockRendersExplicitClassProgressionTable(t *testing.T) { + ctx := &wikiContext{ + wikiDataProviders: map[string]wikiDataProviderDefinition{ + "ClassProgression": {Source: "class_progression", Levels: "1-2"}, + }, + } + page := wikiTemplatePage{ + PageID: "classes:fighter", + Category: "classes", + Key: "classes:fighter", + Title: "Fighter", + Row: map[string]any{ + "HitDie": "10", + "AttackBonusTable": "cls_atk_1", + }, + } + + template := `
    + + + + + {{#each ClassProgression}} + + {{/each}} + +
    LvlBABHP
    {{Level|ordinal}}{{BAB}}{{HPMin}}-{{HPMax}}
    ` + + got, err := ctx.renderWikiTemplateString("classes.html", template, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, want := range []string{ + ``, + ``, + ``, + } { + if !strings.Contains(got, want) { + t.Fatalf("expected rendered output to contain %q:\n%s", want, got) + } + } +} + +func TestWikiTemplateClassProgressionProviderProjectsConfiguredClassFeats(t *testing.T) { + root := t.TempDir() + dataPath := filepath.Join(root, "data.yaml") + writeFile(t, dataPath, ` +providers: + ClassProgression: + source: class_progression + levels: 1-4 + class_feats: + fields: + GrantedFeats: + when: "{{GrantedOnLevel > 0 && List == 3}}" + AvailableFeats: + when: "{{GrantedOnLevel > 0 && List != 3}}" +`+"\n") + providers, err := loadWikiDataProviders(dataPath) + if err != nil { + t.Fatalf("load data providers: %v", err) + } + ctx := testWikiTableContext(t, "") + ctx.wikiDataPath = dataPath + ctx.wikiDataProviders = providers + ctx.featRows["feat:weapon_focus"] = map[string]any{"FEAT": map[string]any{"tlk": map[string]any{"text": "Weapon Focus"}}} + ctx.classFeatTables["classes/feats:fighter"] = wikiTable{OutputStem: "cls_feat_fight", Rows: []map[string]any{ + {"FeatIndex": map[string]any{"id": "feat:literate"}, "GrantedOnLevel": "1", "List": "3"}, + {"FeatIndex": map[string]any{"id": "feat:weapon_specialization"}, "GrantedOnLevel": "4", "List": "1"}, + {"FeatIndex": map[string]any{"id": "feat:weapon_focus"}, "GrantedOnLevel": "4", "List": "1"}, + {"FeatIndex": map[string]any{"id": "feat:weapon_proficiency_simple"}, "GrantedOnLevel": "1", "List": "3"}, + }} + page := wikiTemplatePage{ + PageID: "classes:fighter", + Category: "classes", + Key: "classes:fighter", + Title: "Fighter", + Row: map[string]any{ + "AttackBonusTable": "cls_atk_1", + "BonusFeatsTable": map[string]any{"table": "classes/bonusfeats:fighter"}, + "FeatsTable": map[string]any{"table": "classes/feats:fighter"}, + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `{{#each ClassProgression}} +{{Level}}|{{GrantedFeats|join:", "|wiki}}|{{AvailableFeats|join:", "|wiki}}|{{BonusFeat}} +{{/each}}`, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, expected := range []string{ + `1|[[Feats/Literate|Literate]], [[Feats/Simple_Weapon_Proficiency|Simple Weapon Proficiency]]||1`, + `4||[[Feats/Weapon_Specialization|Weapon Specialization]], [[Feats/Weapon_Focus|Weapon Focus]]|1`, + } { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in rendered progression:\n%s", expected, got) + } + } +} + +func TestWikiTemplateFactsProviderRendersYAMLDeclaredFactRows(t *testing.T) { + root := t.TempDir() + dataPath := filepath.Join(root, "data.yaml") + writeFile(t, dataPath, ` +providers: + FeatFacts: + source: facts + facts: + rows: + - label: Prerequisites + value: "{{feat_prerequisites()}}" + - label: Minimum Attack Bonus + value: "{{MINATTACKBONUS}}" + - label: Armor Check Penalty + value: "{{yes_no(ArmorCheckPenalty)}}" + - label: Missing Optional + value: "{{MissingField}}" +`+"\n") + providers, err := loadWikiDataProviders(dataPath) + if err != nil { + t.Fatalf("load data providers: %v", err) + } + ctx := testWikiTableContext(t, "") + ctx.wikiDataPath = dataPath + ctx.wikiDataProviders = providers + ctx.featRows["feat:power_attack"] = map[string]any{"FEAT": map[string]any{"tlk": map[string]any{"text": "Power Attack"}}} + ctx.featIDToKey[28] = "feat:power_attack" + ctx.skillRows = map[string]map[string]any{"skills:athletics": {"Name": map[string]any{"tlk": map[string]any{"text": "Athletics"}}}} + ctx.skillIDToKey = map[int]string{3: "skills:athletics"} + page := wikiTemplatePage{ + PageID: "feat:cleave", + Category: "feat", + Key: "feat:cleave", + Title: "Cleave", + Row: map[string]any{ + "MINSTR": "13", + "PREREQFEAT1": 28, + "REQSKILL": 3, + "ReqSkillMinRanks": "4", + "MINATTACKBONUS": "2", + "ArmorCheckPenalty": "1", + }, + } + + got, err := ctx.renderWikiTemplateString("feat.html", `
    1st+11-10
    2nd+22-20
    + + {{#each FeatFacts}} + + {{/each}} + +
    {{Label}}{{Value|wiki}}
    `, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, expected := range []string{ + `PrerequisitesStr 13, [[Feats/Power_Attack|Power Attack]], [[Skills/Athletics|Athletics]] 4 ranks`, + `Minimum Attack Bonus2`, + `Armor Check PenaltyYes`, + } { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in rendered facts:\n%s", expected, got) + } + } + if strings.Contains(got, "Missing Optional") { + t.Fatalf("expected empty optional fact to be omitted:\n%s", got) + } +} + +func TestWikiTemplateClassFactsCanUseDerivedClassHelpers(t *testing.T) { + root := t.TempDir() + dataPath := filepath.Join(root, "data.yaml") + writeFile(t, dataPath, ` +providers: + ClassFacts: + source: facts + facts: + rows: + - label: Hit Die + value: "{{\"d\" + HitDie + \" (\" + HitDie + \" per level)\"}}" + - label: Proficiencies + value: "{{join(class_proficiencies(), \", \")}}" + - label: Skill Points at 1st Level + value: "{{\"(\" + SkillPointBase + \" + Int modifier) x4\"}}" + - label: Skill Points at Each Additional Level + value: "{{SkillPointBase + \" + Int modifier\"}}" + - label: Saves + value: "{{class_saves()}}" + - label: Base Attack Bonus + value: "{{class_bab()}}" + - label: Spellcasting + value: "{{\"To cast a spell, a \" + text(Lower) + \" must have a \" + ability_name(SpellcastingAbil) + \" score of 10 + the spell's level. For example, to cast a 2nd-level spell, a \" + text(Lower) + \" must have a \" + ability_name(SpellcastingAbil) + \" of 12.\"}}" + when: "{{SpellCaster == 1}}" + ClassBonusFeats: + source: facts + facts: + rows: + - label: Bonus Feats + value: "{{join(class_selectable_feats(), \", \")}}" +`+"\n") + providers, err := loadWikiDataProviders(dataPath) + if err != nil { + t.Fatalf("load data providers: %v", err) + } + ctx := testWikiTableContext(t, "") + ctx.wikiDataPath = dataPath + ctx.wikiDataProviders = providers + ctx.featRows["feat:armor_proficiency_light"] = map[string]any{"FEAT": map[string]any{"tlk": map[string]any{"text": "Light Armor Proficiency"}}} + ctx.featRows["feat:shield_proficiency"] = map[string]any{"FEAT": map[string]any{"tlk": map[string]any{"text": "Shield Proficiency"}}} + ctx.featRows["feat:extra_music"] = map[string]any{"FEAT": map[string]any{"tlk": map[string]any{"text": "Extra Music"}}} + ctx.featRows["feat:lingering_song"] = map[string]any{"FEAT": map[string]any{"tlk": map[string]any{"text": "Lingering Song"}}} + ctx.classFeatTables["classes/feats:bard"] = wikiTable{OutputStem: "cls_feat_bard", Rows: []map[string]any{ + {"FeatIndex": map[string]any{"id": "feat:literate"}, "GrantedOnLevel": "1", "List": "3"}, + {"FeatIndex": map[string]any{"id": "feat:armor_proficiency_light"}, "GrantedOnLevel": "1", "List": "3"}, + {"FeatIndex": map[string]any{"id": "feat:shield_proficiency"}, "GrantedOnLevel": "1", "List": "3"}, + {"FeatIndex": map[string]any{"id": "feat:extra_music"}, "GrantedOnLevel": "-1", "List": "1"}, + {"FeatIndex": map[string]any{"id": "feat:lingering_song"}, "GrantedOnLevel": "-1", "List": "2"}, + }} + ctx.classSaveTables["classes/saves:reflex_will"] = wikiTable{OutputStem: "cls_savthr_rw", Rows: []map[string]any{ + {"Level": "1", "FortSave": "0", "RefSave": "2", "WillSave": "2"}, + }} + page := wikiTemplatePage{ + PageID: "classes:bard", + Category: "classes", + Key: "classes:bard", + Title: "Bard", + Row: map[string]any{ + "HitDie": "6", + "SkillPointBase": "8", + "Lower": map[string]any{"tlk": map[string]any{"text": "bard"}}, + "SpellCaster": "1", + "SpellcastingAbil": "CHA", + "AttackBonusTable": "cls_atk_2", + "FeatsTable": map[string]any{"table": "classes/feats:bard"}, + "SavingThrowTable": map[string]any{"table": "classes/saves:reflex_will"}, + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", ` +{{#each ClassFacts}}{{/each}} +
    {{Label}}{{Value|wiki}}
    +{{#each ClassBonusFeats}}

    {{Value|wiki}}

    {{/each}}`, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, expected := range []string{ + `Hit Died6 (6 per level)`, + `Proficiencies[[Feats/Light_Armor_Proficiency|Light Armor Proficiency]], [[Feats/Shield_Proficiency|Shield Proficiency]]`, + `Skill Points at 1st Level(8 + Int modifier) x4`, + `Skill Points at Each Additional Level8 + Int modifier`, + `SavesHigh Reflex and Will`, + `Base Attack Bonus+3/4`, + `SpellcastingTo cast a spell, a bard must have a Charisma score of 10 + the spell's level. For example, to cast a 2nd-level spell, a bard must have a Charisma of 12.`, + `

    [[Feats/Extra_Music|Extra Music]], [[Feats/Lingering_Song|Lingering Song]]

    `, + } { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in rendered class facts:\n%s", expected, got) + } + } +} + +func TestWikiTemplateExpressionCanRenderIndefiniteArticles(t *testing.T) { + ctx := &wikiContext{} + page := wikiTemplatePage{ + PageID: "classes:adept", + Category: "classes", + Key: "classes:adept", + Title: "Adept", + Row: map[string]any{ + "Lower": map[string]any{"tlk": map[string]any{"text": "adept"}}, + "SpellcastingAbil": "INT", + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `{{indefinite_article(text(Lower))}} {{text(Lower)}} / {{indefinite_article(ability_name(SpellcastingAbil))}} {{ability_name(SpellcastingAbil)}}`, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + if got != "an adept / an Intelligence" { + t.Fatalf("expected vowel-sound articles, got %q", got) + } +} + +func TestWikiTemplateClassProgressionProviderUsesDefaultNWNAttackSequences(t *testing.T) { + ctx := &wikiContext{ + wikiDataPath: "test-data.yaml", + wikiDataProviders: map[string]wikiDataProviderDefinition{ + "ClassProgression": {Source: "class_progression", Levels: "1-16"}, + }, + } + page := wikiTemplatePage{ + PageID: "classes:fighter", + Category: "classes", + Key: "classes:fighter", + Title: "Fighter", + Row: map[string]any{ + "AttackBonusTable": "cls_atk_1", + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `{{#each ClassProgression}}{{Level}}:{{BABValue}}:{{AttackCount}}:{{AttackSequence}} +{{/each}}`, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, expected := range []string{ + "1:1:1:+1", + "6:6:2:+6/+1", + "11:11:3:+11/+6/+1", + "16:16:4:+16/+11/+6/+1", + } { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in rendered progression:\n%s", expected, got) + } + } +} + +func TestWikiTemplateClassProgressionProviderSupportsFixedAttacksAndOverrides(t *testing.T) { + root := t.TempDir() + dataPath := filepath.Join(root, "data.yaml") + writeFile(t, dataPath, ` +providers: + ClassProgression: + source: class_progression + levels: 1-12 + attacks: + mode: fixed + attacks: 2 + step: -5 + overrides: + - classes: ["classes:fighter"] + levels: 11 + attacks: 3 +`+"\n") + providers, err := loadWikiDataProviders(dataPath) + if err != nil { + t.Fatalf("load data providers: %v", err) + } + ctx := &wikiContext{ + wikiDataPath: dataPath, + wikiDataProviders: providers, + } + page := wikiTemplatePage{ + PageID: "classes:fighter", + Category: "classes", + Key: "classes:fighter", + Title: "Fighter", + Row: map[string]any{ + "AttackBonusTable": "cls_atk_1", + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `{{#each ClassProgression}}{{Level}}:{{AttackCount}}:{{AttackSequence}} +{{/each}}`, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, expected := range []string{ + "1:2:+1/-4", + "6:2:+6/+1", + "11:3:+11/+6/+1", + "12:2:+12/+7", + } { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in rendered progression:\n%s", expected, got) + } + } +} + +func TestWikiTemplateCountHelperSupportsListGrammar(t *testing.T) { + ctx := &wikiContext{} + page := wikiTemplatePage{ + PageID: "classes:test", + Category: "classes", + Key: "classes:test", + Title: "Test Class", + Row: map[string]any{ + "AvailableFeats": []any{"One", "Two"}, + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `{{AvailableFeats|join:", "}} {{if(count(AvailableFeats) > 1, "become", "becomes")}} available.`, page) + if err != nil { + t.Fatalf("render plural template: %v", err) + } + if got != "One, Two become available." { + t.Fatalf("unexpected plural grammar: %q", got) + } + + page.Row["AvailableFeats"] = []any{"One"} + got, err = ctx.renderWikiTemplateString("classes.html", `{{AvailableFeats|join:", "}} {{if(count(AvailableFeats) > 1, "become", "becomes")}} available.`, page) + if err != nil { + t.Fatalf("render singular template: %v", err) + } + if got != "One becomes available." { + t.Fatalf("unexpected singular grammar: %q", got) + } +} + +func TestWikiDataProvidersRejectEmptyClassFeatProjectionCondition(t *testing.T) { + root := t.TempDir() + dataPath := filepath.Join(root, "data.yaml") + writeFile(t, dataPath, ` +providers: + ClassProgression: + source: class_progression + class_feats: + fields: + AvailableFeats: + when: "" +`+"\n") + + _, err := loadWikiDataProviders(dataPath) + if err == nil { + t.Fatal("expected invalid class feat projection condition") + } + for _, expected := range []string{"data.yaml", "ClassProgression", "AvailableFeats"} { + if !strings.Contains(err.Error(), expected) { + t.Fatalf("expected error to contain %q, got %v", expected, err) + } + } +} + +func TestWikiDataProvidersRejectInvalidAttackOverrideConfig(t *testing.T) { + root := t.TempDir() + dataPath := filepath.Join(root, "data.yaml") + writeFile(t, dataPath, ` +providers: + ClassProgression: + source: class_progression + attacks: + mode: fixed + attacks: 2 + overrides: + - classes: ["classes:fighter"] + levels: 11 + attacks: 3 + add: 1 +`+"\n") + + _, err := loadWikiDataProviders(dataPath) + if err == nil { + t.Fatal("expected invalid attack override config") + } + for _, expected := range []string{"data.yaml", "ClassProgression", "attacks", "override"} { + if !strings.Contains(err.Error(), expected) { + t.Fatalf("expected error to contain %q, got %v", expected, err) + } + } +} + +func TestWikiDataProvidersRejectInvalidColumnProviderConfig(t *testing.T) { + root := t.TempDir() + dataPath := filepath.Join(root, "data.yaml") + writeFile(t, dataPath, ` +providers: + ClassSkills: + source: class_skills + BrokenColumns: + source: columns + provider: ClassSkills + columns: 0 + items_per_column: 2 +`+"\n") + + _, err := loadWikiDataProviders(dataPath) + if err == nil { + t.Fatal("expected invalid column provider config") + } + for _, expected := range []string{"data.yaml", "BrokenColumns", "columns"} { + if !strings.Contains(err.Error(), expected) { + t.Fatalf("expected error to contain %q, got %v", expected, err) + } + } +} + +func TestWikiTemplateEachBlockRendersClassSkillsAsIndividualListItems(t *testing.T) { + ctx := &wikiContext{ + wikiDataProviders: map[string]wikiDataProviderDefinition{ + "ClassSkills": {Source: "class_skills"}, + }, + classSkillTables: map[string]wikiTable{ + "classes/skills:fighter": { + Rows: []map[string]any{ + {"SkillIndex": "1", "ClassSkill": "1"}, + {"SkillIndex": "2", "ClassSkill": "1"}, + {"SkillIndex": "3", "ClassSkill": "0"}, + }, + }, + }, + skillIDToKey: map[int]string{ + 1: "skills:athletics", + 2: "skills:parry", + 3: "skills:secret", + }, + skillRows: map[string]map[string]any{ + "skills:athletics": {"Name": "Athletics", "KeyAbility": "STR"}, + "skills:parry": {"Name": "Parry", "KeyAbility": "DEX"}, + "skills:secret": {"Name": "Secret", "KeyAbility": "INT"}, + }, + } + page := wikiTemplatePage{ + PageID: "classes:fighter", + Category: "classes", + Key: "classes:fighter", + Title: "Fighter", + Row: map[string]any{ + "SkillsTable": map[string]any{"table": "classes/skills:fighter"}, + }, + } + + template := `
      +{{#each ClassSkills}} +
    • {{SkillKey|link:SkillName|wiki}} ({{SkillAbility|lower|title}})
      • +{{/each}} +
    ` + + got, err := ctx.renderWikiTemplateString("classes.html", template, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, want := range []string{ + `
  • [[Skills/Athletics|Athletics]] (Str)
    • `, + `
  • [[Skills/Parry|Parry]] (Dex)
    • `, + } { + if !strings.Contains(got, want) { + t.Fatalf("expected rendered output to contain %q:\n%s", want, got) + } + } + for _, reject := range []string{"Athletics, ", "Secret"} { + if strings.Contains(got, reject) { + t.Fatalf("expected rendered output not to contain %q:\n%s", reject, got) + } + } +} + +func TestWikiTemplateColumnProviderRendersNestedItemsAndForcedEmptyColumns(t *testing.T) { + ctx := &wikiContext{ + wikiDataProviders: map[string]wikiDataProviderDefinition{ + "ClassSkills": { + Source: "class_skills", + }, + "ClassSkillColumns": { + Source: "columns", + Provider: "ClassSkills", + Columns: 3, + ItemsPerColumn: 2, + ForceColumns: true, + }, + }, + classSkillTables: map[string]wikiTable{ + "classes/skills:fighter": { + Rows: []map[string]any{ + {"SkillIndex": "1", "ClassSkill": "1"}, + {"SkillIndex": "2", "ClassSkill": "1"}, + {"SkillIndex": "3", "ClassSkill": "1"}, + }, + }, + }, + skillIDToKey: map[int]string{ + 1: "skills:athletics", + 2: "skills:parry", + 3: "skills:ride", + }, + skillRows: map[string]map[string]any{ + "skills:athletics": {"Name": "Athletics"}, + "skills:parry": {"Name": "Parry"}, + "skills:ride": {"Name": "Ride"}, + }, + } + page := wikiTemplatePage{ + PageID: "classes:fighter", + Category: "classes", + Key: "classes:fighter", + Title: "Fighter", + Row: map[string]any{ + "SkillsTable": map[string]any{"table": "classes/skills:fighter"}, + }, + } + + template := `
    +{{#each ClassSkillColumns}} +
      + {{#each Items}} +
    • {{SkillKey|link:SkillName|wiki}}
      • + {{/each}} +
    +{{/each}} +
    ` + + got, err := ctx.renderWikiTemplateString("classes.html", template, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, want := range []string{ + `
      `, + `
    • [[Skills/Athletics|Athletics]]
      • `, + `
    • [[Skills/Parry|Parry]]
      • `, + `
        `, + `
      • [[Skills/Ride|Ride]]
        • `, + `
          `, + } { + if !strings.Contains(got, want) { + t.Fatalf("expected rendered output to contain %q:\n%s", want, got) + } + } + if gotCount := strings.Count(got, `
            {{GrantedFeats|list:"wiki-list wiki-inline-list"}}`, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, want := range []string{ + `
              `, + `
            • [[Feats/Literate|Literate]]
              • `, + `
            • [[Feats/Armor_Proficiency_light|Armor Proficiency (light)]]
              • `, + } { + if !strings.Contains(got, want) { + t.Fatalf("expected rendered output to contain %q:\n%s", want, got) + } + } + if strings.Contains(got, "Literate]], [[") { + t.Fatalf("expected list filter not to render comma-separated links:\n%s", got) + } +} + +func TestWikiTemplateCanRenderRacialFactsAndFeatListInHTML(t *testing.T) { + ctx := &wikiContext{ + wikiDataProviders: map[string]wikiDataProviderDefinition{ + "RaceFeats": {Source: "race_feats"}, + }, + classIDToKey: map[int]string{7: "classes:paladin"}, + featIDToKey: map[int]string{ + 10: "feat:darkvision", + 11: "feat:daylight:aasimar", + }, + classRows: map[string]map[string]any{ + "classes:paladin": {"Name": "Paladin"}, + }, + featRows: map[string]map[string]any{ + "feat:darkvision": {"FEAT": "Darkvision"}, + "feat:daylight:aasimar": {"FEAT": "Daylight (aasimar)"}, + }, + visibleWikiKeys: map[string]bool{ + "classes:paladin": true, + "feat:darkvision": true, + "feat:daylight:aasimar": true, + }, + } + page := wikiTemplatePage{ + PageID: "racialtypes:aasimar", + Category: "racialtypes", + Key: "racialtypes:aasimar", + Title: "Aasimar", + Row: map[string]any{ + "WISAdjust": "2", + "CHAAdjust": "2", + "Favored": "7", + "FeatsTable": []any{"10", "11"}, + }, + } + + template := ` + + + {{#if present(Favored)}} + + {{/if}} + +
              Ability Adjustments{{ability_adjustments()}}
              Favored Class{{Favored|ref:"classes"|wiki}}
              +
                +{{#each RaceFeats}} +
              • {{FeatKey|link:FeatName|wiki}}
                • +{{/each}} +
              ` + + got, err := ctx.renderWikiTemplateString("racialtypes.html", template, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, want := range []string{ + `Ability Adjustments+2 Wisdom, +2 Charisma`, + `Favored Class[[Classes/Paladin|Paladin]]`, + `
            • [[Feats/Darkvision|Darkvision]]
              • `, + `
            • [[Feats/Daylight_aasimar|Daylight (aasimar)]]
              • `, + } { + if !strings.Contains(got, want) { + t.Fatalf("expected rendered output to contain %q:\n%s", want, got) + } + } +} + +func TestWikiTemplateRacialOptionalFactsRenderOnlyWhenPresent(t *testing.T) { + ctx := &wikiContext{} + template := `{{#if present(ECL)}}ECL={{ECL}}{{/if}} +{{#if present(PreferredAlignments)}}Preferred={{PreferredAlignments}}{{/if}} +{{#if present(StartLanguages)}}Start={{StartLanguages}}{{/if}} +{{#if present(BonusLanguages)}}Bonus={{BonusLanguages}}{{/if}}` + page := wikiTemplatePage{ + PageID: "racialtypes:human", + Category: "racialtypes", + Key: "racialtypes:human", + Title: "Human", + Row: map[string]any{ + "ECL": 0, + "PreferredAlignments": "0x00", + "StartLanguages": "regional", + "BonusLanguages": "all", + }, + } + + got, err := ctx.renderWikiTemplateString("racialtypes.html", template, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + for _, want := range []string{ + "ECL=0", + "Preferred=0x00", + "Start=regional", + "Bonus=all", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected rendered output to contain %q:\n%s", want, got) + } + } + + page.Row = map[string]any{ + "ECL": nil, + "PreferredAlignments": "", + "StartLanguages": "****", + } + got, err = ctx.renderWikiTemplateString("racialtypes.html", template, page) + if err != nil { + t.Fatalf("render template with absent values: %v", err) + } + for _, unwanted := range []string{ + "ECL=", + "Preferred=", + "Start=", + "Bonus=", + } { + if strings.Contains(got, unwanted) { + t.Fatalf("expected rendered output to omit %q for absent values:\n%s", unwanted, got) + } + } +} + +func TestWikiTemplateFormatsPreferredAlignments(t *testing.T) { + ctx := &wikiContext{} + template := `{{alignments(PreferredAlignments)}}` + tests := []struct { + name string + value any + want string + }{ + {name: "encoded list", value: "0x120C01", want: "Lawful Evil, Chaotic Good, True Neutral"}, + {name: "authored list", value: []any{"le", "CG", "tn"}, want: "Lawful Evil, Chaotic Good, True Neutral"}, + {name: "authored object alignments", value: map[string]any{"alignments": []any{"ce", "ne"}}, want: "Chaotic Evil, Neutral Evil"}, + {name: "legacy authored object list", value: map[string]any{"list": []any{"ce", "ne"}}, want: "Chaotic Evil, Neutral Evil"}, + {name: "zero", value: 0, want: "Any"}, + {name: "encoded zero", value: "0x00", want: "Any"}, + {name: "null", value: nil, want: "Any"}, + {name: "missing", value: nil, want: "Any"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + row := map[string]any{} + if test.name != "missing" { + row["PreferredAlignments"] = test.value + } + page := wikiTemplatePage{ + PageID: "racialtypes:human", + Category: "racialtypes", + Key: "racialtypes:human", + Title: "Human", + Row: row, + } + got, err := ctx.renderWikiTemplateString("racialtypes.html", template, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + if got != test.want { + t.Fatalf("expected %q, got %q", test.want, got) + } + }) + } +} + +func TestWikiTemplateFormatsPreferredAlignmentsWithConfiguredLinks(t *testing.T) { + ctx := &wikiContext{alignmentLinkPattern: "Guides/Alignment/{alignment}"} + page := wikiTemplatePage{ + PageID: "racialtypes:tiefling", + Category: "racialtypes", + Key: "racialtypes:tiefling", + Title: "Tiefling", + Row: map[string]any{ + "PreferredAlignments": map[string]any{"alignments": []any{"le", "ne", "ce"}}, + }, + } + + got, err := ctx.renderWikiTemplateString("racialtypes.html", `{{alignments(PreferredAlignments)}}`, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + want := "[[Guides/Alignment/Lawful_Evil|Lawful Evil]], [[Guides/Alignment/Neutral_Evil|Neutral Evil]], [[Guides/Alignment/Chaotic_Evil|Chaotic Evil]]" + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} + +func TestWikiTemplateFormatsPreferredAlignmentsWithConfiguredRootAnchors(t *testing.T) { + ctx := &wikiContext{ + alignmentLinkPattern: "/wiki/Guides/Alignment/{alignment}", + alignmentLinkFormat: "html_anchor", + } + page := wikiTemplatePage{ + PageID: "racialtypes:tiefling", + Category: "racialtypes", + Key: "racialtypes:tiefling", + Title: "Tiefling", + Row: map[string]any{ + "PreferredAlignments": map[string]any{"alignments": []any{"le"}}, + }, + } + + got, err := ctx.renderWikiTemplateString("racialtypes.html", `{{alignments(PreferredAlignments)}}`, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + want := `Lawful Evil` + if got != want { + t.Fatalf("expected %q, got %q", want, got) + } +} + +func TestWikiTemplateDescriptionFiltersExtractParagraphsAndSections(t *testing.T) { + ctx := &wikiContext{} + page := wikiTemplatePage{ + PageID: "feat:test", + Category: "feat", + Key: "feat:test", + Title: "Test Feat", + Row: map[string]any{ + "DESCRIPTION": map[string]any{"tlk": map[string]any{"text": strings.Join([]string{ + "Type of Feat: General", + "", + "Prerequisite: Dexterity 13+", + "", + "Specifics: You gain a +2 dodge bonus.", + "The bonus applies only while unarmored.", + "", + "Use: Automatic.", + }, "\n")}}, + }, + } + + tests := []struct { + name string + template string + want string + reject string + }{ + { + name: "first paragraph", + template: `
              {{DESCRIPTION|text|paragraph:first}}
              `, + want: `
              Type of Feat: General
              `, + reject: `Prerequisite`, + }, + { + name: "specifics after marker as html", + template: `
              {{DESCRIPTION|text|after:"Specifics:"|before:"Use:"|trim|html}}
              `, + want: `

              You gain a +2 dodge bonus.
              The bonus applies only while unarmored.

              `, + reject: `Use: Automatic`, + }, + { + name: "first and last paragraphs", + template: `
              {{DESCRIPTION|text|paragraphs:"first,last"|html}}
              `, + want: `

              Type of Feat: General

              Use: Automatic.

              `, + reject: `Specifics`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ctx.renderWikiTemplateString("feat.html", tt.template, page) + if err != nil { + t.Fatalf("render template: %v", err) + } + if !strings.Contains(got, tt.want) { + t.Fatalf("expected output to contain %q:\n%s", tt.want, got) + } + if tt.reject != "" && strings.Contains(got, tt.reject) { + t.Fatalf("expected output not to contain %q:\n%s", tt.reject, got) + } + }) + } +} + +func TestWikiTemplateRendersConfiguredClassProgressionTable(t *testing.T) { + ctx := testWikiTableContext(t, ` +tables: + ClassProgression: + source: class_progression + class: "wiki-table wiki-class-progression" + rows: + levels: 1-4 + when: "{{Level >= 1 && Level <= 4}}" + columns: + - key: level + label: Lvl + value: "{{ordinal(Level)}}" + - key: bab + label: BAB + value: "{{BAB}}" + - key: saves + label: Saves + children: + - key: fort + label: Fort + value: "{{FortSave}}" + - key: reflex + label: Ref + value: "{{RefSave}}" + - key: will + label: Will + value: "{{WillSave}}" + - key: feats + label: Feats + value: "{{join(GrantedFeats, \", \")}}" + empty: "" + allow_wiki_markup: true + - key: hp_range + label: HP range + value: "{{range(HPMin, HPMax)}}" + - key: notes + label: Notes + value: "{{if(Level == 4 && BonusFeat != \"0\", link(\"feat:weapon_specialization\", \"Weapon Specialization\") + \" first available.\" + if(present(Notes), \" \" + Notes, \"\"), Notes)}}" + empty: "" + allow_wiki_markup: true +`) + page := wikiTemplatePage{ + PageID: "classes:fighter", + Category: "classes", + Key: "classes:fighter", + Title: "Fighter", + Row: map[string]any{ + "AttackBonusTable": "cls_atk_1", + "BonusFeatsTable": map[string]any{"table": "classes/bonusfeats:fighter"}, + "FeatsTable": map[string]any{"table": "classes/feats:fighter"}, + "HitDie": 10, + "SavingThrowTable": map[string]any{"table": "classes/saves:fortitude"}, + "meta": map[string]any{ + "wiki": map[string]any{ + "progression_notes": map[string]any{ + "4": "Manual level note.", + }, + }, + }, + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `

              {{title}}

              +{{table:ClassProgression}}`, page) + if err != nil { + t.Fatalf("render table template: %v", err) + } + for _, expected := range []string{ + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + } { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in rendered table:\n%s", expected, got) + } + } +} + +func TestWikiTemplateRendersConfiguredClassProgressionSpellColumns(t *testing.T) { + ctx := testWikiTableContext(t, ` +tables: + ClassProgression: + source: class_progression + class: "wiki-table wiki-class-progression" + rows: + levels: 1-2 + columns: + - key: level + label: Lvl + value: "{{Level}}" + - key: per_day + label: Base spells per day + when: "{{HasSpellsPerDay}}" + children: + - key: per_day_0 + label: 0 + value: "{{SpellsPerDay0}}" + empty: "-" + - key: per_day_1 + label: 1 + value: "{{SpellsPerDay1}}" + empty: "-" + - key: known + label: Known spells + when: "{{HasSpellsKnown}}" + children: + - key: known_0 + label: 0 + value: "{{SpellsKnown0}}" + empty: "-" + - key: known_1 + label: 1 + value: "{{SpellsKnown1}}" + empty: "-" +`) + ctx.classSpellGainTables["classes/spellsgained:bard"] = wikiTable{Rows: []map[string]any{ + {"Level": 1, "SpellLevel0": 2, "SpellLevel1": 1}, + {"Level": 2, "SpellLevel0": 3, "SpellLevel1": nil}, + }} + ctx.classSpellKnownTables["classes/spellsknown:bard"] = wikiTable{Rows: []map[string]any{ + {"Level": 1, "SpellLevel0": 4, "SpellLevel1": 2}, + {"Level": 2, "SpellLevel0": 5, "SpellLevel1": 3}, + }} + page := wikiTemplatePage{ + PageID: "classes:bard", + Category: "classes", + Key: "classes:bard", + Title: "Bard", + Row: map[string]any{ + "SpellCaster": 1, + "SpellGainTable": map[string]any{"table": "classes/spellsgained:bard"}, + "SpellKnownTable": map[string]any{"table": "classes/spellsknown:bard"}, + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `{{table:ClassProgression}}`, page) + if err != nil { + t.Fatalf("render table template: %v", err) + } + for _, expected := range []string{ + ``, + ``, + ``, + ``, + } { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in rendered table:\n%s", expected, got) + } + } +} + +func TestWikiTemplateRendersClassProgressionSpellcastingAdvancementNotes(t *testing.T) { + ctx := testWikiTableContext(t, ` +tables: + ClassProgression: + source: class_progression + rows: + levels: 1-4 + columns: + - key: level + label: Lvl + value: "{{Level}}" + - key: notes + label: Notes + value: "{{SpellcastingAdvancementNote + if(present(SpellcastingAdvancementNote) && present(Notes), \" \", \"\") + Notes}}" + empty: "" +`) + page := wikiTemplatePage{ + PageID: "classes:pale_master", + Category: "classes", + Key: "classes:pale_master", + Title: "Pale Master", + Row: map[string]any{ + "SpellCaster": 0, + "ArcSpellLvlMod": 2, + "DivSpellLvlMod": 0, + "meta": map[string]any{ + "wiki": map[string]any{ + "progression_notes": map[string]any{ + "3": "Manual level note.", + }, + }, + }, + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `{{table:ClassProgression}}`, page) + if err != nil { + t.Fatalf("render table template: %v", err) + } + for _, expected := range []string{ + ``, + ``, + ``, + ``, + } { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in rendered table:\n%s", expected, got) + } + } +} + +func TestWikiTemplateHidesUnusedConfiguredClassSpellColumns(t *testing.T) { + ctx := testWikiTableContext(t, ` +tables: + ClassProgression: + source: class_progression + class: "wiki-table wiki-class-progression" + rows: + levels: 1-2 + columns: + - key: level + label: Lvl + value: "{{Level}}" + - key: per_day + label: Base spells per day + when: "{{HasSpellsPerDay}}" + children: + - key: per_day_0 + label: 0 + value: "{{SpellsPerDay0}}" + when: "{{SpellsPerDay0}}" + - key: per_day_1 + label: 1 + value: "{{SpellsPerDay1}}" + when: "{{SpellsPerDay1}}" + - key: per_day_2 + label: 2 + value: "{{SpellsPerDay2}}" + when: "{{SpellsPerDay2}}" +`) + ctx.classSpellGainTables["classes/spellsgained:bard"] = wikiTable{Rows: []map[string]any{ + {"Level": 1, "SpellLevel0": 2, "SpellLevel1": 1, "SpellLevel2": nil}, + {"Level": 2, "SpellLevel0": 3, "SpellLevel1": 2, "SpellLevel2": nil}, + }} + page := wikiTemplatePage{ + PageID: "classes:bard", + Category: "classes", + Key: "classes:bard", + Title: "Bard", + Row: map[string]any{ + "SpellCaster": 1, + "SpellGainTable": map[string]any{"table": "classes/spellsgained:bard"}, + }, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `{{table:ClassProgression}}`, page) + if err != nil { + t.Fatalf("render table template: %v", err) + } + if !strings.Contains(got, ``) { + t.Fatalf("expected only two visible spell columns:\n%s", got) + } + if strings.Contains(got, ``) { + t.Fatalf("expected unused spell level 2 column to be hidden:\n%s", got) + } +} + +func TestWikiTemplateOmitsSpellColumnsForPreparedAndNonCasterClasses(t *testing.T) { + ctx := testWikiTableContext(t, ` +tables: + ClassProgression: + source: class_progression + rows: + levels: 1 + columns: + - key: level + label: Lvl + value: "{{Level}}" + - key: per_day + label: Base spells per day + when: "{{HasSpellsPerDay}}" + children: + - key: per_day_0 + label: 0 + value: "{{SpellsPerDay0}}" + - key: known + label: Known spells + when: "{{HasSpellsKnown}}" + children: + - key: known_0 + label: 0 + value: "{{SpellsKnown0}}" +`) + ctx.classSpellGainTables["classes/spellsgained:cleric"] = wikiTable{Rows: []map[string]any{ + {"Level": 1, "SpellLevel0": 3}, + }} + prepared := wikiTemplatePage{ + PageID: "classes:cleric", + Category: "classes", + Key: "classes:cleric", + Title: "Cleric", + Row: map[string]any{ + "SpellCaster": 1, + "SpellGainTable": map[string]any{"table": "classes/spellsgained:cleric"}, + }, + } + nonCaster := wikiTemplatePage{ + PageID: "classes:fighter", + Category: "classes", + Key: "classes:fighter", + Title: "Fighter", + Row: map[string]any{"SpellCaster": 0}, + } + + gotPrepared, err := ctx.renderWikiTemplateString("classes.html", `{{table:ClassProgression}}`, prepared) + if err != nil { + t.Fatalf("render prepared caster table: %v", err) + } + if !strings.Contains(gotPrepared, `Base spells per day`) || strings.Contains(gotPrepared, `Known spells`) { + t.Fatalf("expected prepared caster to render per-day but not known columns:\n%s", gotPrepared) + } + gotNonCaster, err := ctx.renderWikiTemplateString("classes.html", `{{table:ClassProgression}}`, nonCaster) + if err != nil { + t.Fatalf("render non-caster table: %v", err) + } + if strings.Contains(gotNonCaster, `Base spells per day`) || strings.Contains(gotNonCaster, `Known spells`) { + t.Fatalf("expected non-caster to omit spell columns:\n%s", gotNonCaster) + } +} + +func TestWikiTemplateRendersClassSpellbookProviderBySpellLevel(t *testing.T) { + root := t.TempDir() + dataPath := filepath.Join(root, "data.yaml") + writeFile(t, dataPath, ` +providers: + ClassSpellbook: + source: class_spellbook +`+"\n") + providers, err := loadWikiDataProviders(dataPath) + if err != nil { + t.Fatalf("load data providers: %v", err) + } + ctx := testWikiTableContext(t, "") + ctx.wikiDataPath = dataPath + ctx.wikiDataProviders = providers + ctx.spellRows = map[string]map[string]any{ + "spells:light": {"Name": map[string]any{"tlk": map[string]any{"text": "Light"}}}, + "spells:cure_light_wounds": {"Name": map[string]any{"tlk": map[string]any{"text": "Cure Light Wounds"}}}, + "spells:bless": {"Name": map[string]any{"tlk": map[string]any{"text": "Bless"}}}, + } + ctx.classSpellbooks = map[string]classSpellbookSpec{ + "classes:acolyte": { + Class: "classes:acolyte", + Levels: map[int][]string{ + 0: {"spells:light"}, + 1: {"spells:cure_light_wounds", "spells:bless"}, + }, + }, + } + page := wikiTemplatePage{ + PageID: "classes:acolyte", + Category: "classes", + Key: "classes:acolyte", + Title: "Acolyte", + Row: map[string]any{"SpellCaster": 1}, + } + + got, err := ctx.renderWikiTemplateString("classes.html", `{{#each ClassSpellbook}} +

              {{SpellLevelLabel}}

              {{Spells|list:"wiki-list wiki-spellbook-list"}} +{{/each}}`, page) + if err != nil { + t.Fatalf("render spellbook provider: %v", err) + } + for _, expected := range []string{ + `

              0-level spells

              • [[Spells/Light|Light]]
              `, + `

              1st-level spells

              • [[Spells/Bless|Bless]]
                • `, + `
              • [[Spells/Cure_Light_Wounds|Cure Light Wounds]]
                • `, + } { + if !strings.Contains(got, expected) { + t.Fatalf("expected %q in rendered spellbook:\n%s", expected, got) + } + } +} + +func TestLoadWikiTablesFromDirsSupportsCanonicalAndLegacyClassTableDirs(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "bonusfeats")) + mkdirAll(t, filepath.Join(root, "bfeat")) + writeFile(t, filepath.Join(root, "bonusfeats", "fighter.json"), `{ + "key": "classes/bonusfeats:fighter", + "output": "cls_bfeat_fight.2da", + "rows": [{"id": 1, "Bonus": "1"}] +}`+"\n") + writeFile(t, filepath.Join(root, "bfeat", "commoner.json"), `{ + "key": "classes/bonusfeats:commoner", + "output": "cls_bfeat_comm.2da", + "rows": [{"id": 1, "Bonus": "0"}] +}`+"\n") + + tables, err := loadWikiTablesFromDirs(filepath.Join(root, "bonusfeats"), filepath.Join(root, "bfeat")) + if err != nil { + t.Fatalf("load wiki tables from aliases: %v", err) + } + for _, key := range []string{"classes/bonusfeats:fighter", "cls_bfeat_fight", "classes/bonusfeats:commoner", "cls_bfeat_comm"} { + if _, ok := tables[key]; !ok { + t.Fatalf("expected table key %q in merged alias tables: %#v", key, tables) + } + } +} + +func TestWikiTableMissingFieldReportsTableAndColumn(t *testing.T) { + ctx := testWikiTableContext(t, ` +tables: + Broken: + source: class_progression + rows: + levels: 1 + columns: + - key: missing + label: Missing + value: "{{NoSuchField}}" +`) + page := wikiTemplatePage{ + PageID: "classes:fighter", + Category: "classes", + Key: "classes:fighter", + Row: map[string]any{ + "AttackBonusTable": "cls_atk_1", + "HitDie": 10, + "SavingThrowTable": map[string]any{"table": "classes/saves:fortitude"}, + }, + } + + _, err := ctx.renderWikiTemplateString("classes.html", `{{table:Broken}}`, page) + if err == nil { + t.Fatal("expected missing field error") + } + for _, expected := range []string{`classes:fighter`, `table Broken`, `column missing`, `NoSuchField`} { + if !strings.Contains(err.Error(), expected) { + t.Fatalf("expected error to contain %q, got %v", expected, err) + } + } +} + +func TestWikiTableRejectsImpossibleNestedHeader(t *testing.T) { + _, err := parseWikiTableDefinitions([]byte(` +tables: + Broken: + source: class_progression + rows: + levels: 1 + columns: + - key: bad + label: Bad + value: "{{Level}}" + children: + - key: child + label: Child + value: "{{Level}}" +`), "test-tables.yaml") + if err == nil || !strings.Contains(err.Error(), "cannot have both value and children") { + t.Fatalf("expected impossible nested header error, got %v", err) + } +} + +func TestWikiVisibilityPolicyRejectsNullLikeFeatsAndHiddenReferences(t *testing.T) { + root := testProjectRoot(t) + for _, dir := range []string{ + filepath.Join(root, "topdata", "data", "classes", "core"), + filepath.Join(root, "topdata", "data", "classes", "feats"), + filepath.Join(root, "topdata", "data", "classes", "skills"), + filepath.Join(root, "topdata", "data", "feat"), + filepath.Join(root, "topdata", "data", "skills"), + filepath.Join(root, "topdata", "wiki"), + filepath.Join(root, "topdata", "wiki", "templates", "pages"), + } { + mkdirAll(t, dir) + } + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "wiki", "templates", "pages", "classes.html"), `

                {{title}}

                +{{format:SkillList}} +{{table:FeatProgression}}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "wiki", "tables.yaml"), ` +tables: + FeatProgression: + source: class_feats + columns: + - key: feat + label: Feat + value: "{{link(FeatKey, FeatName)}}" + allow_wiki_markup: true +`+"\n") + writeFile(t, filepath.Join(root, "topdata", "wiki", "visibility.yaml"), ` +reference_sets: + class_feats: + target_dataset: feat + sources: + - dataset: classes + table_field: FeatsTable + table_kind: class_feats + row_ref_field: FeatIndex +datasets: + classes: + eligibility: + when: "{{all_present(Name, Description)}}" + include: + - when: "{{PlayerClass == 1}}" + feat: + eligibility: + when: "{{all_present(FEAT, DESCRIPTION)}}" + include: + - when: "{{ALLCLASSESCANUSE == 1}}" + - reference_set: class_feats + exclude: + - when: "{{PreReqEpic == 1}}" + skills: + eligibility: + when: "{{all_present(Name, Description)}}" + include: + - when: "{{true}}" + exclude: + - when: "{{HideFromLevelUp == 1}}" +`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp"], + "rows": [ + {"id": 0, "key": "skills:athletics", "Label": "Athletics", "Name": {"tlk": {"text": "Athletics"}}, "Description": {"tlk": {"text": "Visible skill."}}, "HideFromLevelUp": "0"}, + {"id": 1, "key": "skills:secret", "Label": "Secret", "Name": {"tlk": {"text": "Secret"}}, "Description": {"tlk": {"text": "Hidden skill."}}, "HideFromLevelUp": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0,"skills:secret":1}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION", "ALLCLASSESCANUSE", "PreReqEpic"], + "rows": [ + {"id": 0, "key": "feat:visible", "LABEL": "Visible", "FEAT": {"tlk": {"text": "Visible Feat"}}, "DESCRIPTION": {"tlk": {"text": "Visible feat description."}}, "ALLCLASSESCANUSE": "1", "PreReqEpic": "0"}, + {"id": 1, "key": "feat:epic", "LABEL": "Epic", "FEAT": {"tlk": {"text": "Epic Feat"}}, "DESCRIPTION": {"tlk": {"text": "Epic feat description."}}, "ALLCLASSESCANUSE": "1", "PreReqEpic": "1"}, + {"id": 2, "key": "feat:nullfeat", "LABEL": "Null", "FEAT": "****", "DESCRIPTION": {"tlk": {"text": "Null feat description."}}, "ALLCLASSESCANUSE": "1", "PreReqEpic": "0"}, + {"id": 3, "key": "feat:class_only", "LABEL": "ClassOnly", "FEAT": {"tlk": {"text": "Class Feat"}}, "DESCRIPTION": {"tlk": {"text": "Class feat description."}}, "ALLCLASSESCANUSE": "0", "PreReqEpic": "0"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:visible":0,"feat:epic":1,"feat:nullfeat":2,"feat:class_only":3}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "base.json"), `{ + "output": "classes.2da", + "columns": ["Label", "Name", "Description", "PlayerClass", "SkillsTable", "FeatsTable"], + "rows": [ + {"id": 0, "key": "classes:fighter", "Label": "Fighter", "Name": {"tlk": {"text": "Fighter"}}, "Description": {"tlk": {"text": "Fighter text."}}, "PlayerClass": "1", "SkillsTable": {"table": "classes/skills:fighter"}, "FeatsTable": {"table": "classes/feats:fighter"}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "core", "lock.json"), `{"classes:fighter":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "skills", "fighter.json"), `{ + "key": "classes/skills:fighter", + "output": "cls_skill_fight.2da", + "columns": ["SkillIndex", "ClassSkill"], + "rows": [ + {"SkillIndex": {"id": "skills:athletics"}, "ClassSkill": "1"}, + {"SkillIndex": {"id": "skills:secret"}, "ClassSkill": "1"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "classes", "feats", "fighter.json"), `{ + "key": "classes/feats:fighter", + "output": "cls_feat_fight.2da", + "columns": ["FeatIndex", "GrantedOnLevel", "List"], + "rows": [ + {"FeatIndex": {"id": "feat:class_only"}, "GrantedOnLevel": "1", "List": "3"}, + {"FeatIndex": {"id": "feat:epic"}, "GrantedOnLevel": "1", "List": "3"}, + {"FeatIndex": {"id": "feat:nullfeat"}, "GrantedOnLevel": "1", "List": "3"} + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.TopData.Wiki.VisibilityFile = "visibility.yaml" + + if _, err := buildWiki(proj, BuildResult{}, true, nil); err != nil { + t.Fatalf("buildWiki failed: %v", err) + } + for _, path := range []string{ + filepath.Join(root, ".cache", "wiki", "pages", "skills", "secret.html"), + filepath.Join(root, ".cache", "wiki", "pages", "feat", "epic.html"), + filepath.Join(root, ".cache", "wiki", "pages", "feat", "nullfeat.html"), + } { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("expected hidden or ineligible wiki page %s to be omitted, got %v", path, err) + } + } + for _, path := range []string{ + filepath.Join(root, ".cache", "wiki", "pages", "feat", "Visible_Feat.html"), + filepath.Join(root, ".cache", "wiki", "pages", "feat", "Class_Feat.html"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected visible wiki page %s: %v", path, err) + } + } + classPage, err := os.ReadFile(filepath.Join(root, ".cache", "wiki", "pages", "classes", "Fighter.html")) + if err != nil { + t.Fatalf("read class page: %v", err) + } + classText := string(classPage) + for _, hidden := range []string{"Secret", "Epic Feat", "Null"} { + if strings.Contains(classText, hidden) { + t.Fatalf("expected hidden %q reference to be omitted from class page:\n%s", hidden, classText) + } + } + for _, visible := range []string{"[[Skills/Athletics|Athletics]]", "[[Feats/Class_Feat|Class Feat]]"} { + if !strings.Contains(classText, visible) { + t.Fatalf("expected visible %q reference in class page:\n%s", visible, classText) + } + } + indexRaw, err := os.ReadFile(filepath.Join(root, ".cache", "wiki", "page-index.json")) + if err != nil { + t.Fatalf("read page index: %v", err) + } + if strings.Contains(string(indexRaw), `"page_id": "skills:secret"`) || + strings.Contains(string(indexRaw), `"page_id": "feat:epic"`) || + strings.Contains(string(indexRaw), `"page_id": "feat:nullfeat"`) { + t.Fatalf("expected hidden and ineligible pages out of page index:\n%s", string(indexRaw)) + } +} + +func TestWikiVisibilityPredicatesTreatNullLikeFieldsAsAbsent(t *testing.T) { + policy, err := parseWikiVisibilityDefinitions([]byte(` +datasets: + feat: + eligibility: + when: "{{all_present(FEAT, DESCRIPTION)}}" + include: + - when: "{{true}}" +`), "visibility.yaml") + if err != nil { + t.Fatalf("parse visibility policy: %v", err) + } + ctx := &wikiContext{} + for name, row := range map[string]map[string]any{ + "missing": {"DESCRIPTION": "Description"}, + "null": {"FEAT": nil, "DESCRIPTION": "Description"}, + "nwn-null": {"FEAT": "****", "DESCRIPTION": "Description"}, + "empty": {"FEAT": "", "DESCRIPTION": "Description"}, + "present": {"FEAT": "Feat", "DESCRIPTION": "Description"}, + "missing-desc": {"FEAT": "Feat"}, + } { + got, err := ctx.evalWikiVisibilityPredicate(policy.Datasets["feat"].Eligibility.When, row) + if err != nil { + t.Fatalf("%s predicate failed: %v", name, err) + } + if want := name == "present"; got != want { + t.Fatalf("%s expected eligibility %v, got %v", name, want, got) + } + } +} + +func TestWikiVisibilityPolicyRejectsInvalidDatasetAndReferenceSet(t *testing.T) { + tests := []struct { + name string + yaml string + want string + }{ + { + name: "unknown dataset", + yaml: ` +datasets: + traps: + include: + - when: "{{true}}" +`, + want: `unknown dataset "traps"`, + }, + { + name: "unknown reference set", + yaml: ` +datasets: + feat: + include: + - reference_set: missing +`, + want: `unknown reference_set "missing"`, + }, + { + name: "bad table kind", + yaml: ` +reference_sets: + feats: + target_dataset: feat + sources: + - dataset: classes + table_field: FeatsTable + table_kind: unsupported + row_ref_field: FeatIndex +datasets: + feat: + include: + - reference_set: feats +`, + want: `unsupported table_kind`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := parseWikiVisibilityDefinitions([]byte(tt.yaml), "visibility.yaml") + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("expected %q error, got %v", tt.want, err) + } + }) + } +} + +func TestWikiVisibilityCanExcludeUnreachableFeatPrerequisites(t *testing.T) { + policy, err := parseWikiVisibilityDefinitions([]byte(` +datasets: + feat: + include: + - when: "{{true}}" + exclude: + - when: "{{hidden_ref(PREREQFEAT1, \"feat\")}}" + - when: "{{hidden_ref(REQSKILL, \"skills\")}}" + - when: "{{all_hidden_refs(OrReqFeat0, OrReqFeat1, OrReqFeat2, OrReqFeat3, OrReqFeat4, \"feat\")}}" +`), "visibility.yaml") + if err != nil { + t.Fatalf("parse visibility policy: %v", err) + } + ctx := &wikiContext{ + visibilityPolicy: &policy, + visibleWikiKeys: map[string]bool{ + "feat:visible": true, + "feat:hidden": false, + "skills:visible": true, + "skills:hidden": false, + }, + featIDToKey: map[int]string{1: "feat:visible", 2: "feat:hidden"}, + skillIDToKey: map[int]string{1: "skills:visible", 2: "skills:hidden"}, + } + definition := policy.Datasets["feat"] + featRow := func(values map[string]any) map[string]any { + row := map[string]any{"FEAT": "Test Feat", "DESCRIPTION": "Description"} + for key, value := range values { + row[key] = value + } + return row + } + for name, tt := range map[string]struct { + row map[string]any + want bool + }{ + "visible required feat": { + row: featRow(map[string]any{"PREREQFEAT1": 1}), + want: true, + }, + "hidden required feat": { + row: featRow(map[string]any{"PREREQFEAT1": 2}), + want: false, + }, + "missing required feat is unreachable": { + row: featRow(map[string]any{"PREREQFEAT1": 999}), + want: false, + }, + "visible required skill": { + row: featRow(map[string]any{"REQSKILL": "skills:visible"}), + want: true, + }, + "hidden required skill": { + row: featRow(map[string]any{"REQSKILL": "skills:hidden"}), + want: false, + }, + "null prerequisite fields are ignored": { + row: featRow(map[string]any{"PREREQFEAT1": "****", "REQSKILL": "****"}), + want: true, + }, + "or group with visible option remains reachable": { + row: featRow(map[string]any{"OrReqFeat0": 2, "OrReqFeat1": 1}), + want: true, + }, + "or group with only hidden options is unreachable": { + row: featRow(map[string]any{"OrReqFeat0": 2, "OrReqFeat1": "feat:missing"}), + want: false, + }, + "empty or group is ignored": { + row: featRow(map[string]any{"OrReqFeat0": "****", "OrReqFeat1": "****"}), + want: true, + }, + } { + got, err := ctx.evaluateWikiVisibility("feat", "feat:"+name, tt.row, definition, nil) + if err != nil { + t.Fatalf("%s visibility evaluation failed: %v", name, err) + } + if got != tt.want { + t.Fatalf("%s expected visibility %v, got %v", name, tt.want, got) + } + } +} + +func TestWikiVisibilityIndexUsesBuiltVisibilityForReferenceHelpers(t *testing.T) { + policy, err := parseWikiVisibilityDefinitions([]byte(` +datasets: + feat: + include: + - when: "{{true}}" + exclude: + - when: "{{hidden_ref(PREREQFEAT1, \"feat\")}}" +`), "visibility.yaml") + if err != nil { + t.Fatalf("parse visibility policy: %v", err) + } + ctx := &wikiContext{ + visibilityPolicy: &policy, + featRows: map[string]map[string]any{ + "feat:zbase": { + "FEAT": "Base Feat", + "DESCRIPTION": "Description", + }, + "feat:dependent": { + "FEAT": "Dependent Feat", + "DESCRIPTION": "Description", + "PREREQFEAT1": 1, + }, + "feat:chain_dependent": { + "FEAT": "Chain Dependent Feat", + "DESCRIPTION": "Description", + "PREREQFEAT1": 2, + }, + "feat:missing_prereq": { + "FEAT": "Missing Prereq Feat", + "DESCRIPTION": "Description", + "PREREQFEAT1": 999, + }, + }, + featIDToKey: map[int]string{1: "feat:zbase", 2: "feat:dependent"}, + } + visible, err := ctx.buildWikiVisibilityIndex(&policy) + if err != nil { + t.Fatalf("build visibility index: %v", err) + } + if !visible["feat:zbase"] { + t.Fatalf("expected base feat to be visible") + } + if !visible["feat:dependent"] { + t.Fatalf("expected feat with visible prerequisite to remain visible") + } + if !visible["feat:chain_dependent"] { + t.Fatalf("expected feat with transitive visible prerequisite to remain visible") + } + if visible["feat:missing_prereq"] { + t.Fatalf("expected feat with missing prerequisite to be hidden") + } +} + +func TestWikiVisibilityMetadataOverrideCannotBypassEligibility(t *testing.T) { + policy, err := parseWikiVisibilityDefinitions([]byte(` +datasets: + feat: + eligibility: + when: "{{present(FEAT)}}" + include: + - when: "{{ALLCLASSESCANUSE == 1}}" + exclude: + - when: "{{PreReqEpic == 1}}" +`), "visibility.yaml") + if err != nil { + t.Fatalf("parse visibility policy: %v", err) + } + ctx := &wikiContext{} + definition := policy.Datasets["feat"] + for name, tt := range map[string]struct { + row map[string]any + want bool + }{ + "exclude overridden": { + row: map[string]any{ + "FEAT": "Visible", + "DESCRIPTION": "Description", + "PreReqEpic": "1", + "ALLCLASSESCANUSE": "1", + "meta": map[string]any{"wiki": map[string]any{"generate": "1"}}, + }, + want: true, + }, + "eligibility preserved": { + row: map[string]any{ + "FEAT": "****", + "DESCRIPTION": "Description", + "ALLCLASSESCANUSE": "1", + "meta": map[string]any{"wiki": map[string]any{"generate": "1"}}, + }, + want: false, + }, + } { + got, err := ctx.evaluateWikiVisibility("feat", "feat:"+name, tt.row, definition, nil) + if err != nil { + t.Fatalf("%s visibility evaluation failed: %v", name, err) + } + if got != tt.want { + t.Fatalf("%s expected visibility %v, got %v", name, tt.want, got) + } + } +} + +func testWikiTableContext(t *testing.T, yamlSource string) *wikiContext { + t.Helper() + ctx := &wikiContext{ + featRows: map[string]map[string]any{}, + featIDToKey: map[int]string{}, + classFeatTables: map[string]wikiTable{}, + classBonusTables: map[string]wikiTable{}, + classSaveTables: map[string]wikiTable{}, + classSpellGainTables: map[string]wikiTable{}, + classSpellKnownTables: map[string]wikiTable{}, + classSpellbooks: map[string]classSpellbookSpec{}, + wikiTablesPath: "test-tables.yaml", + } + ctx.featRows["feat:literate"] = map[string]any{"FEAT": map[string]any{"tlk": map[string]any{"text": "Literate"}}} + ctx.featRows["feat:weapon_proficiency_simple"] = map[string]any{"FEAT": map[string]any{"tlk": map[string]any{"text": "Simple Weapon Proficiency"}}} + ctx.featRows["feat:weapon_specialization"] = map[string]any{"FEAT": map[string]any{"tlk": map[string]any{"text": "Weapon Specialization"}}} + ctx.classFeatTables["cls_feat_fight"] = wikiTable{OutputStem: "cls_feat_fight", Rows: []map[string]any{ + {"FeatIndex": map[string]any{"id": "feat:literate"}, "GrantedOnLevel": "1", "List": "3"}, + {"FeatIndex": map[string]any{"id": "feat:weapon_proficiency_simple"}, "GrantedOnLevel": "1", "List": "3"}, + }} + ctx.classFeatTables["classes/feats:fighter"] = ctx.classFeatTables["cls_feat_fight"] + ctx.classBonusTables["cls_bfeat_fight"] = wikiTable{OutputStem: "cls_bfeat_fight", Rows: []map[string]any{ + {"id": 1, "Bonus": "1"}, {"id": 2, "Bonus": "1"}, {"id": 3, "Bonus": "0"}, {"id": 4, "Bonus": "1"}, + }} + ctx.classBonusTables["classes/bonusfeats:fighter"] = ctx.classBonusTables["cls_bfeat_fight"] + ctx.classSaveTables["cls_savthr_fort"] = wikiTable{OutputStem: "cls_savthr_fort", Rows: []map[string]any{ + {"Level": "1", "FortSave": "2", "RefSave": "0", "WillSave": "0"}, + {"Level": "2", "FortSave": "3", "RefSave": "0", "WillSave": "0"}, + {"Level": "3", "FortSave": "3", "RefSave": "1", "WillSave": "1"}, + {"Level": "4", "FortSave": "4", "RefSave": "1", "WillSave": "1"}, + }} + ctx.classSaveTables["classes/saves:fortitude"] = ctx.classSaveTables["cls_savthr_fort"] + tables, err := parseWikiTableDefinitions([]byte(yamlSource), ctx.wikiTablesPath) + if err != nil { + t.Fatalf("parse table definitions: %v", err) + } + ctx.wikiTableDefinitions = tables + return ctx +} + +func TestWikiManagedHashIgnoresPreservedSectionBody(t *testing.T) { + first := strings.Join([]string{ + ``, + ``, + `

                Acolyte

                `, + ``, + `

                Original user edit.

                `, + ``, + `

                Generated.

                `, + ``, + }, "\n") + second := strings.Replace(first, "

                Original user edit.

                ", "

                Changed by a user.

                ", 1) + + if computeManagedHash(first) != computeManagedHash(second) { + t.Fatalf("expected preserved section body not to affect managed hash") + } +} + +func TestWikiSourceDigestIncludesTemplates(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "data", "skills")) + mkdirAll(t, filepath.Join(root, "wiki", "templates", "pages")) + writeFile(t, filepath.Join(root, "data", "skills", "base.json"), `{"rows":[]}`+"\n") + templatePath := filepath.Join(root, "wiki", "templates", "pages", "skills.html") + writeFile(t, templatePath, `

                {{title}}

                `+"\n") + proj := &project.Project{ + Root: root, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + TopData: project.TopDataConfig{Source: ".", Wiki: project.TopDataWikiConfig{Source: "wiki"}}, + }, + } + + before, err := computeWikiSourceDigest(proj) + if err != nil { + t.Fatalf("compute initial digest: %v", err) + } + writeFile(t, templatePath, `

                {{title}}

                Changed

                `+"\n") + after, err := computeWikiSourceDigest(proj) + if err != nil { + t.Fatalf("compute changed digest: %v", err) + } + if before == after { + t.Fatalf("expected template edit to change wiki source digest") + } +} + +func TestBuildWikiHonorsConfiguredWikiPaths(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + mkdirAll(t, filepath.Join(root, "docs", "wiki", "page-fragments")) + mkdirAll(t, filepath.Join(root, "docs", "wiki", "config")) + mkdirAll(t, filepath.Join(root, "docs", "wiki", "sections")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "columns": ["Label", "Name"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}} + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + writeFile(t, filepath.Join(root, "docs", "wiki", "page-fragments", "skills.html"), `

                {{title}}

                `+"\n") + writeFile(t, filepath.Join(root, "docs", "wiki", "config", "providers.yaml"), "providers: {}\n") + writeFile(t, filepath.Join(root, "docs", "wiki", "config", "paths.yaml"), "page_paths: {}\n") + writeFile(t, filepath.Join(root, "docs", "wiki", "config", "tables.yaml"), "tables: {}\n") + writeFile(t, filepath.Join(root, "docs", "wiki", "config", "visibility.yaml"), "datasets: {}\n") + writeFile(t, filepath.Join(root, "docs", "wiki", "sections", "defaults.yaml"), ` +manual_sections: + - id: user_top + heading: Custom Top +`) + + statusPages := false + proj := &project.Project{ + Root: root, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: project.PathConfig{Build: "build", Cache: ".cache"}, + TopData: project.TopDataConfig{ + Source: "topdata", + Build: ".cache", + Wiki: project.TopDataWikiConfig{ + Source: "docs/wiki", + DataFile: "config/providers.yaml", + PagePathsFile: "config/paths.yaml", + TablesFile: "config/tables.yaml", + VisibilityFile: "config/visibility.yaml", + PageTemplatesDir: "page-fragments", + ManualSectionsFile: "sections/defaults.yaml", + PageIndexFile: "manifest/pages.json", + StatusPages: &statusPages, + }, + }, + }, + } + + result, err := buildWiki(proj, BuildResult{}, true, nil) + if err != nil { + t.Fatalf("build wiki with configured paths: %v", err) + } + if result.PageCount != 1 { + t.Fatalf("expected generated skill page, got %d", result.PageCount) + } + pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "Athletics.html") + pageRaw, err := os.ReadFile(pagePath) + if err != nil { + t.Fatalf("read configured-path wiki page: %v", err) + } + if !strings.Contains(string(pageRaw), "Custom Top") { + t.Fatalf("expected configured manual section file to be used, got:\n%s", pageRaw) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "manifest", "pages.json")); err != nil { + t.Fatalf("expected configured page-index file: %v", err) + } +} + +func TestWikiPageIDForKeyNormalizesSlashSeparatorsOnly(t *testing.T) { + got := wikiPageIDForKey("feat:special/attacks_bull_rush") + if want := "feat:special_attacks_bull_rush"; got != want { + t.Fatalf("expected normalized page ID %q, got %q", want, got) + } + if got := wikiPageIDForKey("feat:bardic_music_inspire_competence"); got != "feat:bardic_music_inspire_competence" { + t.Fatalf("expected underscore to remain in page ID leaf, got %q", got) + } + if got := wikiPageIDForKey("feat:bardic_music/inspire_competence"); got != "feat:bardic_music_inspire_competence" { + t.Fatalf("expected slash-authored key to stay in one page ID leaf, got %q", got) + } +} + +func TestWikiPageOutputRelPathUsesCanonicalTitleLeaf(t *testing.T) { + if got, want := wikiPageOutputRelPath("feat:bardic_music_inspire_competence", "Inspire Competence"), filepath.FromSlash("feat/Inspire_Competence.html"); got != want { + t.Fatalf("expected underscore page ID to stay in one file leaf, got %q want %q", got, want) + } + if got, want := wikiPageOutputRelPath(wikiPageIDForKey("feat:bardic_music/inspire_competence"), "Inspire Competence"), filepath.FromSlash("feat/Inspire_Competence.html"); got != want { + t.Fatalf("expected slash-authored key to stay in one file leaf, got %q want %q", got, want) + } +} + +func TestSaveWikiPageIndexRejectsDuplicateOutputPaths(t *testing.T) { + err := saveWikiPageIndex(filepath.Join(t.TempDir(), "page-index.json"), wikiPageIndex{ + Version: wikiGeneratorVersion, + Pages: []wikiPageIndexEntry{ + {PageID: "feat:special:attacks:bull:rush", OutputPath: "pages/feat/special/attacks/bull/rush.html"}, + {PageID: "feat:special/attacks:bull:rush", OutputPath: "pages/feat/special/attacks/bull/rush.html"}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "duplicate wiki page-index output_path") { + t.Fatalf("expected duplicate output_path validation error, got %v", err) + } +} + +func TestSaveWikiPageIndexRejectsDuplicatePublicPaths(t *testing.T) { + err := saveWikiPageIndex(filepath.Join(t.TempDir(), "page-index.json"), wikiPageIndex{ + Version: wikiGeneratorVersion, + Pages: []wikiPageIndexEntry{ + {PageID: "spells:darkness", PublicPath: "Spells/Darkness", OutputPath: "pages/spells/darkness.html"}, + {PageID: "spells:gwildshape:driderdarkness", PublicPath: "Spells/Darkness", OutputPath: "pages/spells/gwildshape/driderdarkness.html"}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "duplicate wiki page-index public_path") { + t.Fatalf("expected duplicate public_path validation error, got %v", err) + } +} + +func TestSaveWikiPageIndexRejectsFoldedDuplicatePublicPaths(t *testing.T) { + err := saveWikiPageIndex(filepath.Join(t.TempDir(), "page-index.json"), wikiPageIndex{ + Version: wikiGeneratorVersion, + Pages: []wikiPageIndexEntry{ + {PageID: "spells:fear", PublicPath: "Magic/Fear", OutputPath: "pages/spells/fear.html"}, + {PageID: "spells:accented_fear", PublicPath: "magic/fear", OutputPath: "pages/spells/accented_fear.html"}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "duplicate wiki page-index public_path") { + t.Fatalf("expected folded duplicate public_path validation error, got %v", err) + } +} + +func TestPublicWikiPathSplitsTitlePathWithoutWhitespaceAroundSeparator(t *testing.T) { + ctx := &wikiContext{namespaceTitles: map[string]string{"spells": "Magic"}} + if got := ctx.publicWikiPath("spells", "Parent::Child"); got != "Magic/Parent/Child" { + t.Fatalf("expected NodeBB-style title path split, got %q", got) + } + if got := wikiPageOutputRelPath("spells:parent_child", "Parent::Child"); got != filepath.FromSlash("spells/Parent/Child.html") { + t.Fatalf("expected title-based output path split, got %q", got) + } +} + +func TestParseWikiPagePathDeclarationsRejectsLegacySlugOverrides(t *testing.T) { + _, err := parseWikiPagePathDeclarations([]byte(` +page_paths: + slug_overrides: + - page_id: "spells:pdk:fear" + slug: "pdk-fear" +`), "wiki.yaml") + if err == nil || !strings.Contains(err.Error(), `page_paths.slug_overrides is retired`) { + t.Fatalf("expected retired slug override error, got %v", err) + } +} + +func TestCanonicalWikiSegmentPreservesCaseAndUnderscoreSpaces(t *testing.T) { + tests := map[string]string{ + `Inspire Competence`: "Inspire_Competence", + `Grandmaster's Battle Momentum`: "Grandmasters_Battle_Momentum", + `Bigby’s Clenched Fist`: "Bigbys_Clenched_Fist", + `"Quoted" ‘Curly’ “Title”`: "Quoted_Curly_Title", + `Hardiness vs. Enchantments`: "Hardiness_vs_Enchantments", + `Clairaudience/Clairvoyance`: "Clairaudience_Clairvoyance", + `Moon-on-a-Stick`: "Moon_on_a_Stick", + `Élite & Noble Houses`: "Elite_Noble_Houses", + `Rock 'n' Roll`: "Rock_n_Roll", + `Melf's Acid Arrow`: "Melfs_Acid_Arrow", + `Alpha!@#$%^&*()[]-=_+/?.,<>` + "`" + `~|\Omega`: "Alpha_Omega", + `Æther Œuvre Øresund Straße Þorn Łódź Đelta`: "Aether_Oeuvre_Oresund_Strasse_Thorn_Lodz_Delta", + } + for input, want := range tests { + if got := canonicalWikiSegment(input); got != want { + t.Fatalf("canonicalWikiSegment(%q) = %q, want %q", input, got, want) + } + } +} + +func TestCanonicalWikiSegmentFoldedKeyMatchesPluginFixture(t *testing.T) { + tests := []struct { + source string + canonical string + folded string + }{ + {source: "Inspire Competence", canonical: "Inspire_Competence", folded: "inspire competence"}, + {source: "Grandmaster's Battle Momentum", canonical: "Grandmasters_Battle_Momentum", folded: "grandmasters battle momentum"}, + {source: "Æther Œuvre Øresund Straße Þorn Łódź Đelta", canonical: "Aether_Oeuvre_Oresund_Strasse_Thorn_Lodz_Delta", folded: "aether oeuvre oresund strasse thorn lodz delta"}, + } + for _, tt := range tests { + if got := canonicalWikiSegment(tt.source); got != tt.canonical { + t.Fatalf("canonicalWikiSegment(%q) = %q, want %q", tt.source, got, tt.canonical) + } + if got := canonicalWikiSegmentFoldedKey(tt.source); got != tt.folded { + t.Fatalf("canonicalWikiSegmentFoldedKey(%q) = %q, want %q", tt.source, got, tt.folded) + } + } +} + +func TestWikiStatusPagesEmitCanonicalLinks(t *testing.T) { + ctx := &wikiContext{} + report := renderWikiStatusReportPage(map[string]map[string]any{ + "skills": { + "status": wikiStatusVanilla, + "new": 0, + "modified": 0, + "vanilla": 1, + "total": 1, + }, + }, []string{"skills"}, ctx) + if !strings.Contains(report, "[[Skills|Skills]]") || strings.Contains(report, "[[skills:index|Skills]]") { + t.Fatalf("expected status report namespace link to use canonical public path, got:\n%s", report) + } + + listing := renderWikiStatusListingPage("Skills Vanilla Pages", []string{"skills:inspire_competence", "feat:grandmaster_battle_momentum"}, map[string]string{ + "skills:inspire_competence": "Inspire Competence", + "feat:grandmaster_battle_momentum": "Grandmaster's Battle Momentum", + }, ctx) + for _, want := range []string{ + "[[Skills/Inspire_Competence|Inspire Competence]]", + "[[Feats/Grandmasters_Battle_Momentum|Grandmaster's Battle Momentum]]", + } { + if !strings.Contains(listing, want) { + t.Fatalf("expected status listing canonical link %q, got:\n%s", want, listing) + } + } + if strings.Contains(listing, "[[skills:inspire_competence|") || strings.Contains(listing, "[[feat:grandmaster_battle_momentum|") { + t.Fatalf("expected status listing to omit generated page IDs as public targets, got:\n%s", listing) + } + + customCtx := &wikiContext{ + namespaceTitles: map[string]string{ + "spells": "Magic", + }, + } + customReport := renderWikiStatusReportPage(map[string]map[string]any{ + "spells": { + "status": wikiStatusNew, + "new": 1, + "modified": 0, + "vanilla": 0, + "total": 1, + }, + }, []string{"spells"}, customCtx) + if !strings.Contains(customReport, "[[Magic|Magic]]") || strings.Contains(customReport, "[[Spells|Spells]]") { + t.Fatalf("expected custom namespace status report link to use declared canonical title, got:\n%s", customReport) + } + customListing := renderWikiStatusListingPage("Magic New Pages", []string{"spells:fear"}, map[string]string{ + "spells:fear": "Fear", + }, customCtx) + if !strings.Contains(customListing, "[[Magic/Fear|Fear]]") || strings.Contains(customListing, "[[Spells/Fear|Fear]]") { + t.Fatalf("expected custom namespace status listing link to use declared canonical title, got:\n%s", customListing) + } +} + +func TestPublicWikiTargetUsesCanonicalNamespaceAndTitlePath(t *testing.T) { + ctx := &wikiContext{ + namespaceTitles: map[string]string{ + "feat": "Feats", + }, + namespacePaths: map[string]string{ + "feat": "Wiki/Feats", + }, + rowsByKey: map[string]map[string]any{ + "feat:hardiness_versus_enchantments": {"FEAT": map[string]any{"tlk": map[string]any{"text": "Hardiness vs. Enchantments"}}}, + }, + } + if got, want := ctx.publicWikiTargetForKey("feat:hardiness_versus_enchantments", "Display Alias"), "Wiki/Feats/Hardiness_vs_Enchantments"; got != want { + t.Fatalf("expected canonical title path %q, got %q", want, got) + } +} + +func TestPublicWikiTargetUsesTargetTitleBeforeDisplayLabel(t *testing.T) { + ctx := &wikiContext{ + namespaceTitles: map[string]string{ + "feat": "Feats", + }, + rowsByKey: map[string]map[string]any{ + "feat:hardiness:versus:enchantments": {"FEAT": "Hardiness vs. Enchantments"}, + }, + } + if got, want := ctx.publicWikiTargetForKey("feat:hardiness:versus:enchantments", "Display Alias"), "Feats/Hardiness_vs_Enchantments"; got != want { + t.Fatalf("expected target title slug %q, got %q", want, got) + } +} + +func TestBuildAndPackageBuildsWikiOnlyWhenRequested(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp", "Untrained", "KeyAbility", "ArmorCheckPenalty", "Constant"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, + "Description": {"tlk": {"key": "skills:athletics.description", "text": "Skill text."}}, + "HideFromLevelUp": "0", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "Constant": "SKILL_ATHLETICS" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + result, err := BuildAndPackageWithOptions(proj, BuildAndPackageOptions{}, nil) + if err != nil { + t.Fatalf("BuildAndPackageWithOptions failed: %v", err) + } + if result.WikiOutputDir != "" || result.WikiPages != 0 { + t.Fatalf("expected wiki generation to be skipped by default, got dir=%q pages=%d", result.WikiOutputDir, result.WikiPages) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); !os.IsNotExist(err) { + t.Fatalf("expected no wiki state without explicit wiki build, got %v", err) + } + + result, err = BuildAndPackageWithOptions(proj, BuildAndPackageOptions{Force: true, BuildWiki: true}, nil) + if err != nil { + t.Fatalf("BuildAndPackageWithOptions with wiki failed: %v", err) + } + generatedHTMLCount := countGeneratedWikiHTMLFiles(t, filepath.Join(root, ".cache", "wiki", "pages")) + if result.WikiPages != generatedHTMLCount { + t.Fatalf("expected wiki page count to match generated HTML files when requested, got result=%d files=%d", result.WikiPages, generatedHTMLCount) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil { + t.Fatalf("expected wiki state after explicit wiki build: %v", err) + } +} + +func TestBuildNativeRegeneratesWikiWhenTLKTextChanges(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + basePath := filepath.Join(root, "topdata", "data", "skills", "base.json") + writeFile(t, basePath, `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp", "Untrained", "KeyAbility", "ArmorCheckPenalty", "Constant"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, + "Description": {"tlk": {"key": "skills:athletics.description", "text": "Original text."}}, + "HideFromLevelUp": "0", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "Constant": "SKILL_ATHLETICS" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.Autogen.Consumers = nil + + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("initial BuildNative failed: %v", err) + } + writeFile(t, basePath, `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp", "Untrained", "KeyAbility", "ArmorCheckPenalty", "Constant"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, + "Description": {"tlk": {"key": "skills:athletics.description", "text": "Updated text."}}, + "HideFromLevelUp": "0", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "Constant": "SKILL_ATHLETICS" + } + ] +}`+"\n") + + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("BuildNative after text change failed: %v", err) + } + pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "Athletics.html") + got, err := os.ReadFile(pagePath) + if err != nil { + t.Fatalf("read regenerated page: %v", err) + } + if !strings.Contains(string(got), "Updated text.") { + t.Fatalf("expected regenerated page to contain updated description:\n%s", string(got)) + } +} + +func TestBuildPackageIgnoresWikiSourcesForFreshness(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "skills")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{ + "output": "skills.2da", + "columns": ["Label", "Name", "Description", "HideFromLevelUp", "Untrained", "KeyAbility", "ArmorCheckPenalty", "Constant"], + "rows": [ + { + "id": 0, + "key": "skills:athletics", + "Label": "Athletics", + "Name": {"tlk": {"key": "skills:athletics.name", "text": "Athletics"}}, + "Description": {"tlk": {"key": "skills:athletics.description", "text": "Skill text."}}, + "HideFromLevelUp": "0", + "Untrained": "1", + "KeyAbility": "STR", + "ArmorCheckPenalty": "1", + "Constant": "SKILL_ATHLETICS" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "skills", "lock.json"), `{"skills:athletics":0}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + proj.Config.Autogen.Consumers = nil + + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "Athletics.html") + newTime := time.Now().Add(2 * time.Second) + if err := os.Chtimes(pagePath, newTime, newTime); err != nil { + t.Fatalf("touch wiki page: %v", err) + } + + if _, err := BuildPackage(proj, nil); err != nil { + t.Fatalf("expected wiki output to be ignored for freshness, got %v", err) + } +} diff --git a/internal/topdata/wiki_page_paths.go b/internal/topdata/wiki_page_paths.go new file mode 100644 index 0000000..78dbc90 --- /dev/null +++ b/internal/topdata/wiki_page_paths.go @@ -0,0 +1,77 @@ +package topdata + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" +) + +type wikiPagePathDocument struct { + PagePaths wikiPagePaths `json:"page_paths" yaml:"page_paths"` +} + +type wikiPagePaths struct { + SlugOverrides []wikiPageSlugOverride `json:"slug_overrides" yaml:"slug_overrides"` +} + +type wikiPageSlugOverride struct { + PageID string `json:"page_id" yaml:"page_id"` + Slug string `json:"slug" yaml:"slug"` +} + +func loadWikiPagePathDeclarations(path string) (map[string]string, error) { + raw, err := os.ReadFile(path) + if os.IsNotExist(err) { + return map[string]string{}, nil + } + if err != nil { + return nil, err + } + return parseWikiPagePathDeclarations(raw, path) +} + +func parseWikiPagePathDeclarations(raw []byte, path string) (map[string]string, error) { + var doc wikiPagePathDocument + if err := yaml.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("parse wiki page paths %s: %w", path, err) + } + if len(doc.PagePaths.SlugOverrides) > 0 { + return nil, fmt.Errorf("parse wiki page paths %s: page_paths.slug_overrides is retired; generated public wiki paths are title-derived", path) + } + overrides := map[string]string{} + for _, override := range doc.PagePaths.SlugOverrides { + pageID := strings.TrimSpace(override.PageID) + slug := strings.TrimSpace(override.Slug) + if pageID == "" { + return nil, fmt.Errorf("parse wiki page paths %s: slug override has empty page_id", path) + } + if !isCanonicalWikiSlug(slug) { + return nil, fmt.Errorf("parse wiki page paths %s: page_id %q has invalid slug %q", path, pageID, override.Slug) + } + if _, ok := overrides[pageID]; ok { + return nil, fmt.Errorf("parse wiki page paths %s: duplicate page_id %q", path, pageID) + } + overrides[pageID] = slug + } + return overrides, nil +} + +func isCanonicalWikiSlug(value string) bool { + if value == "" || value != strings.ToLower(value) { + return false + } + parts := strings.Split(value, "-") + for _, part := range parts { + if part == "" { + return false + } + for _, r := range part { + if !((r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')) { + return false + } + } + } + return true +} diff --git a/internal/topdata/wiki_preserve.go b/internal/topdata/wiki_preserve.go new file mode 100644 index 0000000..f5f67d6 --- /dev/null +++ b/internal/topdata/wiki_preserve.go @@ -0,0 +1,54 @@ +package topdata + +import "strings" + +func normalizePreservedSectionBodies(content string) string { + var out strings.Builder + offset := 0 + for { + startRel := strings.Index(content[offset:], manualStartMarkerPrefix) + if startRel < 0 { + out.WriteString(content[offset:]) + break + } + start := offset + startRel + startEndRel := strings.Index(content[start:], "-->") + if startEndRel < 0 { + out.WriteString(content[offset:]) + break + } + startMarker := content[start : start+startEndRel+len("-->")] + id := markerID(startMarker) + if id == "" { + out.WriteString(content[offset : start+len(startMarker)]) + offset = start + len(startMarker) + continue + } + endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\"" + endRel := strings.Index(content[start+len(startMarker):], endPrefix) + if endRel < 0 { + out.WriteString(content[offset:]) + break + } + end := start + len(startMarker) + endRel + endEndRel := strings.Index(content[end:], "-->") + if endEndRel < 0 { + out.WriteString(content[offset:]) + break + } + endMarker := content[end : end+endEndRel+len("-->")] + out.WriteString(content[offset:start]) + out.WriteString(startMarker) + out.WriteString("\n\n") + out.WriteString(endMarker) + offset = end + len(endMarker) + } + return out.String() +} + +func defaultWikiManualSections() []wikiManualSection { + return []wikiManualSection{ + {ID: "user_top", Alias: "UserTopSection", Heading: "", InitialHTML: "

                "}, + {ID: "user_bottom", Alias: "UserBottomSection", Heading: "", InitialHTML: "

                "}, + } +} diff --git a/internal/topdata/wiki_slug.go b/internal/topdata/wiki_slug.go new file mode 100644 index 0000000..5eb6574 --- /dev/null +++ b/internal/topdata/wiki_slug.go @@ -0,0 +1,210 @@ +package topdata + +import ( + "html" + "strings" + "unicode" + + "golang.org/x/text/unicode/norm" +) + +var reservedCanonicalWikiFirstSegments = map[string]struct{}{ + "admin": {}, + "api": {}, + "category": {}, + "compose": {}, + "edit": {}, + "namespace": {}, + "search": {}, +} + +func canonicalWikiSegment(value string) string { + value = html.UnescapeString(strings.TrimSpace(value)) + var b strings.Builder + lastSeparator := false + for _, r := range norm.NFKD.String(value) { + transliterated := transliterateWikiSlugRune(unicode.ToLower(r)) + switch { + case (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9'): + b.WriteRune(r) + lastSeparator = false + case r == '_': + if b.Len() > 0 && !lastSeparator { + b.WriteRune(r) + lastSeparator = true + } + case unicode.Is(unicode.Mn, r): + continue + case isWikiSlugJoinerPunctuation(r): + continue + case transliterated != "": + b.WriteString(matchWikiTransliterationCase(r, transliterated)) + lastSeparator = false + default: + if b.Len() > 0 && !lastSeparator { + b.WriteByte('_') + lastSeparator = true + } + } + } + return strings.Trim(b.String(), "_") +} + +func canonicalWikiSegmentFoldedKey(value string) string { + return strings.Join(strings.Fields(strings.ToLower(strings.ReplaceAll(canonicalWikiSegment(value), "_", " "))), " ") +} + +func canonicalWikiSlashPath(value string) string { + segments := []string{} + for _, part := range strings.Split(strings.Trim(strings.TrimSpace(value), "/"), "/") { + canonical := canonicalWikiSegment(part) + if canonical != "" { + segments = append(segments, canonical) + } + } + return strings.Join(segments, "/") +} + +func canonicalWikiSlashPathFoldedKey(value string) string { + segments := []string{} + for _, part := range strings.Split(strings.Trim(strings.TrimSpace(value), "/"), "/") { + folded := canonicalWikiSegmentFoldedKey(part) + if folded != "" { + segments = append(segments, folded) + } + } + return strings.Join(segments, "/") +} + +func canonicalWikiSlashPathSegments(value string) []string { + segments := []string{} + for _, part := range strings.Split(strings.Trim(strings.TrimSpace(value), "/"), "/") { + canonical := canonicalWikiSegment(part) + if canonical != "" { + segments = append(segments, canonical) + } + } + return segments +} + +func isReservedCanonicalWikiFirstSegment(path string) bool { + segments := canonicalWikiSlashPathSegments(path) + if len(segments) == 0 { + return false + } + first := strings.ToLower(segments[0]) + if isCanonicalWikiNumericSegment(first) { + return true + } + _, reserved := reservedCanonicalWikiFirstSegments[first] + return reserved +} + +func isCanonicalWikiNumericSegment(segment string) bool { + if segment == "" { + return false + } + for _, r := range segment { + if r < '0' || r > '9' { + return false + } + } + return true +} + +func isNestedCanonicalWikiPath(path string) bool { + return len(canonicalWikiSlashPathSegments(path)) > 2 +} + +func matchWikiTransliterationCase(r rune, value string) string { + if unicode.IsUpper(r) { + if len(value) == 1 { + return strings.ToUpper(value) + } + return strings.ToUpper(value[:1]) + value[1:] + } + return value +} + +func canonicalWikiPath(segments ...string) string { + out := []string{} + for _, segment := range segments { + canonical := canonicalWikiSegment(segment) + if canonical != "" { + out = append(out, canonical) + } + } + return strings.Join(out, "/") +} + +func slugifyWikiText(value string, fallback string) string { + value = html.UnescapeString(strings.TrimSpace(value)) + var b strings.Builder + lastDash := false + for _, r := range norm.NFKD.String(strings.ToLower(value)) { + transliterated := transliterateWikiSlugRune(r) + switch { + case r >= 'a' && r <= 'z': + b.WriteRune(r) + lastDash = false + case r >= '0' && r <= '9': + b.WriteRune(r) + lastDash = false + case unicode.Is(unicode.Mn, r): + continue + case isWikiSlugJoinerPunctuation(r): + continue + case transliterated != "": + b.WriteString(transliterated) + lastDash = false + default: + if b.Len() > 0 && !lastDash { + b.WriteByte('-') + lastDash = true + } + } + } + slug := strings.Trim(b.String(), "-") + if slug == "" { + return fallback + } + return slug +} + +func isWikiSlugJoinerPunctuation(r rune) bool { + switch r { + case '\'', '"', '`', '´', '‘', '’', '‚', '‛', '“', '”', '„', '‟', '‹', '›', '«', '»', '′', '″', '＇', '＂': + return true + default: + return false + } +} + +func transliterateWikiSlugRune(r rune) string { + switch r { + case 'æ': + return "ae" + case 'œ': + return "oe" + case 'ø': + return "o" + case 'ß': + return "ss" + case 'þ': + return "th" + case 'ð', 'đ': + return "d" + case 'ł': + return "l" + case 'ħ': + return "h" + case 'ı': + return "i" + case 'ŋ': + return "n" + case 'ŧ': + return "t" + default: + return "" + } +} diff --git a/internal/topdata/wiki_tables.go b/internal/topdata/wiki_tables.go new file mode 100644 index 0000000..4a6a184 --- /dev/null +++ b/internal/topdata/wiki_tables.go @@ -0,0 +1,1749 @@ +package topdata + +import ( + "fmt" + "html" + "os" + "slices" + "strconv" + "strings" + "unicode" + + "gopkg.in/yaml.v3" +) + +type wikiTableDefinitionsDocument struct { + Tables map[string]wikiTableDefinition `json:"tables" yaml:"tables"` +} + +type wikiTableDefinition struct { + Source string `json:"source" yaml:"source"` + Class string `json:"class" yaml:"class"` + When string `json:"when" yaml:"when"` + Rows wikiTableRowsConfig `json:"rows" yaml:"rows"` + ClassFeats wikiDataProviderClassFeatConfig `json:"class_feats" yaml:"class_feats"` + Attacks wikiDataProviderAttacksConfig `json:"attacks" yaml:"attacks"` + Columns []wikiTableColumn `json:"columns" yaml:"columns"` +} + +type wikiTableRowsConfig struct { + Levels string `json:"levels" yaml:"levels"` + When string `json:"when" yaml:"when"` +} + +type wikiTableColumn struct { + Key string `json:"key" yaml:"key"` + Label string `json:"label" yaml:"label"` + Value string `json:"value" yaml:"value"` + Empty string `json:"empty" yaml:"empty"` + Class string `json:"class" yaml:"class"` + Style string `json:"style" yaml:"style"` + HeaderClass string `json:"header_class" yaml:"header_class"` + HeaderStyle string `json:"header_style" yaml:"header_style"` + AllowWikiMarkup bool `json:"allow_wiki_markup" yaml:"allow_wiki_markup"` + When string `json:"when" yaml:"when"` + Children []wikiTableColumn `json:"children" yaml:"children"` +} + +type wikiTableRenderContext struct { + Page wikiTemplatePage + TableName string + Path string +} + +func loadWikiTableDefinitions(path string) (map[string]wikiTableDefinition, error) { + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return map[string]wikiTableDefinition{}, nil + } + return nil, fmt.Errorf("read wiki table definitions %s: %w", path, err) + } + return parseWikiTableDefinitions(raw, path) +} + +func parseWikiTableDefinitions(raw []byte, path string) (map[string]wikiTableDefinition, error) { + var doc wikiTableDefinitionsDocument + if err := yaml.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("parse wiki table definitions %s: %w", path, err) + } + if len(doc.Tables) == 0 { + return map[string]wikiTableDefinition{}, nil + } + for name, def := range doc.Tables { + if strings.TrimSpace(def.Source) == "" { + return nil, fmt.Errorf("wiki table definitions %s: table %s missing source", path, name) + } + if strings.TrimSpace(def.Source) != "class_progression" { + if len(def.ClassFeats.Fields) > 0 { + return nil, fmt.Errorf("wiki table definitions %s: table %s class feat projections require source class_progression", path, name) + } + if strings.TrimSpace(def.Attacks.Mode) != "" { + return nil, fmt.Errorf("wiki table definitions %s: table %s attacks config requires source class_progression", path, name) + } + } + if strings.TrimSpace(def.Source) == "class_progression" { + for field, projection := range def.ClassFeats.Fields { + if strings.TrimSpace(field) == "" { + return nil, fmt.Errorf("wiki table definitions %s: table %s has class feat projection without field name", path, name) + } + if strings.TrimSpace(projection.When) == "" { + return nil, fmt.Errorf("wiki table definitions %s: table %s class feat projection %s missing when", path, name, field) + } + } + if err := validateWikiDataProviderAttacks(path, name, def.Attacks); err != nil { + return nil, err + } + } + if len(def.Columns) == 0 { + return nil, fmt.Errorf("wiki table definitions %s: table %s missing columns", path, name) + } + for _, col := range def.Columns { + if err := validateWikiTableColumn(path, name, col); err != nil { + return nil, err + } + } + } + return doc.Tables, nil +} + +func validateWikiTableColumn(path, tableName string, col wikiTableColumn) error { + if strings.TrimSpace(col.Key) == "" { + return fmt.Errorf("wiki table definitions %s: table %s has column without key", path, tableName) + } + if strings.TrimSpace(col.Value) != "" && len(col.Children) > 0 { + return fmt.Errorf("wiki table definitions %s: table %s column %s cannot have both value and children", path, tableName, col.Key) + } + for _, child := range col.Children { + if err := validateWikiTableColumn(path, tableName, child); err != nil { + return err + } + } + return nil +} + +func (ctx *wikiContext) renderConfiguredWikiTable(templatePath, tableName string, page wikiTemplatePage) (string, error) { + def, ok := ctx.wikiTableDefinitions[tableName] + if !ok { + return "", fmt.Errorf("render wiki template %s for %s: missing table definition %q in %s", templatePath, page.PageID, tableName, ctx.wikiTablesPath) + } + renderCtx := wikiTableRenderContext{Page: page, TableName: tableName, Path: ctx.wikiTablesPath} + if strings.TrimSpace(def.When) != "" { + value, err := ctx.evalWikiTableTemplate(def.When, page.Row, renderCtx) + if err != nil { + return "", err + } + if !truthyWikiTableValue(value) { + return "", nil + } + } + rows, err := ctx.wikiTableSourceRows(def, page) + if err != nil { + return "", fmt.Errorf("render wiki table %s for %s: %w", tableName, page.PageID, err) + } + if strings.TrimSpace(def.Rows.When) != "" { + filtered := rows[:0] + for _, row := range rows { + value, err := ctx.evalWikiTableTemplate(def.Rows.When, row, renderCtx) + if err != nil { + return "", fmt.Errorf("render wiki table %s for %s row condition %q: %w", tableName, page.PageID, def.Rows.When, err) + } + if truthyWikiTableValue(value) { + filtered = append(filtered, row) + } + } + rows = filtered + } + if len(rows) == 0 { + return "", nil + } + columns := visibleWikiTableColumns(def.Columns, aggregateWikiTableVisibilityRow(rows), ctx, renderCtx) + if len(columns) == 0 { + return "", nil + } + headerRows, leaves := buildWikiTableHeaders(columns) + if len(leaves) == 0 { + return "", nil + } + var out strings.Builder + class := strings.TrimSpace(def.Class) + if class != "" { + out.WriteString(`
              LvlSaves1st+12[[Feats/Literate|Literate]], [[Feats/Simple_Weapon_Proficiency|Simple Weapon Proficiency]]1-10[[Feats/Weapon_Specialization|Weapon Specialization]] first available. Manual level note.Base spells per dayKnown spells
              12142
              23-53
              1+1 level of existing arcane spellcasting class
              2
              3+1 level of existing arcane spellcasting class Manual level note.
              4
              Base spells per day2
              `) + } else { + out.WriteString("
              ") + } + out.WriteString("\n \n") + for _, headerRow := range headerRows { + out.WriteString(" ") + for _, cell := range headerRow { + out.WriteString(cell) + } + out.WriteString("\n") + } + out.WriteString(" \n \n") + for _, row := range rows { + out.WriteString(" ") + for _, col := range leaves { + cell, err := ctx.renderWikiTableCell(col, row, renderCtx) + if err != nil { + return "", err + } + out.WriteString(cell) + } + out.WriteString("\n") + } + out.WriteString(" \n
              ") + return out.String(), nil +} + +func visibleWikiTableColumns(columns []wikiTableColumn, row map[string]any, ctx *wikiContext, renderCtx wikiTableRenderContext) []wikiTableColumn { + out := []wikiTableColumn{} + for _, col := range columns { + if strings.TrimSpace(col.When) != "" { + value, err := ctx.evalWikiTableTemplate(col.When, row, renderCtx) + if err != nil || !truthyWikiTableValue(value) { + continue + } + } + if len(col.Children) > 0 { + col.Children = visibleWikiTableColumns(col.Children, row, ctx, renderCtx) + if len(col.Children) == 0 { + continue + } + } + out = append(out, col) + } + return out +} + +func aggregateWikiTableVisibilityRow(rows []map[string]any) map[string]any { + out := map[string]any{} + for _, row := range rows { + for key, value := range row { + if _, exists := out[key]; exists { + continue + } + if truthyWikiTableValue(value) { + out[key] = value + } + } + } + return out +} + +type wikiTableHeaderCell struct { + HTML string +} + +func buildWikiTableHeaders(columns []wikiTableColumn) ([][]string, []wikiTableColumn) { + hasGroups := false + for _, col := range columns { + if len(col.Children) > 0 { + hasGroups = true + break + } + } + first := []string{} + second := []string{} + leaves := []wikiTableColumn{} + for _, col := range columns { + label := html.EscapeString(defaultStringValue(col.Label, col.Key)) + headerClass := attrClass(col.HeaderClass) + headerStyle := attrStyle(col.HeaderStyle) + if len(col.Children) == 0 { + leaves = append(leaves, col) + rowspan := "" + if hasGroups { + rowspan = ` rowspan="2"` + } + first = append(first, ``+label+``) + continue + } + childLeaves := leafWikiTableColumns(col.Children) + first = append(first, ``+label+``) + for _, child := range col.Children { + childLabel := html.EscapeString(defaultStringValue(child.Label, child.Key)) + second = append(second, ``+childLabel+``) + } + leaves = append(leaves, childLeaves...) + } + if hasGroups { + return [][]string{first, second}, leaves + } + return [][]string{first}, leaves +} + +func leafWikiTableColumns(columns []wikiTableColumn) []wikiTableColumn { + out := []wikiTableColumn{} + for _, col := range columns { + if len(col.Children) == 0 { + out = append(out, col) + continue + } + out = append(out, leafWikiTableColumns(col.Children)...) + } + return out +} + +func (ctx *wikiContext) renderWikiTableCell(col wikiTableColumn, row map[string]any, renderCtx wikiTableRenderContext) (string, error) { + value, err := ctx.evalWikiTableTemplate(col.Value, row, renderCtx) + if err != nil { + return "", fmt.Errorf("render wiki table %s for %s column %s expression %q: %w", renderCtx.TableName, renderCtx.Page.PageID, col.Key, col.Value, err) + } + text := stringifyWikiTableValue(value) + if strings.TrimSpace(text) == "" { + text = col.Empty + } + if !col.AllowWikiMarkup { + text = html.EscapeString(text) + } + return `` + text + ``, nil +} + +func attrClass(class string) string { + class = strings.TrimSpace(class) + if class == "" { + return "" + } + return ` class="` + html.EscapeString(class) + `"` +} + +func attrStyle(style string) string { + style = strings.TrimSpace(style) + if style == "" { + return "" + } + return ` style="` + html.EscapeString(style) + `"` +} + +func defaultStringValue(value, fallback string) string { + if strings.TrimSpace(value) != "" { + return value + } + return fallback +} + +func (ctx *wikiContext) wikiTableSourceRows(def wikiTableDefinition, page wikiTemplatePage) ([]map[string]any, error) { + switch strings.TrimSpace(def.Source) { + case "class_progression": + if page.Category != "classes" { + return nil, nil + } + return ctx.classProgressionRows(def.Rows.Levels, def.ClassFeats.Fields, def.Attacks, page.Key, page.Row) + case "class_feats": + if page.Category != "classes" { + return nil, nil + } + return ctx.classFeatRows(page.Row), nil + case "spell_progression": + if page.Category != "classes" { + return nil, nil + } + if !hasSpellProgression(page.Row) { + return nil, nil + } + return []map[string]any{}, nil + case "class_summary": + if page.Category != "classes" { + return nil, nil + } + return ctx.classSummaryRows(page.Row), nil + default: + return nil, fmt.Errorf("unknown source %q", def.Source) + } +} + +func (ctx *wikiContext) classProgressionRows(levels string, projections map[string]wikiDataProviderProjectionField, attacks wikiDataProviderAttacksConfig, classKey string, classRow map[string]any) ([]map[string]any, error) { + start, end, err := parseLevelRange(levels) + if err != nil { + return nil, err + } + saveRows := map[string]map[string]any{} + if table := ctx.tableForValue(fieldValue(classRow, "SavingThrowTable"), ctx.classSaveTables); table != nil { + for _, row := range table.Rows { + level := stringValue(row, "Level") + if level != "" { + saveRows[level] = row + } + } + } + if len(projections) == 0 { + projections = map[string]wikiDataProviderProjectionField{ + "GrantedFeats": {When: "{{GrantedOnLevel > 0}}"}, + } + } + classFeatFields := map[string]map[int][]string{} + for field := range projections { + classFeatFields[field] = map[int][]string{} + } + if table := ctx.tableForValue(fieldValue(classRow, "FeatsTable"), ctx.classFeatTables); table != nil { + for _, featRow := range table.Rows { + level, err := strconv.Atoi(stringValue(featRow, "GrantedOnLevel")) + if err != nil || level < start || level > end { + continue + } + name := ctx.renderReference(fieldValue(featRow, "FeatIndex"), ctx.featIDToKey, ctx.resolveFeatName) + if name == "" { + continue + } + renderCtx := wikiTableRenderContext{TableName: "class_progression", Path: ctx.wikiDataPath} + for field, projection := range projections { + value, err := ctx.evalWikiTableTemplate(projection.When, featRow, renderCtx) + if err != nil { + return nil, fmt.Errorf("class progression field %s condition %q: %w", field, projection.When, err) + } + if truthyWikiTableValue(value) { + classFeatFields[field][level] = append(classFeatFields[field][level], name) + } + } + } + } + bonusByLevel := map[int]string{} + if table := ctx.tableForValue(fieldValue(classRow, "BonusFeatsTable"), ctx.classBonusTables); table != nil { + for _, bonusRow := range table.Rows { + level, err := asInt(fieldValue(bonusRow, "id")) + if err != nil { + continue + } + bonusByLevel[level] = stringValue(bonusRow, "Bonus") + } + } + hitDie, _ := asInt(fieldValue(classRow, "HitDie")) + spellsPerDayByLevel := classSpellProgressionByLevel(ctx.tableForValue(fieldValue(classRow, "SpellGainTable"), ctx.classSpellGainTables)) + spellsKnownByLevel := classSpellProgressionByLevel(ctx.tableForValue(fieldValue(classRow, "SpellKnownTable"), ctx.classSpellKnownTables)) + hasPerDay := hasClassSpellProgressionTable(classRow, ctx.classSpellGainTables, "SpellGainTable") + hasKnown := hasClassSpellProgressionTable(classRow, ctx.classSpellKnownTables, "SpellKnownTable") + out := []map[string]any{} + for level := start; level <= end; level++ { + row := cloneRowMap(classRow) + row["Level"] = level + bab := classBABAtLevel(stringValue(classRow, "AttackBonusTable"), level) + row["BABValue"] = bab + row["BAB"] = formatSignedInt(bab) + attackCount := classAttackCount(attacks, classKey, bab, level) + if attackCount < 1 { + attackCount = 1 + } + countField := defaultStringValue(attacks.CountField, "AttackCount") + sequenceField := defaultStringValue(attacks.Field, "AttackSequence") + row[countField] = attackCount + row[sequenceField] = formatAttackSequence(bab, attackCount, attackStep(attacks)) + if saveRow := saveRows[strconv.Itoa(level)]; saveRow != nil { + for key, value := range saveRow { + row[key] = value + } + } + for field, values := range classFeatFields { + row[field] = values[level] + } + row["BonusFeat"] = bonusByLevel[level] + row["HPMin"] = level + if hitDie > 0 { + row["HPMax"] = level * hitDie + } + if !presentWikiValue(fieldValue(row, "Notes")) { + if note := ctx.classProgressionNote(classRow, level); note != "" { + row["Notes"] = note + } + } + if _, ok := row["Notes"]; !ok { + row["Notes"] = "" + } + row["HasSpellProgression"] = hasSpellProgression(classRow) + row["HasSpellsPerDay"] = hasPerDay + row["HasSpellsKnown"] = hasKnown + row["SpellcastingAdvancementNote"] = classSpellcastingAdvancementNote(classRow, level) + populateClassSpellProgressionFields(row, "SpellsPerDay", spellsPerDayByLevel[level]) + populateClassSpellProgressionFields(row, "SpellsKnown", spellsKnownByLevel[level]) + out = append(out, row) + } + return out, nil +} + +func classSpellcastingAdvancementNote(classRow map[string]any, level int) string { + if level < 1 || isClassSpellcaster(classRow) { + return "" + } + notes := []string{} + if advancesSpellcastingAtLevel(classRow, "ArcSpellLvlMod", level) { + notes = append(notes, "+1 level of existing arcane spellcasting class") + } + if advancesSpellcastingAtLevel(classRow, "DivSpellLvlMod", level) { + notes = append(notes, "+1 level of existing divine spellcasting class") + } + return strings.Join(notes, "; ") +} + +func advancesSpellcastingAtLevel(classRow map[string]any, field string, level int) bool { + mod, err := asInt(fieldValue(classRow, field)) + if err != nil || mod <= 0 { + return false + } + return roundedSpellcastingLevelAdvancement(level, mod) > roundedSpellcastingLevelAdvancement(level-1, mod) +} + +func roundedSpellcastingLevelAdvancement(level, mod int) int { + if level <= 0 || mod <= 0 { + return 0 + } + return (level*2 + mod) / (mod * 2) +} + +func (ctx *wikiContext) classProgressionNote(classRow map[string]any, level int) string { + meta, err := parseExistingMetadata(classRow) + if err != nil || meta == nil { + return "" + } + wiki, _ := meta["wiki"].(map[string]any) + if wiki == nil { + return "" + } + raw := wiki["progression_notes"] + if raw == nil { + return "" + } + switch typed := raw.(type) { + case map[string]any: + for _, key := range []string{strconv.Itoa(level), formatLevelNoteKey(level)} { + if value, ok := typed[key]; ok { + return strings.TrimSpace(ctx.resolveTextValue(value, nil)) + } + } + case []any: + if level > 0 && level <= len(typed) { + return strings.TrimSpace(ctx.resolveTextValue(typed[level-1], nil)) + } + } + return "" +} + +func formatLevelNoteKey(level int) string { + return "level_" + strconv.Itoa(level) +} + +func classSpellProgressionByLevel(table *wikiTable) map[int]map[int]string { + out := map[int]map[int]string{} + if table == nil { + return out + } + for _, tableRow := range table.Rows { + level, err := asInt(fieldValue(tableRow, "Level")) + if err != nil || level < 1 { + continue + } + values := map[int]string{} + for key, value := range tableRow { + if !strings.HasPrefix(key, "SpellLevel") { + continue + } + spellLevel, err := strconv.Atoi(strings.TrimPrefix(key, "SpellLevel")) + if err != nil || spellLevel < 0 { + continue + } + values[spellLevel] = stringifyWikiTableValue(value) + } + out[level] = values + } + return out +} + +func populateClassSpellProgressionFields(row map[string]any, prefix string, values map[int]string) { + ordered := []string{} + for _, spellLevel := range sortedIntMapKeys(values) { + value := values[spellLevel] + row[prefix+strconv.Itoa(spellLevel)] = value + ordered = append(ordered, value) + } + row[prefix+"ByLevel"] = ordered +} + +func sortedIntMapKeys(values map[int]string) []int { + keys := make([]int, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +func parseLevelRange(raw string) (int, int, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return 1, 20, nil + } + if !strings.Contains(raw, "-") { + level, err := strconv.Atoi(raw) + if err != nil || level < 1 { + return 0, 0, fmt.Errorf("bad row range %q", raw) + } + return level, level, nil + } + left, right, _ := strings.Cut(raw, "-") + start, err1 := strconv.Atoi(strings.TrimSpace(left)) + end, err2 := strconv.Atoi(strings.TrimSpace(right)) + if err1 != nil || err2 != nil || start < 1 || end < start { + return 0, 0, fmt.Errorf("bad row range %q", raw) + } + return start, end, nil +} + +func classBABAtLevel(table string, level int) int { + switch strings.ToLower(strings.TrimSpace(table)) { + case "cls_atk_1": + return level + case "cls_atk_2": + return (level * 3) / 4 + case "cls_atk_3": + return level / 2 + default: + return 0 + } +} + +func classAttackCount(cfg wikiDataProviderAttacksConfig, classKey string, bab, level int) int { + count := baseClassAttackCount(cfg, bab) + for _, override := range cfg.Overrides { + if !wikiAttackOverrideMatches(override, classKey, level) { + continue + } + if override.Attacks > 0 { + count = override.Attacks + continue + } + count += override.Add + } + if count < 1 { + return 1 + } + return count +} + +func baseClassAttackCount(cfg wikiDataProviderAttacksConfig, bab int) int { + switch strings.TrimSpace(cfg.Mode) { + case "fixed": + if cfg.Attacks > 0 { + return cfg.Attacks + } + case "bab_thresholds": + return attackCountFromThresholds(cfg.Thresholds, bab) + case "": + return attackCountFromThresholds(defaultWikiAttackThresholds(), bab) + } + return 1 +} + +func attackCountFromThresholds(thresholds []wikiDataProviderAttackThreshold, bab int) int { + count := 1 + for _, threshold := range thresholds { + if threshold.Attacks > 0 && bab >= threshold.MinBAB { + count = threshold.Attacks + } + } + return count +} + +func defaultWikiAttackThresholds() []wikiDataProviderAttackThreshold { + return []wikiDataProviderAttackThreshold{ + {MinBAB: 1, Attacks: 1}, + {MinBAB: 6, Attacks: 2}, + {MinBAB: 11, Attacks: 3}, + {MinBAB: 16, Attacks: 4}, + } +} + +func wikiAttackOverrideMatches(override wikiDataProviderAttackOverrideRule, classKey string, level int) bool { + foundClass := false + for _, candidate := range override.Classes { + if strings.TrimSpace(candidate) == classKey { + foundClass = true + break + } + } + if !foundClass { + return false + } + levels, err := parseWikiAttackLevels(override.Levels) + if err != nil { + return false + } + _, ok := levels[level] + return ok +} + +func attackStep(cfg wikiDataProviderAttacksConfig) int { + if cfg.Step == 0 { + return -5 + } + return cfg.Step +} + +func formatAttackSequence(bab, attacks, step int) string { + if attacks < 1 { + attacks = 1 + } + values := make([]string, 0, attacks) + current := bab + for i := 0; i < attacks; i++ { + values = append(values, formatSignedInt(current)) + current += step + } + return strings.Join(values, "/") +} + +func formatSignedInt(value int) string { + if value > 0 { + return "+" + strconv.Itoa(value) + } + return strconv.Itoa(value) +} + +func (ctx *wikiContext) classFeatRows(classRow map[string]any) []map[string]any { + table := ctx.tableForValue(fieldValue(classRow, "FeatsTable"), ctx.classFeatTables) + if table == nil { + return nil + } + out := []map[string]any{} + for _, source := range table.Rows { + row := cloneRowMap(source) + key := resolveReferenceKey(fieldValue(source, "FeatIndex"), ctx.featIDToKey) + if key == "" { + key = resolveReferenceKey(fieldValue(source, "FeatIndex"), nil) + } + row["FeatKey"] = key + row["FeatName"] = ctx.resolveFeatName(key) + out = append(out, row) + } + return out +} + +func (ctx *wikiContext) classProficiencyLinks(classRow map[string]any) []string { + rows := ctx.classFilteredFeatLinks(classRow, func(row map[string]any, key, name string) bool { + level, err := asInt(fieldValue(row, "GrantedOnLevel")) + if err != nil || level != 1 { + return false + } + text := strings.ToLower(key + " " + name) + return strings.Contains(text, "proficiency") + }) + return rows +} + +func (ctx *wikiContext) classSelectableFeatLinks(classRow map[string]any) []string { + return ctx.classFilteredFeatLinks(classRow, func(row map[string]any, key, name string) bool { + list, err := asInt(fieldValue(row, "List")) + return err == nil && (list == 1 || list == 2) + }) +} + +func (ctx *wikiContext) classFilteredFeatLinks(classRow map[string]any, keep func(row map[string]any, key, name string) bool) []string { + table := ctx.tableForValue(fieldValue(classRow, "FeatsTable"), ctx.classFeatTables) + if table == nil { + return nil + } + out := []string{} + seen := map[string]struct{}{} + for _, source := range table.Rows { + key := resolveReferenceKey(fieldValue(source, "FeatIndex"), ctx.featIDToKey) + if key == "" { + key = resolveReferenceKey(fieldValue(source, "FeatIndex"), nil) + } + if key == "" { + continue + } + name := ctx.resolveFeatName(key) + if !keep(source, key, name) { + continue + } + link := ctx.renderReference(map[string]any{"id": key}, nil, ctx.resolveFeatName) + if link == "" { + continue + } + if _, ok := seen[link]; ok { + continue + } + seen[link] = struct{}{} + out = append(out, link) + } + return out +} + +func (ctx *wikiContext) classSaveSummary(classRow map[string]any) string { + table := ctx.tableForValue(fieldValue(classRow, "SavingThrowTable"), ctx.classSaveTables) + if table == nil { + return "" + } + var levelOne map[string]any + for _, row := range table.Rows { + level, err := asInt(fieldValue(row, "Level")) + if err == nil && level == 1 { + levelOne = row + break + } + } + if levelOne == nil { + return "" + } + high := []string{} + for _, save := range []struct { + field string + label string + }{ + {"FortSave", "Fortitude"}, + {"RefSave", "Reflex"}, + {"WillSave", "Will"}, + } { + value, err := asInt(fieldValue(levelOne, save.field)) + if err == nil && value > 0 { + high = append(high, save.label) + } + } + if len(high) == 0 { + return "Low" + } + return "High " + formatWikiEnglishList(high) +} + +func classBABLabel(table string) string { + switch strings.ToLower(strings.TrimSpace(table)) { + case "cls_atk_1", "atk_1": + return "+1/1" + case "cls_atk_2", "atk_2": + return "+3/4" + case "cls_atk_3", "atk_3": + return "+1/2" + default: + return "" + } +} + +func wikiAbilityName(value string) string { + switch strings.ToUpper(strings.TrimSpace(value)) { + case "STR": + return "Strength" + case "DEX": + return "Dexterity" + case "CON": + return "Constitution" + case "INT": + return "Intelligence" + case "WIS": + return "Wisdom" + case "CHA": + return "Charisma" + default: + return strings.TrimSpace(value) + } +} + +func wikiIndefiniteArticle(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "a" + } + switch strings.ToLower(value[:1]) { + case "a", "e", "i", "o", "u": + return "an" + default: + return "a" + } +} + +func formatWikiEnglishList(values []string) string { + switch len(values) { + case 0: + return "" + case 1: + return values[0] + case 2: + return values[0] + " and " + values[1] + default: + return strings.Join(values[:len(values)-1], ", ") + ", and " + values[len(values)-1] + } +} + +func (ctx *wikiContext) classSummaryRows(classRow map[string]any) []map[string]any { + return []map[string]any{ + {"Label": "Hit Die", "Value": "d" + stringValue(classRow, "HitDie")}, + {"Label": "Skill Points", "Value": stringValue(classRow, "SkillPointBase")}, + {"Label": "Primary Ability", "Value": stringValue(classRow, "PrimaryAbil")}, + } +} + +func isClassSpellcaster(row map[string]any) bool { + return stringValue(row, "SpellCaster") == "1" +} + +func hasSpellProgression(row map[string]any) bool { + return isClassSpellcaster(row) && (hasTableReference(fieldValue(row, "SpellGainTable")) || hasTableReference(fieldValue(row, "SpellKnownTable"))) +} + +func hasClassSpellProgressionTable(row map[string]any, tables map[string]wikiTable, field string) bool { + return isClassSpellcaster(row) && ctxHasTableValue(row, tables, field) +} + +func hasTableReference(value any) bool { + switch typed := value.(type) { + case string: + return strings.TrimSpace(typed) != "" && strings.TrimSpace(typed) != nullValue + case map[string]any: + tableKey, _ := typed["table"].(string) + return strings.TrimSpace(tableKey) != "" + default: + return false + } +} + +func ctxHasTableValue(row map[string]any, tables map[string]wikiTable, field string) bool { + switch value := fieldValue(row, field).(type) { + case string: + if _, ok := tables[value]; ok { + return true + } + _, ok := tables[outputStem(value)] + return ok + case map[string]any: + tableKey, _ := value["table"].(string) + if tableKey == "" { + return false + } + if _, ok := tables[tableKey]; ok { + return true + } + _, ok := tables[outputStem(tableKey)] + return ok + default: + return false + } +} + +func (ctx *wikiContext) classSpellbookRows(classKey string, classRow map[string]any) []map[string]any { + spec, ok := ctx.classSpellbooks[classKey] + if !ok || !ctx.hasClassSpellbook(classKey, classRow) { + return nil + } + out := []map[string]any{} + for _, level := range sortedIntKeys(spec.Levels) { + spells := ctx.renderClassSpellbookLevel(spec.Levels[level]) + if len(spells) == 0 { + continue + } + out = append(out, map[string]any{ + "SpellLevel": level, + "SpellLevelLabel": spellbookLevelLabel(level), + "Spells": spells, + }) + } + return out +} + +func (ctx *wikiContext) hasClassSpellbook(classKey string, classRow map[string]any) bool { + if ctx == nil || !isClassSpellcaster(classRow) { + return false + } + spec, ok := ctx.classSpellbooks[classKey] + if !ok { + return false + } + for _, spells := range spec.Levels { + if len(spells) > 0 { + return true + } + } + return false +} + +func (ctx *wikiContext) renderClassSpellbookLevel(spellKeys []string) []string { + type renderedSpell struct { + Name string + Link string + } + rendered := []renderedSpell{} + for _, spellKey := range spellKeys { + name := ctx.resolveSpellName(spellKey) + if name == "" { + name = spellKey + } + link := ctx.renderReference(map[string]any{"id": spellKey}, nil, ctx.resolveSpellName) + if link == "" { + continue + } + rendered = append(rendered, renderedSpell{Name: name, Link: link}) + } + slices.SortFunc(rendered, func(a, b renderedSpell) int { + if cmp := strings.Compare(strings.ToLower(a.Name), strings.ToLower(b.Name)); cmp != 0 { + return cmp + } + return strings.Compare(a.Link, b.Link) + }) + out := make([]string, 0, len(rendered)) + for _, spell := range rendered { + out = append(out, spell.Link) + } + return out +} + +func spellbookLevelLabel(level int) string { + if level == 0 { + return "0-level spells" + } + return ordinalWikiTableValue(level) + "-level spells" +} + +func (ctx *wikiContext) evalWikiTableTemplate(template string, row map[string]any, renderCtx wikiTableRenderContext) (any, error) { + template = strings.TrimSpace(template) + if strings.HasPrefix(template, "{{") && strings.HasSuffix(template, "}}") { + expr := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(template, "{{"), "}}")) + return evalWikiTableExpression(expr, row, ctx, renderCtx) + } + if strings.Contains(template, "{{") { + return nil, fmt.Errorf("mixed literal/expression templates are not supported") + } + return template, nil +} + +func evalWikiTableExpression(expr string, row map[string]any, ctx *wikiContext, renderCtx wikiTableRenderContext) (any, error) { + parser := wikiExprParser{tokens: tokenizeWikiExpr(expr), row: row, ctx: ctx, renderCtx: renderCtx} + value, err := parser.parseComparison() + if err != nil { + return nil, err + } + if parser.peek().kind != wikiExprEOF { + return nil, fmt.Errorf("unexpected token %q", parser.peek().text) + } + return value, nil +} + +type wikiExprTokenKind int + +const ( + wikiExprEOF wikiExprTokenKind = iota + wikiExprIdent + wikiExprString + wikiExprNumber + wikiExprLParen + wikiExprRParen + wikiExprComma + wikiExprPlus + wikiExprEq + wikiExprNE + wikiExprGT + wikiExprGE + wikiExprLT + wikiExprLE + wikiExprAnd + wikiExprOr +) + +type wikiExprToken struct { + kind wikiExprTokenKind + text string +} + +func tokenizeWikiExpr(expr string) []wikiExprToken { + tokens := []wikiExprToken{} + for i := 0; i < len(expr); { + r := rune(expr[i]) + if unicode.IsSpace(r) { + i++ + continue + } + switch expr[i] { + case '(': + tokens = append(tokens, wikiExprToken{kind: wikiExprLParen, text: "("}) + i++ + case ')': + tokens = append(tokens, wikiExprToken{kind: wikiExprRParen, text: ")"}) + i++ + case ',': + tokens = append(tokens, wikiExprToken{kind: wikiExprComma, text: ","}) + i++ + case '+': + tokens = append(tokens, wikiExprToken{kind: wikiExprPlus, text: "+"}) + i++ + case '>': + if i+1 < len(expr) && expr[i+1] == '=' { + tokens = append(tokens, wikiExprToken{kind: wikiExprGE, text: ">="}) + i += 2 + } else { + tokens = append(tokens, wikiExprToken{kind: wikiExprGT, text: ">"}) + i++ + } + case '<': + if i+1 < len(expr) && expr[i+1] == '=' { + tokens = append(tokens, wikiExprToken{kind: wikiExprLE, text: "<="}) + i += 2 + } else { + tokens = append(tokens, wikiExprToken{kind: wikiExprLT, text: "<"}) + i++ + } + case '!': + if i+1 < len(expr) && expr[i+1] == '=' { + tokens = append(tokens, wikiExprToken{kind: wikiExprNE, text: "!="}) + i += 2 + } else { + tokens = append(tokens, wikiExprToken{kind: wikiExprEOF, text: expr[i:]}) + return tokens + } + case '&': + if i+1 < len(expr) && expr[i+1] == '&' { + tokens = append(tokens, wikiExprToken{kind: wikiExprAnd, text: "&&"}) + i += 2 + } else { + tokens = append(tokens, wikiExprToken{kind: wikiExprEOF, text: expr[i:]}) + return tokens + } + case '|': + if i+1 < len(expr) && expr[i+1] == '|' { + tokens = append(tokens, wikiExprToken{kind: wikiExprOr, text: "||"}) + i += 2 + } else { + tokens = append(tokens, wikiExprToken{kind: wikiExprEOF, text: expr[i:]}) + return tokens + } + case '=': + if i+1 < len(expr) && expr[i+1] == '=' { + tokens = append(tokens, wikiExprToken{kind: wikiExprEq, text: "=="}) + i += 2 + } else { + tokens = append(tokens, wikiExprToken{kind: wikiExprEOF, text: expr[i:]}) + return tokens + } + case '"': + j := i + 1 + var b strings.Builder + for ; j < len(expr); j++ { + if expr[j] == '\\' && j+1 < len(expr) { + j++ + b.WriteByte(expr[j]) + continue + } + if expr[j] == '"' { + break + } + b.WriteByte(expr[j]) + } + if j >= len(expr) { + tokens = append(tokens, wikiExprToken{kind: wikiExprEOF, text: expr[i:]}) + return tokens + } + tokens = append(tokens, wikiExprToken{kind: wikiExprString, text: b.String()}) + i = j + 1 + default: + if unicode.IsDigit(r) || expr[i] == '-' { + j := i + 1 + for j < len(expr) && unicode.IsDigit(rune(expr[j])) { + j++ + } + tokens = append(tokens, wikiExprToken{kind: wikiExprNumber, text: expr[i:j]}) + i = j + continue + } + if unicode.IsLetter(r) || expr[i] == '_' { + j := i + 1 + for j < len(expr) { + rr := rune(expr[j]) + if !unicode.IsLetter(rr) && !unicode.IsDigit(rr) && expr[j] != '_' && expr[j] != '.' { + break + } + j++ + } + tokens = append(tokens, wikiExprToken{kind: wikiExprIdent, text: expr[i:j]}) + i = j + continue + } + tokens = append(tokens, wikiExprToken{kind: wikiExprEOF, text: expr[i:]}) + return tokens + } + } + tokens = append(tokens, wikiExprToken{kind: wikiExprEOF}) + return tokens +} + +type wikiExprParser struct { + tokens []wikiExprToken + pos int + row map[string]any + ctx *wikiContext + renderCtx wikiTableRenderContext + allowMissingFields bool +} + +func (p *wikiExprParser) peek() wikiExprToken { + if p.pos >= len(p.tokens) { + return wikiExprToken{kind: wikiExprEOF} + } + return p.tokens[p.pos] +} + +func (p *wikiExprParser) take(kind wikiExprTokenKind) (wikiExprToken, bool) { + token := p.peek() + if token.kind != kind { + return token, false + } + p.pos++ + return token, true +} + +func (p *wikiExprParser) parseComparison() (any, error) { + return p.parseOr() +} + +func (p *wikiExprParser) parseOr() (any, error) { + left, err := p.parseAnd() + if err != nil { + return nil, err + } + for p.peek().kind == wikiExprOr { + p.pos++ + right, err := p.parseAnd() + if err != nil { + return nil, err + } + left = truthyWikiTableValue(left) || truthyWikiTableValue(right) + } + return left, nil +} + +func (p *wikiExprParser) parseAnd() (any, error) { + left, err := p.parseRelational() + if err != nil { + return nil, err + } + for p.peek().kind == wikiExprAnd { + p.pos++ + right, err := p.parseRelational() + if err != nil { + return nil, err + } + left = truthyWikiTableValue(left) && truthyWikiTableValue(right) + } + return left, nil +} + +func (p *wikiExprParser) parseRelational() (any, error) { + left, err := p.parseConcat() + if err != nil { + return nil, err + } + switch p.peek().kind { + case wikiExprEq, wikiExprNE, wikiExprGT, wikiExprGE, wikiExprLT, wikiExprLE: + op := p.peek().kind + p.pos++ + right, err := p.parseConcat() + if err != nil { + return nil, err + } + switch op { + case wikiExprEq: + return stringifyWikiTableValue(left) == stringifyWikiTableValue(right), nil + case wikiExprNE: + return stringifyWikiTableValue(left) != stringifyWikiTableValue(right), nil + case wikiExprGT: + return numericWikiTableValue(left) > numericWikiTableValue(right), nil + case wikiExprGE: + return numericWikiTableValue(left) >= numericWikiTableValue(right), nil + case wikiExprLT: + return numericWikiTableValue(left) < numericWikiTableValue(right), nil + case wikiExprLE: + return numericWikiTableValue(left) <= numericWikiTableValue(right), nil + } + return false, nil + default: + return left, nil + } +} + +func (p *wikiExprParser) parseConcat() (any, error) { + left, err := p.parsePrimary() + if err != nil { + return nil, err + } + for p.peek().kind == wikiExprPlus { + p.pos++ + right, err := p.parsePrimary() + if err != nil { + return nil, err + } + left = stringifyWikiTableValue(left) + stringifyWikiTableValue(right) + } + return left, nil +} + +func (p *wikiExprParser) parsePrimary() (any, error) { + token := p.peek() + switch token.kind { + case wikiExprString: + p.pos++ + return token.text, nil + case wikiExprNumber: + p.pos++ + value, err := strconv.Atoi(token.text) + if err != nil { + return nil, fmt.Errorf("invalid number %q", token.text) + } + return value, nil + case wikiExprIdent: + p.pos++ + if p.peek().kind == wikiExprLParen { + p.pos++ + args := []any{} + if p.peek().kind != wikiExprRParen { + previousAllowMissing := p.allowMissingFields + if wikiExprHelperAllowsMissingArgs(token.text) { + p.allowMissingFields = true + } + for { + arg, err := p.parseComparison() + if err != nil { + p.allowMissingFields = previousAllowMissing + return nil, err + } + args = append(args, arg) + if _, ok := p.take(wikiExprComma); !ok { + break + } + } + p.allowMissingFields = previousAllowMissing + } + if _, ok := p.take(wikiExprRParen); !ok { + return nil, fmt.Errorf("missing closing parenthesis for %s", token.text) + } + return p.callHelper(token.text, args) + } + return p.resolveField(token.text) + case wikiExprLParen: + p.pos++ + value, err := p.parseComparison() + if err != nil { + return nil, err + } + if _, ok := p.take(wikiExprRParen); !ok { + return nil, fmt.Errorf("missing closing parenthesis") + } + return value, nil + default: + return nil, fmt.Errorf("unexpected token %q", token.text) + } +} + +func wikiExprHelperAllowsMissingArgs(name string) bool { + switch name { + case "present", "all_present", "any_present", "alignments": + return true + default: + return false + } +} + +func (p *wikiExprParser) resolveField(name string) (any, error) { + name = strings.TrimPrefix(name, "row.") + if name == "HasSpellProgression" { + return hasSpellProgression(p.renderCtx.Page.Row), nil + } + if name == "HasSpellsPerDay" { + return hasClassSpellProgressionTable(p.renderCtx.Page.Row, p.ctx.classSpellGainTables, "SpellGainTable"), nil + } + if name == "HasSpellsKnown" { + return hasClassSpellProgressionTable(p.renderCtx.Page.Row, p.ctx.classSpellKnownTables, "SpellKnownTable"), nil + } + if name == "HasClassSpellbook" { + return p.ctx.hasClassSpellbook(p.renderCtx.Page.Key, p.renderCtx.Page.Row), nil + } + switch name { + case "true": + return true, nil + case "false": + return false, nil + } + if value, ok := lookupField(p.row, name); ok { + return value, nil + } + if value, ok := lookupField(p.renderCtx.Page.Row, name); ok { + return value, nil + } + if isClassSpellProgressionFieldName(name) { + return nil, nil + } + if p.allowMissingFields { + return nil, nil + } + return nil, fmt.Errorf("missing field %q", name) +} + +func isClassSpellProgressionFieldName(name string) bool { + for _, prefix := range []string{"SpellsPerDay", "SpellsKnown"} { + suffix := strings.TrimPrefix(name, prefix) + if suffix == name || suffix == "ByLevel" || suffix == "" { + continue + } + if _, err := strconv.Atoi(suffix); err == nil { + return true + } + } + return false +} + +func (p *wikiExprParser) callHelper(name string, args []any) (any, error) { + switch name { + case "ordinal": + if len(args) != 1 { + return nil, fmt.Errorf("ordinal expects 1 argument") + } + return ordinalWikiTableValue(numericWikiTableValue(args[0])), nil + case "range": + if len(args) != 2 { + return nil, fmt.Errorf("range expects 2 arguments") + } + left := stringifyWikiTableValue(args[0]) + right := stringifyWikiTableValue(args[1]) + if left == "" || right == "" { + return "", nil + } + return left + "-" + right, nil + case "link": + if len(args) == 1 { + key := stringifyWikiTableValue(args[0]) + if p.ctx != nil && !p.ctx.canReferenceWikiKey(key) { + return "", nil + } + if p.ctx != nil { + if target := p.ctx.publicWikiTargetForKey(key, ""); target != "" { + key = target + } + } + return "[[" + key + "]]", nil + } + if len(args) == 2 { + key := stringifyWikiTableValue(args[0]) + label := stringifyWikiTableValue(args[1]) + if key == "" { + return label, nil + } + if p.ctx != nil && !p.ctx.canReferenceWikiKey(key) { + return "", nil + } + if p.ctx != nil { + if target := p.ctx.publicWikiTargetForKey(key, label); target != "" { + key = target + } + } + return "[[" + key + "|" + label + "]]", nil + } + return nil, fmt.Errorf("link expects 1 or 2 arguments") + case "if": + if len(args) != 3 { + return nil, fmt.Errorf("if expects 3 arguments") + } + if truthyWikiTableValue(args[0]) { + return args[1], nil + } + return args[2], nil + case "join": + if len(args) != 2 { + return nil, fmt.Errorf("join expects 2 arguments") + } + sep := stringifyWikiTableValue(args[1]) + switch typed := args[0].(type) { + case []string: + return strings.Join(typed, sep), nil + case []any: + parts := make([]string, 0, len(typed)) + for _, value := range typed { + if text := stringifyWikiTableValue(value); text != "" { + parts = append(parts, text) + } + } + return strings.Join(parts, sep), nil + default: + return stringifyWikiTableValue(args[0]), nil + } + case "count": + if len(args) != 1 { + return nil, fmt.Errorf("count expects 1 argument") + } + return countWikiTemplateValue(args[0]), nil + case "nonzero": + if len(args) != 1 { + return nil, fmt.Errorf("nonzero expects 1 argument") + } + if numericWikiTableValue(args[0]) == 0 { + return "", nil + } + return args[0], nil + case "eq": + if len(args) != 2 { + return nil, fmt.Errorf("eq expects 2 arguments") + } + return stringifyWikiTableValue(args[0]) == stringifyWikiTableValue(args[1]), nil + case "field": + if len(args) != 1 { + return nil, fmt.Errorf("field expects 1 argument") + } + return p.resolveField(stringifyWikiTableValue(args[0])) + case "default": + if len(args) != 2 { + return nil, fmt.Errorf("default expects 2 arguments") + } + if strings.TrimSpace(stringifyWikiTableValue(args[0])) == "" { + return args[1], nil + } + return args[0], nil + case "present": + if len(args) != 1 { + return nil, fmt.Errorf("present expects 1 argument") + } + return presentWikiValue(args[0]), nil + case "all_present": + if len(args) == 0 { + return nil, fmt.Errorf("all_present expects at least 1 argument") + } + for _, value := range args { + if !presentWikiValue(value) { + return false, nil + } + } + return true, nil + case "any_present": + if len(args) == 0 { + return nil, fmt.Errorf("any_present expects at least 1 argument") + } + for _, value := range args { + if presentWikiValue(value) { + return true, nil + } + } + return false, nil + case "alignments": + if len(args) != 1 { + return nil, fmt.Errorf("alignments expects 1 argument") + } + return p.ctx.formatWikiTemplateAlignments(args[0]) + case "yes_no": + if len(args) != 1 { + return nil, fmt.Errorf("yes_no expects 1 argument") + } + return yesNoValue(stringifyWikiTableValue(args[0])), nil + case "text": + if len(args) != 1 { + return nil, fmt.Errorf("text expects 1 argument") + } + return p.ctx.resolveTextValue(args[0], nil), nil + case "ref": + if len(args) != 2 { + return nil, fmt.Errorf("ref expects 2 arguments") + } + return p.ctx.renderReference(args[0], p.ctx.wikiIDMapForDataset(stringifyWikiTableValue(args[1])), p.ctx.wikiNameResolverForDataset(stringifyWikiTableValue(args[1]))), nil + case "feat_prerequisites": + if len(args) != 0 { + return nil, fmt.Errorf("feat_prerequisites expects 0 arguments") + } + return p.ctx.renderFeatPrerequisites(p.renderCtx.Page.Row), nil + case "required_feats": + if len(args) != 0 { + return nil, fmt.Errorf("required_feats expects 0 arguments") + } + return p.ctx.renderReferenceList(p.ctx.wikiReqFeats(p.renderCtx.Page.Row)), nil + case "ability_adjustments": + if len(args) != 0 { + return nil, fmt.Errorf("ability_adjustments expects 0 arguments") + } + return formatAbilityAdjustments(p.renderCtx.Page.Row), nil + case "ability_name": + if len(args) != 1 { + return nil, fmt.Errorf("ability_name expects 1 argument") + } + return wikiAbilityName(stringifyWikiTableValue(args[0])), nil + case "indefinite_article": + if len(args) != 1 { + return nil, fmt.Errorf("indefinite_article expects 1 argument") + } + return wikiIndefiniteArticle(stringifyWikiTableValue(args[0])), nil + case "class_bab": + if len(args) != 0 { + return nil, fmt.Errorf("class_bab expects 0 arguments") + } + return classBABLabel(stringValue(p.renderCtx.Page.Row, "AttackBonusTable")), nil + case "class_proficiencies": + if len(args) != 0 { + return nil, fmt.Errorf("class_proficiencies expects 0 arguments") + } + return p.ctx.classProficiencyLinks(p.renderCtx.Page.Row), nil + case "class_saves": + if len(args) != 0 { + return nil, fmt.Errorf("class_saves expects 0 arguments") + } + return p.ctx.classSaveSummary(p.renderCtx.Page.Row), nil + case "class_selectable_feats": + if len(args) != 0 { + return nil, fmt.Errorf("class_selectable_feats expects 0 arguments") + } + return p.ctx.classSelectableFeatLinks(p.renderCtx.Page.Row), nil + case "hidden_ref": + if len(args) != 2 { + return nil, fmt.Errorf("hidden_ref expects 2 arguments") + } + hidden, err := p.hiddenWikiReference(args[0], stringifyWikiTableValue(args[1])) + if err != nil { + return nil, err + } + return hidden, nil + case "all_hidden_refs": + if len(args) < 2 { + return nil, fmt.Errorf("all_hidden_refs expects at least 2 arguments") + } + targetDataset := stringifyWikiTableValue(args[len(args)-1]) + checked := false + for _, value := range args[:len(args)-1] { + if !presentWikiValue(value) { + continue + } + checked = true + hidden, err := p.hiddenWikiReference(value, targetDataset) + if err != nil { + return nil, err + } + if !hidden { + return false, nil + } + } + return checked, nil + default: + return nil, fmt.Errorf("unknown helper %q", name) + } +} + +func countWikiTemplateValue(value any) int { + switch typed := value.(type) { + case nil: + return 0 + case []string: + return len(typed) + case []any: + count := 0 + for _, item := range typed { + if presentWikiValue(item) { + count++ + } + } + return count + case map[string]any: + return len(typed) + default: + if presentWikiValue(value) { + return 1 + } + return 0 + } +} + +func (p *wikiExprParser) hiddenWikiReference(value any, targetDataset string) (bool, error) { + targetDataset = strings.TrimSpace(targetDataset) + if !isWikiVisibilityDataset(targetDataset) { + return false, fmt.Errorf("unknown reference dataset %q", targetDataset) + } + if !presentWikiValue(value) { + return false, nil + } + if p.ctx == nil { + return false, nil + } + key := resolveReferenceKey(value, p.ctx.wikiIDMapForDataset(targetDataset)) + if key == "" { + return true, nil + } + if !strings.HasPrefix(key, targetDataset+":") { + return true, nil + } + return !p.ctx.canReferenceWikiKey(key), nil +} + +func presentWikiValue(value any) bool { + if isNullLike(value) { + return false + } + if text, ok := value.(string); ok { + return strings.TrimSpace(text) != "" + } + return value != nil +} + +func ordinalWikiTableValue(value int) string { + suffix := "th" + if value%100 < 11 || value%100 > 13 { + switch value % 10 { + case 1: + suffix = "st" + case 2: + suffix = "nd" + case 3: + suffix = "rd" + } + } + return strconv.Itoa(value) + suffix +} + +func stringifyWikiTableValue(value any) string { + switch typed := value.(type) { + case nil: + return "" + case string: + if typed == nullValue { + return "" + } + return typed + case bool: + if typed { + return "true" + } + return "" + case int: + return strconv.Itoa(typed) + case float64: + return strconv.Itoa(int(typed)) + case []string: + return strings.Join(typed, ", ") + case []any: + parts := make([]string, 0, len(typed)) + for _, value := range typed { + if text := stringifyWikiTableValue(value); text != "" { + parts = append(parts, text) + } + } + return strings.Join(parts, ", ") + default: + return fmt.Sprint(typed) + } +} + +func numericWikiTableValue(value any) int { + switch typed := value.(type) { + case int: + return typed + case float64: + return int(typed) + case string: + parsed, _ := strconv.Atoi(strings.TrimSpace(typed)) + return parsed + default: + parsed, _ := strconv.Atoi(strings.TrimSpace(stringifyWikiTableValue(value))) + return parsed + } +} + +func truthyWikiTableValue(value any) bool { + switch typed := value.(type) { + case bool: + return typed + case nil: + return false + case string: + trimmed := strings.TrimSpace(strings.ToLower(typed)) + return trimmed != "" && trimmed != "0" && trimmed != "false" && trimmed != nullValue + case int: + return typed != 0 + case float64: + return typed != 0 + default: + return strings.TrimSpace(stringifyWikiTableValue(value)) != "" + } +} diff --git a/internal/topdata/wiki_template.go b/internal/topdata/wiki_template.go new file mode 100644 index 0000000..3ae30c6 --- /dev/null +++ b/internal/topdata/wiki_template.go @@ -0,0 +1,433 @@ +package topdata + +import ( + "fmt" + "html" + "os" + "path/filepath" + "strings" +) + +type wikiTemplatePage struct { + PageID string + Category string + Key string + Title string + Status string + Row map[string]any + ManualSections []wikiManualSection +} + +func (ctx *wikiContext) renderWikiPageTemplate(category string, page wikiTemplatePage) (string, error) { + name := wikiTemplateName(category) + source, path := ctx.loadWikiTemplateSource(name) + if source == "" && name != "default.html" { + source, path = ctx.loadWikiTemplateSource("default.html") + } + if source == "" { + source = builtinWikiTemplate(category) + path = "builtin:" + name + } + return ctx.renderWikiTemplateString(path, source, page) +} + +func wikiTemplateName(category string) string { + switch category { + case "classes", "feat", "skills", "spells", "racialtypes", "baseitems": + return category + ".html" + default: + return "default.html" + } +} + +func (ctx *wikiContext) loadWikiTemplateSource(name string) (string, string) { + if strings.TrimSpace(ctx.templateDir) == "" { + return "", "" + } + path := filepath.Join(ctx.templateDir, filepath.FromSlash(name)) + raw, err := os.ReadFile(path) + if err != nil { + return "", path + } + return string(raw), path +} + +func builtinWikiTemplate(category string) string { + switch category { + case "classes": + return `

              {{title}}

              +{{UserTopSection}} +{{format:StatusCallout}} +{{format:Description}} +{{format:FactsTable}} +{{format:SkillList}} +{{format:FeatProgression}} +{{format:UserBottomFallback}}` + case "racialtypes": + return `

              {{title}}

              +{{UserTopSection}} +{{format:StatusCallout}} +{{format:Description}} +{{format:FactsTable}} +{{format:RaceNameForms}} +{{format:UserBottomFallback}}` + default: + return `

              {{title}}

              +{{UserTopSection}} +{{format:StatusCallout}} +{{format:Description}} +{{format:FactsTable}} +{{format:UserBottomFallback}}` + } +} + +func (ctx *wikiContext) renderWikiTemplateString(path, source string, page wikiTemplatePage) (string, error) { + state := &wikiTemplateRenderState{ + path: path, + page: page, + seenSections: map[string]struct{}{}, + active: true, + } + rendered, offset, closed, err := ctx.renderWikiTemplateRange(source, 0, "", state) + if err != nil { + return "", err + } + if closed { + return "", fmt.Errorf("render wiki template %s for %s: unexpected closing block", path, page.PageID) + } + if offset != len(source) { + return "", fmt.Errorf("render wiki template %s for %s: template parse stopped early", path, page.PageID) + } + return strings.TrimSpace(rendered), nil +} + +type wikiTemplateRenderState struct { + path string + page wikiTemplatePage + row map[string]any + dot any + seenSections map[string]struct{} + active bool +} + +func (ctx *wikiContext) renderWikiTemplateRange(source string, offset int, stopBlock string, state *wikiTemplateRenderState) (string, int, bool, error) { + var out strings.Builder + for { + start := strings.Index(source[offset:], "{{") + if start < 0 { + if stopBlock != "" { + return "", offset, false, fmt.Errorf("render wiki template %s for %s: missing {{/%s}}", state.path, state.page.PageID, stopBlock) + } + if state.active { + out.WriteString(source[offset:]) + } + return out.String(), len(source), false, nil + } + start += offset + end := strings.Index(source[start+2:], "}}") + if end < 0 { + return "", offset, false, fmt.Errorf("render wiki template %s for %s: unclosed placeholder", state.path, state.page.PageID) + } + end += start + 2 + if state.active { + out.WriteString(source[offset:start]) + } + token := strings.TrimSpace(source[start+2 : end]) + if strings.HasPrefix(token, "/") { + name := strings.TrimSpace(strings.TrimPrefix(token, "/")) + if stopBlock == "" || name != stopBlock { + return "", offset, false, fmt.Errorf("render wiki template %s for %s: unexpected {{/%s}}", state.path, state.page.PageID, name) + } + return out.String(), end + 2, true, nil + } + if strings.HasPrefix(token, "#each ") { + providerName := strings.TrimSpace(strings.TrimPrefix(token, "#each ")) + body, nextOffset, err := extractWikiTemplateBlock(source, end+2, "each", state.path, state.page.PageID) + if err != nil { + return "", offset, false, err + } + if state.active { + items, err := ctx.resolveWikiTemplateEachItems(providerName, state) + if err != nil { + return "", offset, false, fmt.Errorf("render wiki template %s for %s: %w", state.path, state.page.PageID, err) + } + for _, item := range items { + child := *state + child.row = item.row + child.dot = item.dot + rendered, childOffset, closed, err := ctx.renderWikiTemplateRange(body, 0, "", &child) + if err != nil { + return "", offset, false, err + } + if closed || childOffset != len(body) { + return "", offset, false, fmt.Errorf("render wiki template %s for %s: malformed each body", state.path, state.page.PageID) + } + out.WriteString(rendered) + } + } + offset = nextOffset + continue + } + if strings.HasPrefix(token, "#if ") { + expr := strings.TrimSpace(strings.TrimPrefix(token, "#if ")) + body, nextOffset, err := extractWikiTemplateBlock(source, end+2, "if", state.path, state.page.PageID) + if err != nil { + return "", offset, false, err + } + if state.active { + value, err := ctx.evalWikiTemplateExpression(expr, state) + if err != nil { + return "", offset, false, err + } + if truthyWikiTableValue(value) { + child := *state + rendered, childOffset, closed, err := ctx.renderWikiTemplateRange(body, 0, "", &child) + if err != nil { + return "", offset, false, err + } + if closed || childOffset != len(body) { + return "", offset, false, fmt.Errorf("render wiki template %s for %s: malformed if body", state.path, state.page.PageID) + } + out.WriteString(rendered) + } + } + offset = nextOffset + continue + } + if strings.HasPrefix(token, "#with ") { + expr := strings.TrimSpace(strings.TrimPrefix(token, "#with ")) + body, nextOffset, err := extractWikiTemplateBlock(source, end+2, "with", state.path, state.page.PageID) + if err != nil { + return "", offset, false, err + } + if state.active { + value, err := ctx.evalWikiTemplateExpression(expr, state) + if err != nil { + return "", offset, false, err + } + if truthyWikiTableValue(value) { + child := *state + child.dot = value + rendered, childOffset, closed, err := ctx.renderWikiTemplateRange(body, 0, "", &child) + if err != nil { + return "", offset, false, err + } + if closed || childOffset != len(body) { + return "", offset, false, fmt.Errorf("render wiki template %s for %s: malformed with body", state.path, state.page.PageID) + } + out.WriteString(rendered) + } + } + offset = nextOffset + continue + } + if !state.active { + offset = end + 2 + continue + } + rendered, sectionID, err := ctx.renderWikiTemplateTokenWithState(token, state) + if err != nil { + return "", offset, false, err + } + if sectionID != "" { + if _, exists := state.seenSections[sectionID]; exists { + return "", offset, false, fmt.Errorf("render wiki template %s for %s: duplicate preserved section %q", state.path, state.page.PageID, sectionID) + } + state.seenSections[sectionID] = struct{}{} + } + out.WriteString(rendered) + offset = end + 2 + } +} + +type wikiTemplateEachItem struct { + row map[string]any + dot any +} + +func (ctx *wikiContext) resolveWikiTemplateEachItems(name string, state *wikiTemplateRenderState) ([]wikiTemplateEachItem, error) { + if value, ok := ctx.resolveWikiTemplateScopedValue(name, state); ok { + return wikiTemplateEachItemsFromValue(value), nil + } + rows, err := ctx.wikiDataProviderRows(name, state.page) + if err != nil { + return nil, err + } + items := make([]wikiTemplateEachItem, 0, len(rows)) + for _, row := range rows { + items = append(items, wikiTemplateEachItem{row: row, dot: row}) + } + return items, nil +} + +func wikiTemplateEachItemsFromValue(value any) []wikiTemplateEachItem { + switch typed := value.(type) { + case []map[string]any: + items := make([]wikiTemplateEachItem, 0, len(typed)) + for _, row := range typed { + items = append(items, wikiTemplateEachItem{row: row, dot: row}) + } + return items + case []any: + items := make([]wikiTemplateEachItem, 0, len(typed)) + for _, value := range typed { + if row, ok := value.(map[string]any); ok { + items = append(items, wikiTemplateEachItem{row: row, dot: row}) + continue + } + items = append(items, wikiTemplateEachItem{dot: value}) + } + return items + case []string: + items := make([]wikiTemplateEachItem, 0, len(typed)) + for _, value := range typed { + items = append(items, wikiTemplateEachItem{dot: value}) + } + return items + default: + return nil + } +} + +func (ctx *wikiContext) renderWikiTemplateTokenWithState(token string, state *wikiTemplateRenderState) (string, string, error) { + if strings.HasPrefix(token, "format:") || strings.HasPrefix(token, "table:") || strings.HasPrefix(token, "field:") || strings.HasPrefix(token, "section:") { + rendered, sectionID, err := ctx.renderWikiTemplateToken(state.path, token, state.page) + return rendered, sectionID, err + } + if strings.Contains(token, "|") || strings.Contains(token, "(") || token == "." || strings.HasPrefix(token, "page.") { + value, err := ctx.evalWikiTemplateExpression(token, state) + if err != nil { + return "", "", err + } + return stringifyWikiTemplateOutput(value), "", nil + } + if state.row != nil { + if _, ok := ctx.resolveWikiTemplateScopedValue(token, state); ok { + value, err := ctx.evalWikiTemplateExpression(token, state) + if err != nil { + return "", "", err + } + return stringifyWikiTemplateOutput(value), "", nil + } + } + rendered, sectionID, err := ctx.renderWikiTemplateToken(state.path, token, state.page) + return rendered, sectionID, err +} + +func (ctx *wikiContext) renderWikiTemplateToken(path, token string, page wikiTemplatePage) (string, string, error) { + switch token { + case "title": + return html.EscapeString(page.Title), "", nil + case "page_id": + return html.EscapeString(page.PageID), "", nil + case "status": + return html.EscapeString(page.Status), "", nil + } + if section, ok := wikiManualSectionForAlias(page.ManualSections, token); ok { + return renderWikiManualSection(section), section.ID, nil + } + if strings.HasPrefix(token, "section:") { + sectionName := strings.TrimSpace(strings.TrimPrefix(token, "section:")) + if section, ok := wikiManualSectionForAlias(page.ManualSections, sectionName); ok { + return renderWikiManualSection(section), section.ID, nil + } + return "", "", nil + } + if strings.HasPrefix(token, "format:") { + rendered, err := ctx.renderWikiFormatter(strings.TrimSpace(strings.TrimPrefix(token, "format:")), page) + return rendered, "", err + } + if strings.HasPrefix(token, "table:") { + rendered, err := ctx.renderConfiguredWikiTable(path, strings.TrimSpace(strings.TrimPrefix(token, "table:")), page) + return rendered, "", err + } + if strings.HasPrefix(token, "field:") { + rendered, err := ctx.renderExplicitWikiField(path, strings.TrimSpace(strings.TrimPrefix(token, "field:")), page) + return rendered, "", err + } + rendered, err := ctx.renderRequiredWikiField(path, token, page) + return rendered, "", err +} + +func wikiManualSectionForAlias(sections []wikiManualSection, alias string) (wikiManualSection, bool) { + for _, section := range sections { + if section.Alias == alias { + return section, true + } + } + switch alias { + case "UserTopSection": + for _, section := range sections { + if section.ID == "user_top" { + return section, true + } + } + case "UserBottomSection": + for _, section := range sections { + if section.ID == "user_bottom" { + return section, true + } + } + } + return wikiManualSection{}, false +} + +func renderWikiManualSection(section wikiManualSection) string { + return strings.Join([]string{ + "", + strings.TrimSpace(section.InitialHTML), + "", + }, "\n") +} + +func (ctx *wikiContext) renderExplicitWikiField(path, spec string, page wikiTemplatePage) (string, error) { + fieldSpec, defaultValue, hasDefault := strings.Cut(spec, "|default=") + fieldSpec = strings.TrimSpace(fieldSpec) + value, ok := ctx.resolveWikiTemplateField(fieldSpec, page) + if !ok { + if hasDefault { + return html.EscapeString(defaultValue), nil + } + return "", fmt.Errorf("render wiki template %s for %s: missing field %q", path, page.PageID, fieldSpec) + } + return html.EscapeString(value), nil +} + +func (ctx *wikiContext) renderRequiredWikiField(path, fieldSpec string, page wikiTemplatePage) (string, error) { + value, ok := ctx.resolveWikiTemplateField(fieldSpec, page) + if !ok { + return "", fmt.Errorf("render wiki template %s for %s: missing field %q", path, page.PageID, fieldSpec) + } + return html.EscapeString(value), nil +} + +func (ctx *wikiContext) resolveWikiTemplateField(fieldSpec string, page wikiTemplatePage) (string, bool) { + fieldSpec = strings.TrimSpace(fieldSpec) + if strings.HasSuffix(fieldSpec, ".text") { + base := strings.TrimSuffix(fieldSpec, ".text") + value, ok := lookupField(page.Row, base) + if !ok { + return "", false + } + text := ctx.resolveTextValue(value, nil) + return text, text != "" + } + value, ok := lookupField(page.Row, fieldSpec) + if !ok || value == nil { + return "", false + } + switch typed := value.(type) { + case string: + if typed == "" || typed == nullValue { + return "", false + } + return typed, true + case int: + return fmt.Sprintf("%d", typed), true + case float64: + return fmt.Sprintf("%d", int(typed)), true + default: + text := ctx.resolveTextValue(typed, nil) + return text, text != "" + } +} diff --git a/internal/topdata/wiki_template_expr.go b/internal/topdata/wiki_template_expr.go new file mode 100644 index 0000000..267febc --- /dev/null +++ b/internal/topdata/wiki_template_expr.go @@ -0,0 +1,629 @@ +package topdata + +import ( + "fmt" + "html" + "strconv" + "strings" + "unicode" +) + +type wikiTemplateHTML string + +func extractWikiTemplateBlock(source string, offset int, blockName, path, pageID string) (string, int, error) { + depth := 1 + bodyStart := offset + for offset < len(source) { + start := strings.Index(source[offset:], "{{") + if start < 0 { + break + } + start += offset + end := strings.Index(source[start+2:], "}}") + if end < 0 { + return "", offset, fmt.Errorf("render wiki template %s for %s: unclosed placeholder", path, pageID) + } + end += start + 2 + token := strings.TrimSpace(source[start+2 : end]) + if token == "#"+blockName || strings.HasPrefix(token, "#"+blockName+" ") { + depth++ + } else if token == "/"+blockName { + depth-- + if depth == 0 { + return source[bodyStart:start], end + 2, nil + } + } + offset = end + 2 + } + return "", offset, fmt.Errorf("render wiki template %s for %s: missing {{/%s}}", path, pageID, blockName) +} + +func (ctx *wikiContext) evalWikiTemplateExpression(expr string, state *wikiTemplateRenderState) (any, error) { + parts := splitWikiTemplatePipes(expr) + if len(parts) == 0 { + return "", nil + } + value, err := ctx.evalWikiTemplateBase(strings.TrimSpace(parts[0]), state) + if err != nil { + return nil, err + } + for _, rawFilter := range parts[1:] { + value, err = ctx.applyWikiTemplateFilter(value, strings.TrimSpace(rawFilter), state) + if err != nil { + return nil, err + } + } + return value, nil +} + +func splitWikiTemplatePipes(expr string) []string { + parts := []string{} + start := 0 + inQuote := false + escaped := false + for i := 0; i < len(expr); i++ { + switch { + case escaped: + escaped = false + case expr[i] == '\\' && inQuote: + escaped = true + case expr[i] == '"': + inQuote = !inQuote + case expr[i] == '|' && !inQuote: + parts = append(parts, expr[start:i]) + start = i + 1 + } + } + parts = append(parts, expr[start:]) + return parts +} + +func (ctx *wikiContext) evalWikiTemplateBase(expr string, state *wikiTemplateRenderState) (any, error) { + if expr == "." { + return state.dot, nil + } + if value, ok := ctx.resolveWikiTemplateScopedValue(expr, state); ok { + return value, nil + } + renderCtx := wikiTableRenderContext{Page: state.page, Path: state.path} + row := state.page.Row + if state.row != nil { + row = state.row + } + parser := wikiExprParser{tokens: tokenizeWikiExpr(expr), row: row, ctx: ctx, renderCtx: renderCtx} + value, err := parser.parseComparison() + if err != nil { + return nil, fmt.Errorf("render wiki template %s for %s expression %q: %w", state.path, state.page.PageID, expr, err) + } + if parser.peek().kind != wikiExprEOF { + return nil, fmt.Errorf("render wiki template %s for %s expression %q: unexpected token %q", state.path, state.page.PageID, expr, parser.peek().text) + } + return value, nil +} + +func (ctx *wikiContext) resolveWikiTemplateScopedValue(name string, state *wikiTemplateRenderState) (any, bool) { + name = strings.TrimSpace(name) + if strings.HasPrefix(name, "page.") { + field := strings.TrimPrefix(name, "page.") + switch field { + case "title", "Title": + return state.page.Title, true + case "page_id", "PageID": + return state.page.PageID, true + case "status", "Status": + return state.page.Status, true + case "category", "Category": + return state.page.Category, true + case "key", "Key": + return state.page.Key, true + default: + if value, ok := lookupField(state.page.Row, field); ok { + return value, true + } + return nil, false + } + } + if state.row != nil { + if value, ok := lookupField(state.row, name); ok { + return value, true + } + } + if value, ok := lookupField(state.page.Row, name); ok { + return value, true + } + switch name { + case "title": + return state.page.Title, true + case "page_id": + return state.page.PageID, true + case "status": + return state.page.Status, true + } + return nil, false +} + +func (ctx *wikiContext) applyWikiTemplateFilter(value any, filter string, state *wikiTemplateRenderState) (any, error) { + name, argText, hasArg := strings.Cut(filter, ":") + name = strings.TrimSpace(name) + args, err := parseWikiTemplateFilterArgs(argText) + if hasArg && err != nil { + return nil, fmt.Errorf("render wiki template %s for %s filter %q: %w", state.path, state.page.PageID, filter, err) + } + switch name { + case "text": + return ctx.resolveTextValue(value, nil), nil + case "trim": + return strings.TrimSpace(stringifyWikiTemplatePlain(value)), nil + case "after": + return afterFirstMarker(stringifyWikiTemplatePlain(value), firstArg(args)), nil + case "before": + return beforeFirstMarker(stringifyWikiTemplatePlain(value), firstArg(args)), nil + case "between": + text := stringifyWikiTemplatePlain(value) + return beforeFirstMarker(afterFirstMarker(text, firstArg(args)), secondArg(args)), nil + case "paragraph": + return selectWikiTemplateParagraphs(stringifyWikiTemplatePlain(value), firstArg(args)), nil + case "paragraphs": + return selectWikiTemplateParagraphs(stringifyWikiTemplatePlain(value), firstArg(args)), nil + case "linebreaks": + return wikiTemplateHTML(strings.ReplaceAll(html.EscapeString(stringifyWikiTemplatePlain(value)), "\n", "
              ")), nil + case "html": + return wikiTemplateHTML(renderWikiTemplatePlainHTML(stringifyWikiTemplatePlain(value))), nil + case "wiki": + return wikiTemplateHTML(renderWikiLinks(stringifyWikiTemplatePlain(value))), nil + case "lower": + return strings.ToLower(stringifyWikiTemplatePlain(value)), nil + case "title": + return strings.Title(stringifyWikiTemplatePlain(value)), nil + case "sentence": + text := stringifyWikiTemplatePlain(value) + for i, r := range text { + if unicode.IsLetter(r) { + return text[:i] + string(unicode.ToUpper(r)) + text[i+len(string(r)):], nil + } + } + return text, nil + case "join": + return joinWikiTemplateValue(value, firstArg(args)), nil + case "count": + return countWikiTemplateValue(value), nil + case "alignments": + return ctx.formatWikiTemplateAlignments(value) + case "list": + return wikiTemplateHTML(renderWikiTemplateList(value, firstArg(args))), nil + case "ordinal": + return ordinalWikiTableValue(numericWikiTableValue(value)), nil + case "default": + if strings.TrimSpace(stringifyWikiTemplatePlain(value)) == "" { + return firstArg(args), nil + } + return value, nil + case "ref": + return ctx.renderTemplateReference(value, firstArg(args)), nil + case "link": + return ctx.renderTemplateLink(value, firstArg(args), state), nil + default: + return nil, fmt.Errorf("render wiki template %s for %s: unknown filter %q", state.path, state.page.PageID, name) + } +} + +func parseWikiTemplateFilterArgs(raw string) ([]string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + args := []string{} + for len(raw) > 0 { + raw = strings.TrimSpace(raw) + if !strings.HasPrefix(raw, `"`) { + if idx := strings.Index(raw, ","); idx >= 0 { + args = append(args, strings.TrimSpace(raw[:idx])) + raw = raw[idx+1:] + continue + } + args = append(args, strings.TrimSpace(raw)) + break + } + var b strings.Builder + escaped := false + i := 1 + for ; i < len(raw); i++ { + if escaped { + b.WriteByte(raw[i]) + escaped = false + continue + } + if raw[i] == '\\' { + escaped = true + continue + } + if raw[i] == '"' { + break + } + b.WriteByte(raw[i]) + } + if i >= len(raw) { + return nil, fmt.Errorf("unterminated quoted argument") + } + args = append(args, b.String()) + raw = strings.TrimSpace(raw[i+1:]) + if strings.HasPrefix(raw, ",") { + raw = raw[1:] + } else if raw != "" { + return nil, fmt.Errorf("unexpected argument text %q", raw) + } + } + return args, nil +} + +func firstArg(args []string) string { + if len(args) == 0 { + return "" + } + return args[0] +} + +func secondArg(args []string) string { + if len(args) < 2 { + return "" + } + return args[1] +} + +func afterFirstMarker(text, marker string) string { + if marker == "" { + return text + } + idx := strings.Index(text, marker) + if idx < 0 { + return "" + } + return text[idx+len(marker):] +} + +func beforeFirstMarker(text, marker string) string { + if marker == "" { + return text + } + idx := strings.Index(text, marker) + if idx < 0 { + return text + } + return text[:idx] +} + +func selectWikiTemplateParagraphs(text, selector string) string { + paragraphs := splitWikiTemplateParagraphs(text) + if len(paragraphs) == 0 { + return "" + } + switch strings.TrimSpace(selector) { + case "first": + return paragraphs[0] + case "last": + return paragraphs[len(paragraphs)-1] + case "all", "": + return strings.Join(paragraphs, "\n\n") + default: + selected := []string{} + for _, part := range strings.Split(selector, ",") { + switch strings.TrimSpace(part) { + case "first": + selected = append(selected, paragraphs[0]) + case "last": + selected = append(selected, paragraphs[len(paragraphs)-1]) + default: + if idx, err := strconv.Atoi(strings.TrimSpace(part)); err == nil && idx > 0 && idx <= len(paragraphs) { + selected = append(selected, paragraphs[idx-1]) + } + } + } + return strings.Join(selected, "\n\n") + } +} + +func splitWikiTemplateParagraphs(text string) []string { + normalized := strings.ReplaceAll(text, "\r\n", "\n") + chunks := strings.Split(normalized, "\n\n") + out := []string{} + for _, chunk := range chunks { + if trimmed := strings.TrimSpace(chunk); trimmed != "" { + out = append(out, trimmed) + } + } + return out +} + +func renderWikiTemplatePlainHTML(text string) string { + paragraphs := splitWikiTemplateParagraphs(text) + if len(paragraphs) == 0 { + return "" + } + out := strings.Builder{} + for _, paragraph := range paragraphs { + escaped := html.EscapeString(paragraph) + escaped = strings.ReplaceAll(escaped, "\n", "
              ") + out.WriteString("

              ") + out.WriteString(escaped) + out.WriteString("

              ") + } + return out.String() +} + +func joinWikiTemplateValue(value any, sep string) string { + switch typed := value.(type) { + case []string: + return strings.Join(typed, sep) + case []any: + parts := make([]string, 0, len(typed)) + for _, item := range typed { + if text := stringifyWikiTemplatePlain(item); text != "" { + parts = append(parts, text) + } + } + return strings.Join(parts, sep) + default: + return stringifyWikiTemplatePlain(value) + } +} + +func (ctx *wikiContext) formatWikiTemplateAlignments(value any) (any, error) { + values, err := wikiTemplateAlignmentValues(value) + if err != nil { + return "", err + } + if len(values) == 0 || (len(values) == 1 && values[0] == 0) { + return "Any", nil + } + labels := make([]string, 0, len(values)) + for _, value := range values { + if value == 0 { + continue + } + label, ok := wikiTemplateAlignmentLabel(value) + if !ok { + return "", fmt.Errorf("unknown alignment value 0x%02X", value) + } + labels = append(labels, ctx.renderWikiTemplateAlignmentLabel(label)) + } + if len(labels) == 0 { + return "Any", nil + } + rendered := strings.Join(labels, ", ") + if ctx != nil && strings.TrimSpace(ctx.alignmentLinkFormat) == "html_anchor" { + return wikiTemplateHTML(rendered), nil + } + return rendered, nil +} + +func (ctx *wikiContext) renderWikiTemplateAlignmentLabel(label string) string { + pattern := "" + format := "wiki_markup" + if ctx != nil { + pattern = strings.TrimSpace(ctx.alignmentLinkPattern) + if strings.TrimSpace(ctx.alignmentLinkFormat) != "" { + format = strings.TrimSpace(ctx.alignmentLinkFormat) + } + } + if pattern == "" { + return label + } + target := strings.ReplaceAll(pattern, "{alignment}", wikiTemplateAlignmentTargetSegment(label)) + target = strings.ReplaceAll(target, "{label}", label) + if strings.TrimSpace(target) == "" { + return label + } + if format == "html_anchor" { + return `` + html.EscapeString(label) + `` + } + return "[[" + target + "|" + label + "]]" +} + +func wikiTemplateAlignmentTargetSegment(label string) string { + return strings.ReplaceAll(strings.TrimSpace(label), " ", "_") +} + +func wikiTemplateAlignmentValues(value any) ([]int, error) { + if isNullLike(value) { + return []int{0}, nil + } + switch typed := value.(type) { + case map[string]any: + _, hasAlignments := typed["alignments"] + _, hasList := typed["list"] + if hasAlignments && hasList { + return nil, fmt.Errorf("alignment list object cannot contain both alignments and list") + } + for key := range typed { + if key != "alignments" && key != "list" { + return nil, fmt.Errorf("alignment list object contains unsupported key %q", key) + } + } + if hasAlignments { + return wikiTemplateAlignmentValues(typed["alignments"]) + } + return wikiTemplateAlignmentValues(typed["list"]) + case string: + trimmed := strings.TrimSpace(typed) + if trimmed == "" || trimmed == "0" || strings.EqualFold(trimmed, "0x00") { + return []int{0}, nil + } + if strings.HasPrefix(strings.ToLower(trimmed), "0x") { + hexText := trimmed[2:] + if len(hexText)%2 != 0 { + return nil, fmt.Errorf("alignment hex list %q must contain two-character chunks", typed) + } + values := make([]int, 0, len(hexText)/2) + for index := 0; index < len(hexText); index += 2 { + parsed, err := strconv.ParseInt(hexText[index:index+2], 16, 0) + if err != nil { + return nil, fmt.Errorf("alignment hex list %q contains invalid chunk %q", typed, hexText[index:index+2]) + } + values = append(values, int(parsed)) + } + return values, nil + } + if strings.Contains(trimmed, ",") { + parts := strings.Split(trimmed, ",") + values := make([]int, 0, len(parts)) + for _, part := range parts { + parsed, err := parseConfiguredAlignmentValue(part) + if err != nil { + return nil, err + } + values = append(values, parsed) + } + return values, nil + } + parsed, err := parseConfiguredAlignmentValue(trimmed) + if err != nil { + return nil, err + } + return []int{parsed}, nil + case []any: + values := make([]int, 0, len(typed)) + for _, item := range typed { + parsed, err := parseConfiguredAlignmentValue(item) + if err != nil { + return nil, err + } + values = append(values, parsed) + } + return values, nil + default: + parsed, err := parseConfiguredAlignmentValue(value) + if err != nil { + return nil, err + } + return []int{parsed}, nil + } +} + +func wikiTemplateAlignmentLabel(value int) (string, bool) { + switch value { + case 0x0A: + return "Lawful Good", true + case 0x09: + return "Neutral Good", true + case 0x0C: + return "Chaotic Good", true + case 0x03: + return "Lawful Neutral", true + case 0x01: + return "True Neutral", true + case 0x05: + return "Chaotic Neutral", true + case 0x12: + return "Lawful Evil", true + case 0x11: + return "Neutral Evil", true + case 0x14: + return "Chaotic Evil", true + default: + return "", false + } +} + +func renderWikiTemplateList(value any, class string) string { + items := wikiTemplateListItems(value) + if len(items) == 0 { + return "" + } + var out strings.Builder + out.WriteString("") + for _, item := range items { + out.WriteString("
            • ") + out.WriteString(renderWikiLinks(item)) + out.WriteString("
              • ") + } + out.WriteString("
            ") + return out.String() +} + +func wikiTemplateListItems(value any) []string { + switch typed := value.(type) { + case []string: + out := make([]string, 0, len(typed)) + for _, item := range typed { + if text := strings.TrimSpace(item); text != "" { + out = append(out, text) + } + } + return out + case []any: + out := make([]string, 0, len(typed)) + for _, item := range typed { + if text := strings.TrimSpace(stringifyWikiTemplatePlain(item)); text != "" { + out = append(out, text) + } + } + return out + default: + if text := strings.TrimSpace(stringifyWikiTemplatePlain(value)); text != "" { + return []string{text} + } + return nil + } +} + +func (ctx *wikiContext) renderTemplateReference(value any, dataset string) string { + return ctx.renderReference(value, ctx.wikiIDMapForDataset(dataset), ctx.wikiNameResolverForDataset(dataset)) +} + +func (ctx *wikiContext) wikiNameResolverForDataset(dataset string) func(string) string { + switch dataset { + case "classes": + return ctx.resolveClassName + case "feat": + return ctx.resolveFeatName + case "skills": + return ctx.resolveSkillName + case "spells": + return ctx.resolveSpellName + case "racialtypes": + return ctx.resolveRaceName + case "baseitems": + return ctx.resolveBaseItemName + default: + return func(string) string { return "" } + } +} + +func (ctx *wikiContext) renderTemplateLink(value any, labelField string, state *wikiTemplateRenderState) string { + key := stringifyWikiTemplatePlain(value) + label := labelField + if state.row != nil { + if raw, ok := lookupField(state.row, labelField); ok { + label = stringifyWikiTemplatePlain(raw) + } + } + if label == "" { + label = key + } + if key == "" { + return label + } + if !ctx.canReferenceWikiKey(key) { + return "" + } + if target := ctx.publicWikiTargetForKey(key, label); target != "" { + key = target + } + return "[[" + key + "|" + label + "]]" +} + +func stringifyWikiTemplateOutput(value any) string { + if htmlValue, ok := value.(wikiTemplateHTML); ok { + return string(htmlValue) + } + return html.EscapeString(stringifyWikiTemplatePlain(value)) +} + +func stringifyWikiTemplatePlain(value any) string { + if htmlValue, ok := value.(wikiTemplateHTML); ok { + return string(htmlValue) + } + return stringifyWikiTableValue(value) +} diff --git a/internal/topdata/wiki_templates/classes.txt b/internal/topdata/wiki_templates/classes.txt new file mode 100644 index 0000000..73f5dc2 --- /dev/null +++ b/internal/topdata/wiki_templates/classes.txt @@ -0,0 +1,17 @@ +====== {{name}} ====== + +{{status}} + +{{description}} + +{{facts}} + +===== Progression ===== + +{{progression}} + + +**Auto-generated content.** This section is rebuilt from game data on each release. Edits above this line will be lost on the next build. + + +===== Notes ===== diff --git a/internal/topdata/wiki_templates/feat.txt b/internal/topdata/wiki_templates/feat.txt new file mode 100644 index 0000000..cdfa835 --- /dev/null +++ b/internal/topdata/wiki_templates/feat.txt @@ -0,0 +1,13 @@ +====== {{name}} ====== + +{{status}} + +{{description}} + +{{facts}} + + +**Auto-generated content.** This section is rebuilt from game data on each release. Edits above this line will be lost on the next build. + + +===== Notes ===== diff --git a/internal/topdata/wiki_templates/items.txt b/internal/topdata/wiki_templates/items.txt new file mode 100644 index 0000000..cdfa835 --- /dev/null +++ b/internal/topdata/wiki_templates/items.txt @@ -0,0 +1,13 @@ +====== {{name}} ====== + +{{status}} + +{{description}} + +{{facts}} + + +**Auto-generated content.** This section is rebuilt from game data on each release. Edits above this line will be lost on the next build. + + +===== Notes ===== diff --git a/internal/topdata/wiki_templates/races.txt b/internal/topdata/wiki_templates/races.txt new file mode 100644 index 0000000..4eaf273 --- /dev/null +++ b/internal/topdata/wiki_templates/races.txt @@ -0,0 +1,15 @@ +====== {{name}} ====== + +{{status}} + +{{nameforms}} + +{{description}} + +{{facts}} + + +**Auto-generated content.** This section is rebuilt from game data on each release. Edits above this line will be lost on the next build. + + +===== Notes ===== diff --git a/internal/topdata/wiki_templates/skills.txt b/internal/topdata/wiki_templates/skills.txt new file mode 100644 index 0000000..cdfa835 --- /dev/null +++ b/internal/topdata/wiki_templates/skills.txt @@ -0,0 +1,13 @@ +====== {{name}} ====== + +{{status}} + +{{description}} + +{{facts}} + + +**Auto-generated content.** This section is rebuilt from game data on each release. Edits above this line will be lost on the next build. + + +===== Notes ===== diff --git a/internal/topdata/wiki_templates/spells.txt b/internal/topdata/wiki_templates/spells.txt new file mode 100644 index 0000000..cdfa835 --- /dev/null +++ b/internal/topdata/wiki_templates/spells.txt @@ -0,0 +1,13 @@ +====== {{name}} ====== + +{{status}} + +{{description}} + +{{facts}} + + +**Auto-generated content.** This section is rebuilt from game data on each release. Edits above this line will be lost on the next build. + + +===== Notes ===== diff --git a/internal/topdata/wiki_visibility.go b/internal/topdata/wiki_visibility.go new file mode 100644 index 0000000..fff9e5c --- /dev/null +++ b/internal/topdata/wiki_visibility.go @@ -0,0 +1,407 @@ +package topdata + +import ( + "fmt" + "os" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +type wikiVisibilityDefinitions struct { + ReferenceSets map[string]wikiVisibilityReferenceSet `json:"reference_sets" yaml:"reference_sets"` + Datasets map[string]wikiVisibilityDataset `json:"datasets" yaml:"datasets"` +} + +type wikiVisibilityDataset struct { + Eligibility wikiVisibilityPredicate `json:"eligibility" yaml:"eligibility"` + Include []wikiVisibilityRule `json:"include" yaml:"include"` + Exclude []wikiVisibilityRule `json:"exclude" yaml:"exclude"` +} + +type wikiVisibilityPredicate struct { + When string `json:"when" yaml:"when"` +} + +type wikiVisibilityRule struct { + When string `json:"when" yaml:"when"` + ReferenceSet string `json:"reference_set" yaml:"reference_set"` +} + +type wikiVisibilityReferenceSet struct { + TargetDataset string `json:"target_dataset" yaml:"target_dataset"` + Sources []wikiVisibilityReferenceSource `json:"sources" yaml:"sources"` +} + +type wikiVisibilityReferenceSource struct { + Dataset string `json:"dataset" yaml:"dataset"` + TableField string `json:"table_field" yaml:"table_field"` + TableKind string `json:"table_kind" yaml:"table_kind"` + RowRefField string `json:"row_ref_field" yaml:"row_ref_field"` + ExpandMasterFeat bool `json:"expand_masterfeat" yaml:"expand_masterfeat"` +} + +func loadWikiVisibilityDefinitions(path string) (*wikiVisibilityDefinitions, error) { + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read wiki visibility definitions %s: %w", path, err) + } + policy, err := parseWikiVisibilityDefinitions(raw, path) + if err != nil { + return nil, err + } + return &policy, nil +} + +func parseWikiVisibilityDefinitions(raw []byte, path string) (wikiVisibilityDefinitions, error) { + var policy wikiVisibilityDefinitions + if err := yaml.Unmarshal(raw, &policy); err != nil { + return wikiVisibilityDefinitions{}, fmt.Errorf("parse wiki visibility definitions %s: %w", path, err) + } + if policy.ReferenceSets == nil { + policy.ReferenceSets = map[string]wikiVisibilityReferenceSet{} + } + if policy.Datasets == nil { + policy.Datasets = map[string]wikiVisibilityDataset{} + } + for dataset, definition := range policy.Datasets { + if !isWikiVisibilityDataset(dataset) { + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: unknown dataset %q", path, dataset) + } + if strings.TrimSpace(definition.Eligibility.When) != "" { + if err := validateWikiVisibilityExpression(definition.Eligibility.When); err != nil { + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: dataset %s eligibility: %w", path, dataset, err) + } + } + for _, rules := range [][]wikiVisibilityRule{definition.Include, definition.Exclude} { + for _, rule := range rules { + if strings.TrimSpace(rule.When) == "" && strings.TrimSpace(rule.ReferenceSet) == "" { + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: dataset %s rule requires when or reference_set", path, dataset) + } + if strings.TrimSpace(rule.When) != "" { + if err := validateWikiVisibilityExpression(rule.When); err != nil { + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: dataset %s rule: %w", path, dataset, err) + } + } + if set := strings.TrimSpace(rule.ReferenceSet); set != "" { + referenceSet, ok := policy.ReferenceSets[set] + if !ok { + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: dataset %s references unknown reference_set %q", path, dataset, set) + } + if referenceSet.TargetDataset != dataset { + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: dataset %s cannot use reference_set %q targeting %s", path, dataset, set, referenceSet.TargetDataset) + } + } + } + } + } + for name, referenceSet := range policy.ReferenceSets { + if !isWikiVisibilityDataset(referenceSet.TargetDataset) { + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: reference_set %s has unknown target_dataset %q", path, name, referenceSet.TargetDataset) + } + if len(referenceSet.Sources) == 0 { + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: reference_set %s requires sources", path, name) + } + for _, source := range referenceSet.Sources { + if !isWikiVisibilityDataset(source.Dataset) { + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: reference_set %s has unknown source dataset %q", path, name, source.Dataset) + } + if strings.TrimSpace(source.TableField) == "" || strings.TrimSpace(source.RowRefField) == "" { + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: reference_set %s source %s requires table_field and row_ref_field", path, name, source.Dataset) + } + switch source.TableKind { + case "class_feats", "class_bonus_feats", "race_feats": + default: + return wikiVisibilityDefinitions{}, fmt.Errorf("wiki visibility definitions %s: reference_set %s source %s has unsupported table_kind %q", path, name, source.Dataset, source.TableKind) + } + } + } + return policy, nil +} + +func isWikiVisibilityDataset(name string) bool { + switch name { + case "baseitems", "classes", "feat", "racialtypes", "skills", "spells": + return true + default: + return false + } +} + +func validateWikiVisibilityExpression(expression string) error { + parser := wikiExprParser{ + tokens: tokenizeWikiExpr(wikiExpressionBody(expression)), + row: map[string]any{}, + allowMissingFields: true, + } + if _, err := parser.parseComparison(); err != nil { + return err + } + if parser.peek().kind != wikiExprEOF { + return fmt.Errorf("unexpected token %q", parser.peek().text) + } + return nil +} + +func wikiExpressionBody(expression string) string { + expression = strings.TrimSpace(expression) + if strings.HasPrefix(expression, "{{") && strings.HasSuffix(expression, "}}") { + return strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(expression, "{{"), "}}")) + } + return expression +} + +func (ctx *wikiContext) loadWikiVisibility(path string) error { + policy, err := loadWikiVisibilityDefinitions(path) + if err != nil { + return err + } + ctx.visibilityPolicy = policy + if policy == nil { + ctx.visibleWikiKeys = nil + return nil + } + ctx.visibleWikiKeys, err = ctx.buildWikiVisibilityIndex(policy) + return err +} + +func (ctx *wikiContext) buildWikiVisibilityIndex(policy *wikiVisibilityDefinitions) (map[string]bool, error) { + visible := map[string]bool{} + for _, dataset := range []string{"classes", "racialtypes", "skills", "spells", "baseitems", "feat"} { + definition, ok := policy.Datasets[dataset] + if !ok { + continue + } + rows := ctx.wikiRowsForDataset(dataset) + for _, key := range sortedKeys(rows) { + row := rows[key] + include, err := ctx.evaluateWikiVisibilityWithoutSets(dataset, key, row, definition) + if err != nil { + return nil, err + } + visible[key] = include + } + } + referenceSets := ctx.collectWikiVisibilityReferenceSets(policy, visible) + previousVisible := ctx.visibleWikiKeys + ctx.visibleWikiKeys = visible + defer func() { + ctx.visibleWikiKeys = previousVisible + }() + for pass := 0; pass <= len(visible); pass++ { + changed := false + for _, dataset := range []string{"classes", "racialtypes", "skills", "spells", "baseitems", "feat"} { + definition, ok := policy.Datasets[dataset] + if !ok { + continue + } + rows := ctx.wikiRowsForDataset(dataset) + for _, key := range sortedKeys(rows) { + include, err := ctx.evaluateWikiVisibility(dataset, key, rows[key], definition, referenceSets) + if err != nil { + return nil, err + } + if visible[key] != include { + changed = true + } + visible[key] = include + } + } + if !changed { + return visible, nil + } + } + return nil, fmt.Errorf("wiki visibility index did not converge") +} + +func (ctx *wikiContext) evaluateWikiVisibilityWithoutSets(dataset, key string, row map[string]any, definition wikiVisibilityDataset) (bool, error) { + return ctx.evaluateWikiVisibility(dataset, key, row, definition, map[string]map[string]struct{}{}) +} + +func (ctx *wikiContext) evaluateWikiVisibility(dataset, key string, row map[string]any, definition wikiVisibilityDataset, referenceSets map[string]map[string]struct{}) (bool, error) { + if row == nil || !ctx.wikiRowCanRender(dataset, row) { + return false, nil + } + if expression := strings.TrimSpace(definition.Eligibility.When); expression != "" { + eligible, err := ctx.evalWikiVisibilityPredicate(expression, row) + if err != nil { + return false, fmt.Errorf("evaluate wiki visibility dataset %s eligibility for %s: %w", dataset, key, err) + } + if !eligible { + return false, nil + } + } + included := len(definition.Include) == 0 + for _, rule := range definition.Include { + match, err := ctx.matchWikiVisibilityRule(rule, key, row, referenceSets) + if err != nil { + return false, fmt.Errorf("evaluate wiki visibility dataset %s include for %s: %w", dataset, key, err) + } + included = included || match + } + excluded := false + for _, rule := range definition.Exclude { + match, err := ctx.matchWikiVisibilityRule(rule, key, row, referenceSets) + if err != nil { + return false, fmt.Errorf("evaluate wiki visibility dataset %s exclude for %s: %w", dataset, key, err) + } + excluded = excluded || match + } + switch ctx.wikiGenerateOverride(row) { + case "0": + return false, nil + case "1": + return true, nil + default: + return included && !excluded, nil + } +} + +func (ctx *wikiContext) matchWikiVisibilityRule(rule wikiVisibilityRule, key string, row map[string]any, referenceSets map[string]map[string]struct{}) (bool, error) { + match := false + if expression := strings.TrimSpace(rule.When); expression != "" { + value, err := ctx.evalWikiVisibilityPredicate(expression, row) + if err != nil { + return false, err + } + match = match || value + } + if set := strings.TrimSpace(rule.ReferenceSet); set != "" { + _, ok := referenceSets[set][key] + match = match || ok + } + return match, nil +} + +func (ctx *wikiContext) evalWikiVisibilityPredicate(expression string, row map[string]any) (bool, error) { + parser := wikiExprParser{ + tokens: tokenizeWikiExpr(wikiExpressionBody(expression)), + row: row, + ctx: ctx, + allowMissingFields: true, + } + value, err := parser.parseComparison() + if err != nil { + return false, err + } + if parser.peek().kind != wikiExprEOF { + return false, fmt.Errorf("unexpected token %q", parser.peek().text) + } + return truthyWikiTableValue(value), nil +} + +func (ctx *wikiContext) collectWikiVisibilityReferenceSets(policy *wikiVisibilityDefinitions, visible map[string]bool) map[string]map[string]struct{} { + out := map[string]map[string]struct{}{} + names := make([]string, 0, len(policy.ReferenceSets)) + for name := range policy.ReferenceSets { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + keys := map[string]struct{}{} + for _, source := range policy.ReferenceSets[name].Sources { + for key, row := range ctx.wikiRowsForDataset(source.Dataset) { + if !visible[key] { + continue + } + table := ctx.tableForValue(fieldValue(row, source.TableField), ctx.wikiVisibilityTables(source.TableKind)) + if table == nil { + continue + } + for _, tableRow := range table.Rows { + refKey := resolveReferenceKey(fieldValue(tableRow, source.RowRefField), ctx.wikiIDMapForDataset(policy.ReferenceSets[name].TargetDataset)) + if refKey == "" { + refKey = resolveReferenceKey(fieldValue(tableRow, source.RowRefField), nil) + } + if refKey == "" { + continue + } + if source.ExpandMasterFeat && strings.HasPrefix(refKey, "masterfeats:") { + for _, featKey := range ctx.masterFeatGroupKeys(refKey) { + keys[featKey] = struct{}{} + } + continue + } + keys[refKey] = struct{}{} + } + } + } + out[name] = keys + } + return out +} + +func (ctx *wikiContext) wikiRowsForDataset(dataset string) map[string]map[string]any { + switch dataset { + case "baseitems": + return ctx.baseitemRows + case "classes": + return ctx.classRows + case "feat": + return ctx.featRows + case "racialtypes": + return ctx.raceRows + case "skills": + return ctx.skillRows + case "spells": + return ctx.spellRows + default: + return nil + } +} + +func (ctx *wikiContext) wikiIDMapForDataset(dataset string) map[int]string { + switch dataset { + case "classes": + return ctx.classIDToKey + case "feat": + return ctx.featIDToKey + case "racialtypes": + return ctx.raceIDToKey + case "skills": + return ctx.skillIDToKey + default: + return nil + } +} + +func (ctx *wikiContext) wikiVisibilityTables(kind string) map[string]wikiTable { + switch kind { + case "class_feats": + return ctx.classFeatTables + case "class_bonus_feats": + return ctx.classBonusTables + case "race_feats": + return ctx.raceFeatTables + default: + return nil + } +} + +func (ctx *wikiContext) masterFeatGroupKeys(group string) []string { + keys := []string{} + for key, row := range ctx.featRows { + if masterfeatGroupKey(fieldValue(row, "MASTERFEAT"), ctx.masterfeatRows) == group { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys +} + +func (ctx *wikiContext) canReferenceWikiKey(key string) bool { + if key == "" || ctx.visibilityPolicy == nil { + return true + } + visible, configured := ctx.visibleWikiKeys[key] + if configured { + return visible + } + dataset := strings.SplitN(key, ":", 2)[0] + _, configured = ctx.visibilityPolicy.Datasets[dataset] + return !configured +} diff --git a/internal/topdata/wingmodel_migrate.go b/internal/topdata/wingmodel_migrate.go new file mode 100644 index 0000000..395d1bc --- /dev/null +++ b/internal/topdata/wingmodel_migrate.go @@ -0,0 +1,86 @@ +package topdata + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func importLegacyWingmodel(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { + _ = legacyTLK + + legacyDir := filepath.Join(referenceBuilderDir, "data", "wingmodel") + if _, err := os.Stat(legacyDir); err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + targetDir := filepath.Join(dataDir, "wingmodel") + targetBasePath := filepath.Join(targetDir, "base.json") + targetLockPath := filepath.Join(targetDir, "lock.json") + targetModulesDir := filepath.Join(targetDir, "modules") + targetModulePaths := []string{ + filepath.Join(targetModulesDir, "add.json"), + filepath.Join(targetModulesDir, "ovr_wingprefixes.json"), + } + if fileExists(targetBasePath) && fileExists(targetLockPath) { + allModulesPresent := true + for _, path := range targetModulePaths { + if !fileExists(path) { + allModulesPresent = false + break + } + } + if allModulesPresent { + return 0, nil + } + } + + baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) + if err != nil { + return 0, err + } + baseObj["output"] = "wingmodel.2da" + + lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) + if err != nil { + return 0, err + } + + moduleNames := []string{"add.json", "ovr_wingprefixes.json"} + moduleObjs := make([]map[string]any, 0, len(moduleNames)) + for _, name := range moduleNames { + obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", name)) + if err != nil { + return 0, err + } + moduleObjs = append(moduleObjs, obj) + } + + if err := os.MkdirAll(targetModulesDir, 0o755); err != nil { + return 0, err + } + + writes := []struct { + path string + obj map[string]any + }{ + {path: targetBasePath, obj: baseObj}, + {path: targetLockPath, obj: lockObj}, + {path: targetModulePaths[0], obj: moduleObjs[0]}, + {path: targetModulePaths[1], obj: moduleObjs[1]}, + } + for _, write := range writes { + raw, err := json.MarshalIndent(write.obj, "", " ") + if err != nil { + return 0, err + } + raw = append(raw, '\n') + if err := os.WriteFile(write.path, raw, 0o644); err != nil { + return 0, err + } + } + return len(writes), nil +} diff --git a/internal/validator/validator.go b/internal/validator/validator.go new file mode 100644 index 0000000..a31674d --- /dev/null +++ b/internal/validator/validator.go @@ -0,0 +1,546 @@ +package validator + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +type Severity string + +const ( + SeverityError Severity = "error" + SeverityWarning Severity = "warning" +) + +type Diagnostic struct { + Path string + Message string + Severity Severity +} + +type Report struct { + Diagnostics []Diagnostic + SourceCount int + DecompiledModels []string +} + +type loadedDocument struct { + Path string + ResRef string + Extension string + Document gff.Document +} + +type assetOccurrence struct { + Path string + Group string +} + +type assetGroupResolver struct { + configs []project.HAKConfig + defaultGroup string +} + +func ValidateProject(p *project.Project) Report { + report := Report{} + resourceIndex := map[string]string{} + scriptIndex := map[string]string{} + assetIndex := map[string]string{} + assetOccurrences := map[string][]assetOccurrence{} + documents := make([]loadedDocument, 0, len(p.Inventory.SourceFiles)) + resolver := newAssetGroupResolver(p) + effective := p.EffectiveConfig() + + for _, rel := range p.Inventory.SourceFiles { + if hasUppercaseResourceName(rel) { + report.add(rel, "resource filenames should be lowercase", SeverityWarning) + } + abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) + document, resref, extension, err := loadDocument(abs) + if err != nil { + report.add(rel, err.Error(), SeverityError) + continue + } + report.SourceCount++ + documents = append(documents, loadedDocument{ + Path: rel, + ResRef: resref, + Extension: extension, + Document: document, + }) + + key := strings.ToLower(resref) + "." + extension + if previous, exists := resourceIndex[key]; exists { + report.add(rel, fmt.Sprintf("duplicate resource %s also defined by %s", key, previous), SeverityError) + } else { + resourceIndex[key] = rel + } + + validateDocumentStructure(&report, rel, extension, document, effective.Validation.RequiredFields) + } + + for _, rel := range p.Inventory.ScriptFiles { + if hasUppercaseResourceName(rel) { + report.add(rel, "resource filenames should be lowercase", SeverityWarning) + } + base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))) + if previous, exists := scriptIndex[base]; exists { + report.add(rel, fmt.Sprintf("duplicate script resource %s also defined by %s", base, previous), SeverityError) + } else { + scriptIndex[base] = rel + } + } + + for _, rel := range p.Inventory.AssetFiles { + musicSource := isMusicSourceAsset(p, rel) + if hasUppercaseResourceName(rel) && !musicSource { + report.add(rel, "resource filenames should be lowercase", SeverityWarning) + } + base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))) + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(rel)), ".") + if extension == "mdl" { + if isTextMDL(filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) { + report.DecompiledModels = append(report.DecompiledModels, rel) + } + } + key := strings.ToLower(base) + "." + extension + if _, exists := assetIndex[key]; !exists { + assetIndex[key] = rel + } + if musicSource { + continue + } + + group, err := resolver.groupFor(rel) + if err != nil { + report.add(rel, err.Error(), SeverityError) + continue + } + + assetOccurrences[key] = append(assetOccurrences[key], assetOccurrence{ + Path: rel, + Group: group, + }) + } + + classifyAssetDuplicates(&report, assetOccurrences) + + for _, document := range documents { + validateReferences(&report, document, resourceIndex, scriptIndex, assetIndex, effective.Validation.BuiltinScriptPrefixes) + } + + slices.SortFunc(report.Diagnostics, func(a, b Diagnostic) int { + if cmp := strings.Compare(a.Path, b.Path); cmp != 0 { + return cmp + } + if cmp := strings.Compare(string(a.Severity), string(b.Severity)); cmp != 0 { + return cmp + } + return strings.Compare(a.Message, b.Message) + }) + slices.Sort(report.DecompiledModels) + + return report +} + +func (r Report) SummaryLines(maxPaths int) []string { + return summarizeDiagnostics(r.Diagnostics, maxPaths) +} + +func (r Report) DecompiledSummaryLine(maxPaths int) string { + if len(r.DecompiledModels) == 0 { + return "" + } + return formatSummaryLine("info", "ASCII/decompiled .mdl files detected", r.DecompiledModels, maxPaths) +} + +func (r *Report) add(path, message string, severity Severity) { + r.Diagnostics = append(r.Diagnostics, Diagnostic{Path: path, Message: message, Severity: severity}) +} + +func (r Report) HasErrors() bool { + return r.ErrorCount() > 0 +} + +func (r Report) ErrorCount() int { + count := 0 + for _, diagnostic := range r.Diagnostics { + if diagnostic.Severity == SeverityError { + count++ + } + } + return count +} + +func (r Report) WarningCount() int { + count := 0 + for _, diagnostic := range r.Diagnostics { + if diagnostic.Severity == SeverityWarning { + count++ + } + } + return count +} + +func summarizeDiagnostics(diagnostics []Diagnostic, maxPaths int) []string { + type key struct { + severity Severity + message string + } + groups := map[key][]string{} + order := make([]key, 0) + for _, diagnostic := range diagnostics { + k := key{severity: diagnostic.Severity, message: diagnostic.Message} + if _, ok := groups[k]; !ok { + order = append(order, k) + } + groups[k] = append(groups[k], diagnostic.Path) + } + slices.SortFunc(order, func(a, b key) int { + if a.severity != b.severity { + if a.severity == SeverityError { + return -1 + } + return 1 + } + return strings.Compare(a.message, b.message) + }) + lines := make([]string, 0, len(order)) + for _, k := range order { + paths := append([]string(nil), groups[k]...) + slices.Sort(paths) + paths = slices.Compact(paths) + lines = append(lines, formatSummaryLine(string(k.severity), k.message, paths, maxPaths)) + } + return lines +} + +func formatSummaryLine(prefix, message string, paths []string, maxPaths int) string { + if len(paths) == 0 { + return fmt.Sprintf("%s: %s", prefix, message) + } + if maxPaths <= 0 { + maxPaths = 3 + } + display := paths + if len(display) > maxPaths { + display = display[:maxPaths] + } + location := strings.Join(display, ", ") + if len(paths) > len(display) { + location += fmt.Sprintf(", and %d more", len(paths)-len(display)) + } + return fmt.Sprintf("%s: %s [%s]", prefix, message, location) +} + +func isTextMDL(path string) bool { + data, err := os.ReadFile(path) + if err != nil || len(data) == 0 { + return false + } + if len(data) > 4096 { + data = data[:4096] + } + if strings.IndexByte(string(data), 0) >= 0 { + return false + } + printable := 0 + for _, b := range data { + if b == '\n' || b == '\r' || b == '\t' || (b >= 32 && b <= 126) { + printable++ + } + } + ratio := float64(printable) / float64(len(data)) + lower := strings.ToLower(string(data)) + return strings.Contains(lower, "newmodel ") || strings.Contains(lower, "donemodel") || ratio >= 0.9 +} + +func loadDocument(path string) (gff.Document, string, string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return gff.Document{}, "", "", fmt.Errorf("read file: %w", err) + } + + stem := strings.TrimSuffix(filepath.Base(path), ".json") + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(stem)), ".") + if extension == "" { + return gff.Document{}, "", "", fmt.Errorf("missing resource extension before .json") + } + resref := strings.ToLower(strings.TrimSuffix(stem, "."+extension)) + + var document gff.Document + if err := json.Unmarshal(raw, &document); err != nil { + return gff.Document{}, "", "", fmt.Errorf("parse json: %w", err) + } + return document, resref, extension, nil +} + +func validateDocumentStructure(report *Report, path, extension string, document gff.Document, requiredFields map[string][]string) { + expectedType := strings.ToUpper(extension) + if strings.TrimSpace(document.FileType) != "" && strings.TrimSpace(document.FileType) != expectedType { + report.add(path, fmt.Sprintf("file_type %q does not match expected %q", document.FileType, expectedType), SeverityError) + } + + required := requiredFields[extension] + if len(required) > 0 { + fields := map[string]struct{}{} + for _, field := range document.Root.Fields { + fields[field.Label] = struct{}{} + } + for _, label := range required { + if _, ok := fields[label]; !ok { + report.add(path, fmt.Sprintf("missing required field %q", label), SeverityError) + } + } + } + + validateUniqueLabels(report, path, "root", document.Root) +} + +func validateUniqueLabels(report *Report, path, scope string, s gff.Struct) { + seen := map[string]struct{}{} + for _, field := range s.Fields { + if _, exists := seen[field.Label]; exists { + report.add(path, fmt.Sprintf("duplicate field label %q in %s", field.Label, scope), SeverityError) + } else { + seen[field.Label] = struct{}{} + } + + switch value := field.Value.(type) { + case gff.Struct: + validateUniqueLabels(report, path, scope+"."+field.Label, value) + case gff.ListValue: + for index, item := range value { + validateUniqueLabels(report, path, fmt.Sprintf("%s.%s[%d]", scope, field.Label, index), item) + } + } + } +} + +func validateReferences(report *Report, document loadedDocument, resources, scripts, assets map[string]string, builtinScriptPrefixes []string) { + walkFields(document.Document.Root, func(field gff.Field) { + value, ok := fieldStringValue(field.Value) + if !ok || value == "" { + return + } + + switch { + case isScriptField(field.Label): + if isBuiltinScript(value, builtinScriptPrefixes) { + return + } + if _, exists := scripts[strings.ToLower(value)]; !exists { + report.add(document.Path, fmt.Sprintf("missing script reference %q from field %q", value, field.Label), SeverityWarning) + } + case field.Label == "Conversation": + key := strings.ToLower(value) + ".dlg" + if _, exists := resources[key]; !exists { + report.add(document.Path, fmt.Sprintf("missing dialog reference %q", key), SeverityError) + } + case field.Label == "Mod_Entry_Area": + key := strings.ToLower(value) + ".are" + if _, exists := resources[key]; !exists { + report.add(document.Path, fmt.Sprintf("missing area reference %q", key), SeverityError) + } + case field.Label == "Model": + if !hasAsset(strings.ToLower(value), []string{"mdl"}, assets) { + report.add(document.Path, fmt.Sprintf("missing model asset for %q", value), SeverityWarning) + } + case field.Label == "Sound": + if !hasAsset(strings.ToLower(value), []string{"wav"}, assets) { + report.add(document.Path, fmt.Sprintf("missing sound asset for %q", value), SeverityWarning) + } + } + }) +} + +func classifyAssetDuplicates(report *Report, occurrences map[string][]assetOccurrence) { + keys := make([]string, 0, len(occurrences)) + for key := range occurrences { + keys = append(keys, key) + } + slices.Sort(keys) + + for _, key := range keys { + entries := occurrences[key] + if len(entries) < 2 { + continue + } + + groups := make([]string, 0, len(entries)) + groupSet := map[string]struct{}{} + paths := make([]string, 0, len(entries)) + for _, entry := range entries { + paths = append(paths, entry.Path) + if _, exists := groupSet[entry.Group]; exists { + continue + } + groupSet[entry.Group] = struct{}{} + groups = append(groups, entry.Group) + } + slices.Sort(groups) + slices.Sort(paths) + + if len(groups) == 1 { + report.add(entries[0].Path, fmt.Sprintf("duplicate asset resource %s appears multiple times in hak group %q (%s); build will fail if duplicates land in the same generated hak", key, groups[0], strings.Join(paths, ", ")), SeverityError) + continue + } + + report.add(entries[0].Path, fmt.Sprintf("duplicate asset resource %s is split across hak groups %s (%s); later hak order overrides earlier at runtime", key, strings.Join(groups, ", "), strings.Join(paths, ", ")), SeverityWarning) + } +} + +func newAssetGroupResolver(p *project.Project) assetGroupResolver { + configs := make([]project.HAKConfig, len(p.Config.HAKs)) + copy(configs, p.Config.HAKs) + slices.SortFunc(configs, func(a, b project.HAKConfig) int { + if a.Priority != b.Priority { + if a.Priority < b.Priority { + return -1 + } + return 1 + } + return strings.Compare(a.Name, b.Name) + }) + return assetGroupResolver{configs: configs, defaultGroup: p.Config.Module.ResRef} +} + +func (r assetGroupResolver) groupFor(rel string) (string, error) { + if len(r.configs) == 0 { + return r.defaultGroup, nil + } + for _, cfg := range r.configs { + if matchesAnyPattern(rel, cfg.Include) { + return cfg.Name, nil + } + } + return "", fmt.Errorf("asset does not match any hak include pattern") +} + +func matchesAnyPattern(path string, patterns []string) bool { + for _, pattern := range patterns { + if matchPathPattern(filepath.ToSlash(path), filepath.ToSlash(pattern)) { + return true + } + } + return false +} + +func matchPathPattern(path, pattern string) bool { + pathSegs := strings.Split(strings.Trim(path, "/"), "/") + patternSegs := strings.Split(strings.Trim(pattern, "/"), "/") + return matchSegments(pathSegs, patternSegs) +} + +func matchSegments(pathSegs, patternSegs []string) bool { + if len(patternSegs) == 0 { + return len(pathSegs) == 0 + } + if patternSegs[0] == "**" { + if matchSegments(pathSegs, patternSegs[1:]) { + return true + } + if len(pathSegs) > 0 { + return matchSegments(pathSegs[1:], patternSegs) + } + return false + } + if len(pathSegs) == 0 { + return false + } + ok, err := filepath.Match(patternSegs[0], pathSegs[0]) + if err != nil || !ok { + return false + } + return matchSegments(pathSegs[1:], patternSegs[1:]) +} + +func walkFields(s gff.Struct, visit func(gff.Field)) { + for _, field := range s.Fields { + visit(field) + switch value := field.Value.(type) { + case gff.Struct: + walkFields(value, visit) + case gff.ListValue: + for _, item := range value { + walkFields(item, visit) + } + } + } +} + +func hasUppercaseResourceName(rel string) bool { + base := filepath.Base(rel) + return base != strings.ToLower(base) +} + +func fieldStringValue(value gff.Value) (string, bool) { + switch typed := value.(type) { + case gff.StringValue: + return string(typed), true + case gff.ResRefValue: + return string(typed), true + default: + return "", false + } +} + +func isScriptField(label string) bool { + return strings.HasPrefix(label, "On") || strings.HasSuffix(label, "Script") || strings.Contains(label, "Script") +} + +func hasAsset(name string, extensions []string, assets map[string]string) bool { + for _, extension := range extensions { + if _, exists := assets[name+"."+extension]; exists { + return true + } + } + return false +} + +func isMusicSourceAsset(p *project.Project, rel string) bool { + rel = filepath.ToSlash(strings.TrimSpace(rel)) + ext := strings.ToLower(filepath.Ext(rel)) + effective := p.EffectiveConfig() + for _, dataset := range effective.Music.Datasets { + if !music.PathIsUnder(dataset.Source, rel) { + continue + } + for _, candidate := range dataset.ConvertExtensions { + if ext == strings.ToLower(strings.TrimSpace(candidate)) { + return true + } + } + } + if music.PathIsUnder("envi/music", rel) { + for _, candidate := range effective.Music.ConvertExtensions { + if ext == strings.ToLower(strings.TrimSpace(candidate)) { + return true + } + } + } + return false +} + +func isBuiltinScript(name string, builtinScriptPrefixes []string) bool { + lower := strings.ToLower(name) + for _, prefix := range builtinScriptPrefixes { + if strings.HasPrefix(lower, prefix) { + return true + } + } + return false +} + +func ResourceTypeForSourceExtension(extension string) (uint16, bool) { + return erf.ResourceTypeForExtension(extension) +} diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go new file mode 100644 index 0000000..35841b8 --- /dev/null +++ b/internal/validator/validator_test.go @@ -0,0 +1,356 @@ +package validator + +import ( + "os" + "path/filepath" + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func TestValidateProjectWarnsForCrossHAKDuplicateAssets(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "low")) + mustMkdir(t, filepath.Join(root, "assets", "high")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Assets", + "resref": "testassets" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { "name": "low", "priority": 1, "max_bytes": 0, "split": false, "include": ["low/**"] }, + { "name": "high", "priority": 2, "max_bytes": 0, "split": false, "include": ["high/**"] } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "low", "shared.mdl"), "low") + mustWriteFile(t, filepath.Join(root, "assets", "high", "shared.mdl"), "high") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if report.HasErrors() { + t.Fatalf("expected warnings only, got errors: %#v", report.Diagnostics) + } + if report.WarningCount() != 1 { + t.Fatalf("expected 1 warning, got %d (%#v)", report.WarningCount(), report.Diagnostics) + } +} + +func TestValidateProjectErrorsForSameHAKDuplicateAssets(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "core", "a")) + mustMkdir(t, filepath.Join(root, "assets", "core", "b")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Assets", + "resref": "testassets" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { "name": "core", "priority": 1, "max_bytes": 0, "split": false, "include": ["core/**"] } + ] +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "a", "shared.mdl"), "one") + mustWriteFile(t, filepath.Join(root, "assets", "core", "b", "shared.mdl"), "two") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if !report.HasErrors() { + t.Fatalf("expected same-hak duplicate to be an error, got %#v", report.Diagnostics) + } + if report.ErrorCount() != 1 { + t.Fatalf("expected 1 error, got %d (%#v)", report.ErrorCount(), report.Diagnostics) + } + if report.WarningCount() != 0 { + t.Fatalf("expected no warnings, got %d (%#v)", report.WarningCount(), report.Diagnostics) + } +} + +func TestValidateProjectWarnsForUppercaseResourceNames(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "blueprints", "items")) + mustMkdir(t, filepath.Join(root, "assets", "vfx")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json"), `{ + "file_type": "UTI ", + "file_version": "V3.2", + "root": {"struct_type": 0, "fields": []} +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "vfx", "SPELL_FIRE.tga"), "fire") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if report.WarningCount() < 2 { + t.Fatalf("expected lowercase warnings, got %#v", report.Diagnostics) + } +} + +func TestValidateProjectWarnsForMissingScriptReferences(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "dialogs")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "dialogs", "merchant.dlg.json"), `{ + "file_type": "DLG ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Script", + "type": "ResRef", + "value": "open_store" + } + ] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if report.HasErrors() { + t.Fatalf("expected warnings only, got errors: %#v", report.Diagnostics) + } + if report.WarningCount() == 0 { + t.Fatalf("expected missing-script warning, got %#v", report.Diagnostics) + } +} + +func TestValidateProjectUsesConfiguredBuiltinScriptPrefixes(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "dialogs")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +validation: + builtin_script_prefixes: + - custom_ +`) + mustWriteFile(t, filepath.Join(root, "src", "dialogs", "merchant.dlg.json"), `{ + "file_type": "DLG ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Script", + "type": "ResRef", + "value": "custom_open_store" + } + ] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if report.HasErrors() || report.WarningCount() != 0 { + t.Fatalf("expected configured script prefix to suppress warning, got %#v", report.Diagnostics) + } +} + +func TestValidateProjectUsesConfiguredRequiredFields(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +validation: + required_fields: + ifo: [] +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if report.HasErrors() { + t.Fatalf("expected configured required fields to suppress IFO error, got %#v", report.Diagnostics) + } +} + +func TestValidateProjectReportsDecompiledModelsWithoutFailingValidation(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets", "models")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "models", "plain_text.mdl"), "newmodel plain_text\nsetsupermodel plain_text NULL\nbeginmodelgeom plain_text\nendmodelgeom plain_text\ndonemodel plain_text\n") + if err := os.WriteFile(filepath.Join(root, "assets", "models", "binary_ok.mdl"), []byte{0x00, 0x01, 0x02, 0x03}, 0o644); err != nil { + t.Fatalf("write binary mdl: %v", err) + } + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if report.HasErrors() { + t.Fatalf("expected no validation errors, got %#v", report.Diagnostics) + } + if len(report.DecompiledModels) != 1 { + t.Fatalf("expected 1 decompiled model, got %#v", report.DecompiledModels) + } + if report.DecompiledModels[0] != "models/plain_text.mdl" { + t.Fatalf("unexpected decompiled model path: %#v", report.DecompiledModels) + } +} + +func mustMkdir(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } +} + +func mustWriteFile(t *testing.T, path, data string) { + t.Helper() + if err := os.WriteFile(path, []byte(data), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} diff --git a/scripts/crucible-smoke.sh b/scripts/crucible-smoke.sh index 4dc600d..a497b71 100755 --- a/scripts/crucible-smoke.sh +++ b/scripts/crucible-smoke.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash # Smoke test the built Crucible binaries: # - the dispatcher reports version + lists builders -# - every builder fails closed (exit 70) until it is wired (Phase 5 scaffold) +# - an unwired builder (no migrated logic yet) fails closed with exit 70 +# - a wired builder rejects a missing subcommand with a usage error (exit 64, +# NOT 70) and exits cleanly under --help — proof it delegates to the migrated +# nwn-tool surface rather than faking an artifact # - each standalone shim agrees with the dispatcher set -euo pipefail @@ -13,17 +16,16 @@ bin=bin "${bin}/crucible" version >/dev/null "${bin}/crucible" list >/dev/null -builders=(depot hak module topdata wiki) -for b in "${builders[@]}"; do +# Keep in sync with internal/dispatch.Registry (Wired flag). +unwired=(depot) +wired=(hak module topdata wiki) + +exit_of() { set +e; "$@" >/dev/null 2>&1; local c=$?; set -e; echo "${c}"; } + +for b in "${unwired[@]}"; do [[ -x "${bin}/crucible-${b}" ]] || { echo "missing ${bin}/crucible-${b}" >&2; exit 1; } - - set +e - "${bin}/crucible" "${b}" >/dev/null 2>&1 - via_dispatch=$? - "${bin}/crucible-${b}" >/dev/null 2>&1 - via_shim=$? - set -e - + via_dispatch="$(exit_of "${bin}/crucible" "${b}")" + via_shim="$(exit_of "${bin}/crucible-${b}")" if [[ "${via_dispatch}" -ne 70 ]]; then echo "FAIL: crucible ${b} exit=${via_dispatch}, want 70 (unwired must fail closed)" >&2 exit 1 @@ -34,4 +36,30 @@ for b in "${builders[@]}"; do fi done -echo "smoke ok: dispatcher + ${#builders[@]} builders fail closed as expected" +for b in "${wired[@]}"; do + [[ -x "${bin}/crucible-${b}" ]] || { echo "missing ${bin}/crucible-${b}" >&2; exit 1; } + + # --help is a clean exit for a wired builder. + help_code="$(exit_of "${bin}/crucible" "${b}" --help)" + if [[ "${help_code}" -ne 0 ]]; then + echo "FAIL: crucible ${b} --help exit=${help_code}, want 0 (wired builder)" >&2 + exit 1 + fi + + # No subcommand is a usage error, and crucially NOT the unwired 70. + via_dispatch="$(exit_of "${bin}/crucible" "${b}")" + via_shim="$(exit_of "${bin}/crucible-${b}")" + for pair in "dispatch:${via_dispatch}" "shim:${via_shim}"; do + code="${pair#*:}" + if [[ "${code}" -eq 70 ]]; then + echo "FAIL: ${pair%%:*} crucible ${b} still fails closed (70) but is wired" >&2 + exit 1 + fi + if [[ "${code}" -ne 64 ]]; then + echo "FAIL: ${pair%%:*} crucible ${b} exit=${code}, want 64 (missing subcommand)" >&2 + exit 1 + fi + done +done + +echo "smoke ok: ${#unwired[@]} unwired fail closed (70), ${#wired[@]} wired delegate to nwn-tool logic"