Phase 5: scaffold Crucible (sow-tools) — dispatcher + builder shims, container, PR-first CI, fail-closed (D11, D19).
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
|
||||
[*.go]
|
||||
indent_style = tab
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
[*.sh]
|
||||
indent_size = 2
|
||||
|
||||
[*.{json,yml,yaml}]
|
||||
indent_size = 2
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
@@ -0,0 +1,40 @@
|
||||
# Build the Crucible image. PR-first (D7): PRs build only; push to main also
|
||||
# publishes registry.westgate.pw/sow/crucible:<git-sha>. Never a mutable tag.
|
||||
name: build-image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
env:
|
||||
REGISTRY: registry.westgate.pw
|
||||
IMAGE: sow/crucible
|
||||
|
||||
jobs:
|
||||
build-image:
|
||||
runs-on: docker
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with: { fetch-depth: 0 }
|
||||
|
||||
- name: Resolve tag
|
||||
id: tag
|
||||
run: echo "sha=$(git rev-parse --short=12 HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build image
|
||||
run: |
|
||||
docker build \
|
||||
--build-arg GIT_SHA=${{ steps.tag.outputs.sha }} \
|
||||
-f docker/Dockerfile \
|
||||
-t ${REGISTRY}/${IMAGE}:${{ steps.tag.outputs.sha }} .
|
||||
|
||||
# Publish only on main (post-merge). PRs verify the build but never push.
|
||||
- name: Publish image
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
env:
|
||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: |
|
||||
echo "${REGISTRY_PASSWORD}" | docker login "${REGISTRY}" -u "${REGISTRY_USER}" --password-stdin
|
||||
docker push ${REGISTRY}/${IMAGE}:${{ steps.tag.outputs.sha }}
|
||||
@@ -1,179 +0,0 @@
|
||||
name: Release Tool
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: self-hosted
|
||||
env:
|
||||
CHECKOUT_PATH: ${{ vars.SOW_TOOLS_CHECKOUT_PATH }}
|
||||
|
||||
steps:
|
||||
- name: Sync tools checkout to tag
|
||||
env:
|
||||
TAG_NAME: ${{ github.ref_name }}
|
||||
EVENT_SHA: ${{ github.sha }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CHECKOUT_PATH="${CHECKOUT_PATH:-/home/ubuntu/tools-checkout}"
|
||||
CLONE_URL="https://x-access-token:${GITHUB_TOKEN}@gitea.westgate.pw/ShadowsOverWestgate/sow-tools.git"
|
||||
export GIT_TERMINAL_PROMPT=0
|
||||
|
||||
if [[ ! -d "${CHECKOUT_PATH}/.git" ]]; then
|
||||
mkdir -p "$(dirname "${CHECKOUT_PATH}")"
|
||||
echo "Cloning tools checkout into ${CHECKOUT_PATH}"
|
||||
timeout 120 git clone --filter=blob:none "${CLONE_URL}" "${CHECKOUT_PATH}"
|
||||
fi
|
||||
|
||||
cd "${CHECKOUT_PATH}"
|
||||
git remote set-url origin "${CLONE_URL}"
|
||||
echo "Fetching tools refs"
|
||||
timeout 120 git fetch origin --prune --force --tags
|
||||
TAG_SHA="$(git rev-list -n1 "refs/tags/${TAG_NAME}")"
|
||||
if [[ -n "${EVENT_SHA}" && "${TAG_SHA}" != "${EVENT_SHA}" ]]; then
|
||||
echo "Fetched tag ${TAG_NAME} resolves to ${TAG_SHA}, but the workflow event SHA is ${EVENT_SHA}."
|
||||
echo "Refusing to build a release from a stale or mismatched local tag."
|
||||
exit 1
|
||||
fi
|
||||
git reset --hard "refs/tags/${TAG_NAME}"
|
||||
git clean -ffdx
|
||||
echo "Release ${TAG_NAME} will build commit $(git rev-parse HEAD)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends curl jq
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.26'
|
||||
cache: false
|
||||
|
||||
- name: Build sow-toolkit binaries
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CHECKOUT_PATH="${CHECKOUT_PATH:-/home/ubuntu/tools-checkout}"
|
||||
cd "${CHECKOUT_PATH}"
|
||||
rm -f ./tools/sow-toolkit ./tools/sow-toolkit.exe
|
||||
mkdir -p tools .cache/go-build
|
||||
GOCACHE="${PWD}/.cache/go-build" go build -o ./tools/sow-toolkit ./cmd/nwn-tool
|
||||
GOOS=windows GOARCH=amd64 GOCACHE="${PWD}/.cache/go-build" go build -o ./tools/sow-toolkit.exe ./cmd/nwn-tool
|
||||
go version -m ./tools/sow-toolkit
|
||||
|
||||
- name: Pack release bundles
|
||||
id: pack
|
||||
env:
|
||||
TAG_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CHECKOUT_PATH="${CHECKOUT_PATH:-/home/ubuntu/tools-checkout}"
|
||||
cd "${CHECKOUT_PATH}"
|
||||
linux_binary="tools/sow-toolkit"
|
||||
windows_binary="tools/sow-toolkit.exe"
|
||||
if [[ ! -x "${linux_binary}" ]]; then
|
||||
echo "Missing built tool: ${linux_binary}"
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -f "${windows_binary}" ]]; then
|
||||
echo "Missing built tool: ${windows_binary}"
|
||||
exit 1
|
||||
fi
|
||||
linux_archive="/tmp/sow-toolkit-linux-amd64-${TAG_NAME}.tar.gz"
|
||||
windows_archive="/tmp/sow-toolkit-windows-amd64-${TAG_NAME}.exe"
|
||||
tar -czf "${linux_archive}" -C tools sow-toolkit
|
||||
cp "${windows_binary}" "${windows_archive}"
|
||||
echo "linux_archive=${linux_archive}" >> "$GITHUB_OUTPUT"
|
||||
echo "linux_asset_name=sow-toolkit-linux-amd64.tar.gz" >> "$GITHUB_OUTPUT"
|
||||
echo "windows_archive=${windows_archive}" >> "$GITHUB_OUTPUT"
|
||||
echo "windows_asset_name=sow-toolkit-windows-amd64.exe" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create release and upload artifacts
|
||||
env:
|
||||
GITEA_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITEA_SERVER_URL: ${{ github.server_url }}
|
||||
GITEA_REPO: ${{ github.repository }}
|
||||
LINUX_ARCHIVE: ${{ steps.pack.outputs.linux_archive }}
|
||||
LINUX_ASSET_NAME: ${{ steps.pack.outputs.linux_asset_name }}
|
||||
WINDOWS_ARCHIVE: ${{ steps.pack.outputs.windows_archive }}
|
||||
WINDOWS_ASSET_NAME: ${{ steps.pack.outputs.windows_asset_name }}
|
||||
TAG_NAME: ${{ steps.pack.outputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${GITEA_SERVER_URL%/}/api/v1"
|
||||
OWNER="${GITEA_REPO%%/*}"
|
||||
NAME="${GITEA_REPO#*/}"
|
||||
AUTH=(-H "Authorization: token ${GITEA_API_TOKEN}" -H "Accept: application/json")
|
||||
RESP_FILE=/tmp/gitea-release.json
|
||||
|
||||
CODE=$(curl -sS -o "${RESP_FILE}" -w "%{http_code}" "${AUTH[@]}" \
|
||||
"${API}/repos/${OWNER}/${NAME}/releases/tags/${TAG_NAME}")
|
||||
|
||||
if [[ "${CODE}" == "200" ]]; then
|
||||
RELEASE_ID=$(jq -r '.id' "${RESP_FILE}")
|
||||
else
|
||||
BODY=$(jq -n --arg t "${TAG_NAME}" \
|
||||
'{tag_name: $t, name: $t, body: "sow-toolkit binary release built by Gitea Actions."}')
|
||||
CODE=$(curl -sS -o "${RESP_FILE}" -w "%{http_code}" "${AUTH[@]}" \
|
||||
-X POST -H "Content-Type: application/json" \
|
||||
-d "${BODY}" \
|
||||
"${API}/repos/${OWNER}/${NAME}/releases")
|
||||
if [[ "${CODE}" != "201" && "${CODE}" != "200" ]]; then
|
||||
echo "Create release failed (HTTP ${CODE}):"
|
||||
cat "${RESP_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
RELEASE_ID=$(jq -r '.id' "${RESP_FILE}")
|
||||
fi
|
||||
|
||||
for name in "${LINUX_ASSET_NAME}" "${WINDOWS_ASSET_NAME}"; do
|
||||
ASSET_ID=$(jq -r --arg name "${name}" '.assets[]? | select(.name == $name) | .id' "${RESP_FILE}" | head -n1)
|
||||
if [[ -n "${ASSET_ID}" ]]; then
|
||||
curl -sS -f "${AUTH[@]}" -X DELETE \
|
||||
"${API}/repos/${OWNER}/${NAME}/releases/${RELEASE_ID}/assets/${ASSET_ID}"
|
||||
fi
|
||||
done
|
||||
|
||||
curl -sS -f "${AUTH[@]}" -X POST \
|
||||
-F "attachment=@${LINUX_ARCHIVE}" \
|
||||
"${API}/repos/${OWNER}/${NAME}/releases/${RELEASE_ID}/assets?name=${LINUX_ASSET_NAME}"
|
||||
|
||||
curl -sS -f "${AUTH[@]}" -X POST \
|
||||
-F "attachment=@${WINDOWS_ARCHIVE}" \
|
||||
"${API}/repos/${OWNER}/${NAME}/releases/${RELEASE_ID}/assets?name=${WINDOWS_ASSET_NAME}"
|
||||
|
||||
- name: Cleanup temporary files
|
||||
if: always()
|
||||
env:
|
||||
TAG_NAME: ${{ github.ref_name }}
|
||||
run: |
|
||||
rm -f \
|
||||
"/tmp/sow-toolkit-linux-amd64-${TAG_NAME}.tar.gz" \
|
||||
"/tmp/sow-toolkit-windows-amd64-${TAG_NAME}.exe" \
|
||||
/tmp/gitea-release.json
|
||||
|
||||
- name: Prune old releases (keep latest 1)
|
||||
env:
|
||||
GITEA_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITEA_SERVER_URL: ${{ github.server_url }}
|
||||
GITEA_REPO: ${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${GITEA_SERVER_URL%/}/api/v1"
|
||||
OWNER="${GITEA_REPO%%/*}"
|
||||
NAME="${GITEA_REPO#*/}"
|
||||
AUTH=(-H "Authorization: token ${GITEA_API_TOKEN}" -H "Accept: application/json")
|
||||
curl -sS "${AUTH[@]}" \
|
||||
"${API}/repos/${OWNER}/${NAME}/releases?limit=50&page=1" \
|
||||
-o /tmp/all-releases.json
|
||||
jq -r '.[1:][].id' /tmp/all-releases.json | while read -r rid; do
|
||||
[[ -n "${rid}" ]] || continue
|
||||
curl -sS -f "${AUTH[@]}" -X DELETE \
|
||||
"${API}/repos/${OWNER}/${NAME}/releases/${rid}"
|
||||
done
|
||||
rm -f /tmp/all-releases.json
|
||||
@@ -0,0 +1,40 @@
|
||||
# Tag a Crucible release: re-tag the already-built immutable image and attach
|
||||
# per-platform binary bundles. Tag-gated; never PR-triggered (deploy/release is
|
||||
# not a PR concern, D7). The :<sha> image from build-image is the source of truth.
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
env:
|
||||
REGISTRY: registry.westgate.pw
|
||||
IMAGE: sow/crucible
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: nixos/nix:latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with: { fetch-depth: 0 }
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
nix develop --command bash -c '
|
||||
set -euo pipefail
|
||||
GOOS=linux GOARCH=amd64 scripts/build-binaries.sh
|
||||
mkdir -p dist
|
||||
tar -C bin -czf "dist/crucible-linux-amd64-${GITHUB_REF_NAME}.tar.gz" .
|
||||
'
|
||||
|
||||
- name: Upload release artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: crucible-${{ github.ref_name }}
|
||||
path: dist/*
|
||||
|
||||
# TODO(phase-6): re-tag registry.westgate.pw/sow/crucible:<sha> as :<tag>
|
||||
# and pin it from sow-platform releases/*.yml once the registry is live.
|
||||
@@ -0,0 +1,27 @@
|
||||
# Lint + unit tests for the Crucible suite. PR-first: PRs + push to main (D7).
|
||||
name: test
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: docker
|
||||
container:
|
||||
image: nixos/nix:latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with: { fetch-depth: 0 }
|
||||
- name: vet + test + lint
|
||||
run: |
|
||||
nix develop --command bash -c '
|
||||
set -euo pipefail
|
||||
go vet ./...
|
||||
go test ./...
|
||||
shellcheck scripts/*.sh
|
||||
yamllint .gitea
|
||||
'
|
||||
- name: binary smoke (fail-closed contract)
|
||||
run: nix develop --command make smoke
|
||||
+16
-8
@@ -1,12 +1,20 @@
|
||||
# Local Go build cache
|
||||
# Build outputs — Crucible binaries are CI artifacts / image layers, never
|
||||
# committed (D19; replaces the old habit of committing nwn-tool/sow-toolkit).
|
||||
/bin/
|
||||
*.exe
|
||||
nwn-tool
|
||||
sow-toolkit
|
||||
|
||||
# Go / build cache
|
||||
.cache/
|
||||
.cache/**
|
||||
|
||||
# Built tool output
|
||||
/tools/sow-toolkit
|
||||
/tools/sow-toolkit.exe
|
||||
.bin/
|
||||
.worktrees/
|
||||
# nix build symlink
|
||||
result
|
||||
result-*
|
||||
|
||||
# OS/editor noise
|
||||
# OS / editor
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
*.swp
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
external-sources=true
|
||||
source-path=SCRIPTDIR
|
||||
@@ -0,0 +1,8 @@
|
||||
extends: relaxed
|
||||
rules:
|
||||
line-length: disable
|
||||
comments:
|
||||
min-spaces-from-content: 1
|
||||
document-start: disable
|
||||
truthy:
|
||||
check-keys: false
|
||||
@@ -1,274 +1,57 @@
|
||||
---
|
||||
description:
|
||||
description: sow-tools / Crucible — the build/conversion/sync toolchain repo.
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# Toolkit Agent Guide
|
||||
# sow-tools / Crucible Agent Guide
|
||||
|
||||
`toolkit/` owns the shared `sow-toolkit` Go binary consumed by `module/` and
|
||||
`assets/`. It does not own authored game content — it owns the build, extract,
|
||||
validate, compare, topdata, wiki, music, and changelog machinery that those
|
||||
repos invoke through wrapper scripts.
|
||||
This repo owns the **builder logic** for the migration: one Go module
|
||||
(`gitea.westgate.pw/ShadowsOverWestgate/sow-tools`) producing the `crucible`
|
||||
dispatcher and the `crucible-<name>` binaries (D11). Read
|
||||
[`../AGENTS.md`](../AGENTS.md) (migration hub) and
|
||||
[`../../KICKOFF_PROMPT.md`](../../KICKOFF_PROMPT.md) first.
|
||||
|
||||
## 1. Project Overview
|
||||
## What this repo owns / does not own
|
||||
|
||||
Repository name: `sow-tools`
|
||||
Go module: `gitea.westgate.pw/ShadowsOverWestgate/sow-tools`
|
||||
Language: Go 1.26+
|
||||
Entry point: `cmd/nwn-tool/`
|
||||
Built binary: `tools/sow-toolkit` (Linux) / `tools/sow-toolkit.exe` (Windows)
|
||||
Owns: build/extract/validate/compare pipeline, ERF/HAK packing, topdata 2da/tlk
|
||||
compilation, wiki rendering/deploy, depot blob verify, music conversion,
|
||||
changelog. Does **not** own authored game content (that is `sow-module` /
|
||||
`sow-topdata` / `sow-assets-manifest`) or any production deploy authority (that
|
||||
is `sow-platform`).
|
||||
|
||||
What this repo owns:
|
||||
## Scaffold rules (Phase 5)
|
||||
|
||||
- the `sow-toolkit` CLI implementation
|
||||
- all build, extract, compare, and validate pipeline logic
|
||||
- native topdata validation, build, compare, conversion, and packaging
|
||||
- wiki rendering and NodeBB deployment
|
||||
- music scanning, BMU generation, credits, and normalization
|
||||
- changelog generation from Gitea API
|
||||
- config loading and effective-config inspection
|
||||
- HAK archive reading and writing (ERF format)
|
||||
- GFF JSON conversion
|
||||
1. **Source is not transplanted.** The migration hard rules forbid copying the
|
||||
`internal/` packages from `gitea/sow-tools` automatically. The operator
|
||||
migrates them at cutover; see [`docs/migration-from-nwn-tool.md`](docs/migration-from-nwn-tool.md).
|
||||
2. **Fail closed, never fake.** Unwired builders exit `70`. Do not stub a
|
||||
builder to emit a placeholder artifact.
|
||||
3. **Binaries are not committed.** They are CI artifacts / image layers (D19).
|
||||
`/bin/`, `*.exe`, `nwn-tool`, `sow-toolkit` are gitignored.
|
||||
4. **No home-dir / `NWN_ROOT` guessing.** Builders take roots explicitly via
|
||||
flag or env. See [`docs/consumer-contract.md`](docs/consumer-contract.md).
|
||||
5. **The registry is the command surface.** `internal/dispatch.Registry` is the
|
||||
single source of truth; keep it in sync with `cmd/` and
|
||||
[`docs/command-surface.md`](docs/command-surface.md). Adding a builder = a
|
||||
`cmd/crucible-<name>/main.go` shim + a `Registry` entry + a doc row.
|
||||
|
||||
What this repo does not own:
|
||||
## Wiring a builder (operator cutover)
|
||||
|
||||
- authored module source (`src/`, `topdata/`) — that is `module/`
|
||||
- authored binary asset source (`content/`) — that is `assets/`
|
||||
- user-facing wrapper commands — those live in the consumer repos
|
||||
1. Migrate the relevant `internal/` package(s) from `gitea/sow-tools`.
|
||||
2. Register a handler and flip the builder off the unwired path in
|
||||
`internal/dispatch`.
|
||||
3. Add tests; keep outputs deterministic (same input → same bytes).
|
||||
4. `make check` must stay green; `make smoke` is updated to expect a wired exit.
|
||||
|
||||
## 2. Repository Layout
|
||||
|
||||
```text
|
||||
cmd/nwn-tool/ CLI entrypoint (flag parsing, command dispatch)
|
||||
internal/app/ command wiring, console UX, verbosity model
|
||||
internal/changelog/ changelog rendering from Gitea API
|
||||
internal/erf/ ERF/MOD/HAK reading and writing
|
||||
internal/gff/ canonical GFF JSON and binary conversion
|
||||
internal/music/ music scanning, BMU build, credits, normalization
|
||||
internal/pipeline/ build, extract, compare, and manifest workflows
|
||||
internal/project/ config loading, effective config, repo scanning
|
||||
internal/topdata/ native topdata, wiki, and autogen workflows
|
||||
internal/validator/ validation and diagnostics
|
||||
tools/ built development binary output
|
||||
docs/ additional implementation-level documentation
|
||||
```
|
||||
|
||||
Key topdata-adjacent contracts in `internal/topdata/` (read before editing
|
||||
topdata logic):
|
||||
|
||||
- `internal/topdata/FEAT_GENERATED_FAMILIES_CONTRACT.md`
|
||||
- `internal/topdata/FAMILY_EXPANSION_CONTRACT.md`
|
||||
- `internal/topdata/MASTERFEATS_CONTRACT.md`
|
||||
- `internal/topdata/INHERITANCE_CONTRACT.md`
|
||||
- `internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md`
|
||||
- `internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md`
|
||||
|
||||
## 3. Relevant Links And Documentation
|
||||
|
||||
Read these before making significant changes:
|
||||
|
||||
- `README.md` — full command surface, consumer resolution, release automation
|
||||
- `CONFIGURATION_HARDENING_CONTRACT.md` — YAML authority, config refactor progress, outstanding hardening work
|
||||
- `CONFIGURATION_REFACTOR_CONTRACT.md` — config normalization scope
|
||||
- `LOG_REFACTOR_CONTRACT.md` — logging verbosity model, phase/summary presentation, remaining command gaps
|
||||
- `TOPDATA_HARDENING_CONTRACT.md` — active topdata hardening priorities and findings
|
||||
- `WIKI_DEPLOYMENT_CONTRACT.md` — wiki build/deploy phase, current state, immediate goals
|
||||
- `CHANGELOG_AUTOMATION_CONTRACT.md` — changelog generator contract and consumer ownership
|
||||
- `MUSIC_REFACTOR_CONTRACT.md` — music pipeline refactor status and dataset model
|
||||
- `MODEL_COMPILATION_ON_BUILDTIME_CONTRACT.md` — outstanding model compilation build-time contract
|
||||
- `SCRIPT_WRAPPER_CONTRACT.md` — wrapper contract status (largely complete)
|
||||
- `module/topdata/wiki/TEMPLATE_AUTHORITY_CONTRACT.md` in `sow-module` — wiki template/YAML ownership rules; mandatory before editing wiki rendering
|
||||
|
||||
## 4. Full Command Surface
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
sow-toolkit build # build module ERF from src/
|
||||
sow-toolkit build-module # alias; builds module with HAK manifest refresh
|
||||
sow-toolkit build-haks # build HAKs and haks.json manifest
|
||||
[--hak <name>] # build one HAK by name
|
||||
[--archive <name>] # build one archive
|
||||
[--source-manifest <path>] # seed split state from prior manifest
|
||||
[--plan-only] # show build plan without writing
|
||||
[--skip-music] # skip music conversion for this run
|
||||
[--music-dataset <id>] # process a single music dataset
|
||||
sow-toolkit extract [<archive>...] # extract built archive back into src/
|
||||
sow-toolkit validate # validate project structure and config
|
||||
sow-toolkit compare # compare built output against extracted source
|
||||
sow-toolkit apply-hak-manifest # apply haks.json to module HAK list
|
||||
[<manifest-path>]
|
||||
|
||||
sow-toolkit config validate # validate nwn-tool.yaml against schema
|
||||
sow-toolkit config effective # dump effective merged config
|
||||
sow-toolkit config inspect <key> # inspect one config value
|
||||
sow-toolkit config explain <key> # explain a config key's meaning and source
|
||||
sow-toolkit config sources # show config file discovery order
|
||||
|
||||
sow-toolkit music list-datasets # list configured music datasets
|
||||
sow-toolkit music scan # scan and report music sources
|
||||
sow-toolkit music build # convert and package music BMUs
|
||||
sow-toolkit music credits # generate music credits
|
||||
sow-toolkit music validate # validate music dataset config
|
||||
sow-toolkit music manifest # emit music manifest
|
||||
sow-toolkit music normalize # normalize music metadata
|
||||
|
||||
sow-toolkit validate-topdata # validate topdata authoring
|
||||
sow-toolkit build-topdata # compile topdata + optional wiki
|
||||
[--force] # force rebuild even if cache is fresh
|
||||
[--wiki] # also run wiki build
|
||||
sow-toolkit build-top-package # package sow_top.hak + sow_tlk.tlk only
|
||||
[--force]
|
||||
sow-toolkit compare-topdata # native self-check for topdata pipeline
|
||||
sow-toolkit convert-topdata # 2DA/JSON/module conversion utilities
|
||||
<2da-to-json|2da-to-module|json-to-2da> ...
|
||||
|
||||
sow-toolkit build-wiki # render topdata wiki into .cache/wiki/
|
||||
[--force]
|
||||
sow-toolkit deploy-wiki # deploy rendered wiki to NodeBB
|
||||
[--source-dir <path>]
|
||||
[--endpoint <url>]
|
||||
[--token <token>]
|
||||
[--version <ver>]
|
||||
[--namespace <ns>]
|
||||
[--category <namespace=cid>]
|
||||
[--manifest <path>]
|
||||
[--stale-policy <report|archive|purge>]
|
||||
[--dry-run]
|
||||
[--create]
|
||||
[--force]
|
||||
[--reset-managed-namespaces]
|
||||
|
||||
sow-toolkit build-changelog # render release changelog from Gitea API
|
||||
[--config <path>] # defaults to scripts/changelog.json
|
||||
[--output <path>] # defaults to stdout
|
||||
[--current-tag <tag>]
|
||||
[--previous-tag <tag>]
|
||||
[--api-base-url <url>]
|
||||
[--token <token>] # or env: GITEA_API_TOKEN, GITEA_TOKEN, SOW_TOOLS_TOKEN
|
||||
nix develop && make check # vet + test + shellcheck + yamllint
|
||||
make build # cmd/* -> ./bin
|
||||
make smoke # assert fail-closed contract
|
||||
make image # crucible:<sha>
|
||||
```
|
||||
|
||||
Global flags (pass after the command name): `--quiet`, `--verbose`, `--debug`
|
||||
## Git
|
||||
|
||||
## 5. Config Discovery Order
|
||||
|
||||
1. `nwn-tool.yaml`
|
||||
2. `nwn-tool.yml`
|
||||
3. `nwn-tool.json` (legacy, prints migration warning)
|
||||
|
||||
YAML is canonical. Inspect effective config with:
|
||||
|
||||
```bash
|
||||
./scripts/run-nwn-tool.sh config effective --json
|
||||
./scripts/run-nwn-tool.sh config explain topdata.package_hak
|
||||
./scripts/run-nwn-tool.sh config explain autogen.cache.root
|
||||
./scripts/run-nwn-tool.sh config explain topdata.wiki.page_templates_dir
|
||||
```
|
||||
|
||||
## 6. Building The Toolkit
|
||||
|
||||
Normal build (produces `tools/sow-toolkit`):
|
||||
|
||||
```bash
|
||||
./build-tool.sh # Linux/macOS
|
||||
.\build-tool.ps1 # Windows
|
||||
```
|
||||
|
||||
Both wrappers use `.cache/go-build` and skip rebuild when sources are older than
|
||||
the existing binary.
|
||||
|
||||
Or run directly without building:
|
||||
|
||||
```bash
|
||||
GOCACHE="$PWD/.cache/go-build" go run ./cmd/nwn-tool --help
|
||||
```
|
||||
|
||||
After building, test it from a consumer repo:
|
||||
|
||||
```bash
|
||||
cd ../module
|
||||
SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./validate-topdata.sh
|
||||
```
|
||||
|
||||
## 7. Key Tests
|
||||
|
||||
Run the full suite:
|
||||
|
||||
```bash
|
||||
go test ./...
|
||||
```
|
||||
|
||||
Determinism test — mandatory when changing topdata ID allocation, lockfile
|
||||
persistence, generated families, or null-and-relocate behavior:
|
||||
|
||||
```bash
|
||||
go test ./internal/topdata -run TestBuildNativeKeepsGeneratedIDsStableAcrossRepeatedBuilds
|
||||
```
|
||||
|
||||
App-level tests (CLI behavior, compact vs. verbose output):
|
||||
|
||||
```bash
|
||||
go test ./internal/app/...
|
||||
```
|
||||
|
||||
Changelog tests:
|
||||
|
||||
```bash
|
||||
go test ./internal/changelog/...
|
||||
```
|
||||
|
||||
## 8. Consumer Resolution
|
||||
|
||||
Consumer repos resolve the toolkit binary in this order:
|
||||
|
||||
1. Pinned local binary: `tools/sow-toolkit` / `tools/sow-toolkit.exe`
|
||||
2. Dev override via `SOW_TOOLS_DEV_BINARY=<path>`
|
||||
3. Auto-install from latest `sow-tools` Gitea release
|
||||
|
||||
Use `SOW_TOOLS_DEV_BINARY` to override for one invocation without replacing
|
||||
the installed binary:
|
||||
|
||||
```bash
|
||||
cd ../module
|
||||
SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-topdata.sh
|
||||
```
|
||||
|
||||
## 9. Release Artifacts
|
||||
|
||||
This repo publishes:
|
||||
|
||||
- `sow-toolkit-linux-amd64.tar.gz`
|
||||
- `sow-toolkit-windows-amd64.exe`
|
||||
|
||||
Consumer repos install them with `scripts/install-tool.sh` or
|
||||
`scripts/install-tool.ps1`.
|
||||
|
||||
## 10. Wiki Ownership Rule
|
||||
|
||||
When a task touches `internal/topdata/wiki*`:
|
||||
|
||||
1. Read `module/topdata/wiki/TEMPLATE_AUTHORITY_CONTRACT.md` in `sow-module` first.
|
||||
2. Page structure, displayed values, headings, and wording belong in
|
||||
`module/topdata/wiki/templates/` and `module/topdata/wiki/data.yaml` /
|
||||
`tables.yaml`.
|
||||
3. Toolkit code owns only generic rendering mechanics, expression helpers,
|
||||
validation, deterministic ordering, and compatibility fallbacks.
|
||||
4. Do not mention Shadows Over Westgate-specific categories, row keys, labels,
|
||||
or wording in toolkit Go code unless they are compatibility shims.
|
||||
|
||||
## 11. Repository Working Rules
|
||||
|
||||
### Git
|
||||
|
||||
- Never create new branches or commit anything.
|
||||
- Allow the user to do it manually.
|
||||
- Suggest a single commit message, but do not commit yourself.
|
||||
- Never reset, discard, or overwrite user changes unless explicitly asked.
|
||||
|
||||
### Implementation Discipline
|
||||
|
||||
- YAML is the authoritative config format. JSON is for generated artifacts only.
|
||||
- All repository-shaped behavior must flow through `nwn-tool.yaml` declarations,
|
||||
not through toolkit-internal hardcoded assumptions.
|
||||
- New config fields require validation; invalid config must fail early and clearly.
|
||||
- Outputs must be deterministic across runs given the same inputs.
|
||||
- Run `go test ./...` before claiming a change is complete.
|
||||
- Check `config effective` in a consumer repo when changing config-loading logic.
|
||||
Never commit, branch, or push. Suggest a commit message; let the operator do it.
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
# Changelog Automation Contract
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- The shared implementation is present in `internal/changelog/` and is exposed
|
||||
as `sow-toolkit build-changelog`.
|
||||
- Consumer configs exist in both `module/scripts/changelog.json` and
|
||||
`assets/scripts/changelog.json`.
|
||||
- CLI parsing and generator behavior are covered by
|
||||
`internal/app/app_test.go` and `internal/changelog/changelog_test.go`.
|
||||
- `module/scripts/release-all.sh` already stages `build/CHANGELOG.md`.
|
||||
|
||||
Remaining work:
|
||||
|
||||
- confirm and document the `assets/` release workflow uses the shared command in
|
||||
the same way as `module/`
|
||||
- build the deferred cross-repo NodeBB aggregation bot if combined public release
|
||||
posts are still desired
|
||||
|
||||
## Scope
|
||||
|
||||
This contract governs release changelog generation across:
|
||||
|
||||
- `toolkit/` as the implementation owner
|
||||
- `module/` as a changelog-producing consumer
|
||||
- `assets/` as a changelog-producing consumer
|
||||
|
||||
The immediate goal is to produce stable per-repo `CHANGELOG.md` release artifacts
|
||||
that can later be merged by a NodeBB bot into one combined release post.
|
||||
|
||||
## Ownership Split
|
||||
|
||||
- `toolkit/`
|
||||
- owns the shared changelog generator implementation
|
||||
- owns the CLI contract exposed through `sow-toolkit`
|
||||
- owns the config schema used by consumer repos
|
||||
- `module/`
|
||||
- owns its changelog section mapping under `scripts/changelog.json`
|
||||
- owns invoking changelog generation during its tag release flow
|
||||
- `assets/`
|
||||
- owns its changelog section mapping under `scripts/changelog.json`
|
||||
- owns invoking changelog generation during its tag release flow
|
||||
|
||||
## Shared CLI Contract
|
||||
|
||||
The canonical generator entrypoint is:
|
||||
|
||||
```bash
|
||||
sow-toolkit build-changelog
|
||||
```
|
||||
|
||||
Supported flags:
|
||||
|
||||
- `--config <path>`: repo-local JSON section config
|
||||
- `--output <path>`: write rendered Markdown to a file; otherwise stdout
|
||||
- `--current-tag <tag>`: optional explicit current tag
|
||||
- `--previous-tag <tag>`: optional explicit previous tag
|
||||
- `--api-base-url <url>`: optional explicit Gitea API base URL
|
||||
- `--token <token>`: optional explicit Gitea API token
|
||||
|
||||
If `--current-tag` is omitted, the generator must resolve the exact tag at
|
||||
`HEAD`. If `--previous-tag` is omitted, it must select the next older tag by
|
||||
`creatordate`.
|
||||
|
||||
## Config Contract
|
||||
|
||||
Each consumer repo provides `scripts/changelog.json` with:
|
||||
|
||||
- `repo_url`: canonical web URL for pull request and commit links
|
||||
- `categories[]`
|
||||
- `categories[].title`
|
||||
- `categories[].sections[]`
|
||||
- `categories[].sections[].title`
|
||||
- `categories[].sections[].paths[]`
|
||||
|
||||
Paths are repo-relative path filters passed through to `git log`.
|
||||
|
||||
## Author Contract
|
||||
|
||||
Rendered changelog entries must support both first-parent pull-request-style
|
||||
pushes and direct first-parent pushes between the selected tags.
|
||||
|
||||
For pull-request-style commits that include a `(#123)` suffix, the generator
|
||||
must resolve author names from the Gitea pull request API rather than trusting
|
||||
the merge commit author field.
|
||||
|
||||
Expected endpoint shape:
|
||||
|
||||
- `GET /api/v1/repos/{owner}/{repo}/pulls/{index}`
|
||||
- prefer `user.full_name`
|
||||
- fall back to `user.login` only if `full_name` is empty
|
||||
|
||||
## Release Contract
|
||||
|
||||
For each version-tagged release workflow:
|
||||
|
||||
1. generate `build/CHANGELOG.md` for the current tag;
|
||||
2. upload `CHANGELOG.md` as a release asset on the versioned release;
|
||||
3. keep the file name stable across repos so later automation can fetch it
|
||||
without repo-specific asset naming logic.
|
||||
|
||||
The changelog is a staged build artifact, not a committed source-controlled file.
|
||||
This avoids detached-tag push complexity while preserving a stable artifact for
|
||||
later aggregation.
|
||||
|
||||
For direct pushes without a pull number suffix, the generator must render the
|
||||
commit using its commit hash link and commit author without calling the pull
|
||||
request API.
|
||||
|
||||
## Output Contract
|
||||
|
||||
The Markdown output must remain simple and merge-friendly:
|
||||
|
||||
- title line with current tag and publish date
|
||||
- comparison line naming the previous tag when one exists
|
||||
- category headings
|
||||
- section headings
|
||||
- bullet entries in one of these forms:
|
||||
|
||||
```text
|
||||
- Title ([#123](.../pulls/123)) - From Author Name
|
||||
- Title ([abc1234](.../commit/<sha>)) - From Author Name
|
||||
```
|
||||
|
||||
## Deferred Work
|
||||
|
||||
- add a dedicated bot that fetches `CHANGELOG.md` assets from module and assets
|
||||
releases, merges them, and publishes a single NodeBB release post
|
||||
- decide whether changelog generation should later move from shell wrappers into
|
||||
first-class repo-root user commands
|
||||
@@ -1,71 +0,0 @@
|
||||
# Configuration Hardening Audit
|
||||
|
||||
This audit records the current repository-configuration authority model for
|
||||
`sow-toolkit`.
|
||||
|
||||
## Authority Rules
|
||||
|
||||
- Human-authored root repository configuration is YAML: `nwn-tool.yaml`, then
|
||||
`nwn-tool.yml`.
|
||||
- `nwn-tool.json` is legacy compatibility only. It remains readable during the
|
||||
migration window and is reported as deprecated provenance.
|
||||
- Generated and machine-readable artifacts remain JSON. This includes HAK
|
||||
manifests, canonical GFF JSON, topdata datasets, autogen manifests, caches,
|
||||
wiki state, deploy manifests, and credits inventories.
|
||||
- Toolkit defaults are centralized in `internal/project/effective.go` and are
|
||||
visible through `sow-toolkit config effective`.
|
||||
- Environment variables are treated as runtime overrides and are listed by
|
||||
`sow-toolkit config sources` and `config effective`. Sensitive values are
|
||||
masked.
|
||||
|
||||
## Implemented Schema Expansion
|
||||
|
||||
- `paths.cache`, `paths.tools`
|
||||
- `outputs.module_archive`, `outputs.hak_manifest`, `outputs.hak_archive`
|
||||
- `build.keep_existing_haks`
|
||||
- `inventory.source_extensions`, `inventory.asset_extensions`,
|
||||
`inventory.source_json_pattern`
|
||||
- `validation.profile`, `validation.builtin_script_prefixes`, and
|
||||
`validation.required_fields`
|
||||
- `scripts.source_dir`, `scripts.cache`, `scripts.compiler`
|
||||
- `extract.layout`, `extract.hak_discovery`, `extract.cleanup_stale`
|
||||
- `topdata.compiled_2da_dir`, `topdata.compiled_tlk`, package names, and wiki
|
||||
output/deploy defaults
|
||||
- `autogen.cache.root`, `autogen.cache.max_age`,
|
||||
`autogen.cache.refresh_env`, and release-source env names
|
||||
|
||||
## Remaining Engine Invariants
|
||||
|
||||
These are intentionally still implemented as toolkit behavior rather than
|
||||
repository-specific YAML:
|
||||
|
||||
- Canonical GFF JSON shape.
|
||||
- ERF/MOD/HAK binary serialization.
|
||||
- Supported autogen algorithms: `parts_rows`, `accessory_visualeffects`,
|
||||
`trailing_numeric_suffix`, `model_stem`, and `first_path_segment`.
|
||||
- Native topdata dataset internals and generated JSON dataset formats.
|
||||
- NWN validator resource-reference checks. These are candidates for future
|
||||
validation profiles, but are not repository-specific defaults.
|
||||
|
||||
## Migration Checklist
|
||||
|
||||
For each consumer repository:
|
||||
|
||||
1. Confirm root config is `nwn-tool.yaml` or `nwn-tool.yml`.
|
||||
2. Remove root `nwn-tool.json` unless it is a deliberate test fixture.
|
||||
3. Run `sow-toolkit config effective --json` and record generated output,
|
||||
cache paths, HAK manifest paths, topdata package names, script compiler
|
||||
resolution policy, and active overrides.
|
||||
4. Move wrapper-provided environment behavior into YAML where it is stable
|
||||
repository policy.
|
||||
5. Keep credentials and one-off CI values as runtime overrides.
|
||||
6. Run build/validate workflows and compare generated JSON artifacts.
|
||||
|
||||
## Follow-Up Tickets
|
||||
|
||||
- Remove legacy root JSON loading after all active repositories migrate.
|
||||
- Add validation-profile schema if repositories need to vary validator rules.
|
||||
- Complete the separate music dataset refactor described in
|
||||
`MUSIC_REFACTOR_CONTRACT.md`.
|
||||
- Add static linting for new repository-specific literals in command and
|
||||
pipeline code.
|
||||
@@ -1,804 +0,0 @@
|
||||
Create a long-term multi-repository hardening plan focused on making YAML configuration the authoritative source of truth.
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- This hardening effort is active and partially complete.
|
||||
- YAML authority, strict config loading, and effective-config inspection are
|
||||
implemented.
|
||||
- Major repository behavior now flows through normalized config, including HAK
|
||||
layout, topdata packaging, extract behavior, autogen manifests, scripts, wiki,
|
||||
and music datasets.
|
||||
- The remaining gaps are mostly about reducing legacy compatibility surface,
|
||||
moving more tests to YAML fixtures, and deciding which residual toolkit
|
||||
defaults stay intrinsic versus become explicit schema.
|
||||
|
||||
Primary outstanding work:
|
||||
|
||||
- reduce reliance on `nwn-tool.json` in tests and migration fixtures
|
||||
- keep pruning repository-shaped defaults that still live in code when they
|
||||
should be surfaced as schema-backed settings
|
||||
- maintain docs/examples for `config effective` and the normalized key space as
|
||||
the schema evolves
|
||||
|
||||
# Goal
|
||||
|
||||
Reduce hardcoded behavior across all repositories and toolkit code.
|
||||
|
||||
All behavior that touches repository-specific configuration must be driven by YAML. If a behavior currently exists only as an explicit rule in application code, move it into configuration or create a new configuration option for it.
|
||||
|
||||
# Core principle
|
||||
|
||||
Configuration is fact.
|
||||
|
||||
Code should interpret and enforce configuration, not silently override it.
|
||||
|
||||
# Requirements
|
||||
|
||||
- YAML is the only human-authored repository configuration format.
|
||||
- JSON remains acceptable only for generated artifacts, datasets, manifests, caches, and machine-readable outputs.
|
||||
- Any code path that touches configurable behavior must lock to YAML-defined values.
|
||||
- If YAML defines a value, code must not substitute another value unless explicitly configured as a fallback.
|
||||
- If behavior varies by repository, dataset, build target, asset type, or command, it belongs in YAML.
|
||||
- If no configuration option exists for a behavior, create one.
|
||||
- Remove implicit repository assumptions from code.
|
||||
- Remove hidden defaults unless they are documented toolkit-level defaults.
|
||||
- Make all defaults visible, documented, and overrideable.
|
||||
|
||||
# Audit scope
|
||||
|
||||
Audit all repositories and toolkit code for hardcoded or implicit behavior related to:
|
||||
|
||||
- paths
|
||||
- module metadata
|
||||
- resrefs
|
||||
- HAK names
|
||||
- HAK order
|
||||
- HAK grouping
|
||||
- HAK priority
|
||||
- HAK splitting
|
||||
- HAK size limits
|
||||
- optional HAKs
|
||||
- include/exclude globs
|
||||
- generated manifest names
|
||||
- cache names
|
||||
- release tags
|
||||
- dataset names
|
||||
- autogen producers
|
||||
- autogen consumers
|
||||
- derive rules
|
||||
- music prefixes
|
||||
- credits behavior
|
||||
- extract behavior
|
||||
- topdata behavior
|
||||
- package names
|
||||
- TLK names
|
||||
- build directories
|
||||
- source directories
|
||||
- validation rules
|
||||
- cleanup rules
|
||||
- normalization rules
|
||||
- naming schemes
|
||||
- file extension rules
|
||||
- ignore rules
|
||||
- command-specific behavior
|
||||
- logging verbosity
|
||||
- deterministic ordering rules
|
||||
|
||||
# Hardening rules
|
||||
|
||||
1. YAML wins. If YAML declares a value, the application must use it exactly.
|
||||
|
||||
2. No silent fallbacks. Fallbacks must be explicit, documented, and visible in logs.
|
||||
|
||||
3. No hidden repository assumptions. Repository-specific behavior must not live in toolkit code.
|
||||
|
||||
4. No magic names. Names like `sow_top`, `sow_core`, `content`, `build`, `.cache`, or specific manifest names must come from YAML unless they are generated from a configured rule.
|
||||
|
||||
5. No implicit behavior drift. Existing behavior should remain stable unless the YAML explicitly changes it.
|
||||
|
||||
6. Deterministic by default. Ordering, output paths, generated names, cache keys, manifests, and logs must be stable.
|
||||
|
||||
7. Validation before execution. Invalid or incomplete configuration should fail early with clear errors.
|
||||
|
||||
8. Configuration gaps become schema work. If code needs a behavior that YAML cannot express, extend the schema.
|
||||
|
||||
# Implementation direction
|
||||
|
||||
- Build a central configuration model.
|
||||
- Route all repository behavior through that model.
|
||||
- Add typed config objects or schema validation.
|
||||
- Normalize config once at startup.
|
||||
- Pass normalized config into commands instead of reading globals or hardcoded constants.
|
||||
- Make command behavior depend on normalized config only.
|
||||
- Add explicit toolkit defaults in one place.
|
||||
- Generate an effective configuration view for debugging.
|
||||
- Add a command such as:
|
||||
|
||||
```text
|
||||
toolkit config inspect
|
||||
toolkit config validate
|
||||
toolkit config explain <key>
|
||||
toolkit config effective
|
||||
```
|
||||
|
||||
# Effective config
|
||||
|
||||
Add support for printing the final resolved configuration after defaults, overrides, and dataset-specific settings have been applied.
|
||||
|
||||
This should help verify that:
|
||||
|
||||
- YAML was loaded.
|
||||
- defaults were applied intentionally.
|
||||
- no hidden behavior was used.
|
||||
- paths resolved as expected.
|
||||
- command behavior is explainable.
|
||||
|
||||
# Repository hardening process
|
||||
|
||||
For each repository:
|
||||
|
||||
1. Inventory existing YAML config.
|
||||
2. Inventory generated JSON artifacts separately.
|
||||
3. Search code for hardcoded repository behavior.
|
||||
4. Map each hardcoded behavior to an existing YAML entry.
|
||||
5. If no YAML entry exists, add one.
|
||||
6. Update schema and validation.
|
||||
7. Update code to consume normalized config.
|
||||
8. Add tests proving YAML controls the behavior.
|
||||
9. Add regression tests for existing outputs.
|
||||
10. Document the config option.
|
||||
|
||||
# Testing requirements
|
||||
|
||||
Add tests for:
|
||||
|
||||
- YAML config loading
|
||||
- config validation
|
||||
- effective config generation
|
||||
- deterministic ordering
|
||||
- missing required config
|
||||
- duplicate config entries
|
||||
- invalid paths
|
||||
- invalid HAK definitions
|
||||
- invalid autogen definitions
|
||||
- invalid dataset definitions
|
||||
- command behavior matching YAML
|
||||
- no hardcoded repository names
|
||||
- JSON artifacts remaining generated data only
|
||||
|
||||
Add regression tests where possible:
|
||||
|
||||
- same YAML input produces same build plan
|
||||
- same YAML input produces same HAK order
|
||||
- same YAML input produces same manifest names
|
||||
- same YAML input produces same cache paths
|
||||
- same YAML input produces same autogen behavior
|
||||
|
||||
# Code audit targets
|
||||
|
||||
Look specifically for:
|
||||
|
||||
- string literals that look like paths, names, prefixes, manifests, or cache files
|
||||
- constants that duplicate config values
|
||||
- default values inside command implementations
|
||||
- repository-specific branch logic
|
||||
- command-specific assumptions
|
||||
- filename conventions not declared in config
|
||||
- implicit glob patterns
|
||||
- implicit output names
|
||||
- implicit ordering rules
|
||||
- duplicated config parsing
|
||||
- separate code paths that bypass central config
|
||||
|
||||
# Deliverables
|
||||
|
||||
1. Long-term hardening plan
|
||||
2. Config authority rules
|
||||
3. Audit checklist
|
||||
4. Proposed YAML schema expansions
|
||||
5. Effective config design
|
||||
6. Migration strategy per repository
|
||||
7. Testing strategy
|
||||
8. Documentation strategy
|
||||
9. List of hardcoded assumptions found
|
||||
10. Refactor tickets grouped by risk and priority
|
||||
|
||||
# Acceptance criteria
|
||||
|
||||
- YAML is the only human-authored repository config.
|
||||
- Code does not override YAML silently.
|
||||
- Repository-specific behavior is not hardcoded.
|
||||
- Any behavior touching config is lock-set to the normalized YAML value.
|
||||
- Missing config options are added instead of encoded in application logic.
|
||||
- Generated JSON artifacts remain unchanged unless intentionally modified.
|
||||
- Builds remain deterministic.
|
||||
- Existing repositories continue to build.
|
||||
- Developers can inspect the effective config and understand why each value was used.
|
||||
|
||||
# Current project-needs analysis
|
||||
|
||||
## Current state
|
||||
|
||||
- Root repository configuration is already discovered as:
|
||||
1. `nwn-tool.yaml`
|
||||
2. `nwn-tool.yml`
|
||||
3. legacy `nwn-tool.json`
|
||||
- YAML is decoded with `KnownFields(true)`.
|
||||
- Legacy JSON is decoded with `DisallowUnknownFields()`.
|
||||
- `internal/app.loadProject` prints the loaded config filename and warns when legacy JSON is used.
|
||||
- The root config model currently lives in `internal/project.Config`.
|
||||
- `internal/project.defaultConfig` currently only defaults `paths.build` to `build`.
|
||||
- Generated and machine-readable JSON remains widespread and should stay JSON:
|
||||
- HAK manifest: `build/haks.json`
|
||||
- canonical GFF JSON documents
|
||||
- topdata authored/generated datasets
|
||||
- topdata wiki state
|
||||
- topdata deploy manifest
|
||||
- autogen manifests and release caches
|
||||
- credits inventory
|
||||
- Some behavior is already YAML-driven:
|
||||
- module name/resref/description
|
||||
- module HAK order
|
||||
- source/assets/build paths
|
||||
- HAK names, priorities, split behavior, max bytes, optional flag, include globs
|
||||
- music prefix mapping
|
||||
- topdata source/build/reference/assets/package names
|
||||
- extract ignore extensions and post-extract module deletion
|
||||
- autogen producer/consumer ids, roots, includes, modes, derive rules, release tags, asset names, cache names
|
||||
- Some behavior is still implicit in code and should be made explicit or documented as toolkit defaults.
|
||||
|
||||
## Important naming clarification
|
||||
|
||||
The current repository root config file is `nwn-tool.yaml`, not `config.yaml`.
|
||||
|
||||
Hardening assumption:
|
||||
|
||||
- Keep `nwn-tool.yaml` as the canonical root config filename.
|
||||
- Keep `nwn-tool.yml` as an accepted alternate YAML filename.
|
||||
- Treat `nwn-tool.json` as temporary legacy compatibility only.
|
||||
- Do not rename generated or nested JSON files.
|
||||
- Do not treat topdata dataset JSON as root repository configuration.
|
||||
|
||||
If a future decision changes the canonical name to `config.yaml`, update `project.ConfigFile`, docs, consumer wrappers, tests, and migration scripts together.
|
||||
|
||||
# Current hardcoded assumptions found
|
||||
|
||||
## Project config and path defaults
|
||||
|
||||
- `paths.build` defaults to `build` in `internal/project.defaultConfig`.
|
||||
- `HAKManifestPath` always returns `<build>/haks.json`.
|
||||
- `ModuleArchivePath` always uses `<build>/<module.resref>.mod`.
|
||||
- `HAKArchivePath` always uses `<build>/<hak.name>.hak`.
|
||||
- `SourceDir`, `AssetsDir`, and `BuildDir` directly join configured paths with the repo root and do not yet expose a richer path policy.
|
||||
- `TopDataBuildDir` falls back to `<paths.build>/topdata` when `topdata.build` is empty.
|
||||
- `TopDataUsesRepoCache` special-cases `topdata.build == .cache`.
|
||||
- `TopDataCompiled2DADir` always appends `2da`.
|
||||
- `TopDataCompiledTLKPath` uses `sow_tlk.tlk` either under `build` or topdata build output.
|
||||
- `TopDataWikiRootDir` uses `.cache/wiki` for repo-cache mode and `<topdata.build>/wiki` otherwise.
|
||||
- `TopDataWikiStatePath` always uses `state.json`.
|
||||
- `TopDataPackageHAKName` defaults to `sow_top.hak`.
|
||||
- `TopDataPackageTLKName` defaults to `sow_tlk.tlk`.
|
||||
|
||||
Hardening direction:
|
||||
|
||||
- Split these into documented toolkit defaults and repository-overridable fields.
|
||||
- Keep existing values as defaults only for compatibility.
|
||||
- Show all derived values in effective config output.
|
||||
|
||||
## Inventory and extension rules
|
||||
|
||||
- Source extensions are hardcoded in `project.SourceExtensions`.
|
||||
- Asset extensions are hardcoded in `project.AssetExtensions`.
|
||||
- `Project.Scan` treats `.json` and `.nss` as source inputs.
|
||||
- Source JSON extension detection assumes resource filenames like `<resref>.<ext>.json`.
|
||||
- Validator required fields are hardcoded, currently including `ifo -> Mod_Name`.
|
||||
- Built-in script prefixes are hardcoded in `internal/validator`.
|
||||
- Validator reference checks hardcode specific field labels and expected resource extensions:
|
||||
- `Conversation -> .dlg`
|
||||
- `Mod_Entry_Area -> .are`
|
||||
- `Model -> .mdl`
|
||||
- `Sound -> .wav`
|
||||
|
||||
Hardening direction:
|
||||
|
||||
- Add a `resources` or `inventory` config section for extension policy and source/asset classification.
|
||||
- Keep NWN defaults built in, but make them inspectable and overrideable.
|
||||
- Consider allowing validators to be named profiles, for example `nwn_module`, instead of requiring every repository to spell out base NWN rules.
|
||||
|
||||
## Build and HAK behavior
|
||||
|
||||
- `build-haks` reads/writes the hardcoded manifest name from `Project.HAKManifestPath`.
|
||||
- HAK files are always emitted under `paths.build`.
|
||||
- HAK archive filenames are always derived from `haks[].name`.
|
||||
- `extract` scans all `*.hak` in the build directory rather than only configured/generated HAKs.
|
||||
- `build-haks` has command flags that mutate behavior independently of config:
|
||||
- `--hak`
|
||||
- `--archive`
|
||||
- `--source-manifest`
|
||||
- `--plan-only`
|
||||
- `--quiet`
|
||||
- `--verbose`
|
||||
- `--debug`
|
||||
- `SOW_BUILD_HAKS_KEEP_EXISTING` controls HAK cleanup behavior outside YAML.
|
||||
|
||||
Hardening direction:
|
||||
|
||||
- Add configurable manifest names and cleanup policy.
|
||||
- Define command-line flags as explicit per-run overrides in the effective config model.
|
||||
- Log all active environment overrides.
|
||||
- Consider config-controlled archive discovery for `extract`.
|
||||
|
||||
## Script compiler behavior
|
||||
|
||||
- Compiled scripts are cached under `.cache/ncs`.
|
||||
- Compiler discovery is hardcoded to:
|
||||
- `SOW_NWN_SCRIPT_COMPILER`
|
||||
- `tools/script-compiler/nwn_script_comp`
|
||||
- `tools/script-compiler/nwn_script_comp.exe`
|
||||
- `tools/nwn_script_comp`
|
||||
- `tools/nwn_script_comp.exe`
|
||||
- `PATH`
|
||||
- Script source directory is always `<paths.source>/scripts`.
|
||||
- Compiler environment probing uses `SOW_NWN_USER_DIRECTORY`, `NWN_HOME`, `NWN_USER_DIRECTORY`, `SOW_NWN_ROOT`, and `NWN_ROOT`.
|
||||
|
||||
Hardening direction:
|
||||
|
||||
- Add `tools.script_compiler` and `script.cache` config.
|
||||
- Keep environment variables as explicit override sources.
|
||||
- Include resolved compiler path and cache path in effective config and debug logs.
|
||||
|
||||
## Music behavior
|
||||
|
||||
- Music conversion/scanning is hardcoded around `envi/music`.
|
||||
- Music staging is hardcoded to `.cache/music`.
|
||||
- Credits output is hardcoded to `.cache/credits`.
|
||||
- Credits overlay filename is hardcoded to `CREDITS.md`.
|
||||
- FFmpeg/ffprobe discovery is environment/PATH based.
|
||||
- BMU output naming rules and 16-character NWN stem limits are hardcoded.
|
||||
|
||||
Hardening direction:
|
||||
|
||||
- Apply `MUSIC_REFACTOR_CONTRACT.md`.
|
||||
- Convert music behavior to dataset configuration.
|
||||
- Keep old `music.prefixes` only as compatibility shorthand until repositories migrate.
|
||||
|
||||
## Topdata behavior
|
||||
|
||||
- Topdata source layout is partly convention-based:
|
||||
- `data`
|
||||
- `tlk`
|
||||
- `base_dialog.json`
|
||||
- `lock.json`
|
||||
- `.tlk_state.json`
|
||||
- `templates`
|
||||
- `assets`
|
||||
- Topdata compiled output subdirectories are hardcoded:
|
||||
- `2da`
|
||||
- `wiki`
|
||||
- `wiki/pages`
|
||||
- `wiki/state.json`
|
||||
- Wiki generation has hardcoded managed namespaces and status labels.
|
||||
- Wiki deploy defaults are hardcoded:
|
||||
- `.wiki_deploy_manifest.json`
|
||||
- `Auto-generated from native builder`
|
||||
- managed namespaces including `classes`, `feat`, `itemtypes`, `races`, `skills`, `spells`, `meta`
|
||||
- Topdata package fallback names are `sow_top.hak` and `sow_tlk.tlk`.
|
||||
- Autogen released manifest cache path is `.cache/<cache_name>`.
|
||||
- Autogen released manifest max age is one hour.
|
||||
- `SOW_AUTOGEN_MANIFEST_REFRESH` forces refresh outside YAML.
|
||||
- Released autogen manifest discovery derives a Shadows Over Westgate assets repo spec from git remote and environment overrides.
|
||||
- Supported autogen modes are hardcoded:
|
||||
- `parts_rows`
|
||||
- `accessory_visualeffects`
|
||||
- Supported derive kinds are hardcoded:
|
||||
- `trailing_numeric_suffix`
|
||||
- `model_stem`
|
||||
- Supported derive group source is hardcoded:
|
||||
- `first_path_segment`
|
||||
|
||||
Hardening direction:
|
||||
|
||||
- Separate topdata toolkit defaults from repository-specific defaults.
|
||||
- Add explicit config for output subpaths, wiki generation/deploy defaults, autogen cache policy, and release source.
|
||||
- Keep dataset JSON shape unchanged.
|
||||
- Treat topdata dataset internals as authored data, not root YAML config, unless the behavior genuinely varies by repository.
|
||||
|
||||
## Extract behavior
|
||||
|
||||
- Extract output layout is hardcoded:
|
||||
- `.nss` -> `<paths.source>/scripts/<resref>.nss`
|
||||
- GFF resources -> `<paths.source>/<sourceSubdir(ext)>/<resref>.<ext>.json`
|
||||
- other resources -> `<paths.assets>/<extension>/<resref>.<extension>`
|
||||
- Stale cleanup uses the current project inventory.
|
||||
- Ignore behavior is configurable only by extension.
|
||||
|
||||
Hardening direction:
|
||||
|
||||
- Add extract output layout settings or named extraction profiles.
|
||||
- Keep current layout as `nwn_canonical_json`.
|
||||
- Make cleanup scope explicit and visible.
|
||||
|
||||
## App and logging behavior
|
||||
|
||||
- Command descriptions mention hardcoded paths like `src/`, `assets/`, `build/`, `.cache/2da`, `.cache/wiki`, `sow_top.hak`, and `sow_tlk.tlk`.
|
||||
- Verbosity flags are command-specific, especially for `build-haks`.
|
||||
- Spinner and phase labels are hardcoded in `internal/app/app.go`.
|
||||
- Config load diagnostics are not yet a full effective-config explanation.
|
||||
|
||||
Hardening direction:
|
||||
|
||||
- Update descriptions to avoid implying fixed repository layouts.
|
||||
- Introduce shared global verbosity flags or a documented command-level policy.
|
||||
- Expose effective config and active overrides through `config` commands.
|
||||
|
||||
# Proposed schema expansions
|
||||
|
||||
These are target concepts, not a requirement to implement every field in one pass.
|
||||
|
||||
```yaml
|
||||
toolkit:
|
||||
defaults_profile: nwn
|
||||
generated_artifacts:
|
||||
format: json
|
||||
logging:
|
||||
default_verbosity: normal
|
||||
paths: relative
|
||||
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
cache: .cache
|
||||
tools: tools
|
||||
|
||||
outputs:
|
||||
module_archive: "{module.resref}.mod"
|
||||
hak_manifest: haks.json
|
||||
hak_archive: "{hak.name}.hak"
|
||||
|
||||
inventory:
|
||||
source_extensions:
|
||||
- .are
|
||||
- .dlg
|
||||
- .fac
|
||||
- .git
|
||||
- .ifo
|
||||
- .jrl
|
||||
- .utc
|
||||
- .utd
|
||||
- .ute
|
||||
- .uti
|
||||
- .utm
|
||||
- .utp
|
||||
- .uts
|
||||
- .utt
|
||||
- .utw
|
||||
asset_extensions:
|
||||
- .2da
|
||||
- .bmu
|
||||
- .dds
|
||||
- .mdl
|
||||
- .wav
|
||||
source_json_pattern: "{resref}.{extension}.json"
|
||||
|
||||
validation:
|
||||
profile: nwn_module
|
||||
builtin_script_prefixes:
|
||||
- nw_
|
||||
- x0_
|
||||
- x1_
|
||||
- x2_
|
||||
- x3_
|
||||
- ga_
|
||||
- gc_
|
||||
- gen_
|
||||
- gui_
|
||||
- nwg_
|
||||
- ta_
|
||||
|
||||
scripts:
|
||||
source_dir: scripts
|
||||
cache: "{paths.cache}/ncs"
|
||||
compiler:
|
||||
path: ""
|
||||
env:
|
||||
path: SOW_NWN_SCRIPT_COMPILER
|
||||
nwn_root: SOW_NWN_ROOT
|
||||
nwn_user_directory: SOW_NWN_USER_DIRECTORY
|
||||
search:
|
||||
- "{paths.tools}/script-compiler/nwn_script_comp"
|
||||
- "{paths.tools}/script-compiler/nwn_script_comp.exe"
|
||||
- "{paths.tools}/nwn_script_comp"
|
||||
- "{paths.tools}/nwn_script_comp.exe"
|
||||
- PATH:nwn_script_comp
|
||||
|
||||
extract:
|
||||
layout: nwn_canonical_json
|
||||
ignore_extensions: []
|
||||
cleanup_stale: true
|
||||
delete_module_archive_after_success: false
|
||||
|
||||
topdata:
|
||||
source: topdata
|
||||
build: "{paths.build}/topdata"
|
||||
compiled_2da_dir: 2da
|
||||
compiled_tlk: sow_tlk.tlk
|
||||
package_hak: sow_top.hak
|
||||
package_tlk: sow_tlk.tlk
|
||||
wiki:
|
||||
enabled_by_default: false
|
||||
output_root: wiki
|
||||
pages_dir: pages
|
||||
state_file: state.json
|
||||
managed_namespaces:
|
||||
- classes
|
||||
- feat
|
||||
- itemtypes
|
||||
- races
|
||||
- skills
|
||||
- spells
|
||||
deploy_manifest: .wiki_deploy_manifest.json
|
||||
deploy_edit_summary: Auto-generated from native builder
|
||||
|
||||
autogen:
|
||||
cache:
|
||||
root: "{paths.cache}"
|
||||
max_age: 1h
|
||||
refresh_env: SOW_AUTOGEN_MANIFEST_REFRESH
|
||||
release_source:
|
||||
provider: gitea
|
||||
server_url_env: SOW_ASSETS_SERVER_URL
|
||||
repo_env: SOW_ASSETS_REPO
|
||||
```
|
||||
|
||||
Implementation does not need to preserve exactly this naming, but each hardcoded behavior above should either become:
|
||||
|
||||
- a field in schema,
|
||||
- a documented toolkit default visible in effective config, or
|
||||
- a deliberate non-configurable engine invariant with a clear rationale.
|
||||
|
||||
# Effective config design
|
||||
|
||||
## Commands
|
||||
|
||||
Add config inspection commands:
|
||||
|
||||
```text
|
||||
sow-toolkit config validate
|
||||
sow-toolkit config effective [--json|--yaml]
|
||||
sow-toolkit config inspect [<key>] [--json]
|
||||
sow-toolkit config explain <key>
|
||||
sow-toolkit config sources
|
||||
```
|
||||
|
||||
`config validate` should load, normalize, and validate without scanning/building.
|
||||
|
||||
`config effective` should print the complete normalized config after defaults and compatibility migrations.
|
||||
|
||||
`config explain <key>` should show:
|
||||
|
||||
- final value
|
||||
- source of the value:
|
||||
- toolkit default
|
||||
- root YAML
|
||||
- compatibility migration
|
||||
- environment override
|
||||
- command-line override
|
||||
- whether the value is deprecated
|
||||
- validation rules that apply
|
||||
|
||||
`config sources` should print:
|
||||
|
||||
- loaded config path and format
|
||||
- legacy status
|
||||
- accepted root config candidates
|
||||
- active environment overrides
|
||||
- command-line overrides used for the current invocation
|
||||
|
||||
## Internal model
|
||||
|
||||
Introduce a normalized/effective config layer separate from raw YAML structs.
|
||||
|
||||
Suggested distinction:
|
||||
|
||||
- `RawConfig`: direct YAML representation with pointer fields where inheritance or "unset" matters.
|
||||
- `EffectiveConfig`: fully resolved values consumed by commands.
|
||||
- `ConfigProvenance`: source metadata for inspect/explain commands.
|
||||
- `ConfigDiagnostics`: warnings/errors/deprecations produced during normalization and validation.
|
||||
|
||||
Commands should consume `EffectiveConfig` or methods derived from it, not raw YAML fields.
|
||||
|
||||
## Override rules
|
||||
|
||||
Order of authority:
|
||||
|
||||
1. toolkit built-in defaults
|
||||
2. YAML root config
|
||||
3. compatibility migrations from deprecated YAML fields
|
||||
4. environment overrides
|
||||
5. command-line overrides
|
||||
|
||||
For each override:
|
||||
|
||||
- document whether it is allowed
|
||||
- record provenance
|
||||
- show it in debug/effective output
|
||||
- avoid silent behavior changes
|
||||
|
||||
Environment variables should not be removed immediately, but they should be treated as visible overrides, not hidden behavior.
|
||||
|
||||
# Hardening phases
|
||||
|
||||
## Phase 1: Baseline audit and effective config shell
|
||||
|
||||
- Add a config audit document generated from code search.
|
||||
- Implement `config validate`.
|
||||
- Implement `config effective --json` using current config and current defaults.
|
||||
- Add provenance for:
|
||||
- config filename
|
||||
- legacy JSON use
|
||||
- `paths.build` default
|
||||
- topdata default package names
|
||||
- HAK manifest default name
|
||||
- Do not change build behavior in this phase.
|
||||
|
||||
## Phase 2: Centralize derived paths and names
|
||||
|
||||
- Move path/name derivation out of scattered methods and into effective config.
|
||||
- Cover:
|
||||
- module archive path
|
||||
- HAK archive path template
|
||||
- HAK manifest path
|
||||
- cache root
|
||||
- script cache
|
||||
- topdata compiled dirs
|
||||
- topdata package names
|
||||
- wiki output paths
|
||||
- autogen cache root
|
||||
- Keep old methods as wrappers around effective config during migration.
|
||||
- Add tests proving old and new derivations match.
|
||||
|
||||
## Phase 3: Schema expansion for current hardcoded behavior
|
||||
|
||||
- Add schema fields for outputs, cache roots, inventory extensions, script compiler settings, extract layout, topdata output subpaths, wiki deploy defaults, and autogen cache policy.
|
||||
- Preserve current defaults.
|
||||
- Add strict validation for new fields.
|
||||
- Update README examples.
|
||||
|
||||
## Phase 4: Command integration
|
||||
|
||||
- Update build, extract, validate, compare, topdata, autogen, music, and wiki commands to consume effective config.
|
||||
- Convert environment variables into explicit override records.
|
||||
- Update logs to report configured names/paths instead of hardcoded labels.
|
||||
- Add `--debug` output for effective values used by each command.
|
||||
|
||||
## Phase 5: Repository migrations
|
||||
|
||||
- Update consumer repository YAML to explicitly declare any behavior that is repository-specific.
|
||||
- Remove compatibility shims only after all active repositories have migrated.
|
||||
- Keep generated JSON artifacts stable unless a separate contract says otherwise.
|
||||
- Add regression checks comparing pre/post build plans and generated manifests.
|
||||
|
||||
## Phase 6: Enforcement
|
||||
|
||||
- Add tests or static checks that reject new repository-specific literals in core command paths.
|
||||
- Remove legacy JSON loading after the agreed migration window.
|
||||
- Turn deprecated YAML shorthand warnings into validation errors where appropriate.
|
||||
|
||||
# Repository-by-repository checklist
|
||||
|
||||
For each consumer repository:
|
||||
|
||||
1. Identify root config file and confirm it is `nwn-tool.yaml` or `nwn-tool.yml`.
|
||||
2. Confirm no root `nwn-tool.json` remains except as deliberate migration fixture.
|
||||
3. List all wrapper scripts and environment variables they set.
|
||||
4. Record local tool paths and whether they need YAML fields.
|
||||
5. List generated artifact paths and confirm they are not treated as config.
|
||||
6. Run current build and save:
|
||||
- HAK manifest
|
||||
- module HAK order
|
||||
- topdata package paths
|
||||
- autogen manifest paths
|
||||
- credits inventory paths
|
||||
7. Convert implicit behavior to explicit YAML.
|
||||
8. Run `config effective` and verify values.
|
||||
9. Run build/validate workflows and compare generated outputs.
|
||||
10. Document intentional diffs.
|
||||
|
||||
# Test strategy expansion
|
||||
|
||||
Add tests for:
|
||||
|
||||
- `config validate` succeeds without scanning missing optional trees.
|
||||
- `config effective --json` is deterministic.
|
||||
- `config explain paths.build` identifies toolkit default when omitted.
|
||||
- `config explain topdata.package_hak` identifies YAML source when configured.
|
||||
- legacy `nwn-tool.json` emits warning and provenance.
|
||||
- YAML wins over legacy JSON when both exist.
|
||||
- unknown YAML fields fail before any scan/build side effects.
|
||||
- environment overrides appear in effective config.
|
||||
- command-line overrides appear in effective config for the command invocation.
|
||||
- HAK manifest name can be configured without changing generated JSON schema.
|
||||
- topdata package names can be configured and all command output follows them.
|
||||
- script compiler cache path can be configured.
|
||||
- extract layout profile preserves current output paths.
|
||||
- autogen cache root and max age are configurable.
|
||||
- wiki deploy manifest name is configurable.
|
||||
- source/asset extension defaults match current scanning behavior.
|
||||
- invalid path traversal in output/cache config fails early.
|
||||
- absolute paths are rejected unless the field explicitly allows them.
|
||||
- duplicate or case-colliding generated output paths fail validation.
|
||||
- config-generated effective values are sorted deterministically.
|
||||
|
||||
Regression tests:
|
||||
|
||||
- current minimal YAML still builds to the same default paths.
|
||||
- current Shadows Over Westgate-style topdata package output remains `sow_top.hak` / `sow_tlk.tlk` unless YAML changes it.
|
||||
- current HAK manifest remains `haks.json` unless YAML changes it.
|
||||
- current `.cache` outputs remain stable unless YAML changes them.
|
||||
- old `music.prefixes` compatibility still maps to current behavior until removed.
|
||||
|
||||
# Priority refactor tickets
|
||||
|
||||
## P0: Authority and safety
|
||||
|
||||
- Add effective config command output.
|
||||
- Record config provenance and legacy warnings.
|
||||
- Centralize output/cache path derivation.
|
||||
- Validate path traversal and unsafe output roots.
|
||||
- Make environment overrides visible.
|
||||
|
||||
## P1: Repository-specific names and paths
|
||||
|
||||
- Configure HAK manifest filename.
|
||||
- Configure cache root.
|
||||
- Configure script compiler path/search/cache.
|
||||
- Configure topdata compiled output subpaths.
|
||||
- Configure wiki deploy manifest/edit summary/namespaces.
|
||||
- Configure autogen cache root/max age/release source.
|
||||
|
||||
## P2: Inventory and validation profiles
|
||||
|
||||
- Configure source and asset extension lists.
|
||||
- Configure or profile validator reference rules.
|
||||
- Configure built-in script prefixes.
|
||||
- Configure extract output layout.
|
||||
|
||||
## P3: Migration cleanup
|
||||
|
||||
- Deprecate and remove legacy JSON root config loading.
|
||||
- Deprecate music prefix shorthand after dataset migration.
|
||||
- Replace command descriptions that mention hardcoded defaults.
|
||||
- Add static/lint checks for new repository-specific literals.
|
||||
|
||||
# Documentation strategy
|
||||
|
||||
- Document `nwn-tool.yaml` as the canonical root config file.
|
||||
- Maintain a generated effective-config reference in docs.
|
||||
- Document every toolkit default separately from repository examples.
|
||||
- For every config field, document:
|
||||
- type
|
||||
- default
|
||||
- whether relative paths are repo-relative or output-root-relative
|
||||
- whether absolute paths are allowed
|
||||
- validation rules
|
||||
- generated artifacts affected
|
||||
- migration notes
|
||||
- Keep a clear line between:
|
||||
- human-authored root YAML config
|
||||
- human-authored topdata dataset JSON
|
||||
- generated JSON artifacts
|
||||
- compatibility/legacy files
|
||||
|
||||
# Expanded acceptance criteria
|
||||
|
||||
- `sow-toolkit config effective --json` shows all values that commands use.
|
||||
- Every effective value can be traced to YAML, toolkit default, environment override, compatibility migration, or command override.
|
||||
- Existing hardcoded defaults are either configurable or documented as toolkit defaults.
|
||||
- Command implementations do not invent repository-specific paths after config normalization.
|
||||
- Topdata, HAK, music, extract, autogen, wiki, and script compiler paths are inspectable.
|
||||
- Legacy JSON root config use is visible and eventually removable.
|
||||
- Generated JSON schemas remain stable unless intentionally changed.
|
||||
- Existing consumer repositories build before and after migration.
|
||||
@@ -1,402 +0,0 @@
|
||||
Refactor toolkit configuration from JSON to YAML only.
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- This refactor is largely implemented.
|
||||
- Root config discovery now prefers `nwn-tool.yaml`, then `nwn-tool.yml`, then
|
||||
legacy `nwn-tool.json`.
|
||||
- YAML is canonical in the consumer repos, and the toolkit exposes
|
||||
`config validate|effective|inspect|explain|sources`.
|
||||
- Legacy JSON compatibility still exists by design and is still used heavily in
|
||||
test fixtures.
|
||||
|
||||
Remaining work:
|
||||
|
||||
- continue migrating representative tests from `nwn-tool.json` fixtures to YAML
|
||||
while keeping only targeted legacy coverage
|
||||
- decide when to deprecate or remove legacy shorthand such as `music.prefixes`
|
||||
after consumer configs are fully dataset-based
|
||||
- keep documentation/examples synchronized with the normalized YAML schema as it
|
||||
expands
|
||||
|
||||
## Active Scope
|
||||
|
||||
This contract remains active and is one of the current priorities.
|
||||
|
||||
The implementation gap is no longer "load YAML at all". The real gap is
|
||||
configuration authority drift:
|
||||
|
||||
- some repository behavior is still encoded as toolkit defaults when it should
|
||||
become explicit schema
|
||||
- some tests still model old JSON-era expectations
|
||||
- some topdata behavior is still driven by explicit namespace/path checks rather
|
||||
than generic dataset rules
|
||||
|
||||
## Current Findings
|
||||
|
||||
The most important remaining configuration work is topdata-related:
|
||||
|
||||
- topdata dataset discovery already derives default output names when `output` is
|
||||
absent, but validation still hard-fails for
|
||||
`topdata/data/masterfeats/base.json` when `output` is missing
|
||||
- `internal/topdata/topdata.go` dispatches validation through explicit path
|
||||
checks for `masterfeats`, `feat`, parts overrides, and racialtypes registry
|
||||
- topdata defaults in `internal/project` still mix generic toolkit defaults with
|
||||
behavior that may need clearer schema ownership
|
||||
|
||||
This means the config-refactor track now overlaps directly with topdata
|
||||
hardening. The core question is no longer file format migration; it is whether a
|
||||
behavior belongs in:
|
||||
|
||||
- root YAML
|
||||
- dataset-authored JSON
|
||||
- or a documented intrinsic toolkit invariant
|
||||
|
||||
## Deferred Scope
|
||||
|
||||
Defer unrelated schema expansion for now:
|
||||
|
||||
- model compilation config
|
||||
- broader asset-maintenance policy schema
|
||||
- release-announcement config
|
||||
|
||||
## Immediate Plan
|
||||
|
||||
1. Continue shrinking legacy `nwn-tool.json` coverage down to deliberate
|
||||
compatibility tests.
|
||||
2. Audit hardcoded topdata behaviors and classify each as:
|
||||
- root YAML concern
|
||||
- dataset JSON concern
|
||||
- toolkit invariant
|
||||
3. Remove outdated namespace-specific validation assumptions where generic
|
||||
dataset logic already exists.
|
||||
4. Keep generated artifacts in JSON, but stop using explicit path-based logic
|
||||
when the same rule can be inferred from generic dataset shape.
|
||||
5. Document every remaining intentional toolkit default in `config effective`
|
||||
terms.
|
||||
|
||||
# Context
|
||||
|
||||
Repositories currently use `config.json` for human-authored toolkit configuration.
|
||||
|
||||
However, JSON is also used heavily for generated datasets, manifests, caches, and machine-readable artifacts. This makes repository configuration harder to distinguish from generated data.
|
||||
|
||||
# Goal
|
||||
|
||||
Make YAML the source of truth for all human-authored repository configuration.
|
||||
|
||||
JSON should remain valid for generated data artifacts, but not as the preferred format for repository configuration.
|
||||
|
||||
# Core requirements
|
||||
|
||||
- Replace `config.json` with `config.yaml`.
|
||||
- Load repository configuration from YAML.
|
||||
- Treat YAML as the canonical source-of-truth.
|
||||
- Keep generated datasets, manifests, caches, and machine-readable outputs as JSON.
|
||||
- Do not move dataset artifacts to YAML.
|
||||
- Do not hardcode configuration behavior in code if it can reasonably be expressed in config.
|
||||
- The toolkit should be deterministic.
|
||||
|
||||
# Configuration philosophy
|
||||
|
||||
Configuration should control repository behavior.
|
||||
|
||||
Avoid hardcoded assumptions for:
|
||||
|
||||
- module names
|
||||
- resrefs
|
||||
- paths
|
||||
- HAK names
|
||||
- HAK priority
|
||||
- HAK split behavior
|
||||
- HAK size limits
|
||||
- include/exclude globs
|
||||
- optional HAKs
|
||||
- music prefixes
|
||||
- extract rules
|
||||
- topdata paths/packages
|
||||
- autogen producers
|
||||
- autogen consumers
|
||||
- manifest names
|
||||
- cache names
|
||||
- release tags
|
||||
- dataset names
|
||||
- derivation rules
|
||||
|
||||
If behavior varies by repository, it belongs in YAML.
|
||||
|
||||
# Example target config
|
||||
|
||||
Convert this kind of repository config:
|
||||
|
||||
```json
|
||||
{
|
||||
"module": {
|
||||
"name": "Shadows Over Westgate",
|
||||
"resref": "sow_module",
|
||||
"description": "Shadows Over Westgate asset pipeline.",
|
||||
"hak_order": ["sow_top", "group:sow_over", "group:sow_core"]
|
||||
},
|
||||
"paths": {
|
||||
"assets": "content",
|
||||
"build": "build"
|
||||
},
|
||||
"music": {
|
||||
"prefixes": {
|
||||
"envi/music/westgate": "mus_wg_"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Into:
|
||||
|
||||
```yaml
|
||||
module:
|
||||
name: Shadows Over Westgate
|
||||
resref: sow_module
|
||||
description: Shadows Over Westgate asset pipeline.
|
||||
hak_order:
|
||||
- sow_top
|
||||
- group:sow_over
|
||||
- group:sow_core
|
||||
|
||||
paths:
|
||||
assets: content
|
||||
build: build
|
||||
|
||||
music:
|
||||
prefixes:
|
||||
envi/music/westgate: mus_wg_
|
||||
```
|
||||
|
||||
# Implementation requirements
|
||||
|
||||
- Add YAML config loading.
|
||||
- Prefer `config.yaml`.
|
||||
- Optionally accept `config.yml`.
|
||||
- Remove reliance on `config.json` for repository configuration.
|
||||
- If `config.json` support is kept temporarily, mark it as legacy/deprecated.
|
||||
- If both YAML and JSON configs exist, YAML must win.
|
||||
- Emit a clear warning when legacy JSON config is used.
|
||||
- Preserve existing schema semantics.
|
||||
- Preserve existing defaults only where they are intentional toolkit defaults.
|
||||
- Move repository-specific defaults into YAML.
|
||||
- Keep validation strict.
|
||||
- Fail clearly on malformed YAML or unknown required fields.
|
||||
- Preserve deterministic ordering where configs are serialized, logged, or normalized.
|
||||
|
||||
# Validation requirements
|
||||
|
||||
- Validate YAML against the same schema expectations currently used for JSON.
|
||||
- Ensure required sections are present when needed.
|
||||
- Ensure paths are strings.
|
||||
- Ensure HAK entries have required fields.
|
||||
- Ensure autogen producers/consumers have valid ids, roots, includes, derive rules, and manifest config.
|
||||
- Ensure music prefixes are deterministic and unambiguous.
|
||||
- Ensure duplicate HAK names, producer ids, consumer ids, or dataset ids are rejected.
|
||||
- Ensure priority ordering is deterministic even when priorities match.
|
||||
- Ensure include glob ordering is stable.
|
||||
|
||||
# Migration requirements
|
||||
|
||||
- Convert existing `config.json` files to `config.yaml`.
|
||||
- Preserve all existing values exactly unless a change is intentional.
|
||||
- Update code paths, scripts, tests, and documentation to reference YAML.
|
||||
- Add or update tests for:
|
||||
- loading `config.yaml`
|
||||
- loading `config.yml`
|
||||
- legacy `config.json` fallback, if retained
|
||||
- YAML precedence over JSON
|
||||
- validation failures
|
||||
- deterministic ordering
|
||||
- representative configs matching the current repository examples
|
||||
|
||||
- Do not change generated artifact formats unless explicitly required.
|
||||
|
||||
# Generated artifacts
|
||||
|
||||
Keep these as JSON unless there is a separate reason to change them:
|
||||
|
||||
- datasets
|
||||
- manifests
|
||||
- caches
|
||||
- credits output
|
||||
- build summaries
|
||||
- machine-readable reports
|
||||
|
||||
Rationale:
|
||||
|
||||
- YAML is for human-authored repository configuration.
|
||||
- JSON is for generated or machine-readable data.
|
||||
|
||||
# Deliverables
|
||||
|
||||
1. YAML config loader design
|
||||
2. Updated config schema
|
||||
3. JSON-to-YAML migration plan
|
||||
4. Converted example configs
|
||||
5. Legacy JSON handling policy
|
||||
6. Validation updates
|
||||
7. Test plan
|
||||
8. Documentation updates
|
||||
9. List of code-level assumptions that should be moved into config
|
||||
10. Before/after examples
|
||||
|
||||
# Initial project-needs analysis
|
||||
|
||||
## Current state
|
||||
|
||||
- Main repository configuration is currently modeled by `internal/project.Config`.
|
||||
- The toolkit currently discovers and loads `nwn-tool.json`, not `config.json`.
|
||||
- Config loading is centralized in `internal/project.FindRoot` and `internal/project.Load`.
|
||||
- Config structs currently have JSON tags only.
|
||||
- There is no YAML dependency in `go.mod`.
|
||||
- Generated and machine-readable JSON is widespread and should remain JSON:
|
||||
- build HAK manifest: `build/haks.json`
|
||||
- topdata authored/generated datasets
|
||||
- topdata template conversion metadata: `topdata/templates/config.yaml`
|
||||
- topdata wiki state and deploy manifests
|
||||
- autogen manifest caches under `.cache`
|
||||
- credits inventory output
|
||||
- GFF canonical JSON documents
|
||||
|
||||
## Naming decision needed
|
||||
|
||||
The contract says to replace `config.json` with `config.yaml`, but the current toolkit root config file is `nwn-tool.json`.
|
||||
|
||||
Plan assumption:
|
||||
|
||||
- Treat `nwn-tool.yaml` as the direct YAML replacement for the existing root config.
|
||||
- Optionally accept `nwn-tool.yml`.
|
||||
- Keep temporary legacy fallback to `nwn-tool.json` with a warning.
|
||||
- Do not rename generated or nested metadata files unless they are confirmed to be human-authored repository root configuration.
|
||||
|
||||
If the intended final filename is instead literally `config.yaml`, update the loader plan before implementation.
|
||||
|
||||
## Code areas affected
|
||||
|
||||
- `internal/project/project.go`
|
||||
- config file discovery
|
||||
- YAML/legacy JSON loading
|
||||
- schema tags and strict decode behavior
|
||||
- validation for duplicate names, path fields, include ordering, and required config sections
|
||||
- currently hardcoded defaults such as `build`, `.cache`, `sow_top.hak`, `sow_tlk.tlk`, and `haks.json`
|
||||
- `internal/app/app.go`
|
||||
- project-load diagnostics and logging so commands clearly state which config file was loaded
|
||||
- user-facing descriptions that mention hardcoded paths or outputs
|
||||
- `internal/pipeline/*`
|
||||
- HAK ordering, manifest naming, music cache/credits paths, and music prefix behavior
|
||||
- generated JSON outputs must remain JSON
|
||||
- `internal/topdata/*`
|
||||
- package names, cache names, released manifest metadata, wiki state paths, and autogen consumer cache paths
|
||||
- topdata datasets and manifests must remain JSON
|
||||
- `internal/validator/*`
|
||||
- validation tests and duplicate/ordering rules
|
||||
- potential config control for built-in script prefixes if repository-specific
|
||||
- Tests under `internal/**/*_test.go`
|
||||
- many tests currently write `nwn-tool.json`; representative tests should move to YAML with targeted legacy JSON coverage retained
|
||||
- `README.md` and workflow docs
|
||||
- update root configuration references to YAML
|
||||
- explicitly document JSON artifacts that remain JSON
|
||||
|
||||
## Hardcoded assumptions to audit for config ownership
|
||||
|
||||
Move to YAML if repository-specific:
|
||||
|
||||
- root config filename policy and migration behavior
|
||||
- build output directory and HAK manifest filename
|
||||
- topdata build/cache paths
|
||||
- topdata package HAK/TLK filenames and derived TLK behavior
|
||||
- autogen release tags, asset names, cache names, producers, consumers, roots, includes, modes, and derive rules
|
||||
- released parts manifest repository/server metadata when not a universal toolkit default
|
||||
- HAK names, priorities, split behavior, optional flags, include globs, and module HAK order
|
||||
- music source roots, prefixes, credit overlay names, generated credits paths, and music stem constraints where repository-specific
|
||||
- source/asset include extension sets where repository-specific
|
||||
- validator script-prefix exclusions if they vary by repository
|
||||
|
||||
Keep in code if toolkit-intrinsic:
|
||||
|
||||
- generated JSON encoding for datasets, manifests, caches, GFF documents, and reports
|
||||
- NWN resource format constraints such as resref length limits
|
||||
- deterministic sort rules after config values are loaded
|
||||
- supported autogen derive kinds and consumer modes unless extensibility is implemented
|
||||
|
||||
# Implementation plan
|
||||
|
||||
## Phase 1: Loader and schema foundation
|
||||
|
||||
- Add YAML parsing dependency, likely `gopkg.in/yaml.v3`.
|
||||
- Add YAML tags to all config structs while preserving JSON tags for legacy decode.
|
||||
- Introduce config discovery order:
|
||||
1. `nwn-tool.yaml`
|
||||
2. `nwn-tool.yml`
|
||||
3. legacy `nwn-tool.json`
|
||||
- Return loaded config metadata from `project.Load`, including path, format, and legacy flag.
|
||||
- Make YAML win when both YAML and JSON are present.
|
||||
- Emit a clear legacy warning when `nwn-tool.json` is used.
|
||||
- Emit/log the selected config path for normal command execution.
|
||||
- Fail clearly for malformed YAML and unknown fields.
|
||||
|
||||
## Phase 2: Validation and determinism
|
||||
|
||||
- Preserve existing schema semantics while tightening validation.
|
||||
- Validate required module fields and path field types.
|
||||
- Reject duplicate HAK names, autogen producer IDs, autogen consumer IDs, and dataset IDs where represented in root config.
|
||||
- Validate HAK entries for name, priority, max bytes, include globs, and output-safe names.
|
||||
- Validate autogen manifest filenames and deterministic producer/consumer references.
|
||||
- Validate music prefixes for ambiguous duplicate normalized roots.
|
||||
- Ensure include globs and map-derived outputs are sorted before use where order matters.
|
||||
- Add deterministic tie-breakers for HAK priority sorting.
|
||||
|
||||
## Phase 3: Move repository-specific defaults into YAML
|
||||
|
||||
- Keep only intentional toolkit defaults in `defaultConfig`.
|
||||
- Move Shadows Over Westgate-specific defaults into repo YAML:
|
||||
- top package names
|
||||
- topdata cache/build paths
|
||||
- autogen manifest metadata
|
||||
- music prefixes
|
||||
- HAK ordering and include globs
|
||||
- Add config fields where needed before removing hardcoded assumptions.
|
||||
- Leave generated artifact paths as JSON outputs unless the path itself must be configurable.
|
||||
|
||||
## Phase 4: Migration
|
||||
|
||||
- Convert consumer root configs from `nwn-tool.json` to `nwn-tool.yaml`.
|
||||
- Preserve existing values exactly unless a separate behavior change is documented.
|
||||
- Treat `topdata/templates/config.yaml` as authored converter configuration, distinct from JSON data artifacts and root repository config.
|
||||
- Keep generated datasets, manifests, caches, credits inventory, and GFF JSON unchanged.
|
||||
- Add before/after examples for root config migration.
|
||||
|
||||
## Phase 5: Tests
|
||||
|
||||
- Add focused loader tests for:
|
||||
- `nwn-tool.yaml`
|
||||
- `nwn-tool.yml`
|
||||
- legacy `nwn-tool.json`
|
||||
- YAML precedence over JSON
|
||||
- malformed YAML
|
||||
- unknown YAML fields
|
||||
- Update representative build, validator, pipeline, topdata, and app tests to use YAML root config.
|
||||
- Keep targeted JSON tests only for legacy fallback and generated artifact behavior.
|
||||
- Add validation tests for duplicates, path type failures, deterministic HAK ordering, and ambiguous music prefixes.
|
||||
- Run `go test ./...`.
|
||||
|
||||
## Phase 6: Documentation
|
||||
|
||||
- Update `README.md` repository layout and consumer instructions to reference `nwn-tool.yaml`.
|
||||
- Document legacy JSON fallback and planned removal policy.
|
||||
- Document which JSON files remain intentional generated or machine-readable artifacts.
|
||||
- Document config precedence and command log behavior.
|
||||
|
||||
# Acceptance criteria
|
||||
|
||||
- A repository can build from the chosen YAML root config file (`nwn-tool.yaml` under the current plan, or `config.yaml` if the naming decision changes).
|
||||
- Existing generated JSON datasets and manifests still work.
|
||||
- No repository-specific behavior remains hardcoded if it can be expressed in YAML.
|
||||
- Builds remain deterministic.
|
||||
- Existing behavior is preserved after migration.
|
||||
- Logs clearly state which config file was loaded.
|
||||
@@ -1,215 +0,0 @@
|
||||
You are improving CLI/build logs for a NWN asset toolchain.
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- This work is largely implemented for the current `build-haks` path.
|
||||
- The app layer now exposes `--quiet`, `--verbose`, and `--debug`, and normal
|
||||
mode prints compact music summaries instead of raw per-file mappings.
|
||||
- Detailed mapping output is available behind verbose logging, and app tests
|
||||
cover the compact versus detailed presentation split.
|
||||
|
||||
Remaining work:
|
||||
|
||||
- continue applying the same presentation discipline to other command families
|
||||
where raw pipeline output still leaks through
|
||||
- keep summary wording and phase naming aligned as new generated-asset stages
|
||||
are added
|
||||
|
||||
## Active Scope
|
||||
|
||||
Logging remains an active priority, but no longer for `build-haks` alone.
|
||||
|
||||
The next logging phase should target:
|
||||
|
||||
- `build-topdata`
|
||||
- `validate-topdata`
|
||||
- `compare-topdata`
|
||||
- `build-wiki`
|
||||
- `deploy-wiki`
|
||||
- release wrapper flows that currently interleave raw shell/tool output
|
||||
|
||||
## Current Findings
|
||||
|
||||
The asset-side log refactor proved the presentation model. The remaining gap is
|
||||
uneven command coverage:
|
||||
|
||||
- topdata and wiki commands do not yet have the same polished phase/summarized
|
||||
output contract as `build-haks`
|
||||
- release scripts still emit long shell-oriented logs that are useful for CI
|
||||
diagnosis but not shaped into the same structured presentation model
|
||||
- wrapper-side status lines and toolkit-side progress lines are not yet fully
|
||||
harmonized
|
||||
|
||||
## Deferred Scope
|
||||
|
||||
Defer cosmetic-only logging work that does not improve operator clarity.
|
||||
|
||||
For now, prioritize:
|
||||
|
||||
- structured summaries
|
||||
- deterministic phase labels
|
||||
- stderr/stdout hygiene
|
||||
- parity between topdata/wiki flows and the existing build-haks presentation
|
||||
|
||||
## Immediate Plan
|
||||
|
||||
1. Reuse the existing verbosity model for topdata and wiki command families.
|
||||
2. Define concise normal-mode summaries for topdata build/validate/compare.
|
||||
3. Add deploy/build wiki progress summaries that surface:
|
||||
- local pages scanned
|
||||
- create/update/skip/drift counts
|
||||
- manifest path
|
||||
4. Keep debug mode capable of exposing raw internal progress when diagnosis is
|
||||
required.
|
||||
5. Extend app-layer tests to cover topdata/wiki presentation once those
|
||||
commands adopt the structured output contract.
|
||||
|
||||
Goal: Make the logs cleaner, prettier, less repetitive, and easier to understand, without hiding important diagnostics.
|
||||
|
||||
Current issues:
|
||||
|
||||
- The same credits refresh output appears twice.
|
||||
- Long file-by-file mappings overwhelm the main log.
|
||||
- Some messages are noisy or too implementation-focused.
|
||||
- The final build summary is useful but visually cramped.
|
||||
- Paths are too long for normal human reading unless needed for debugging.
|
||||
- Status prefixes are inconsistent: [credits], [build-haks], run-nwn-tool.sh, [|].
|
||||
|
||||
Desired behavior:
|
||||
|
||||
1. Use clear phase headers.
|
||||
2. Collapse repetitive file listings by default.
|
||||
3. Show counts and summaries in normal mode.
|
||||
4. Keep detailed mappings available behind a verbose/debug flag.
|
||||
5. Avoid printing the same credits scan twice unless something changed.
|
||||
6. Use consistent prefixes and indentation.
|
||||
7. Prefer relative paths when possible.
|
||||
8. Group related output together.
|
||||
9. End with a compact, readable summary.
|
||||
10. Preserve warnings, errors, and actionable diagnostics prominently.
|
||||
|
||||
Suggested output style:
|
||||
|
||||
Build HAKs ────────── • Refreshing credits cache
|
||||
|
||||
- Scanned: envi/music/westgate
|
||||
- Tracks: 38
|
||||
- Output: .cache/credits/envi/music/westgate/credits.json
|
||||
- Aggregate cache updated
|
||||
|
||||
• Validating project • Collecting asset resources • Writing autogen manifests
|
||||
|
||||
- sow-parts-manifest.json
|
||||
- sow-accessory-vfx-manifest.json
|
||||
|
||||
• Planning HAK chunks • Resolving module HAK order • Reusing unchanged HAKs
|
||||
|
||||
- 12 / 12 reused
|
||||
- 65,320 assets total
|
||||
|
||||
• Cleaning stale generated HAKs • Writing HAK manifest
|
||||
|
||||
Summary ─────── Project: Shadows Over Westgate HAK archives: 12 HAK assets: 65,320 Manifest: assets/build/haks.json Credits artifacts: 2 Autogen manifests: 2 Status: complete
|
||||
|
||||
Rules:
|
||||
|
||||
- In normal mode, do not print every `mp3 -> bmu` mapping.
|
||||
- Instead print something like: `Mapped 38 music files. Use --verbose to list mappings.`
|
||||
- In verbose mode, print mappings under an indented section.
|
||||
- If the credits cache is refreshed before and after the build for normalization, make the second refresh terse: `Restored normalized credits cache: unchanged.`
|
||||
- If files changed, say how many changed and where.
|
||||
- Do not remove any warnings or errors.
|
||||
- Do not make failure logs terse; failures should include enough context to debug.
|
||||
- Keep output deterministic so CI diffs remain readable.
|
||||
- Use plain ASCII unless the toolchain already supports Unicode reliably.
|
||||
- Do not introduce color unless color can be disabled with NO_COLOR or --no-color.
|
||||
|
||||
Implementation guidance:
|
||||
|
||||
- Add logging helpers for phases, steps, summaries, verbose details, warnings, and errors.
|
||||
- Add a verbosity level, for example: quiet, normal, verbose, debug.
|
||||
- Route repeated detailed output through verbose/debug logging.
|
||||
- Track counts while scanning instead of printing every scanned item.
|
||||
- Detect unchanged cache writes and report them as unchanged rather than rewriting noisy output.
|
||||
- Normalize absolute paths to project-relative paths in normal output.
|
||||
- Keep machine-readable artifacts unchanged.
|
||||
|
||||
Concrete implementation plan:
|
||||
|
||||
1. CLI presentation layer
|
||||
|
||||
- Keep the pipeline responsible for state and artifact generation.
|
||||
- Move `build-haks` presentation rules into `internal/app/app.go`.
|
||||
- Introduce `--quiet`, `--verbose`, and `--debug` for `build-haks`.
|
||||
- In normal mode, print grouped sections and a compact summary.
|
||||
- In verbose mode, include detailed mapping and per-archive action lines.
|
||||
- In debug mode, preserve raw pipeline progress messages for diagnosis.
|
||||
|
||||
2. Spinner behavior
|
||||
|
||||
- Reuse the existing spinner infrastructure instead of adding a second animation system.
|
||||
- Make the spinner phase-aware so it shows the current high-level step, for example:
|
||||
- `Build HAKs: refreshing credits cache`
|
||||
- `Build HAKs: collecting asset resources`
|
||||
- `Build HAKs: planning HAK chunks`
|
||||
- Only animate on interactive TTY stderr.
|
||||
- Disable spinner animation automatically in CI and non-interactive output.
|
||||
- Always keep stable human-readable logs on stdout.
|
||||
|
||||
3. Pipeline data surfaced to the CLI
|
||||
|
||||
- Extend `pipeline.BuildResult` with log-summary metadata instead of forcing the CLI to parse raw strings.
|
||||
- Return credits refresh summary data:
|
||||
- scanned music directories
|
||||
- converted track count
|
||||
- generated credits artifact paths
|
||||
- per-file source/output mappings
|
||||
- changed-versus-unchanged artifact count
|
||||
- Return HAK archive summary data:
|
||||
- total archive count
|
||||
- reused archive count
|
||||
- written archive count
|
||||
- per-archive actions for verbose mode
|
||||
|
||||
4. Credits artifact handling
|
||||
|
||||
- Stop blindly deleting and rewriting `.cache/credits` on every run.
|
||||
- Render the desired credits outputs in memory first.
|
||||
- Compare desired artifacts against the current on-disk artifacts.
|
||||
- Rewrite only changed files.
|
||||
- Remove stale artifacts that are no longer desired.
|
||||
- Report `unchanged` when the second pass produces identical output.
|
||||
|
||||
5. Output rules
|
||||
|
||||
- Normal mode:
|
||||
- show phase blocks
|
||||
- show counts
|
||||
- show relative paths
|
||||
- hide file-by-file mappings
|
||||
- Verbose mode:
|
||||
- show music mappings under an indented `mappings:` section
|
||||
- show per-archive `reused` or `wrote` details
|
||||
- Quiet mode:
|
||||
- keep only the final summary
|
||||
- Debug mode:
|
||||
- allow raw internal progress lines in addition to the formatted summary
|
||||
|
||||
6. Tests
|
||||
|
||||
- Add app-layer tests for:
|
||||
- compact normal-mode output
|
||||
- verbose mapping output
|
||||
- Add pipeline regression coverage proving:
|
||||
- music conversion still writes the expected credits artifacts
|
||||
- a second identical credits refresh reports zero changed files
|
||||
|
||||
Acceptance checks:
|
||||
|
||||
- `build-haks` emits the new grouped summary format.
|
||||
- `build-haks --verbose` includes per-track mappings and per-archive details.
|
||||
- The spinner reflects phase changes without polluting stdout logs.
|
||||
- Re-running the same build keeps credits refresh output deterministic and reports `unchanged` when applicable.
|
||||
- Existing machine-readable artifacts remain unchanged in schema and location.
|
||||
@@ -1,723 +0,0 @@
|
||||
# Model Compilation Build-Time Contract
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- This contract is still outstanding.
|
||||
- `build-haks` does not currently compile authored ASCII `.mdl` files into
|
||||
binary build artifacts.
|
||||
- Toolkit validation and extraction understand `.mdl`, but there is no
|
||||
config-backed build-time model compiler pipeline in `internal/pipeline/`.
|
||||
|
||||
Open work remains exactly in the areas this contract describes:
|
||||
|
||||
- schema and validation for `models.compile.*`
|
||||
- compiler/tool resolution and staging
|
||||
- generated-asset overlay integration in HAK builds
|
||||
- cache/reuse semantics and regression coverage
|
||||
|
||||
## Scope
|
||||
|
||||
Expand the toolkit so authored ASCII Neverwinter Nights `.mdl` assets can be compiled into binary `.mdl` build artifacts during `sow-toolkit build-haks` without changing the authored source tree.
|
||||
|
||||
This contract covers:
|
||||
|
||||
- `toolkit/` as the implementation owner
|
||||
- `sow-assets/` as the primary consumer repository
|
||||
- any future consumer repo that packages NWN model assets through the same HAK build pipeline
|
||||
|
||||
This contract does not cover:
|
||||
|
||||
- replacing authored ASCII `.mdl` files in source control
|
||||
- introducing a separate standalone shell-only build system as the primary path
|
||||
- downloading or redistributing copyrighted NWN game data
|
||||
- building an in-game runtime compiler workflow
|
||||
|
||||
## Goal
|
||||
|
||||
Make model compilation a first-class, deterministic, configurable build stage in the Go toolkit so generated HAKs contain compiled binary models while authored repositories continue to keep ASCII `.mdl` files as source of truth.
|
||||
|
||||
## Why This Exists
|
||||
|
||||
Repo-local context already shows that ASCII `.mdl` files are an intentional part of authored asset workflows:
|
||||
|
||||
- validation reports ASCII/decompiled models as informational, not erroneous
|
||||
- topdata autogen manifest generation scans authored `.mdl` paths directly
|
||||
- part discovery derives IDs from authored `.mdl` filenames
|
||||
- HAK building already has other generated-asset stages such as NWScript and music conversion
|
||||
|
||||
Upstream NWN documentation also supports compiling before release:
|
||||
|
||||
- `nwn.wiki` documents that ASCII models are slower because the game compiles them at load time and recommends shipping compiled models
|
||||
- `nwnmdlcomp` usage docs confirm ASCII-to-binary compilation as a standard flow
|
||||
- community documentation shows `nwnmdlcomp.exe` expects old NWN registry/data layout and may need a 1.69-era data root
|
||||
|
||||
Therefore the correct implementation is not “replace asset scanning with binary models everywhere”. The correct implementation is:
|
||||
|
||||
1. keep authored ASCII `.mdl` in `paths.assets`
|
||||
2. preserve current validation and manifest derivation behavior against source
|
||||
3. compile only the build payload that becomes generated HAK content
|
||||
|
||||
## Ownership Split
|
||||
|
||||
- `toolkit/`
|
||||
- owns config schema, validation, discovery, cache policy, build staging, and HAK integration
|
||||
- owns Wine execution and NWN data preparation logic
|
||||
- owns tests and user-facing CLI behavior
|
||||
- `sow-assets/`
|
||||
- owns opting into the feature through `nwn-tool.yaml`
|
||||
- owns providing local tool binaries and any fake/minimal NWN data tree it wants to use
|
||||
- owns wrapper scripts and CI invocation details
|
||||
- `sow-module/`
|
||||
- is an indirect consumer only when it consumes generated HAKs
|
||||
- must not need authored-source changes for this feature to work
|
||||
|
||||
## Current Project-Needs Analysis
|
||||
|
||||
## Existing architecture
|
||||
|
||||
Current HAK packaging is Go-native and centered on:
|
||||
|
||||
- `cmd/nwn-tool/main.go`
|
||||
- `internal/app/app.go`
|
||||
- `internal/pipeline/build.go`
|
||||
- `internal/project/*`
|
||||
- `internal/validator/*`
|
||||
|
||||
`build-haks` already does all asset discovery, planning, manifest writing, HAK reuse, and generated-asset preparation inside the toolkit. Adding model compilation as a disconnected shell script would duplicate the existing build graph and create drift.
|
||||
|
||||
## Existing relevant behavior
|
||||
|
||||
- `project.AssetExtensions` already includes `.mdl`
|
||||
- `validator.ValidateProject` detects text `.mdl` and reports them via `Report.DecompiledModels`
|
||||
- `emitValidatorReport` already surfaces that information in CLI output
|
||||
- `collectAssetResources` packages raw files from `paths.assets` directly into HAK resources
|
||||
- `BuildHAKs` already supports generated assets and temporary staging through the music pipeline
|
||||
- script compilation already establishes a precedent for:
|
||||
- config-backed tool resolution
|
||||
- environment discovery
|
||||
- cache directories under `.cache`
|
||||
- build-time generated resources that do not modify authored sources
|
||||
|
||||
## Constraints the plan must preserve
|
||||
|
||||
- authored `.mdl` files remain canonical inputs
|
||||
- autogen and topdata logic must continue to see source `.mdl` paths
|
||||
- validation must still be able to detect ASCII source models
|
||||
- HAK duplicate detection and manifest generation must remain deterministic
|
||||
- build output reuse must not falsely reuse HAKs built from stale compiled model content
|
||||
- consumer repos must be able to opt out cleanly
|
||||
|
||||
## Architecture Decision
|
||||
|
||||
Implement model compilation as a generated-asset overlay inside the existing HAK build pipeline, not as a separate top-level shell build.
|
||||
|
||||
That means:
|
||||
|
||||
- source scanning still indexes authored `.mdl`
|
||||
- a model preparation stage produces compiled build artifacts in cache/staging
|
||||
- `collectAssetResources` substitutes compiled `.mdl` bytes for eligible source models during HAK packaging
|
||||
- non-model assets continue to flow unchanged
|
||||
- the written HAK manifest still records the authored relative asset path, not a cache path
|
||||
|
||||
This mirrors the existing music pipeline more than the script pipeline:
|
||||
|
||||
- source asset exists in `paths.assets`
|
||||
- generated build artifact lives in cache/staging
|
||||
- HAK includes the generated bytes under the original logical resource name
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- no automatic decompilation
|
||||
- no mutation of files under `paths.assets`
|
||||
- no requirement to compile every `.mdl` in every repo by default
|
||||
- no requirement to support Windows-only shell wrappers as the primary interface
|
||||
- no requirement to support arbitrary model compilers on day one beyond `nwnmdlcomp.exe` via Wine
|
||||
- no attempt to “fix” malformed source models automatically
|
||||
|
||||
## Config Contract
|
||||
|
||||
Add a new top-level config section:
|
||||
|
||||
```yaml
|
||||
models:
|
||||
compile:
|
||||
enabled: true
|
||||
source_extensions: [.mdl]
|
||||
include: []
|
||||
exclude: []
|
||||
cache: "{paths.cache}/models"
|
||||
compiler:
|
||||
path: ""
|
||||
env:
|
||||
path: SOW_NWN_MODEL_COMPILER
|
||||
wine: SOW_NWN_WINE
|
||||
winepath: SOW_NWN_WINEPATH
|
||||
prefix: SOW_NWN_WINEPREFIX
|
||||
nwn_data: [SOW_NWN_MODEL_DATA, SOW_NWN_ROOT, NWN_ROOT]
|
||||
search:
|
||||
- "{paths.tools}/nwnmdlcomp/nwnmdlcomp.exe"
|
||||
- "{paths.tools}/nwnmdlcomp.exe"
|
||||
runtime:
|
||||
mode: wine
|
||||
jobs: 1
|
||||
force: false
|
||||
setup_registry: true
|
||||
compiler_cwd: "{paths.tools}/nwnmdlcomp"
|
||||
data:
|
||||
root: ""
|
||||
require_keys: [chitin.key, nwn_base.key, data/nwn_base.key]
|
||||
writable_aliases: [chitin.key, nwn_base.key]
|
||||
policy:
|
||||
compile_ascii_only: true
|
||||
preserve_binary_sources: true
|
||||
fail_on_missing_output: true
|
||||
fail_on_zero_byte_output: true
|
||||
fail_on_warning: false
|
||||
```
|
||||
|
||||
## Config design rules
|
||||
|
||||
- `models.compile.enabled` must default to `false`
|
||||
- the feature must be opt-in
|
||||
- config must be repository-driven, not hardcoded per consumer
|
||||
- search/discovery semantics should match `scripts.compiler` where practical
|
||||
- path templating should reuse the existing effective config path expansion model
|
||||
- include/exclude filters apply to asset-relative paths under `paths.assets`
|
||||
- default jobs should be serial unless parallel execution is explicitly enabled
|
||||
|
||||
## Environment precedence
|
||||
|
||||
Use this precedence for the compiler executable:
|
||||
|
||||
1. `models.compile.compiler.path`
|
||||
2. env var named by `models.compile.compiler.env.path`
|
||||
3. configured search paths
|
||||
|
||||
Use this precedence for NWN data root:
|
||||
|
||||
1. `models.compile.data.root`
|
||||
2. env vars listed in `models.compile.compiler.env.nwn_data`
|
||||
3. no implicit auto-discovery unless explicitly implemented and documented
|
||||
|
||||
Wine tools should resolve as:
|
||||
|
||||
1. env override for `wine`
|
||||
2. `PATH:wine`
|
||||
|
||||
and:
|
||||
|
||||
1. env override for `winepath`
|
||||
2. `PATH:winepath`
|
||||
|
||||
Do not silently guess a fake NWN data tree.
|
||||
|
||||
## Build Contract
|
||||
|
||||
## Eligibility rules
|
||||
|
||||
A model is eligible for build-time compilation when all of the following are true:
|
||||
|
||||
- asset extension is `.mdl`
|
||||
- `models.compile.enabled` is true
|
||||
- path passes include/exclude filters
|
||||
- file is detected as ASCII when `compile_ascii_only` is true
|
||||
|
||||
If a `.mdl` file is already binary and `preserve_binary_sources` is true:
|
||||
|
||||
- do not recompile it
|
||||
- package it as-is
|
||||
- optionally log a verbose skip reason
|
||||
|
||||
## Integration point
|
||||
|
||||
Insert model preparation into `planOrBuildHAKs` before `collectAssetResources` finalizes HAK resource content, alongside other generated-asset preparation.
|
||||
|
||||
Expected flow:
|
||||
|
||||
1. validate project
|
||||
2. prepare music assets if enabled
|
||||
3. prepare compiled model assets if enabled
|
||||
4. collect asset resources using source inventory plus prepared overlays
|
||||
5. plan/write HAKs as today
|
||||
|
||||
## Prepared asset representation
|
||||
|
||||
Introduce a prepared-model result similar in spirit to `preparedMusicAssets`, for example:
|
||||
|
||||
- generated assets keyed by original asset-relative path
|
||||
- skipped source-relative paths
|
||||
- summary counts and loggable actions
|
||||
- cleanup hook for temporary staging if needed
|
||||
|
||||
The HAK layer should continue to see:
|
||||
|
||||
- resource name from the original source path stem
|
||||
- resource type `mdl`
|
||||
- relative manifest path equal to the original source path
|
||||
|
||||
Only the file bytes should differ.
|
||||
|
||||
## Cache and staging contract
|
||||
|
||||
Compiled models are generated artifacts and must live outside the authored asset tree.
|
||||
|
||||
Default cache root:
|
||||
|
||||
```text
|
||||
{paths.cache}/models
|
||||
```
|
||||
|
||||
Suggested cache layout:
|
||||
|
||||
```text
|
||||
.cache/models/
|
||||
compiled/<asset-relative-path>.mdl
|
||||
logs/<asset-relative-path>.log
|
||||
metadata.json
|
||||
wine-prefix/ # only if the repo chooses a toolkit-managed prefix
|
||||
nwn-data-links/ # only if alias files/symlinks are staged
|
||||
```
|
||||
|
||||
The exact layout may differ, but it must support:
|
||||
|
||||
- deterministic mapping from source asset path to compiled output path
|
||||
- per-model logging for failures
|
||||
- incremental stale detection
|
||||
- cleanup without touching authored content
|
||||
|
||||
## Incremental contract
|
||||
|
||||
The plan must support incremental recompilation for persistent cache dirs.
|
||||
|
||||
At minimum, recompile when any of these change:
|
||||
|
||||
- source file content hash changed
|
||||
- source file modtime is newer than compiled output
|
||||
- compiler executable path or content fingerprint changed
|
||||
- NWN data root fingerprint changed if the implementation tracks it
|
||||
- relevant config changed
|
||||
- `--force` or config `force: true` is active
|
||||
- compiled output is missing or zero bytes
|
||||
|
||||
Recommended metadata key:
|
||||
|
||||
- source asset relative path
|
||||
- source SHA-256
|
||||
- output SHA-256
|
||||
- compiler path
|
||||
- compiler binary mtime or hash
|
||||
- NWN data root path
|
||||
- build config fingerprint
|
||||
- compiled timestamp
|
||||
|
||||
If metadata is unavailable or invalid, fail open to recompilation, not reuse.
|
||||
|
||||
## NWN Data Contract
|
||||
|
||||
`nwnmdlcomp.exe` requires a classic NWN data/registry environment.
|
||||
|
||||
The implementation must:
|
||||
|
||||
- validate the configured NWN data root exists
|
||||
- require at least one of:
|
||||
- `chitin.key`
|
||||
- `nwn_base.key`
|
||||
- `data/nwn_base.key`
|
||||
- never download game data
|
||||
- never synthesize copyrighted content
|
||||
|
||||
If only `data/nwn_base.key` exists, the implementation may create a staged alias view for the compiler by:
|
||||
|
||||
- symlinking inside a cache-owned staging dir, or
|
||||
- copying just the key file names into a cache-owned staging dir when symlinks are unavailable
|
||||
|
||||
Do not mutate the user’s real install directory unless they explicitly pointed the config at a writable fake data tree and the implementation clearly documents that behavior.
|
||||
|
||||
## Wine Registry Contract
|
||||
|
||||
When `runtime.setup_registry` is enabled, prepare the Wine registry keys needed by `nwnmdlcomp.exe`:
|
||||
|
||||
```text
|
||||
HKLM\Software\WOW6432Node\BioWare\NWN\Neverwinter
|
||||
Location = <wine path>
|
||||
Path = <wine path>
|
||||
Version = 1.69
|
||||
GUID = 00000000-0000-0000-0000-000000000000
|
||||
Language = 0
|
||||
```
|
||||
|
||||
Implementation rules:
|
||||
|
||||
- convert Linux paths with `winepath -w`
|
||||
- target the Wine prefix selected by config/env
|
||||
- make registry setup idempotent
|
||||
- separate “prepare registry” failure messages from “compile model” failures
|
||||
|
||||
The implementation may either:
|
||||
|
||||
- manage a dedicated toolkit prefix, or
|
||||
- use a user-provided existing prefix
|
||||
|
||||
The contract should prefer an isolated prefix to avoid polluting unrelated Wine state.
|
||||
|
||||
## Compiler Invocation Contract
|
||||
|
||||
Primary supported compiler:
|
||||
|
||||
- `nwnmdlcomp.exe`
|
||||
|
||||
Expected invocation shape:
|
||||
|
||||
```bash
|
||||
WINEDEBUG=-all wine "$COMPILER" -c "$INPUT_WIN" "$OUTPUT_WIN" -e
|
||||
```
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
- invoke through `os/exec`, not shell string concatenation
|
||||
- convert input/output paths with `winepath -w`
|
||||
- allow compiler working directory configuration if supermodel resolution or relative lookup requires it
|
||||
- capture combined stdout/stderr per compile attempt
|
||||
|
||||
## Supermodel and sibling dependency considerations
|
||||
|
||||
The plan must explicitly account for model compilation not being a pure single-file transform.
|
||||
|
||||
Potential dependencies include:
|
||||
|
||||
- `setsupermodel` references
|
||||
- sibling helper models in the same asset family
|
||||
- textures or material references that may not block compilation but do matter for diagnostics
|
||||
|
||||
Minimum contract:
|
||||
|
||||
- compile against a staging layout that preserves original relative structure
|
||||
- ensure each model’s directory context exists in staging
|
||||
- document that some supermodel cases may require staging additional authored `.mdl` siblings into the compiler-visible workspace
|
||||
|
||||
Do not assume “compile one file in isolation” is always sufficient.
|
||||
|
||||
## Staging strategy
|
||||
|
||||
Preferred strategy:
|
||||
|
||||
1. keep source inventory rooted in `paths.assets`
|
||||
2. materialize a compiler workspace under cache that mirrors required relative directories for eligible `.mdl`
|
||||
3. compile outputs into a sibling compiled tree
|
||||
4. package compiled output bytes while preserving source-relative logical names
|
||||
|
||||
This avoids:
|
||||
|
||||
- polluting source trees
|
||||
- output collisions between models with the same basename in different folders
|
||||
- broken relative assumptions inside asset packs
|
||||
|
||||
## Logging Contract
|
||||
|
||||
Progress output should fit existing `ProgressFunc` style.
|
||||
|
||||
Minimum messages:
|
||||
|
||||
- `Preparing model compiler...`
|
||||
- `Preparing model compilation workspace...`
|
||||
- `Compiling model 12/324: part/belt/pfa0_belt018.mdl`
|
||||
- `Reusing compiled model: vfxs/head_accessories/hfx_bandana.mdl`
|
||||
- `Skipping binary model source: creatures/foo.mdl`
|
||||
- `FAILED model compile: placeables/broken_model.mdl`
|
||||
|
||||
At summary level, surface:
|
||||
|
||||
- models considered
|
||||
- compiled
|
||||
- reused
|
||||
- skipped binary
|
||||
- skipped filtered
|
||||
- failed
|
||||
|
||||
Verbose/debug output may include:
|
||||
|
||||
- chosen compiler path
|
||||
- chosen Wine/Wineprefix/Winepath tools
|
||||
- chosen NWN data root
|
||||
- per-model log file location
|
||||
|
||||
## Error Handling Contract
|
||||
|
||||
Model compilation is a build-critical stage once enabled.
|
||||
|
||||
The build must fail when an eligible model compile:
|
||||
|
||||
- exits non-zero
|
||||
- does not produce the expected output file
|
||||
- produces a zero-byte output file
|
||||
- cannot be staged or fingerprinted
|
||||
|
||||
Warnings from the compiler do not fail the build unless future config enables that policy.
|
||||
|
||||
Failure output must include:
|
||||
|
||||
- original source-relative asset path
|
||||
- compiler command context sufficient for debugging
|
||||
- log file path if written
|
||||
|
||||
At the end of a failed run, include a concise failed-path summary.
|
||||
|
||||
## CLI Contract
|
||||
|
||||
The canonical user-facing entrypoint remains:
|
||||
|
||||
```bash
|
||||
sow-toolkit build-haks
|
||||
```
|
||||
|
||||
Add targeted flags only if they materially improve operability and match existing CLI patterns. Recommended flags:
|
||||
|
||||
- `build-haks --force-models`
|
||||
- `build-haks --skip-models`
|
||||
- `build-haks --model <glob-or-prefix>` optional, only if there is a real use case for scoped rebuilds
|
||||
|
||||
Flag rules:
|
||||
|
||||
- flags override config for the current invocation only
|
||||
- `--skip-models` must bypass model preparation even if config is enabled
|
||||
- `--force-models` must invalidate incremental reuse for model outputs only
|
||||
|
||||
Do not introduce a parallel standalone command as the only way to use the feature. A helper/diagnostic subcommand is acceptable later, but build-haks must own the real integration.
|
||||
|
||||
## Validation Contract
|
||||
|
||||
Current validation already reports authored ASCII `.mdl` as informational. That behavior should remain, but the plan should extend validation to cover model compilation readiness when the feature is enabled.
|
||||
|
||||
Recommended validation additions:
|
||||
|
||||
- compiler path resolves or search candidates exist
|
||||
- `wine` and `winepath` are resolvable on non-Windows platforms when mode is `wine`
|
||||
- NWN data root exists and has required keys
|
||||
- cache path is safe and repo-relative unless explicitly documented otherwise
|
||||
- include/exclude glob config is well-formed
|
||||
- jobs is >= 1
|
||||
|
||||
Validation severity:
|
||||
|
||||
- config/discovery failures that make enabled compilation impossible should be errors
|
||||
- authored ASCII `.mdl` detection remains info
|
||||
- binary `.mdl` sources alongside enabled compilation are not errors
|
||||
|
||||
## Manifest And Reuse Contract
|
||||
|
||||
Generated `haks.json` must continue to describe authored asset-relative paths, not cache file paths.
|
||||
|
||||
HAK reuse logic must account for compiled model output bytes. Reuse must not mistakenly treat an unchanged authored path as unchanged content if the compiled artifact changed.
|
||||
|
||||
That means one of:
|
||||
|
||||
- the existing asset `ContentID` for `.mdl` must reflect compiled output hash when compilation is enabled, or
|
||||
- chunk hashing must incorporate the prepared compiled output fingerprint
|
||||
|
||||
Without this, stale HAK reuse would be incorrect.
|
||||
|
||||
## Parallelism Contract
|
||||
|
||||
Support configurable parallel compilation with conservative defaults.
|
||||
|
||||
Requirements:
|
||||
|
||||
- default jobs is `1`
|
||||
- jobs > 1 must not corrupt outputs or logs
|
||||
- each compile must write to a unique output path and unique log path
|
||||
- shared registry/prefix setup must happen before parallel work or under safe synchronization
|
||||
|
||||
If `nwnmdlcomp.exe` or Wine proves unstable under parallel execution:
|
||||
|
||||
- keep serial default
|
||||
- document the risk
|
||||
- allow opting into higher parallelism with clear caveats
|
||||
|
||||
## Testing Contract
|
||||
|
||||
Implementation is not complete without automated tests. Use fake executables and temporary directories rather than requiring a real Wine/NWN environment in unit tests.
|
||||
|
||||
## Unit tests
|
||||
|
||||
Add tests for:
|
||||
|
||||
- config loading and effective config defaults for `models.compile`
|
||||
- compiler path/env/search resolution
|
||||
- Wine tool resolution
|
||||
- NWN data root validation and alias handling
|
||||
- ASCII-vs-binary `.mdl` eligibility logic
|
||||
- include/exclude filtering
|
||||
- incremental reuse decisions
|
||||
- metadata invalidation on source/config/compiler changes
|
||||
- content hashing based on compiled output, not source text alone
|
||||
|
||||
## Pipeline tests
|
||||
|
||||
Add or extend `internal/pipeline/pipeline_test.go` coverage for:
|
||||
|
||||
- HAK build packages compiled `.mdl` bytes instead of source ASCII bytes
|
||||
- non-`.mdl` assets remain unchanged
|
||||
- binary source `.mdl` passes through unchanged
|
||||
- failed model compile aborts build
|
||||
- missing output aborts build
|
||||
- zero-byte output aborts build
|
||||
- `--skip-models` bypasses compilation
|
||||
- `--force-models` rebuilds compiled outputs
|
||||
- unchanged compiled output allows HAK reuse
|
||||
- changed compiled output forces HAK rewrite
|
||||
- duplicate asset collision rules still behave as before
|
||||
|
||||
## Validator tests
|
||||
|
||||
Keep and extend tests proving:
|
||||
|
||||
- ASCII source models are still reported as informational
|
||||
- enabled compilation readiness errors surface when config is invalid
|
||||
|
||||
## Integration/manual acceptance tests
|
||||
|
||||
Document real-world manual checks for a workstation with Wine and `nwnmdlcomp.exe` available.
|
||||
|
||||
### Test 1: readiness validation
|
||||
|
||||
```bash
|
||||
sow-toolkit validate
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
- no model-compiler readiness errors when config/env are correct
|
||||
- ASCII `.mdl` info line still appears for source models
|
||||
|
||||
### Test 2: single-model HAK packaging
|
||||
|
||||
Given an authored ASCII model under `paths.assets`, run:
|
||||
|
||||
```bash
|
||||
sow-toolkit build-haks --force-models
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
- build succeeds
|
||||
- resulting HAK contains a binary `.mdl`
|
||||
- authored source file remains unchanged
|
||||
|
||||
### Test 3: incremental reuse
|
||||
|
||||
Run `build-haks` twice without changes.
|
||||
|
||||
Expected:
|
||||
|
||||
- second run reports model reuse
|
||||
- unchanged HAKs are reused when manifest/chunk fingerprints match
|
||||
|
||||
### Test 4: source change invalidation
|
||||
|
||||
Edit one ASCII `.mdl` source and rebuild.
|
||||
|
||||
Expected:
|
||||
|
||||
- only affected compiled output is rebuilt
|
||||
- affected HAK chunk is rewritten
|
||||
- unrelated HAK chunks remain reusable
|
||||
|
||||
### Test 5: missing NWN data
|
||||
|
||||
Misconfigure `models.compile.data.root` and run build.
|
||||
|
||||
Expected:
|
||||
|
||||
- validation/build fails clearly before or during preparation
|
||||
- error names the missing key/data requirement
|
||||
|
||||
## Documentation Contract
|
||||
|
||||
Update `README.md` to include:
|
||||
|
||||
- why compiled models are shipped even though ASCII remains authored source
|
||||
- the new `models.compile` config section
|
||||
- required local dependencies:
|
||||
- `nwnmdlcomp.exe`
|
||||
- `wine`
|
||||
- `winepath`
|
||||
- NWN data root
|
||||
- environment override behavior
|
||||
- cache/staging behavior
|
||||
- `build-haks` flags related to model compilation
|
||||
- limitations and troubleshooting notes
|
||||
|
||||
Optional but recommended:
|
||||
|
||||
- `tools/nwnmdlcomp/README.md` for consumer repo operators
|
||||
|
||||
## Suggested Implementation Breakdown
|
||||
|
||||
1. Schema and config foundation
|
||||
- add `models.compile` config structs to `internal/project/project.go`
|
||||
- add effective config defaults and path expansion in `internal/project/effective.go`
|
||||
- add project helper methods for compiler/cache/env resolution
|
||||
|
||||
2. Validation foundation
|
||||
- extend project/config validation for enabled compilation readiness
|
||||
- extend CLI validation reporting where needed
|
||||
|
||||
3. Preparation layer
|
||||
- add model compiler resolution, Wine resolution, NWN data staging, registry setup, and incremental metadata logic
|
||||
- add a `preparedModelAssets` representation
|
||||
|
||||
4. Pipeline integration
|
||||
- plug model preparation into `planOrBuildHAKs`
|
||||
- teach `collectAssetResources` to substitute compiled model payloads
|
||||
- ensure `ContentID` and HAK reuse reflect compiled output
|
||||
|
||||
5. CLI polish
|
||||
- add build-haks flags if approved
|
||||
- add progress messages and summary output
|
||||
|
||||
6. Tests
|
||||
- config tests
|
||||
- validator tests
|
||||
- pipeline tests with fake compiler/Wine shims
|
||||
|
||||
7. Docs
|
||||
- repo README updates
|
||||
- optional operator README under `tools/`
|
||||
|
||||
## Open Design Decisions To Resolve Before Coding
|
||||
|
||||
These should be decided explicitly during implementation, not left implicit:
|
||||
|
||||
1. Should the toolkit own a dedicated Wine prefix under cache by default, or require a user-provided prefix?
|
||||
2. Should NWN data alias files be created in a separate staging tree, or should the configured data root be used directly when already compatible?
|
||||
3. Is parallel model compilation stable enough to expose beyond best-effort?
|
||||
4. Do we want dedicated `build-haks` flags now, or only config for the first implementation?
|
||||
5. Should per-model logs always be persisted, or only on failure / debug mode?
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
The feature is complete when all of the following are true:
|
||||
|
||||
- `sow-toolkit build-haks` can package compiled binary `.mdl` content from authored ASCII `.mdl` source when `models.compile.enabled: true`
|
||||
- authored `.mdl` files remain unchanged on disk
|
||||
- validation and topdata/autogen behaviors that depend on authored `.mdl` source still work
|
||||
- generated HAK manifests still reference authored asset-relative paths
|
||||
- HAK reuse logic correctly tracks compiled output changes
|
||||
- the feature is opt-in, deterministic, and tested
|
||||
- failures are actionable and name the offending model paths
|
||||
- README/config docs are updated
|
||||
|
||||
## Relevant References
|
||||
|
||||
- Repo code:
|
||||
- `internal/pipeline/build.go`
|
||||
- `internal/validator/validator.go`
|
||||
- `internal/topdata/autogen.go`
|
||||
- `internal/topdata/parts_discovery.go`
|
||||
- `README.md`
|
||||
- Upstream docs consulted for this contract:
|
||||
- `nwn.wiki` MDL page: compiled models are recommended for release because ASCII loads more slowly
|
||||
- Neverwinter Vault `nwnmdlcomp` page: standard compile/decompile CLI usage
|
||||
- Neverwinter Vault forum discussions: `nwnmdlcomp` expects old NWN registry keys/data layout and some supermodel cases need additional care
|
||||
@@ -1,757 +0,0 @@
|
||||
Refactor the music generation / editing / credits pipeline so it exists entirely inside the shared toolkit and is no longer hardcoded or repository-specific.
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- This refactor is largely implemented.
|
||||
- Music logic now lives under `internal/music/` with toolkit-owned CLI entry
|
||||
points and shared use from `build-haks`.
|
||||
- Dataset-based config is supported and documented; the toolkit exposes
|
||||
`music list-datasets|scan|build|credits|validate|manifest|normalize`.
|
||||
- Incremental credits artifact handling and compact/verbose log presentation are
|
||||
present in the current implementation.
|
||||
|
||||
Remaining work:
|
||||
|
||||
- finish retiring old `music.prefixes`-only assumptions in docs/tests after all
|
||||
consumer configs are fully dataset-native
|
||||
- continue separating any remaining music-specific presentation rules from
|
||||
broader `build-haks` output policy as the build graph evolves
|
||||
|
||||
Goal: Turn the current Shadows Over Westgate-specific music pipeline into a reusable, repository-agnostic toolkit subsystem that can operate across multiple projects and datasets with configuration-driven behavior.
|
||||
|
||||
# Requirements
|
||||
|
||||
## Architecture
|
||||
|
||||
- Move all music pipeline logic into the toolkit itself.
|
||||
- Eliminate repository-specific assumptions, paths, naming conventions, or hardcoded behavior.
|
||||
- Repositories should only provide configuration and assets.
|
||||
- The toolkit should expose reusable commands/services for:
|
||||
- music scanning
|
||||
- metadata extraction
|
||||
- credits generation
|
||||
- track normalization
|
||||
- BMU generation/mapping
|
||||
- music editing/transcoding
|
||||
- manifest generation
|
||||
- cache generation
|
||||
- validation
|
||||
|
||||
## Configuration
|
||||
|
||||
Support configuration at multiple levels:
|
||||
|
||||
1. Global toolkit defaults
|
||||
2. Repository-level configuration
|
||||
3. Dataset-level overrides
|
||||
|
||||
Example hierarchy:
|
||||
|
||||
- toolkit defaults
|
||||
- repo config
|
||||
- dataset config
|
||||
- per-task overrides
|
||||
|
||||
The system should merge configuration predictably.
|
||||
|
||||
## Datasets
|
||||
|
||||
A "dataset" should represent an isolated music collection/pipeline target.
|
||||
|
||||
Examples:
|
||||
|
||||
- envi/music/westgate
|
||||
- modules/oc/music
|
||||
- premium/music/bardpack
|
||||
|
||||
Each dataset should support:
|
||||
|
||||
- independent input/output paths
|
||||
- naming rules
|
||||
- normalization settings
|
||||
- credits settings
|
||||
- encoding/transcoding options
|
||||
- cache behavior
|
||||
- manifest generation
|
||||
- validation rules
|
||||
|
||||
# Desired Features
|
||||
|
||||
## Reusable pipeline stages
|
||||
|
||||
Design the pipeline as composable stages.
|
||||
|
||||
Example:
|
||||
|
||||
- discover
|
||||
- scan metadata
|
||||
- normalize
|
||||
- transcode
|
||||
- generate identifiers
|
||||
- build mappings
|
||||
- generate credits
|
||||
- write manifests
|
||||
- validate outputs
|
||||
|
||||
Stages should:
|
||||
|
||||
- be independently callable
|
||||
- support dry-run
|
||||
- support incremental rebuilds
|
||||
- support verbose/debug logging
|
||||
- expose machine-readable results
|
||||
|
||||
## Incremental behavior
|
||||
|
||||
Avoid unnecessary rebuilds.
|
||||
|
||||
The pipeline should:
|
||||
|
||||
- detect unchanged inputs
|
||||
- reuse generated artifacts
|
||||
- avoid duplicate rescans
|
||||
- avoid rewriting unchanged outputs
|
||||
- report concise summaries
|
||||
|
||||
## Repository independence
|
||||
|
||||
Do not assume:
|
||||
|
||||
- NWN-specific paths
|
||||
- westgate naming
|
||||
- BMU naming formats
|
||||
- specific cache directories
|
||||
- specific manifest names
|
||||
|
||||
All of these should be configurable.
|
||||
|
||||
# CLI design
|
||||
|
||||
Design clean toolkit commands.
|
||||
|
||||
Example direction:
|
||||
|
||||
- toolkit music scan
|
||||
- toolkit music build
|
||||
- toolkit music credits
|
||||
- toolkit music normalize
|
||||
- toolkit music validate
|
||||
|
||||
Or:
|
||||
|
||||
- toolkit dataset build <dataset>
|
||||
|
||||
The exact CLI structure may change if a cleaner abstraction exists.
|
||||
|
||||
# Configuration examples
|
||||
|
||||
Support something like:
|
||||
|
||||
```yaml
|
||||
music:
|
||||
defaults:
|
||||
normalization:
|
||||
enabled: true
|
||||
|
||||
datasets:
|
||||
westgate:
|
||||
source: envi/music/westgate
|
||||
output: build/music/westgate
|
||||
|
||||
naming:
|
||||
scheme: nwn_bmu
|
||||
|
||||
credits:
|
||||
enabled: true
|
||||
output: .cache/credits/westgate
|
||||
|
||||
premium_bardpack:
|
||||
source: premium/music/bardpack
|
||||
|
||||
normalization:
|
||||
loudness: -14
|
||||
|
||||
transcoding:
|
||||
format: ogg
|
||||
```
|
||||
|
||||
# Implementation expectations
|
||||
|
||||
- Separate orchestration from implementation details.
|
||||
- Separate scanning, transformation, and output stages cleanly.
|
||||
- Introduce typed dataset definitions/config models.
|
||||
- Centralize logging and cache management.
|
||||
- Prefer declarative configuration over imperative repository scripts.
|
||||
- Keep generated outputs deterministic.
|
||||
- Preserve compatibility where possible through adapters/shims.
|
||||
- Minimize repository glue code.
|
||||
|
||||
# Migration
|
||||
|
||||
- Migrate existing Shadows Over Westgate behavior onto the new system.
|
||||
- Preserve existing outputs unless intentionally improved.
|
||||
- Add compatibility wrappers if needed.
|
||||
- Ensure old workflows continue functioning during transition.
|
||||
|
||||
# Deliverables
|
||||
|
||||
1. Proposed architecture
|
||||
2. Configuration schema
|
||||
3. CLI design
|
||||
4. Migration strategy
|
||||
5. Refactor plan
|
||||
6. Example configs
|
||||
7. Before/after repository responsibilities
|
||||
8. Suggested module/package layout
|
||||
9. Identification of reusable abstractions
|
||||
10. Example logs/output
|
||||
|
||||
Focus on maintainability, reuse, composability, and repository independence.
|
||||
|
||||
# Initial project-needs analysis
|
||||
|
||||
## Current state
|
||||
|
||||
- Music processing is currently implemented in `internal/pipeline/music.go`.
|
||||
- The music pipeline is not independently callable. It is invoked as part of:
|
||||
- `BuildHAKs` / `planOrBuildHAKs`
|
||||
- `plannedModuleHAKOrder`
|
||||
- The current behavior assumes the repository has an asset tree and that music lives under `envi/music`.
|
||||
- Convertible source formats are currently hardcoded to:
|
||||
- `.mp3`
|
||||
- `.ogg`
|
||||
- Passthrough audio/resource formats are currently hardcoded to:
|
||||
- `.bmu`
|
||||
- `.wav`
|
||||
- Generated BMU-like resources are staged under `.cache/music`, loaded into HAK resources, and then the staging root is deleted.
|
||||
- Generated credits are written under `.cache/credits`.
|
||||
- Credits inventory is written to `.cache/credits/credits.json`.
|
||||
- Authored credit overlays are discovered by walking the whole configured assets directory for `CREDITS.md`.
|
||||
- A per-directory `CREDITS.md` can override probed metadata and manually pin the generated output filename.
|
||||
- FFmpeg and ffprobe are resolved from:
|
||||
- `SOW_FFMPEG`
|
||||
- `SOW_FFPROBE`
|
||||
- `PATH`
|
||||
- The ffmpeg command currently strips metadata and writes MP3-compatible audio bytes to a file named `.bmu`.
|
||||
- Output stems are constrained to 16 characters, lower-case ASCII letters, digits, and underscores.
|
||||
- Generated output names are reserved against every existing asset stem, not just music files.
|
||||
- Current generated credits writes are already incremental: desired files are rendered in memory, unchanged files are not rewritten, and stale generated artifacts are removed.
|
||||
- App-layer output for music is currently coupled to `build-haks` summary output in `internal/app/app.go`.
|
||||
|
||||
## Existing behavior to preserve during migration
|
||||
|
||||
- `build-haks` must continue to convert configured music sources into HAK resources without requiring repository scripts.
|
||||
- Existing Shadows Over Westgate outputs should remain stable by default:
|
||||
- source root: `envi/music`
|
||||
- westgate prefix mapping: `envi/music/westgate -> mus_wg_`
|
||||
- generated output extension: `.bmu`
|
||||
- generated cache inventory: `.cache/credits/credits.json`
|
||||
- generated credits markdown shape and columns
|
||||
- manual `CREDITS.md` overlay semantics
|
||||
- duplicate output-name collision behavior
|
||||
- 16-character NWN resource stem limit
|
||||
- Existing concise/verbose `build-haks` log behavior should keep working.
|
||||
- Existing environment variable overrides for ffmpeg/ffprobe should continue to work as compatibility aliases.
|
||||
- Machine-readable generated artifacts should remain JSON unless there is a separate explicit migration contract.
|
||||
|
||||
## Current code areas affected
|
||||
|
||||
- `internal/project/project.go`
|
||||
- `Config.Music` currently only supports `prefixes`.
|
||||
- `ValidateLayout` calls `validateMusicConfig`.
|
||||
- `AssetExtensions` currently defines which music files appear in project inventory.
|
||||
- `Scan` only scans the configured assets directory, so datasets outside `paths.assets` need a deliberate design.
|
||||
- `internal/pipeline/music.go`
|
||||
- all scanning, naming, metadata, conversion, credits, cache, and cleanup logic is currently one private pipeline helper.
|
||||
- hardcoded assumptions include `envi/music`, `.cache/music`, `.cache/credits`, `CREDITS.md`, BMU output, ffmpeg args, metadata tags, and NWN stem rules.
|
||||
- `internal/pipeline/build.go`
|
||||
- HAK planning/building depends on `prepareMusicAssets`.
|
||||
- plan-only HAK workflows can still invoke music probing/conversion today through HAK planning paths.
|
||||
- `internal/app/app.go`
|
||||
- `build-haks` owns the only user-facing music summary.
|
||||
- New music commands should reuse the same logging/verbosity model instead of inventing unrelated output.
|
||||
- `internal/pipeline/pipeline_test.go`
|
||||
- currently has regression coverage for conversion, generated credits, manual overlay, inventory inclusion, staging cleanup, and unchanged second pass.
|
||||
- `internal/app/app_test.go`
|
||||
- currently checks normal versus verbose mapping output.
|
||||
- `README.md`
|
||||
- currently documents only `music.prefixes`.
|
||||
|
||||
# Design constraints and decisions
|
||||
|
||||
## Dataset roots
|
||||
|
||||
Datasets should explicitly state where they read from and where generated artifacts go.
|
||||
|
||||
Important distinction:
|
||||
|
||||
- `source` is the authored input root for a dataset.
|
||||
- `output` is the generated publish/staging root for transformed audio resources.
|
||||
- `credits.output` is where generated human-readable credits go.
|
||||
- `manifest.output` is where generated machine-readable dataset state goes.
|
||||
- `cache.root` is where reusable implementation caches live.
|
||||
|
||||
Do not assume dataset sources are under `paths.assets`. Support both:
|
||||
|
||||
- asset-relative dataset roots for existing HAK packaging behavior
|
||||
- project-relative dataset roots for future non-HAK or external datasets
|
||||
|
||||
For HAK builds, the pipeline must know whether generated outputs are meant to be injected into the project asset inventory and under which resource-relative path.
|
||||
|
||||
## NWN-specific behavior
|
||||
|
||||
The toolkit can support NWN conventions, but they must be selected by configuration instead of implicit global behavior.
|
||||
|
||||
Suggested built-in profiles:
|
||||
|
||||
- `nwn_bmu`
|
||||
- output extension: `.bmu`
|
||||
- max stem length: 16
|
||||
- allowed stem characters: `[a-z0-9_]`
|
||||
- lower-case output names
|
||||
- default ffmpeg encoder compatible with current behavior
|
||||
- `passthrough`
|
||||
- no renaming unless configured
|
||||
- no transcoding unless configured
|
||||
- `audio_export`
|
||||
- configurable format, stem length, and codec rules without NWN resource constraints
|
||||
|
||||
Do not make `nwn_bmu` the only naming implementation.
|
||||
|
||||
## Configuration merge semantics
|
||||
|
||||
Merge configuration in this order:
|
||||
|
||||
1. toolkit defaults
|
||||
2. repository `music.defaults`
|
||||
3. dataset definition
|
||||
4. command-line/per-task overrides
|
||||
|
||||
Merge rules should be explicit:
|
||||
|
||||
- scalar fields replace inherited values
|
||||
- maps merge by key
|
||||
- lists replace by default unless a field is documented as appendable
|
||||
- empty strings mean "not set", not "clear this inherited field"
|
||||
- booleans that need tri-state inheritance should use pointer/nullable semantics internally
|
||||
- command-line overrides should be represented in result metadata so logs can show effective behavior
|
||||
|
||||
Reject unknown fields via strict YAML decode, matching the rest of the project config philosophy.
|
||||
|
||||
## Compatibility config shape
|
||||
|
||||
The current config:
|
||||
|
||||
```yaml
|
||||
music:
|
||||
prefixes:
|
||||
envi/music/westgate: mus_wg_
|
||||
```
|
||||
|
||||
should be treated as a compatibility shorthand equivalent to:
|
||||
|
||||
```yaml
|
||||
music:
|
||||
datasets:
|
||||
envi_music:
|
||||
source: envi/music
|
||||
package:
|
||||
mode: hak_asset
|
||||
output_root: envi/music
|
||||
naming:
|
||||
scheme: nwn_bmu
|
||||
prefixes:
|
||||
westgate: mus_wg_
|
||||
```
|
||||
|
||||
Exact normalized dataset ids may differ, but the migration must define how old `music.prefixes` maps into the new dataset model.
|
||||
|
||||
## Proposed expanded schema
|
||||
|
||||
Example target:
|
||||
|
||||
```yaml
|
||||
music:
|
||||
tools:
|
||||
ffmpeg: "" # optional; fallback to SOW_FFMPEG then PATH
|
||||
ffprobe: "" # optional; fallback to SOW_FFPROBE then PATH
|
||||
|
||||
defaults:
|
||||
discover:
|
||||
include:
|
||||
- "**/*.mp3"
|
||||
- "**/*.ogg"
|
||||
passthrough:
|
||||
- "**/*.bmu"
|
||||
- "**/*.wav"
|
||||
exclude: []
|
||||
credits_file: CREDITS.md
|
||||
|
||||
naming:
|
||||
scheme: nwn_bmu
|
||||
max_stem_length: 16
|
||||
prefix: ""
|
||||
prefixes: {}
|
||||
drop_words: []
|
||||
reserved_stems_from_assets: true
|
||||
|
||||
metadata:
|
||||
enabled: true
|
||||
prefer_overlay: true
|
||||
probe_tags:
|
||||
title:
|
||||
- title
|
||||
artist:
|
||||
- artist
|
||||
- album_artist
|
||||
- composer
|
||||
rights:
|
||||
- license
|
||||
- license_url
|
||||
- copyright
|
||||
|
||||
transcoding:
|
||||
enabled: true
|
||||
output_extension: .bmu
|
||||
codec: libmp3lame
|
||||
format: mp3
|
||||
sample_rate: 44100
|
||||
channels: 2
|
||||
bitrate: 192k
|
||||
strip_metadata: true
|
||||
ffmpeg_args: []
|
||||
|
||||
credits:
|
||||
enabled: true
|
||||
output: .cache/credits
|
||||
inventory: .cache/credits/credits.json
|
||||
format: nwn_markdown_table
|
||||
|
||||
cache:
|
||||
root: .cache/music
|
||||
mode: content_hash
|
||||
keep_staging: false
|
||||
|
||||
manifest:
|
||||
enabled: true
|
||||
output: .cache/music/manifest.json
|
||||
format: json
|
||||
|
||||
validation:
|
||||
require_credits_for_generated: false
|
||||
reject_unknown_overlay_rows: true
|
||||
reject_output_collisions: true
|
||||
|
||||
datasets:
|
||||
westgate:
|
||||
source: envi/music/westgate
|
||||
output: envi/music/westgate
|
||||
package:
|
||||
mode: hak_asset
|
||||
naming:
|
||||
prefix: mus_wg_
|
||||
```
|
||||
|
||||
Schema names can change during implementation, but the final shape should preserve these concepts.
|
||||
|
||||
# Pipeline architecture
|
||||
|
||||
## Suggested package layout
|
||||
|
||||
Prefer a dedicated internal subsystem instead of expanding `internal/pipeline/music.go`.
|
||||
|
||||
Suggested layout:
|
||||
|
||||
```text
|
||||
internal/music/
|
||||
config.go typed config, defaults, merge, validation
|
||||
dataset.go resolved dataset model and effective settings
|
||||
discover.go source discovery and file classification
|
||||
metadata.go ffprobe integration and metadata normalization
|
||||
credits.go markdown parsing/rendering and inventory generation
|
||||
naming.go naming schemes, collision detection, compatibility prefix logic
|
||||
transcode.go ffmpeg integration and output writing
|
||||
cache.go content hashing and incremental state
|
||||
manifest.go machine-readable dataset/stage manifests
|
||||
validate.go dataset validation diagnostics
|
||||
pipeline.go stage orchestration
|
||||
result.go machine-readable stage/build result types
|
||||
```
|
||||
|
||||
Then keep `internal/pipeline` responsible for HAK/module orchestration and call `internal/music` through a narrow adapter.
|
||||
|
||||
## Stage model
|
||||
|
||||
Each stage should have:
|
||||
|
||||
- typed input
|
||||
- typed output
|
||||
- deterministic ordering
|
||||
- dry-run support where it can avoid writes
|
||||
- changed/unchanged reporting
|
||||
- structured diagnostics
|
||||
- stable machine-readable result data
|
||||
|
||||
Suggested stages:
|
||||
|
||||
1. Resolve datasets and effective config.
|
||||
2. Discover candidate files.
|
||||
3. Load authored overlays.
|
||||
4. Probe source metadata.
|
||||
5. Plan output names and detect collisions.
|
||||
6. Plan transformations.
|
||||
7. Reuse cached outputs or transcode changed files.
|
||||
8. Render generated credits.
|
||||
9. Write dataset manifest/inventory.
|
||||
10. Return generated package resources to the HAK pipeline.
|
||||
11. Validate outputs and report diagnostics.
|
||||
|
||||
## Result model
|
||||
|
||||
Music command results should be structured enough for both CLI output and tests.
|
||||
|
||||
Include:
|
||||
|
||||
- dataset id
|
||||
- source root
|
||||
- output root
|
||||
- files discovered
|
||||
- files converted
|
||||
- files reused from cache
|
||||
- passthrough files
|
||||
- skipped files
|
||||
- generated resources
|
||||
- credit artifacts
|
||||
- manifest paths
|
||||
- changed files
|
||||
- warnings/errors
|
||||
- source-to-output mappings
|
||||
- effective naming/transcoding profile
|
||||
|
||||
Do not require the app layer to parse free-form progress strings.
|
||||
|
||||
# Incremental and cache behavior
|
||||
|
||||
## Content identity
|
||||
|
||||
Incremental rebuilds should be based on content and effective configuration, not only mtimes.
|
||||
|
||||
Cache keys should include:
|
||||
|
||||
- source content hash
|
||||
- source relative path
|
||||
- effective dataset id
|
||||
- effective naming settings
|
||||
- effective transcoding settings
|
||||
- ffmpeg command/profile version
|
||||
- toolkit music pipeline version
|
||||
- manual overlay data that affects output metadata/name
|
||||
|
||||
Changing an output prefix, codec, bitrate, overlay output filename, or metadata rendering rules must invalidate the relevant planned output.
|
||||
|
||||
## Output writes
|
||||
|
||||
- Avoid rewriting unchanged credits, manifests, and transformed outputs.
|
||||
- Write through temporary files and atomic rename where practical.
|
||||
- Preserve deterministic JSON indentation and key ordering.
|
||||
- Remove stale generated artifacts inside owned generated roots.
|
||||
- Never remove authored source files.
|
||||
- Never remove files outside configured owned cache/output roots.
|
||||
|
||||
## Plan-only behavior
|
||||
|
||||
Current HAK planning can trigger conversion work. The refactor should define stricter behavior:
|
||||
|
||||
- `music scan` and HAK `--plan-only` should not transcode by default.
|
||||
- They may probe metadata only if needed and allowed.
|
||||
- They should be able to produce planned mappings without writing transformed audio.
|
||||
- If actual resource size is needed for HAK chunk planning, use cached transformed size when available and report when exact planning requires a build.
|
||||
|
||||
# CLI design expansion
|
||||
|
||||
Add first-class commands while preserving `build-haks` compatibility.
|
||||
|
||||
Suggested commands:
|
||||
|
||||
```text
|
||||
sow-toolkit music list-datasets
|
||||
sow-toolkit music scan [--dataset <id>] [--json]
|
||||
sow-toolkit music build [--dataset <id>] [--dry-run] [--force]
|
||||
sow-toolkit music credits [--dataset <id>] [--check]
|
||||
sow-toolkit music validate [--dataset <id>] [--json]
|
||||
sow-toolkit music manifest [--dataset <id>] [--check]
|
||||
```
|
||||
|
||||
Integration commands:
|
||||
|
||||
```text
|
||||
sow-toolkit build-haks
|
||||
sow-toolkit build-haks --skip-music
|
||||
sow-toolkit build-haks --music-dataset <id>
|
||||
```
|
||||
|
||||
Command output should follow the existing log refactor direction:
|
||||
|
||||
- normal mode: phase summaries and counts
|
||||
- verbose mode: mappings and per-file actions
|
||||
- debug mode: resolved config, tool paths, ffmpeg args, cache keys
|
||||
- quiet mode: final summary only
|
||||
- `--json`: machine-readable result on stdout and human diagnostics on stderr
|
||||
|
||||
# Validation requirements
|
||||
|
||||
Validate repository config:
|
||||
|
||||
- duplicate dataset ids
|
||||
- invalid dataset source/output paths
|
||||
- absolute paths unless explicitly allowed
|
||||
- path traversal outside project root for owned outputs/caches
|
||||
- duplicate or ambiguous prefix mappings
|
||||
- prefix too long for selected naming scheme
|
||||
- unsupported output extension for selected package mode
|
||||
- invalid codec/profile fields
|
||||
- impossible inherited config combinations
|
||||
- `package.mode: hak_asset` without an asset-relative output root
|
||||
- unknown naming/transcoding/credits/manifest schemes
|
||||
|
||||
Validate discovered music:
|
||||
|
||||
- source files with unsupported extensions
|
||||
- duplicate case-insensitive output stems
|
||||
- output name collision with existing assets and generated outputs
|
||||
- manual overlay rows referencing unknown originals or outputs
|
||||
- manual overlay output filenames violating selected naming scheme
|
||||
- metadata probe failures
|
||||
- source paths differing only by case on case-insensitive targets
|
||||
- generated resource names that exceed NWN constraints
|
||||
- generated output extension/resource type compatibility
|
||||
|
||||
Validation severity should distinguish:
|
||||
|
||||
- errors that block deterministic build output
|
||||
- warnings for incomplete credits or missing optional metadata
|
||||
- info for skipped files and unchanged caches
|
||||
|
||||
# Fringe considerations
|
||||
|
||||
## Credits and licensing
|
||||
|
||||
- Preserve authored `CREDITS.md` files as source overlays.
|
||||
- Generated credits should clearly identify generated versus authored sources in inventory.
|
||||
- Missing artist/title/license can be warning-level or error-level per dataset.
|
||||
- Overlay parsing should keep accepting backtick-wrapped filenames.
|
||||
- Markdown parsing should continue to support escaped pipes and `<br>` line breaks.
|
||||
- Generated credits should be deterministic across platforms.
|
||||
- Consider whether aggregate credits inventory should include datasets disabled for build but enabled for credits.
|
||||
|
||||
## Filename and path edge cases
|
||||
|
||||
- Normalize separators to forward slashes in config, manifests, and logs.
|
||||
- Treat paths case-insensitively for resource collision checks.
|
||||
- Reject NUL bytes and unsafe relative paths.
|
||||
- Define behavior for spaces, Unicode, apostrophes, brackets, years, and punctuation in stems.
|
||||
- Keep existing slug/drop-word behavior for `nwn_bmu` compatibility, but make drop words configurable.
|
||||
- Avoid Windows-reserved names and problematic trailing dots/spaces if outputs may be checked out or built on Windows.
|
||||
- Detect collisions after truncation and suffixing.
|
||||
- Validate suffixing still respects max stem length.
|
||||
|
||||
## Audio tooling
|
||||
|
||||
- Keep `SOW_FFMPEG` and `SOW_FFPROBE` as compatibility fallbacks.
|
||||
- Add config-level tool paths for repositories that pin local toolchain binaries.
|
||||
- Expose ffmpeg/ffprobe version in debug output and manifest metadata.
|
||||
- Capture ffmpeg failures with enough context to diagnose source file, dataset, and command profile.
|
||||
- Do not print enormous ffmpeg logs in normal mode unless the command fails.
|
||||
- Decide whether `.ogg -> .bmu` should continue transcoding or become configurable passthrough for datasets that want native OGG output.
|
||||
|
||||
## HAK/resource integration
|
||||
|
||||
- Generated music resources must enter collision detection alongside authored resources.
|
||||
- Source `.mp3`/`.ogg` files that are converted must be omitted from HAK packaging unless config says otherwise.
|
||||
- Passthrough `.bmu` and `.wav` handling should remain distinct from generated outputs.
|
||||
- Resource sorting should stay deterministic and reuse existing HAK resource ordering rules.
|
||||
- Generated music should be represented in build manifests with enough source provenance to debug.
|
||||
- HAK include globs should match generated output paths, not source paths, unless explicitly configured otherwise.
|
||||
|
||||
## Cross-repository behavior
|
||||
|
||||
- A repository with no `music` config should not pay a heavy ffmpeg/ffprobe cost.
|
||||
- A repository with authored `.bmu` files but no generated music config should preserve current passthrough packaging.
|
||||
- Multiple datasets should be able to target different output roots without collisions.
|
||||
- Multiple repositories should be able to use different naming schemes without code changes.
|
||||
- Dataset config should be portable between Linux and Windows path conventions.
|
||||
- The toolkit should not assume Shadows Over Westgate names, HAK names, prefixes, or source layout.
|
||||
|
||||
# Migration strategy
|
||||
|
||||
## Phase 1: Extract without behavior changes
|
||||
|
||||
- Move existing private music functions from `internal/pipeline/music.go` into `internal/music`.
|
||||
- Keep the current `prepareMusicAssets` adapter in `internal/pipeline` to avoid broad HAK changes.
|
||||
- Preserve every existing test expectation.
|
||||
- Add focused unit tests around moved naming, overlay, metadata, and credits functions.
|
||||
|
||||
## Phase 2: Introduce dataset config
|
||||
|
||||
- Extend `project.MusicConfig` with typed defaults and datasets.
|
||||
- Keep `music.prefixes` as deprecated compatibility shorthand.
|
||||
- Implement config normalization from legacy prefixes into a default dataset.
|
||||
- Add strict validation for dataset ids, paths, naming, and prefix constraints.
|
||||
- Update README examples.
|
||||
|
||||
## Phase 3: Add first-class music commands
|
||||
|
||||
- Wire `music scan`, `music build`, `music credits`, and `music validate` into `internal/app`.
|
||||
- Reuse existing verbosity levels and summary formatting.
|
||||
- Add JSON output support if the command result model is ready.
|
||||
- Ensure `build-haks` calls the same service API as the music commands.
|
||||
|
||||
## Phase 4: Incremental generated output cache
|
||||
|
||||
- Replace temporary-only `.cache/music` staging with a content-addressed cache when configured.
|
||||
- Keep cleanup behavior compatible for existing builds unless `cache.keep_staging` is enabled.
|
||||
- Avoid conversion during dry-run/plan-only unless exact output artifacts are explicitly requested.
|
||||
- Add stale generated artifact cleanup scoped only to owned roots.
|
||||
|
||||
## Phase 5: Repository migration
|
||||
|
||||
- Convert Shadows Over Westgate config from `music.prefixes` to explicit dataset config.
|
||||
- Keep compatibility warning for one release window.
|
||||
- Update consumer wrapper scripts only if command names change.
|
||||
- Verify existing generated HAK manifests and credits inventories remain stable or document intentional diffs.
|
||||
|
||||
# Test plan
|
||||
|
||||
Add or preserve tests for:
|
||||
|
||||
- legacy `music.prefixes` compatibility
|
||||
- explicit dataset config equivalent to current Westgate behavior
|
||||
- dataset source outside `envi/music`
|
||||
- multiple datasets with independent prefixes and outputs
|
||||
- no music config with authored `.bmu` passthrough assets
|
||||
- manual overlay output filename validation
|
||||
- manual overlay references unknown source/output
|
||||
- duplicate generated output collisions
|
||||
- collision with existing authored asset stem
|
||||
- max stem length and suffix collision behavior
|
||||
- unchanged second build does not rewrite credits or manifests
|
||||
- dry-run/plan-only does not transcode
|
||||
- missing ffmpeg/ffprobe failure messages
|
||||
- configured ffmpeg/ffprobe paths
|
||||
- normal, verbose, quiet, debug, and JSON CLI output
|
||||
- Windows-style path separators in config normalize deterministically
|
||||
- generated credits and inventory are deterministic
|
||||
|
||||
# Acceptance criteria
|
||||
|
||||
- Music functionality is reusable through a dedicated toolkit subsystem.
|
||||
- No repository-specific path or prefix is hardcoded in music implementation.
|
||||
- Existing `build-haks` behavior remains compatible for Shadows Over Westgate.
|
||||
- First-class music commands can scan, build, generate credits, and validate a configured dataset.
|
||||
- Dataset configuration can express current Westgate behavior and at least one non-Westgate layout.
|
||||
- Generated outputs are deterministic and unchanged files are not rewritten.
|
||||
- Machine-readable artifacts remain JSON.
|
||||
- Logs are concise by default and detailed under verbose/debug.
|
||||
- Tests cover the compatibility path and the new dataset-driven path.
|
||||
@@ -0,0 +1,30 @@
|
||||
SHELL := bash
|
||||
.ONESHELL:
|
||||
.PHONY: check test vet build smoke fmt image
|
||||
|
||||
# Lint + unit tests. Mirrors the `test` CI job; runs green inside `nix develop`.
|
||||
check: vet test
|
||||
shellcheck scripts/*.sh
|
||||
yamllint .gitea
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
# Build every cmd/* binary into ./bin (gitignored — binaries are CI artifacts).
|
||||
build:
|
||||
scripts/build-binaries.sh
|
||||
|
||||
smoke: build
|
||||
scripts/crucible-smoke.sh
|
||||
|
||||
fmt:
|
||||
gofmt -l -w cmd internal
|
||||
|
||||
# Build the Crucible image locally. Requires docker; mirrors build-image CI.
|
||||
IMAGE ?= crucible
|
||||
GIT_SHA ?= $(shell git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)
|
||||
image:
|
||||
docker build --build-arg GIT_SHA=$(GIT_SHA) -f docker/Dockerfile -t $(IMAGE):$(GIT_SHA) .
|
||||
@@ -1,695 +1,68 @@
|
||||
# sow-tools
|
||||
# sow-tools — Crucible
|
||||
|
||||
Shared NWN build tooling for Shadows Over Westgate.
|
||||
|
||||
This repository owns the Go-based `sow-toolkit` binary used by the separate module and assets repositories.
|
||||
|
||||
## Build
|
||||
|
||||
Build the development binary into `tools/sow-toolkit`:
|
||||
|
||||
```bash
|
||||
./build-tool.sh
|
||||
```
|
||||
|
||||
On Windows:
|
||||
|
||||
```powershell
|
||||
.\build-tool.ps1
|
||||
```
|
||||
|
||||
Both wrappers use `.cache/go-build` for the Go build cache and skip rebuilding
|
||||
when the sources are older than the existing development binary.
|
||||
|
||||
Or run directly with Go:
|
||||
|
||||
```bash
|
||||
GOCACHE="$PWD/.cache/go-build" go run ./cmd/nwn-tool --help
|
||||
```
|
||||
|
||||
When changing native topdata id allocation, lockfile persistence, generated
|
||||
families, or null-and-relocate behavior, run the repeat-build determinism test:
|
||||
|
||||
```bash
|
||||
go test ./internal/topdata -run TestBuildNativeKeepsGeneratedIDsStableAcrossRepeatedBuilds
|
||||
```
|
||||
|
||||
That test builds the same topdata project ten times in one process and asserts
|
||||
that relocated/generated ids stay pinned across every pass.
|
||||
|
||||
## Repository Layout
|
||||
Crucible is the Shadows Over Westgate build/conversion/sync toolchain: one Go
|
||||
module producing several small binaries plus a `crucible` dispatcher (D11). It is
|
||||
the **only** repo that owns builder logic; the artifact repos (`sow-module`,
|
||||
`sow-topdata`, `sow-assets-manifest`) invoke Crucible through wrapper scripts and
|
||||
never embed a toolkit.
|
||||
|
||||
```text
|
||||
cmd/nwn-tool/ CLI entrypoint
|
||||
internal/app/ command wiring and console UX
|
||||
internal/erf/ ERF/MOD/HAK reading and writing
|
||||
internal/gff/ canonical GFF JSON and binary conversion
|
||||
internal/pipeline/ build, extract, compare, and manifest workflows
|
||||
internal/project/ config loading and repo scanning
|
||||
internal/topdata/ native topdata, wiki, and autogen workflows
|
||||
internal/validator/validation and diagnostics
|
||||
tools/ built development binary output
|
||||
Repos produce artifacts. sow-platform deploys artifacts.
|
||||
Crucible is how the artifact repos turn source into artifacts.
|
||||
```
|
||||
|
||||
## Commands
|
||||
## Binaries
|
||||
|
||||
The shared CLI exposes these project utilities:
|
||||
| Binary | Dispatcher form | Owns |
|
||||
| ------------------ | ------------------ | ----------------------------------------------- |
|
||||
| `crucible` | — | dispatcher: `crucible <builder> [args]` |
|
||||
| `crucible-depot` | `crucible depot` | content-addressed depot blob verify/move |
|
||||
| `crucible-hak` | `crucible hak` | ERF/HAK pack/unpack + hak manifests |
|
||||
| `crucible-module` | `crucible module` | build/extract/validate/compare the `.mod` |
|
||||
| `crucible-topdata` | `crucible topdata` | compile 2da/tlk topdata + packages |
|
||||
| `crucible-wiki` | `crucible wiki` | render + deploy mechanical wiki pages |
|
||||
|
||||
The dispatcher and the standalone shims share one registry
|
||||
(`internal/dispatch`); the shims exist so consumer wrapper scripts can resolve a
|
||||
single-token command. The full legacy `nwn-tool` command surface and where each
|
||||
command lands is mapped in [`docs/command-surface.md`](docs/command-surface.md).
|
||||
|
||||
## Status (Phase 5 scaffold)
|
||||
|
||||
The suite **builds, vets, tests, and runs**, but every builder is **unwired**:
|
||||
running one fails closed with exit `70` and never fakes an artifact. The internal
|
||||
pipeline/topdata/erf/wiki/music packages from `gitea/sow-tools` are migrated by
|
||||
the operator at cutover — the workspace hard rules forbid transplanting that
|
||||
source automatically. See [`docs/migration-from-nwn-tool.md`](docs/migration-from-nwn-tool.md).
|
||||
|
||||
This matches the downstream skeletons: `sow-module` / `sow-topdata` package
|
||||
scripts already fail closed until they can resolve a Crucible binary.
|
||||
|
||||
## Develop
|
||||
|
||||
Self-contained (D8) — a host with only Nix can run everything:
|
||||
|
||||
```bash
|
||||
sow-toolkit build
|
||||
sow-toolkit build-module
|
||||
sow-toolkit build-haks [--hak <name>] [--archive <name>] [--source-manifest <path>] [--plan-only] [--skip-music] [--music-dataset <id>]
|
||||
sow-toolkit extract [<build-relative-archive-or-glob> ...]
|
||||
sow-toolkit validate
|
||||
sow-toolkit compare
|
||||
sow-toolkit apply-hak-manifest [<manifest-path>]
|
||||
sow-toolkit config <validate|effective|inspect|explain|sources>
|
||||
sow-toolkit music <list-datasets|scan|build|credits|validate|manifest|normalize>
|
||||
sow-toolkit validate-topdata
|
||||
sow-toolkit build-topdata [--force] [--wiki]
|
||||
sow-toolkit build-top-package [--force]
|
||||
sow-toolkit compare-topdata
|
||||
sow-toolkit convert-topdata <2da-to-json|2da-to-module|json-to-2da> ...
|
||||
sow-toolkit build-wiki [--force]
|
||||
sow-toolkit deploy-wiki [--source-dir <path>] [--endpoint <url>] [--token <token>] [--version <ver>] [--namespace <ns>] [--category <namespace=cid>] [--manifest <path>] [--stale-policy <report|archive|purge>] [--dry-run] [--create] [--force] [--reset-managed-namespaces]
|
||||
sow-toolkit build-changelog [--config <path>] [--output <path>] [--current-tag <tag>] [--previous-tag <tag>] [--api-base-url <url>] [--token <token>]
|
||||
nix develop # Go + ffmpeg + shellcheck + yamllint + make
|
||||
make check # go vet + go test + shellcheck + yamllint
|
||||
make build # build every cmd/* into ./bin (gitignored)
|
||||
make smoke # build + assert the fail-closed contract
|
||||
make image # docker build -> crucible:<sha>
|
||||
```
|
||||
|
||||
Global logging flags `--quiet`, `--verbose`, and `--debug` may be passed after
|
||||
the command name. Long options that take values accept either `--flag value` or
|
||||
`--flag=value`; empty inline values such as `--dataset=` are rejected.
|
||||
Binaries are **never committed** — they are CI artifacts / image layers (D19).
|
||||
This retires the old habit of checking in `nwn-tool` / `sow-toolkit`.
|
||||
|
||||
## Repository Configuration
|
||||
## CI
|
||||
|
||||
Human-authored consumer repository configuration is YAML. The toolkit discovers
|
||||
configuration in this order:
|
||||
PR-first (D7): every check runs on pull requests and on push to `main`.
|
||||
|
||||
1. `nwn-tool.yaml`
|
||||
2. `nwn-tool.yml`
|
||||
3. legacy `nwn-tool.json`
|
||||
- `test.yml` — vet, test, shellcheck, yamllint, binary smoke.
|
||||
- `build-image.yml` — build `registry.westgate.pw/sow/crucible:<sha>`; publish
|
||||
only on `main`. PRs build but never push. No mutable tags.
|
||||
- `release.yml` — tag-gated binary bundles; image re-tag is wired in Phase 6.
|
||||
|
||||
YAML is canonical. If both YAML and legacy JSON files are present, the YAML file
|
||||
wins. Legacy `nwn-tool.json` still loads for migration, but commands print a
|
||||
warning and should be updated to `nwn-tool.yaml`.
|
||||
## Consumers
|
||||
|
||||
Example:
|
||||
|
||||
```yaml
|
||||
module:
|
||||
name: Shadows Over Westgate
|
||||
resref: sow_module
|
||||
description: Shadows Over Westgate asset pipeline.
|
||||
hak_order:
|
||||
- sow_top
|
||||
- group:sow_over
|
||||
- group:sow_core
|
||||
|
||||
paths:
|
||||
assets: content
|
||||
build: build
|
||||
|
||||
music:
|
||||
# Legacy shorthand — automatically normalized into a dataset.
|
||||
prefixes:
|
||||
envi/music/westgate: mus_wg_
|
||||
|
||||
# Explicit dataset config (replaces prefixes above).
|
||||
tools:
|
||||
ffmpeg: tools/ffmpeg
|
||||
ffprobe: tools/ffprobe
|
||||
defaults:
|
||||
stage_root: "{paths.cache}/music"
|
||||
credits_root: "{paths.cache}/credits"
|
||||
credits_overlay: CREDITS.md
|
||||
max_stem_length: 16
|
||||
naming_scheme: nwn_bmu
|
||||
output_extension: .bmu
|
||||
convert_extensions: [.flac, .m4a, .mp3, .ogg, .wav]
|
||||
datasets:
|
||||
westgate:
|
||||
source: envi/music/westgate
|
||||
output: envi/music/westgate
|
||||
prefix: mus_wg_
|
||||
package_mode: hak_asset
|
||||
```
|
||||
|
||||
Resolved configuration can be inspected without scanning or building:
|
||||
|
||||
```bash
|
||||
sow-toolkit config validate
|
||||
sow-toolkit config effective --json
|
||||
sow-toolkit config inspect paths.build
|
||||
sow-toolkit config explain topdata.package_hak
|
||||
sow-toolkit config sources
|
||||
```
|
||||
|
||||
`config effective` prints the normalized values consumed by commands, including
|
||||
toolkit defaults, YAML values, legacy status, active environment overrides, and
|
||||
provenance. Sensitive override values such as tokens are reported as set without
|
||||
printing the secret.
|
||||
|
||||
Topdata value encodings under `topdata.value_encodings` are matched by dataset
|
||||
and column. Set `dataset: "*"` to apply an encoding to the same column name in
|
||||
any native topdata dataset; an exact dataset rule takes precedence over the
|
||||
wildcard rule for that column. Supported compact encodings include packed hex
|
||||
lists, external hex-list sidecar tables, alignment set masks, and ID hex lists.
|
||||
Explicit `{"alignments": [...]}` and `{"ids": [...]}` objects are portable in
|
||||
any native topdata dataset even without a column-specific rule; configured
|
||||
rules still override when a column needs custom bounds, widths, or alignment
|
||||
encoding mode. Plain `{"list": [...]}` objects remain config-driven because
|
||||
numeric lists need table-specific output semantics.
|
||||
|
||||
Class spell availability can be authored as JSON spellbooks under
|
||||
`topdata/data/classes/spellbooks/`. Each file names a `classes:*` row, and the
|
||||
native builder reads that row's `SpellTableColumn` to generate the matching
|
||||
`spells.2da` column. Spellbook files are discovered by convention, so ordinary
|
||||
per-class spellbook authoring does not require YAML registration.
|
||||
|
||||
Common configurable defaults:
|
||||
|
||||
```yaml
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
cache: .cache
|
||||
tools: tools
|
||||
|
||||
outputs:
|
||||
module_archive: "{module.resref}.mod"
|
||||
hak_manifest: haks.json
|
||||
hak_archive: "{hak.name}.hak"
|
||||
|
||||
build:
|
||||
keep_existing_haks: false
|
||||
|
||||
generated_assets:
|
||||
topdata_2da:
|
||||
- id: parts
|
||||
source: topdata
|
||||
output: "{paths.cache}/generated-assets/parts-2da"
|
||||
include_datasets: [parts/**]
|
||||
package_root: part
|
||||
autogen:
|
||||
id: parts
|
||||
mode: parts_rows
|
||||
root: part
|
||||
include: ["**/*.mdl"]
|
||||
derive:
|
||||
kind: trailing_numeric_suffix
|
||||
group_from: first_path_segment
|
||||
parts_rows:
|
||||
row_defaults:
|
||||
COSTMODIFIER: "0"
|
||||
acbonus:
|
||||
default:
|
||||
strategy: ascending_row_id_sort_key
|
||||
divisor: 100
|
||||
format: "%.2f"
|
||||
datasets:
|
||||
chest:
|
||||
strategy: fixed
|
||||
value: "0.00"
|
||||
|
||||
inventory:
|
||||
source_extensions: [] # empty means built-in NWN source extensions
|
||||
asset_extensions: [] # empty means built-in NWN asset extensions
|
||||
source_json_pattern: "{resref}.{extension}.json"
|
||||
|
||||
validation:
|
||||
profile: nwn_module
|
||||
builtin_script_prefixes: [nw_, x0_, x1_, x2_, x3_, ga_, gc_, gen_, gui_, nwg_, ta_]
|
||||
required_fields:
|
||||
ifo: [Mod_Name]
|
||||
|
||||
scripts:
|
||||
source_dir: scripts
|
||||
cache: "{paths.cache}/ncs"
|
||||
compiler:
|
||||
path: ""
|
||||
env:
|
||||
path: SOW_NWN_SCRIPT_COMPILER
|
||||
nwn_root: [SOW_NWN_ROOT, NWN_ROOT]
|
||||
nwn_user_directory: [SOW_NWN_USER_DIRECTORY, NWN_HOME, NWN_USER_DIRECTORY]
|
||||
search:
|
||||
- "{paths.tools}/script-compiler/nwn_script_comp"
|
||||
- "{paths.tools}/script-compiler/nwn_script_comp.exe"
|
||||
- "{paths.tools}/nwn_script_comp"
|
||||
- "{paths.tools}/nwn_script_comp.exe"
|
||||
- PATH:nwn_script_comp
|
||||
|
||||
extract:
|
||||
layout: nwn_canonical_json
|
||||
archives:
|
||||
- "{module.resref}.mod"
|
||||
cleanup_stale: true
|
||||
consume_archives: false
|
||||
ignore_extensions: []
|
||||
delete_module_archive_after_success: false # deprecated compatibility alias
|
||||
|
||||
topdata:
|
||||
build: "{paths.build}/topdata"
|
||||
compiled_2da_dir: 2da
|
||||
compiled_tlk: sow_tlk.tlk
|
||||
package_hak: sow_top.hak
|
||||
package_tlk: sow_tlk.tlk
|
||||
value_encodings:
|
||||
- dataset: racialtypes/core
|
||||
column: AvailableSkinColors
|
||||
mode: packed_hex_list
|
||||
min: 0
|
||||
max: 175
|
||||
hex_width: 2
|
||||
row_generation:
|
||||
- namespace: portraits
|
||||
mode: first_null_row
|
||||
minimum_row: 15000
|
||||
wiki:
|
||||
source: topdata/wiki
|
||||
output_root: wiki
|
||||
pages_dir: pages
|
||||
state_file: state.json
|
||||
page_index_file: page-index.json
|
||||
managed_namespaces: [classes, feat, itemtypes, races, skills, spells, meta]
|
||||
deploy_manifest: .wiki_deploy_manifest.json
|
||||
deploy_edit_summary: Auto-generated from native builder
|
||||
namespaces_file: namespaces.yaml
|
||||
data_file: data.yaml
|
||||
page_paths_file: wiki.yaml
|
||||
tables_file: tables.yaml
|
||||
visibility_file: visibility.yaml
|
||||
templates_dir: templates
|
||||
page_templates_dir: templates/pages
|
||||
manual_sections_dir: manual-sections
|
||||
manual_sections_file: manual-sections/default.yaml
|
||||
title_prefix_min_length: 3
|
||||
status_listing_scope: all
|
||||
|
||||
autogen:
|
||||
cache:
|
||||
root: "{paths.cache}"
|
||||
max_age: 1h
|
||||
refresh_env: SOW_AUTOGEN_MANIFEST_REFRESH
|
||||
release_source:
|
||||
provider: gitea
|
||||
server_url_env: SOW_ASSETS_SERVER_URL
|
||||
repo_env: SOW_ASSETS_REPO
|
||||
```
|
||||
|
||||
`topdata.value_encodings` enables configured authoring sugar for compact 2DA
|
||||
cell values. The `packed_hex_list` mode accepts authored values such as
|
||||
`{"list": [0, {"range": [10, 12]}]}` and emits a single `0x...` string with
|
||||
fixed-width hex chunks. The `alignment_hex_list` mode accepts two-letter
|
||||
alignment codes such as `{"list": ["le", "CG"]}` and emits the corresponding
|
||||
fixed-width hex chunks.
|
||||
|
||||
`topdata.value_defaults` applies configured fallback values to missing or
|
||||
null-like cells before 2DA emission:
|
||||
|
||||
```yaml
|
||||
topdata:
|
||||
value_defaults:
|
||||
- dataset: racialtypes/core
|
||||
column: ECL
|
||||
value: 0
|
||||
```
|
||||
|
||||
`topdata.row_generation` controls where newly generated rows are allocated for
|
||||
specific datasets. The default `after_base` mode preserves existing behavior by
|
||||
allocating after the final imported base row. `first_null_row` fills base rows
|
||||
whose canonical key is empty and whose emitted 2DA columns are all null-like
|
||||
(`null`, empty string, or `****`) before extending the table. `minimum_row`
|
||||
optionally prevents generated allocation below a configured row while preserving
|
||||
the selected allocation mode at and above that row.
|
||||
|
||||
`topdata.row_extensions` can extend plain native tables at build time without
|
||||
expanding the authored JSON. The `repeat_last` mode matches configured dataset
|
||||
glob patterns, finds the highest authored value in `level_column`, and appends
|
||||
generated rows through `target_level` by copying the last authored row's emitted
|
||||
columns while incrementing the level and row id. Generated extension rows are
|
||||
build output only and do not update source JSON or lockfiles.
|
||||
|
||||
```yaml
|
||||
topdata:
|
||||
row_extensions:
|
||||
- dataset: classes/spellsgained/*
|
||||
mode: repeat_last
|
||||
level_column: Level
|
||||
target_level: 60
|
||||
```
|
||||
|
||||
Topdata dataset directories may include an optional `global.json`. For
|
||||
base/module datasets, it supports normal `entries`, `overrides`, and `columns`
|
||||
and applies after ordinary modules; global overrides may use `{"match":"all"}`
|
||||
to update every final row. For recursive plain datasets, ancestor manifests
|
||||
apply to descendant tables and may define `injections` with optional
|
||||
`require_present` and `unless_present` conditions. This is the preferred
|
||||
authoring surface for class feat and class skill global rows. The legacy
|
||||
`topdata.class_feat_injections` YAML block still loads for compatibility when no
|
||||
class feat `global.json` applies.
|
||||
|
||||
`topdata.wiki.alignment_links.target_pattern` controls whether formatted
|
||||
alignment names become internal wiki links. The pattern may use `{alignment}`
|
||||
for the title-shaped alignment segment with spaces converted to underscores, and
|
||||
`{label}` for the display label. `alignment_links.link_format` defaults to
|
||||
`wiki_markup`, which emits `[[target|label]]`. Use `html_anchor` when the
|
||||
configured target should be emitted as a root URL anchor instead of being
|
||||
resolved as namespace-relative wiki markup:
|
||||
|
||||
```yaml
|
||||
topdata:
|
||||
wiki:
|
||||
alignment_links:
|
||||
target_pattern: /wiki/Guides/Alignment/{alignment}
|
||||
link_format: html_anchor
|
||||
```
|
||||
|
||||
Generated wiki input and output paths are YAML-configurable. `source` points to
|
||||
the wiki authoring root, `data_file`, `page_paths_file`, `tables_file`,
|
||||
`visibility_file`, `page_templates_dir`, and `manual_sections_file` are resolved
|
||||
under that source root, and `page_index_file`, `pages_dir`, and `state_file` are
|
||||
resolved under the configured wiki output root. `templates_dir` and
|
||||
`manual_sections_dir` remain as compatibility defaults for deriving
|
||||
`page_templates_dir` and `manual_sections_file` when the newer explicit paths
|
||||
are omitted.
|
||||
|
||||
Generated wiki page structure should be owned by repository templates and YAML
|
||||
where possible. `topdata/wiki/data.yaml` can expose named providers for page
|
||||
templates, including `source: facts` providers whose `facts.rows` declare
|
||||
ordered `label` and `value` expressions. Empty and null-like fact values are
|
||||
omitted, so templates can render category fact tables with `{{#each
|
||||
ProviderName}}` while YAML controls which 2DA-backed values are shown.
|
||||
|
||||
`paths.source` and `paths.assets` must name real repository subtrees when they
|
||||
are configured. They may be omitted for repositories that do not own that class
|
||||
of source, but they must not resolve to the repository root (`.`). Output and
|
||||
cache paths must stay relative to the repository root unless a specific setting
|
||||
documents otherwise.
|
||||
|
||||
`generated_assets.topdata_2da` is for HAK repositories that need a small
|
||||
native-topdata subset packaged as HAK resources. It builds only `.2da` output,
|
||||
injects those generated files into `build-haks`, and rejects TLK-backed text
|
||||
because this path does not produce TLK packages. Static generated groups can
|
||||
omit `autogen.mode`; in that case the selected datasets are compiled directly,
|
||||
and `package_root` plus the repository's HAK include patterns determine where
|
||||
the resources are packaged.
|
||||
|
||||
For `cachedmodels_rows` generated assets, the configured autogen scan appends
|
||||
discovered model stems from the configured include globs to the selected
|
||||
`cachedmodels` dataset without `.mdl` extensions and skips models already
|
||||
present in authored rows.
|
||||
|
||||
For `accessory_visualeffects` autogen, producers can publish the group-to-row policy
|
||||
in `autogen.producers[].accessory_visualeffects`, and consumers can override or
|
||||
extend it in `autogen.consumers[].accessory_visualeffects`. The toolkit uses
|
||||
compatibility defaults for older configs and manifests, but YAML can own the
|
||||
group token source, category source, delimiter, case policy, legacy group
|
||||
mapping, model stem prefixes to strip, key/label formats, model column, and
|
||||
default row values. `group_token_source: folder_name` uses the manifest folder
|
||||
group directly instead of a prefix mapping, and `category_from:
|
||||
immediate_parent` can expose the model's category folder in keys and labels.
|
||||
Individual groups can override the model column with `model_column` or write
|
||||
the discovered model stem to several columns with `model_columns`, which is
|
||||
useful for root-node visualeffects. Supported format tokens are `{dataset}`,
|
||||
`{group}`, `{group_raw}`, `{prefix}`, `{category}`, `{category_upper}`,
|
||||
`{category_segment}`, `{category_segment_upper}`, `{subgroup}`, `{stem}`,
|
||||
`{stem_upper}`, `{model_stem}`, and `{delimiter}`.
|
||||
|
||||
For `parts_rows` generated assets, `parts_rows.acbonus` can normalize final
|
||||
ACBONUS values before module overrides are applied. The
|
||||
`ascending_row_id_sort_key` strategy writes toolset ordering values from row
|
||||
IDs so higher part rows can sort before lower rows in the toolset; the legacy
|
||||
`descending_row_id_sort_key` strategy remains available for repositories that
|
||||
already depend on it. Per-dataset policies such as `chest: fixed 0.00` can
|
||||
override the default. Authored source ACBONUS values are not authoritative when
|
||||
this policy is configured, except that authored `****` values remain null
|
||||
unless model discovery activates the row. Dense gaps without authored rows or
|
||||
discovered part models are emitted as `****` null rows. Files under
|
||||
`parts/modules` remain the final override layer.
|
||||
|
||||
Extraction is deliberately guarded because it writes source files from binary
|
||||
archives. `extract.archives` is a list of glob patterns relative to
|
||||
`paths.build`; the default is only the configured module `.mod`. HAK extraction
|
||||
is opt-in, for example:
|
||||
|
||||
```yaml
|
||||
extract:
|
||||
archives:
|
||||
- "{module.resref}.mod"
|
||||
- "sow_over*.hak"
|
||||
```
|
||||
|
||||
Module resources require a configured `paths.source`; HAK assets require a
|
||||
configured `paths.assets`; both roots must resolve to safe subtrees. If a root
|
||||
is unset or resolves to the repository root, extraction fails before writing
|
||||
instead of creating extension directories like `mdl/` or `png/` at the repo top
|
||||
level. `.erf` files are rejected by extraction because they do not contain
|
||||
`module.ifo`, so they cannot safely update module-level area membership.
|
||||
|
||||
`consume_archives: true` deletes the selected build archive files only after the
|
||||
entire extraction succeeds. The older `delete_module_archive_after_success`
|
||||
setting remains as a module-only compatibility alias when `consume_archives` is
|
||||
not set. Stale cleanup is scoped to roots actually touched by extracted
|
||||
resources, so the default module-only extraction does not prune assets.
|
||||
|
||||
`topdata.wiki.status_listing_scope` controls the generated wiki-status detail
|
||||
pages. The default `all` writes both per-namespace listings and aggregate
|
||||
`meta:wikistatus:<status>` listings. Use `namespace` when aggregate status
|
||||
lists would be too large for the target NodeBB post limit. Set
|
||||
`topdata.wiki.status_pages: false` to omit status pages entirely; existing
|
||||
tracked status pages then become stale deploy-manifest entries and can be
|
||||
removed with `deploy-wiki --stale-policy purge`.
|
||||
|
||||
GFF JSON extraction can merge selected fields and lists from the existing source
|
||||
file instead of replacing the whole extracted document. Rules are matched by
|
||||
`target` relative to `paths.source`; files without a matching rule keep the
|
||||
default overwrite behavior. For example:
|
||||
|
||||
```yaml
|
||||
extract:
|
||||
merge:
|
||||
gff_json:
|
||||
- target: module/module.ifo.json
|
||||
preserve_fields:
|
||||
- Mod_Entry_Area
|
||||
- Mod_Entry_X
|
||||
- Mod_Entry_Y
|
||||
- Mod_Entry_Z
|
||||
- Mod_Entry_Dir_X
|
||||
- Mod_Entry_Dir_Y
|
||||
merge_lists:
|
||||
- field: Mod_Area_list
|
||||
key_field: Area_Name
|
||||
strategy: preserve_existing_order_append_new
|
||||
```
|
||||
|
||||
`preserve_fields` copies matching root fields from the existing source document
|
||||
into the extracted document before writing. The
|
||||
`preserve_existing_order_append_new` list strategy keeps existing keyed list
|
||||
order for entries still present in the extracted archive, uses extracted entries
|
||||
as the value source, appends archive-only entries in archive order, and removes
|
||||
existing entries that are no longer present in the archive.
|
||||
|
||||
Music defaults (used when not explicitly configured):
|
||||
|
||||
```yaml
|
||||
music:
|
||||
defaults:
|
||||
stage_root: "{paths.cache}/music"
|
||||
credits_root: "{paths.cache}/credits"
|
||||
credits_overlay: CREDITS.md
|
||||
max_stem_length: 16
|
||||
naming_scheme: nwn_bmu
|
||||
output_extension: .bmu
|
||||
convert_extensions: [.flac, .m4a, .mp3, .ogg, .wav]
|
||||
```
|
||||
|
||||
Music datasets define individual music collections with independent sources and
|
||||
naming prefixes. `source` is the authored input root under `paths.assets`;
|
||||
`output` is the generated HAK asset root matched by HAK include globs. If
|
||||
`output` is omitted, it defaults to `source`. `package_mode: hak_asset` converts
|
||||
configured source extensions into generated HAK resources; `package_mode: none`
|
||||
scans and writes credits without adding generated HAK resources. The legacy
|
||||
`music.prefixes` shorthand is automatically normalized into datasets.
|
||||
`convert_extensions` is configuration-driven; the built-in default set is
|
||||
`.flac`, `.m4a`, `.mp3`, `.ogg`, and `.wav`.
|
||||
Those extensions are discovered from YAML within configured music dataset source
|
||||
roots, so setting them in `music.defaults.convert_extensions` or a dataset
|
||||
override is sufficient; you do not need a parallel
|
||||
`inventory.asset_extensions` override just to make music scanning see them.
|
||||
|
||||
Music can also be run directly:
|
||||
|
||||
```bash
|
||||
sow-toolkit music list-datasets
|
||||
sow-toolkit music scan --dataset westgate
|
||||
sow-toolkit music build --dataset westgate --dry-run
|
||||
sow-toolkit music build --dataset westgate
|
||||
sow-toolkit music credits --dataset westgate --check
|
||||
sow-toolkit music validate --dataset westgate --json
|
||||
sow-toolkit music manifest --dataset westgate
|
||||
sow-toolkit music normalize --dataset westgate
|
||||
```
|
||||
|
||||
`build-haks` uses the same music pipeline. Use `--plan-only` or music
|
||||
`--dry-run` to compute mappings without transcoding. Use `build-haks
|
||||
--skip-music` to package authored assets only, or `build-haks --music-dataset
|
||||
westgate` to limit conversion to a configured dataset.
|
||||
|
||||
Generated and machine-readable artifacts remain JSON, including HAK manifests,
|
||||
topdata datasets, caches, credits inventories, build reports, and canonical GFF
|
||||
documents. Nested metadata such as `topdata/templates/config.yaml` remains
|
||||
valid when it is local workflow configuration rather than root repository
|
||||
configuration.
|
||||
|
||||
## Wiki And Changelog Utilities
|
||||
|
||||
`build-topdata --wiki` or `build-wiki` renders wiki pages from the current
|
||||
native topdata state as Tiptap-compatible NodeBB HTML. Page templates are HTML
|
||||
fragments; they do not need to include a visible title heading because deploy
|
||||
uses generated `page-index.json` metadata for page titles and public paths.
|
||||
`deploy-wiki` publishes those pages to NodeBB. It reads
|
||||
`NODEBB_API_ENDPOINT`, `NODEBB_API_TOKEN`, `GITHUB_REF_NAME`, and
|
||||
`NODEBB_WIKI_CATEGORIES` when equivalent flags are omitted. Category mappings
|
||||
use `namespace=cid`, for example:
|
||||
|
||||
```bash
|
||||
sow-toolkit deploy-wiki --dry-run --namespace spells --category spells=42
|
||||
```
|
||||
|
||||
Namespace declarations in `topdata/wiki/namespaces.yaml` may set
|
||||
`canonical_path` when a generated namespace lives below a fuller NodeBB category
|
||||
path, for example `canonical_path: Wiki/Feats`. The `title` remains the display
|
||||
label, while `canonical_path` drives generated `public_path` metadata, wiki
|
||||
links, page-index entries, and deploy adoption matching.
|
||||
|
||||
When a deploy manifest is missing but matching wiki pages already exist in
|
||||
NodeBB, `deploy-wiki` checks the Westgate Wiki namespace directory API and
|
||||
adopts those topic/post mappings before planning updates. It does the same when
|
||||
a cached manifest mapping points to a post NodeBB no longer has. Canonical
|
||||
`wikiPath`/`canonicalPath` data is preferred during adoption. Nested generated
|
||||
paths require that exact canonical path data; leaf/title compatibility matching
|
||||
is only used where it cannot cross a canonical parent path. `--create` is still
|
||||
required for genuinely new pages that have no existing remote topic. If NodeBB
|
||||
rejects a create because the clean wiki URL already exists, `deploy-wiki`
|
||||
re-queries the namespace by page leaf, adopts the matching topic, and updates it
|
||||
instead. Pass `--debug` when deploying large wiki changes to emit planning and
|
||||
live mutation progress. Updates and stale-page archives acquire a Westgate Wiki
|
||||
edit lock before saving, then send the returned
|
||||
`wikiEditLockToken` with the NodeBB post edit request. Creates, updates, and
|
||||
archives write the same managed HTML to both NodeBB `content` and
|
||||
`sourceContent`; the deploy manifest records `source_content_synced` so older
|
||||
manifests receive a one-time body repair even when the generated hash is
|
||||
unchanged. Stale generated pages are reported by default.
|
||||
`--stale-policy archive` rewrites stale generated pages in place as archived
|
||||
managed content. `--stale-policy purge` permanently removes only stale generated
|
||||
topics already tracked in the wiki deploy manifest. Dry-run that policy before
|
||||
the live run; it never discovers manual wiki pages from NodeBB for purge.
|
||||
`--reset-managed-namespaces` is a stronger destructive mode for slug or path
|
||||
standard migrations: it permanently removes every topic currently listed by the
|
||||
configured managed namespace categories, ignores the previous deploy manifest,
|
||||
and recreates the current generated pages. It requires `--create` when local
|
||||
generated pages are present.
|
||||
|
||||
For newly created NodeBB topics, `topdata.wiki.title_prefix_min_length`
|
||||
controls when the deployer prefixes the namespace in the topic title. The
|
||||
default is `3`, so one- and two-character page titles are created as
|
||||
`Namespace: Title`, while titles with three or more characters keep the page
|
||||
title alone. Existing bot-managed pages with older prefixed titles are renamed
|
||||
on later deploy passes when their current title no longer matches this policy.
|
||||
|
||||
`build-changelog` renders a release changelog from tagged pushes. Pull-request
|
||||
style commits keep their PR links and authors from the Gitea API; direct pushes
|
||||
are rendered as commit links using the commit author. It defaults to
|
||||
`scripts/changelog.json`, reads Gitea tokens from `GITEA_API_TOKEN`,
|
||||
`GITEA_TOKEN`, or `SOW_TOOLS_TOKEN`, and writes to stdout unless `--output` is
|
||||
provided.
|
||||
|
||||
## Intended Consumers
|
||||
|
||||
- `sow-module` uses `sow-toolkit` for `build`, `build-module`, `extract`, `validate`, `compare`, and `apply-hak-manifest`
|
||||
- `sow-assets` uses `sow-toolkit` for `build-haks`, music processing, changelog generation, and validation
|
||||
- validation now fails early if duplicate asset resources would collide inside the same generated HAK
|
||||
- duplicate resources split across different HAK groups remain warnings because later HAK order can still override earlier content at runtime
|
||||
- `sow-module` now also uses `sow-toolkit` for topdata:
|
||||
- `validate-topdata`
|
||||
- `build-topdata`
|
||||
- `build-top-package`
|
||||
- `compare-topdata`
|
||||
- `convert-topdata`
|
||||
- `build-wiki`
|
||||
- `deploy-wiki`
|
||||
|
||||
During development, sibling repositories can point at `../sow-tools/tools/sow-toolkit`
|
||||
or `../sow-tools/tools/sow-toolkit.exe`. For pinned releases, each consumer repo can
|
||||
instead carry its own copy in its local `tools/` directory.
|
||||
|
||||
## Cross-Repository Behavior
|
||||
|
||||
The current ownership split is:
|
||||
|
||||
- `sow-tools`
|
||||
- owns the shared CLI and build logic
|
||||
- owns module build, extract, compare, and HAK manifest application logic
|
||||
- owns topdata validation, build, compare, conversion, and packaging logic
|
||||
- `sow-module`
|
||||
- owns canonical module source under `src/`
|
||||
- owns canonical topdata authored data under `topdata/`
|
||||
- owns the user-facing module and topdata wrapper commands
|
||||
- `sow-assets`
|
||||
- owns canonical binary asset source under `assets/`
|
||||
- owns generated HAK manifests and HAK-side parts 2DA assets
|
||||
- owns the user-facing asset wrapper commands
|
||||
|
||||
Consumer wrapper resolution is intentionally aligned:
|
||||
|
||||
1. Prefer a pinned local binary in `tools/`
|
||||
2. Else prefer a sibling `../sow-tools` checkout for local source development
|
||||
3. Else install the latest published `sow-tools` release artifact
|
||||
|
||||
NWScript compilation behavior is also aligned and configurable under
|
||||
`scripts.compiler`:
|
||||
|
||||
1. Prefer `SOW_NWN_SCRIPT_COMPILER` when explicitly set
|
||||
2. Else prefer a local compiler in `tools/script-compiler/` or `tools/`
|
||||
3. Else use a system-installed `nwn_script_comp` from `PATH`
|
||||
|
||||
When the compiler runs, the toolkit prepares `NWN_HOME` / `NWN_USER_DIRECTORY`
|
||||
automatically and tries to detect `NWN_ROOT` from common Steam locations, including
|
||||
custom Steam library folders, before an explicit override is required.
|
||||
|
||||
|
||||
## Release Automation
|
||||
|
||||
This repo publishes:
|
||||
|
||||
- `sow-toolkit-linux-amd64.tar.gz`
|
||||
- `sow-toolkit-windows-amd64.exe`
|
||||
|
||||
Consumer repos can fetch those binaries into their local `tools/` directory with:
|
||||
|
||||
- `install-tool.sh` on Linux/macOS
|
||||
- `install-tool.ps1` on Windows
|
||||
|
||||
## TopData
|
||||
|
||||
The native topdata system is implemented in this repo and authored in `sow-module`.
|
||||
Generated native topdata lockfiles use matching `.editorconfig` JSON formatting
|
||||
settings when a consumer repository provides them.
|
||||
|
||||
High-level ownership split:
|
||||
|
||||
- `sow-tools`
|
||||
- owns the `sow-toolkit` implementation
|
||||
- owns topdata validation, build, compare, conversion, and dataset-resolution logic
|
||||
- `sow-module`
|
||||
- owns canonical topdata authored data under `topdata/data/`
|
||||
- owns the user-facing topdata workflow scripts
|
||||
|
||||
For a human-readable front-to-back explanation of how the native topdata builder works
|
||||
and how to use it, see:
|
||||
|
||||
- [sow-module/topdata/README.md](../sow-module/topdata/README.md)
|
||||
|
||||
Important topdata contracts still live here under `internal/topdata/` because they are
|
||||
code-adjacent implementation contracts, not user workflow docs.
|
||||
How the artifact repos resolve a Crucible binary (and the `NWN_ROOT` rule) is
|
||||
documented in [`docs/consumer-contract.md`](docs/consumer-contract.md).
|
||||
|
||||
@@ -1,322 +0,0 @@
|
||||
Refactor and harden all shell wrapper scripts across the module and assets repositories.
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- Argument passthrough is solved for the active root wrappers.
|
||||
- `module/` and `assets/` now use repo-root launchers plus
|
||||
`scripts/run-nwn-tool.*` passthrough shims.
|
||||
- Wrapper regression coverage already exists in `module/tests/` and
|
||||
`assets/tests/`.
|
||||
- Active repo-root `.sh` and `.ps1` wrappers are intentionally thin launchers.
|
||||
- The previously stale JSON-config assumption shown in this contract is no
|
||||
longer current; active repos use `nwn-tool.yaml`.
|
||||
|
||||
## Scope Status
|
||||
|
||||
This contract is complete for active root wrappers as of 2026-05-13.
|
||||
|
||||
Completed:
|
||||
|
||||
- root `.sh` wrappers are thin `cd` plus `scripts/run-nwn-tool.sh` launchers
|
||||
- root `.ps1` wrappers are thin `scripts/run-nwn-tool.ps1` launchers
|
||||
- all active root wrappers forward arbitrary arguments transparently
|
||||
- shell and PowerShell wrapper regression coverage includes every active root
|
||||
wrapper in `module/` and `assets/`
|
||||
- stale documentation references to unsupported root wrappers were removed
|
||||
|
||||
## Current Findings
|
||||
|
||||
No active root-wrapper thinning work remains.
|
||||
|
||||
The scripts below are intentionally not root wrappers:
|
||||
|
||||
- `module/scripts/release-all.sh` and `assets/scripts/release-haks.sh` both
|
||||
parse effective config and own release publishing orchestration.
|
||||
- `scripts/run-nwn-tool.*` still own tool installation/update bootstrap. This is
|
||||
shared repo plumbing, not build business logic.
|
||||
|
||||
These scripts remain acceptable under this wrapper contract because they are
|
||||
release/bootstrap plumbing, not user-facing build/extract wrappers. Future
|
||||
release automation cleanup should happen under release or logging contracts, not
|
||||
by adding logic to root wrappers.
|
||||
|
||||
## Maintenance Rules
|
||||
|
||||
1. Keep paired `.sh` and `.ps1` root wrappers thin; do not add new wrappers
|
||||
unless a command is intentionally promoted to a user-facing entrypoint.
|
||||
2. Root wrappers must not parse config, inspect source trees, preflight build
|
||||
products, or duplicate toolkit command decisions.
|
||||
3. `scripts/run-nwn-tool.*` may keep tool install/update bootstrap, but must not
|
||||
perform command-specific build logic.
|
||||
4. Release-entrypoint orchestration belongs in release contracts; do not treat it
|
||||
as a reason to thicken root wrappers.
|
||||
5. If a root wrapper is added or removed, update README command lists and wrapper
|
||||
contract tests in the same change.
|
||||
|
||||
# Historical Problem
|
||||
|
||||
Earlier `.sh` and `.ps1` wrapper scripts did not properly forward arbitrary CLI arguments to the underlying toolkit commands.
|
||||
|
||||
Example:
|
||||
|
||||
- `--verbose`
|
||||
- `--debug`
|
||||
- `--dry-run`
|
||||
- `--config`
|
||||
- `--dataset`
|
||||
- `--force`
|
||||
- future flags
|
||||
|
||||
That passthrough issue is now fixed for active root wrappers.
|
||||
|
||||
Additionally, after recent refactors — including:
|
||||
|
||||
- YAML configuration migration
|
||||
- configuration hardening
|
||||
- toolkit restructuring
|
||||
- command refactors
|
||||
- repository abstraction changes
|
||||
|
||||
— several wrapper scripts became outdated and no longer correctly reflected toolkit behavior.
|
||||
|
||||
Example of a recent error produced:
|
||||
|
||||
```bash
|
||||
$ ./build-haks.sh
|
||||
run-nwn-tool.sh: Updating sow-toolkit to match latest sow-tools release...
|
||||
installed: ${HOME}/Projects/nwnee-shadowsoverwestgate/assets/tools/sow-toolkit
|
||||
run-nwn-tool.sh: Refreshing credits cache before build-haks...
|
||||
Traceback (most recent call last):
|
||||
File "${HOME}/Projects/nwnee-shadowsoverwestgate/assets/./scripts/sync-credits-cache.py", line 61, in <module>
|
||||
main()
|
||||
~~~~^^
|
||||
File "${HOME}/Projects/nwnee-shadowsoverwestgate/assets/./scripts/sync-credits-cache.py", line 46, in main
|
||||
music_configs = load_music_configs()
|
||||
File "${HOME}/Projects/nwnee-shadowsoverwestgate/assets/./scripts/sync-credits-cache.py", line 16, in load_music_configs
|
||||
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||
File "/usr/lib/python3.14/pathlib/__init__.py", line 787, in read_text
|
||||
with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f:
|
||||
~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/usr/lib/python3.14/pathlib/__init__.py", line 771, in open
|
||||
return io.open(self, mode, buffering, encoding, errors, newline)
|
||||
~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
FileNotFoundError: [Errno 2] No such file or directory: '${HOME}/Projects/nwnee-shadowsoverwestgate/assets/nwn-tool.json'
|
||||
```
|
||||
|
||||
# Completed Goal
|
||||
|
||||
Active root wrapper scripts are now:
|
||||
|
||||
- consistent
|
||||
- argument-transparent
|
||||
- future-safe
|
||||
- deterministic
|
||||
- repository-agnostic where possible
|
||||
|
||||
# Core requirements
|
||||
|
||||
## Global argument forwarding
|
||||
|
||||
All wrappers must forward arbitrary arguments transparently to the underlying command.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
./run-tool.sh build-haks --verbose
|
||||
./run-tool.sh music build --dataset westgate --dry-run
|
||||
./extract-module.sh --debug --force
|
||||
```
|
||||
|
||||
Must correctly pass all additional arguments through unchanged.
|
||||
|
||||
PowerShell wrappers must behave equivalently:
|
||||
|
||||
```powershell
|
||||
.\run-tool.ps1 build-haks --verbose
|
||||
```
|
||||
|
||||
No wrapper should:
|
||||
|
||||
- silently swallow arguments
|
||||
- hardcode known flags only
|
||||
- require updating for every new CLI option
|
||||
- partially parse arguments unless absolutely necessary
|
||||
|
||||
Wrappers should behave as thin passthrough layers.
|
||||
|
||||
# Wrapper audit
|
||||
|
||||
Audit all `.sh` and `.ps1` scripts in:
|
||||
|
||||
- module repositories
|
||||
- assets repositories
|
||||
- toolkit-related repositories
|
||||
|
||||
Identify wrappers that:
|
||||
|
||||
- no longer match current toolkit commands
|
||||
- reference outdated config paths
|
||||
- reference old JSON config names
|
||||
- assume old directory layouts
|
||||
- assume deprecated commands
|
||||
- hardcode obsolete arguments
|
||||
- bypass the new normalized YAML config system
|
||||
- duplicate logic now handled in the toolkit
|
||||
|
||||
# Consistency requirements
|
||||
|
||||
All wrappers should follow consistent conventions:
|
||||
|
||||
- argument forwarding
|
||||
- environment setup
|
||||
- working directory resolution
|
||||
- config resolution
|
||||
- logging behavior
|
||||
- error handling
|
||||
- exit code propagation
|
||||
- path normalization
|
||||
- quoting/escaping
|
||||
|
||||
# Behavior requirements
|
||||
|
||||
## Argument passthrough
|
||||
|
||||
Shell wrappers should use safe passthrough semantics.
|
||||
|
||||
Examples:
|
||||
|
||||
- Bash: `"$@"`
|
||||
- PowerShell: `@args`
|
||||
|
||||
Arguments must preserve:
|
||||
|
||||
- ordering
|
||||
- quoting
|
||||
- spaces
|
||||
- special characters
|
||||
|
||||
## Exit codes
|
||||
|
||||
Wrappers must propagate underlying command exit codes correctly.
|
||||
|
||||
No swallowing failures.
|
||||
|
||||
## Logging
|
||||
|
||||
Wrappers should:
|
||||
|
||||
- print concise execution summaries
|
||||
- show the underlying toolkit command being executed
|
||||
- optionally support wrapper-level debug logging
|
||||
- avoid duplicating toolkit logs
|
||||
|
||||
## Configuration awareness
|
||||
|
||||
Wrappers must support:
|
||||
|
||||
- YAML config paths
|
||||
- overridden config locations
|
||||
- repository-local config resolution
|
||||
- future config expansion
|
||||
|
||||
No wrapper should assume:
|
||||
|
||||
- `config.json`
|
||||
- fixed build paths
|
||||
- fixed repository names
|
||||
- fixed datasets
|
||||
- fixed HAK names
|
||||
|
||||
# Repository hardening alignment
|
||||
|
||||
Wrappers must align with the broader configuration hardening effort:
|
||||
|
||||
- no repository-specific hardcoding
|
||||
- no hidden defaults
|
||||
- no implicit assumptions
|
||||
- no duplicated business logic
|
||||
|
||||
Wrappers should invoke toolkit functionality, not reimplement it.
|
||||
|
||||
# Desired architecture
|
||||
|
||||
Wrappers should become:
|
||||
|
||||
- thin launchers
|
||||
- environment bootstrap layers
|
||||
- argument passthrough adapters
|
||||
|
||||
The toolkit itself should contain:
|
||||
|
||||
- validation
|
||||
- command parsing
|
||||
- config loading
|
||||
- business logic
|
||||
- deterministic behavior
|
||||
|
||||
# Cross-platform behavior
|
||||
|
||||
Ensure `.sh` and `.ps1` wrappers behave equivalently where possible.
|
||||
|
||||
Validate:
|
||||
|
||||
- quoting
|
||||
- argument escaping
|
||||
- relative path handling
|
||||
- environment variable handling
|
||||
- error propagation
|
||||
- working directory behavior
|
||||
|
||||
# Migration tasks
|
||||
|
||||
1. Inventory all wrappers.
|
||||
2. Classify active vs obsolete wrappers.
|
||||
3. Update wrappers to support transparent argument forwarding.
|
||||
4. Remove outdated assumptions.
|
||||
5. Align wrappers with YAML config loading.
|
||||
6. Update docs/examples/help text.
|
||||
7. Add regression tests.
|
||||
8. Ensure compatibility with current toolkit commands.
|
||||
9. Remove duplicated wrapper logic where possible.
|
||||
10. Consolidate shared wrapper behavior if appropriate.
|
||||
|
||||
# Testing requirements
|
||||
|
||||
Add tests for:
|
||||
|
||||
- argument passthrough
|
||||
- quoted arguments
|
||||
- multi-argument forwarding
|
||||
- verbose/debug forwarding
|
||||
- config override forwarding
|
||||
- dataset override forwarding
|
||||
- exit code propagation
|
||||
- invalid command handling
|
||||
- PowerShell parity
|
||||
- shell parity
|
||||
- wrapper execution from different working directories
|
||||
|
||||
Examples:
|
||||
|
||||
- `--verbose`
|
||||
- `--debug`
|
||||
- `--dry-run`
|
||||
- `--config path/to/config.yaml`
|
||||
- `--dataset westgate`
|
||||
- `--force`
|
||||
- unknown future flags
|
||||
|
||||
# Acceptance criteria
|
||||
|
||||
- All wrappers forward arbitrary arguments correctly.
|
||||
- Wrappers reflect the current toolkit architecture.
|
||||
- No wrappers assume old JSON config behavior.
|
||||
- No wrappers contain obsolete repository-specific logic.
|
||||
- Wrapper behavior is deterministic and cross-platform.
|
||||
- Exit codes propagate correctly.
|
||||
- Wrapper maintenance burden is minimized.
|
||||
- New CLI flags work automatically without wrapper updates.
|
||||
- The toolkit, not wrappers, owns application logic.
|
||||
@@ -1,221 +0,0 @@
|
||||
# Native Topdata Hardening Contract
|
||||
|
||||
## Scope
|
||||
|
||||
This contract governs the current topdata hardening and bugfixing phase across:
|
||||
|
||||
- `toolkit/` as implementation owner
|
||||
- `module/` as the primary authored topdata consumer
|
||||
|
||||
This is an active-priority contract.
|
||||
|
||||
## Current Priority
|
||||
|
||||
Focus now on:
|
||||
|
||||
1. removing outdated explicit namespace/path hardcoding
|
||||
2. fixing validation/build mismatches
|
||||
3. making native topdata behavior generic where the data model is already generic
|
||||
4. improving diagnostics and operator-facing output for topdata commands
|
||||
|
||||
Defer for now:
|
||||
|
||||
- model compilation
|
||||
- release patch-note automation
|
||||
- unrelated asset-processing expansion
|
||||
- non-topdata feature work that does not unblock wiki deployment
|
||||
|
||||
Wiki deployment itself is a follow-on phase after this hardening pass, because
|
||||
`build-wiki` and `deploy-wiki` depend on topdata outputs and metadata being
|
||||
stable.
|
||||
|
||||
## Problem Statement
|
||||
|
||||
Native topdata has reached the point where most major functionality exists, but
|
||||
too much behavior is still implemented as explicit namespace knowledge instead of
|
||||
generic dataset rules.
|
||||
|
||||
One concrete example already observed:
|
||||
|
||||
- dataset discovery in `internal/topdata/native.go` already derives a default
|
||||
`output` name when `base.json` omits it
|
||||
- validation in `internal/topdata/topdata.go` still hard-fails specifically for
|
||||
`topdata/data/masterfeats/base.json` if `output` is missing
|
||||
|
||||
That mismatch proved the current methodology was stale:
|
||||
|
||||
- the runtime/build side treats `masterfeats` like a generic dataset
|
||||
- the validator still treats it like a special namespace with explicit required
|
||||
metadata
|
||||
|
||||
Current status: the output-name mismatch has been fixed for `masterfeats` and
|
||||
the equivalent `feat` base dataset rule by sharing the derived output-name rule
|
||||
between discovery and validation.
|
||||
|
||||
This same category of problem appears in several other places.
|
||||
|
||||
## Findings From Current Audit
|
||||
|
||||
### 1. Validation dispatch is path-special-cased
|
||||
|
||||
`validateDataObject()` currently routes behavior through explicit path checks for:
|
||||
|
||||
- `masterfeats/base.json`
|
||||
- `masterfeats/lock.json`
|
||||
- `feat/base.json`
|
||||
- `feat/lock.json`
|
||||
- `feat/generated/*`
|
||||
- `parts/overrides/*`
|
||||
- `racialtypes/registry/races/*`
|
||||
|
||||
Some of these are legitimate dataset-shape differences. Others are stale
|
||||
special-cases that should be generalized.
|
||||
|
||||
### 2. `masterfeats` carries dataset-specific invariants
|
||||
|
||||
`validateMasterfeatsBaseFile()` currently enforces:
|
||||
|
||||
- explicit `output == masterfeats.2da` when `output` is authored
|
||||
- specific required columns
|
||||
- `masterfeats:` row keys
|
||||
|
||||
The key-prefix and column requirements may still be valid dataset invariants.
|
||||
The stale requirement that `output` must be authored has been removed; validation
|
||||
now shares the same derived default output-name rule as native discovery.
|
||||
|
||||
### 3. Migration still seeds explicit dataset exceptions
|
||||
|
||||
`feat_migrate.go` explicitly writes:
|
||||
|
||||
- `output = "feat.2da"`
|
||||
- `compare_reference = false`
|
||||
|
||||
This may still be operationally correct, but it means migration continues to
|
||||
materialize special-case control fields instead of relying on a clearer generic
|
||||
contract for dataset reference comparison.
|
||||
|
||||
### 4. Project-level topdata defaults are still mixed
|
||||
|
||||
`internal/project` still owns several topdata defaults directly:
|
||||
|
||||
- build directory derivation
|
||||
- compiled 2DA subdirectory
|
||||
- compiled TLK name
|
||||
- wiki root/pages/state defaults
|
||||
- wiki deploy manifest and edit summary defaults
|
||||
- package HAK/TLK fallback names
|
||||
|
||||
Some of these are valid toolkit defaults. Some should be reviewed as explicit
|
||||
schema-backed config.
|
||||
|
||||
### 5. Wiki generation and deploy depend on stable dataset semantics
|
||||
|
||||
`build-wiki` and `deploy-wiki` are downstream consumers of native topdata state.
|
||||
If dataset rules remain partly explicit and partly generic, wiki generation will
|
||||
keep inheriting brittle assumptions.
|
||||
|
||||
## Hardening Rules
|
||||
|
||||
1. If a rule can be inferred generically from dataset shape, do not hardcode it
|
||||
to a namespace path.
|
||||
2. If a rule is truly dataset-specific, document why it is intrinsic rather than
|
||||
configuration-driven.
|
||||
3. Validation and build behavior must agree. Discovery defaults and validation
|
||||
requirements cannot contradict each other.
|
||||
4. Dataset-authored JSON should remain the source of truth for dataset-level
|
||||
semantics; root YAML should not absorb dataset internals just to avoid code
|
||||
cleanup.
|
||||
5. Error messages must point to the actual invariant being violated, not an
|
||||
outdated historical implementation detail.
|
||||
|
||||
## Immediate Work Items
|
||||
|
||||
### Work Item 1: Remove outdated `masterfeats.output` hard requirement
|
||||
|
||||
Status: implemented.
|
||||
|
||||
`validateMasterfeatsBaseFile()` now:
|
||||
|
||||
- missing `output` is accepted when generic dataset discovery can derive the same
|
||||
output name deterministically
|
||||
- explicit `output` remains allowed
|
||||
- wrong explicit `output` still fails clearly if the dataset truly requires a
|
||||
fixed emitted file name
|
||||
|
||||
This should be test-covered.
|
||||
|
||||
### Work Item 2: Audit path-based validator routing
|
||||
|
||||
Status: implemented for root dataset invariants; family-specific routes remain
|
||||
where the authored shape is not a generic base/module/lock file.
|
||||
|
||||
Classify every explicit route in `validateDataObject()` as one of:
|
||||
|
||||
- generic canonical shape handling
|
||||
- dataset-family behavior that should become generic
|
||||
- deliberate intrinsic special-case with justification
|
||||
|
||||
Reduce the explicit route set where possible.
|
||||
|
||||
Current classification:
|
||||
|
||||
- `masterfeats/base.json`, `masterfeats/lock.json`, `feat/base.json`, and
|
||||
`feat/lock.json` are now handled by a generic dataset invariant table. These
|
||||
remain toolkit invariants because the row key namespaces and required columns
|
||||
are intrinsic to those native datasets, not repository-level YAML choices.
|
||||
- `feat/generated/*` remains an intrinsic dataset-family route. Generated feat
|
||||
files support compact family expansion and generated entries/overrides, which
|
||||
are not generic module shapes.
|
||||
- `parts/overrides/*` remains a dataset-family route for non-module override
|
||||
files outside the standard `modules/` tree.
|
||||
- `racialtypes/registry/races/*` remains a registry-family route because those
|
||||
files describe race registry records, optional feat-table generation, and core
|
||||
row payloads rather than generic row/module documents.
|
||||
|
||||
### Work Item 3: Define topdata invariant boundaries
|
||||
|
||||
For each current explicit topdata behavior, decide whether it belongs in:
|
||||
|
||||
- root YAML config
|
||||
- dataset JSON
|
||||
- toolkit invariant
|
||||
|
||||
Do not move dataset-authored semantics into YAML unless the behavior truly varies
|
||||
at repository level rather than dataset level.
|
||||
|
||||
### Work Item 4: Revisit `compare_reference` ownership
|
||||
|
||||
`feat.2da` is still excluded via `compare_reference = false`. Confirm whether:
|
||||
|
||||
- this remains a temporary rollout flag
|
||||
- or compare coverage needs a more explicit generic contract for generated-family
|
||||
datasets
|
||||
|
||||
### Work Item 5: Bring topdata logging up to current CLI standards
|
||||
|
||||
Coordinate with `LOG_REFACTOR_CONTRACT.md` so topdata commands emit:
|
||||
|
||||
- deterministic phase summaries
|
||||
- concise validation/build/compare totals
|
||||
- clear file/namespace diagnostics
|
||||
|
||||
## Testing Requirements
|
||||
|
||||
Add or update tests for:
|
||||
|
||||
- `masterfeats/base.json` without explicit `output` when the derived output would
|
||||
still be `masterfeats.2da`
|
||||
- explicit wrong `output` for `masterfeats`
|
||||
- validator/build parity for dataset output naming
|
||||
- any reduction in path-special-case validation routing
|
||||
- topdata command output summaries if presentation changes land in the same phase
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- topdata validation no longer rejects data solely because of stale explicit
|
||||
namespace assumptions
|
||||
- build and validation agree on dataset output-name behavior
|
||||
- path-special-casing is reduced or explicitly justified
|
||||
- remaining dataset-specific invariants are documented as such
|
||||
- topdata hardening leaves wiki generation/deployment with a cleaner and more
|
||||
stable contract
|
||||
@@ -1,108 +0,0 @@
|
||||
# Wiki Deployment Contract
|
||||
|
||||
## Scope
|
||||
|
||||
This contract governs the next focused phase after wrappers, config refactor,
|
||||
logging, and topdata hardening:
|
||||
|
||||
- `toolkit/` owns `build-wiki` and `deploy-wiki`
|
||||
- `module/` owns the consuming release workflow and operational configuration
|
||||
|
||||
This work is intentionally sequenced after topdata hardening because the wiki
|
||||
pipeline consumes native topdata outputs and metadata.
|
||||
|
||||
## Current State
|
||||
|
||||
As of 2026-05-13:
|
||||
|
||||
- `build-wiki` exists
|
||||
- `deploy-wiki` exists
|
||||
- deployment targets NodeBB core `/api/v3`
|
||||
- local deploy manifests and NodeBB topic mapping exist
|
||||
- release automation in `module/scripts/release-all.sh` already runs wiki build
|
||||
and deploy steps after release publication
|
||||
- toolkit tests already cover dry-run behavior, topic creation, and manifest
|
||||
writes
|
||||
|
||||
This means wiki deployment is no longer a greenfield feature. The next phase is
|
||||
hardening, integration clarity, and operator experience.
|
||||
|
||||
## Deferred Until Current Priority Work Completes
|
||||
|
||||
Do not prioritize the following until wrappers/config/logging/topdata work is in
|
||||
better shape:
|
||||
|
||||
- new NodeBB release-announcement posting flows
|
||||
- broad wiki feature expansion
|
||||
- unrelated formatting changes to generated page content
|
||||
|
||||
## Immediate Goals For The Wiki Phase
|
||||
|
||||
1. align wiki build/deploy logs with the structured logging contract
|
||||
2. tighten config ownership for wiki defaults and category mappings
|
||||
3. reduce residual hardcoded behavior that belongs in normalized config
|
||||
4. improve release-flow clarity around dry-run, drift, create, and force modes
|
||||
5. verify that topdata hardening does not silently change generated page identity
|
||||
|
||||
## Current Findings
|
||||
|
||||
- `deploy-wiki` already reads managed namespaces from effective config when the
|
||||
caller does not specify them explicitly
|
||||
- deploy manifest naming and edit-summary defaults still come from toolkit
|
||||
defaults in `internal/project/effective.go`
|
||||
- release orchestration already guards publication order correctly, but the
|
||||
operator-facing logs are still more shell-oriented than the newer structured
|
||||
`build-haks` presentation
|
||||
- wiki deployment should be treated as a first-class operational command family,
|
||||
not just a release-script tail step
|
||||
|
||||
## Immediate Plan
|
||||
|
||||
### Phase 1: Logging and UX parity
|
||||
|
||||
- add concise summaries for `build-wiki` and `deploy-wiki`
|
||||
- surface counts for:
|
||||
- pages collected
|
||||
- creates
|
||||
- updates
|
||||
- skips
|
||||
- drifted pages
|
||||
- keep raw debug output available when diagnosis is needed
|
||||
|
||||
### Phase 2: Config clarity
|
||||
|
||||
- audit which wiki defaults should remain toolkit defaults versus explicit repo
|
||||
config
|
||||
- keep namespace/category ownership visible in effective config
|
||||
- ensure deploy-manifest path and edit summary remain discoverable and easy to
|
||||
override
|
||||
|
||||
### Phase 3: Release-flow hardening
|
||||
|
||||
- verify `module/scripts/release-all.sh` remains the authoritative release entry
|
||||
point
|
||||
- keep dry-run before live deploy
|
||||
- ensure failure and drift messages are crisp and actionable
|
||||
- make sure cache restore/save behavior is clearly documented and tested
|
||||
|
||||
### Phase 4: Regression verification after topdata hardening
|
||||
|
||||
- confirm topdata de-hardcoding does not break page IDs or manifest mapping
|
||||
- confirm generated-page identity remains stable across unchanged inputs
|
||||
|
||||
## Testing Requirements
|
||||
|
||||
Add or expand tests for:
|
||||
|
||||
- app-layer output for `build-wiki` and `deploy-wiki`
|
||||
- release-flow sequencing assumptions where wiki deploy must happen only after
|
||||
release assets are published
|
||||
- stable page identity and manifest mapping across repeated runs
|
||||
- config-driven namespace selection and manifest path overrides
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- wiki build/deploy logs match the newer structured CLI standard
|
||||
- wiki defaults and overrides are clearly represented in effective config
|
||||
- release orchestration remains deterministic and easy to reason about
|
||||
- topdata hardening does not regress page identity or deploy manifest behavior
|
||||
@@ -1,41 +0,0 @@
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
Set-Location $repoRoot
|
||||
|
||||
$env:GOCACHE = Join-Path $repoRoot ".cache/go-build"
|
||||
$outputPath = Join-Path $repoRoot "tools/sow-toolkit.exe"
|
||||
|
||||
function Test-ToolSourcesAreNewer {
|
||||
if (-not (Test-Path $outputPath)) {
|
||||
return $true
|
||||
}
|
||||
|
||||
$outputTime = (Get-Item -LiteralPath $outputPath).LastWriteTimeUtc
|
||||
$sourcePaths = @("go.mod", "go.sum", "cmd", "internal")
|
||||
foreach ($sourcePath in $sourcePaths) {
|
||||
$fullPath = Join-Path $repoRoot $sourcePath
|
||||
if (-not (Test-Path $fullPath)) {
|
||||
continue
|
||||
}
|
||||
$newerSource = Get-ChildItem -LiteralPath $fullPath -Recurse -File |
|
||||
Where-Object { $_.LastWriteTimeUtc -gt $outputTime } |
|
||||
Select-Object -First 1
|
||||
if ($newerSource) {
|
||||
return $true
|
||||
}
|
||||
}
|
||||
|
||||
return $false
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $repoRoot "tools") | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path $env:GOCACHE | Out-Null
|
||||
if (-not (Test-ToolSourcesAreNewer)) {
|
||||
Write-Host "sow-toolkit is up to date."
|
||||
exit 0
|
||||
}
|
||||
|
||||
Write-Host "Building sow-toolkit..."
|
||||
go build -o $outputPath ./cmd/nwn-tool
|
||||
@@ -1,31 +0,0 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
output="./tools/sow-toolkit"
|
||||
|
||||
tool_sources_are_newer() {
|
||||
if [ ! -x "$output" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for path in go.mod go.sum cmd internal; do
|
||||
if [ ! -e "$path" ]; then
|
||||
continue
|
||||
fi
|
||||
if [ -n "$(find "$path" -type f -newer "$output" -print -quit)" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
mkdir -p tools .cache/go-build
|
||||
if ! tool_sources_are_newer; then
|
||||
echo "sow-toolkit is up to date."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Building sow-toolkit..."
|
||||
GOCACHE="${PWD}/.cache/go-build" go build -o "$output" ./cmd/nwn-tool
|
||||
@@ -0,0 +1,11 @@
|
||||
// Command crucible-depot is the standalone depot builder (equivalent to
|
||||
// `crucible depot`). A single-token binary keeps consumer wrapper scripts simple.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/dispatch"
|
||||
)
|
||||
|
||||
func main() { os.Exit(dispatch.RunBuilder("depot", os.Args[1:])) }
|
||||
@@ -0,0 +1,10 @@
|
||||
// Command crucible-hak is the standalone HAK builder (equivalent to `crucible hak`).
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/dispatch"
|
||||
)
|
||||
|
||||
func main() { os.Exit(dispatch.RunBuilder("hak", os.Args[1:])) }
|
||||
@@ -0,0 +1,11 @@
|
||||
// Command crucible-module is the standalone module builder (equivalent to
|
||||
// `crucible module`). sow-module resolves this binary to package its .mod.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/dispatch"
|
||||
)
|
||||
|
||||
func main() { os.Exit(dispatch.RunBuilder("module", os.Args[1:])) }
|
||||
@@ -0,0 +1,11 @@
|
||||
// Command crucible-topdata is the standalone topdata builder (equivalent to
|
||||
// `crucible topdata`). sow-topdata resolves this binary to compile 2da/tlk + wiki.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/dispatch"
|
||||
)
|
||||
|
||||
func main() { os.Exit(dispatch.RunBuilder("topdata", os.Args[1:])) }
|
||||
@@ -0,0 +1,10 @@
|
||||
// Command crucible-wiki is the standalone wiki builder (equivalent to `crucible wiki`).
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/dispatch"
|
||||
)
|
||||
|
||||
func main() { os.Exit(dispatch.RunBuilder("wiki", os.Args[1:])) }
|
||||
@@ -0,0 +1,10 @@
|
||||
// Command crucible is the Crucible dispatcher: `crucible <builder> [args]`.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/dispatch"
|
||||
)
|
||||
|
||||
func main() { os.Exit(dispatch.Main(os.Args[1:])) }
|
||||
@@ -1,16 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/app"
|
||||
)
|
||||
|
||||
func main() {
|
||||
code, err := app.Run(os.Args[1:])
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
}
|
||||
os.Exit(code)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
#
|
||||
# Crucible toolchain image: registry.westgate.pw/sow/crucible:<git-sha>
|
||||
#
|
||||
# Reproducible multi-stage build, no on-VPS build. Produces every cmd/* binary
|
||||
# and ships them on a static base. The `crucible` dispatcher is the entrypoint;
|
||||
# consumer CI can also call the standalone crucible-<name> binaries by path.
|
||||
#
|
||||
# NOTE (Phase 5 scaffold): the binaries are pure stdlib and fail closed until
|
||||
# the operator migrates the internal pipeline. When the music pipeline is wired
|
||||
# it needs ffmpeg at runtime — switch the final stage to a debian-slim base with
|
||||
# ffmpeg then (see docs/migration-from-nwn-tool.md).
|
||||
|
||||
FROM golang:1.26-alpine AS build
|
||||
WORKDIR /src
|
||||
RUN apk add --no-cache git
|
||||
# go.sum is optional while the scaffold has no external deps.
|
||||
COPY go.mod go.sum* ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
ARG GIT_SHA=unknown
|
||||
ENV CGO_ENABLED=0
|
||||
RUN set -eux; \
|
||||
mkdir -p /out; \
|
||||
for dir in ./cmd/*/; do \
|
||||
name="$(basename "${dir}")"; \
|
||||
go build -trimpath \
|
||||
-ldflags "-s -w -X gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo.Version=${GIT_SHA}" \
|
||||
-o "/out/${name}" "${dir}"; \
|
||||
done
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:nonroot
|
||||
COPY --from=build /out/ /usr/local/bin/
|
||||
USER nonroot
|
||||
ENTRYPOINT ["/usr/local/bin/crucible"]
|
||||
CMD ["help"]
|
||||
@@ -0,0 +1,49 @@
|
||||
# Crucible command surface
|
||||
|
||||
Crucible re-homes the single `nwn-tool` (a.k.a. `sow-toolkit`) command surface
|
||||
into five builders plus a dispatcher (D11). This table maps every legacy command
|
||||
to its Crucible home. It is the migration contract — keep it in sync with
|
||||
`internal/dispatch.Registry`.
|
||||
|
||||
## Builders
|
||||
|
||||
| Builder | Legacy `nwn-tool` commands subsumed |
|
||||
| ------------------ | ------------------------------------------------------------------------------------- |
|
||||
| `crucible-module` | `build`, `build-module`, `extract`, `validate`, `compare`, `apply-hak-manifest`† |
|
||||
| `crucible-topdata` | `validate-topdata`, `build-topdata`, `build-top-package`, `compare-topdata`, `convert-topdata` |
|
||||
| `crucible-hak` | `build-haks`, `apply-hak-manifest` |
|
||||
| `crucible-wiki` | `build-wiki`, `deploy-wiki` |
|
||||
| `crucible-depot` | (new) depot blob verify/move — was shell `mc`/S3 in `sow-assets-manifest` |
|
||||
|
||||
† `apply-hak-manifest` is HAK-list maintenance on a module; it is reachable from
|
||||
both `crucible-module` and `crucible-hak`. Canonical home is `crucible-hak`.
|
||||
|
||||
## Folded / global commands
|
||||
|
||||
These legacy commands are **not** top-level builders:
|
||||
|
||||
- `music *` (`list-datasets`, `scan`, `build`, `credits`, `validate`, `manifest`,
|
||||
`normalize`) → folds into the **module/hak build pipeline** (music BMUs are
|
||||
packed into HAKs at build time). Exposed as `crucible-module music ...` /
|
||||
`crucible-hak music ...` when wired. Needs `ffmpeg` at runtime.
|
||||
- `config *` (`validate`, `effective`, `inspect`, `explain`, `sources`) → a
|
||||
**global** concern shared by every builder; exposed as a global subcommand
|
||||
group, not its own binary.
|
||||
- `build-changelog` → release tooling; lands as a global `crucible changelog`
|
||||
rather than a content builder.
|
||||
|
||||
## Global commands (implemented in the scaffold)
|
||||
|
||||
```text
|
||||
crucible help | -h usage + builder list
|
||||
crucible version | -V build version (git sha via -ldflags)
|
||||
crucible list builders, one per line: name<TAB>bin<TAB>summary
|
||||
```
|
||||
|
||||
`crucible list` is machine-readable so CI can enumerate builders without parsing
|
||||
help text.
|
||||
|
||||
## Global flags (planned, at wiring time)
|
||||
|
||||
`--quiet`, `--verbose`, `--debug` (the legacy verbosity model), passed after the
|
||||
builder name, e.g. `crucible topdata build-topdata --verbose`.
|
||||
@@ -0,0 +1,50 @@
|
||||
# Crucible consumer contract
|
||||
|
||||
How the artifact repos (`sow-module`, `sow-topdata`, `sow-assets-manifest`)
|
||||
locate and invoke a Crucible builder. The goal: **no local-machine assumptions**
|
||||
(Phase 5 exit criterion). A consumer either finds a Crucible binary or fails
|
||||
closed — it never fakes an artifact.
|
||||
|
||||
## Resolution order (single-token command)
|
||||
|
||||
Each consumer wrapper resolves its builder to one executable token, in order:
|
||||
|
||||
1. **Explicit override** — `$SOW_MODULE_BUILD` / `$SOW_TOPDATA_BUILD` /
|
||||
`$CRUCIBLE` (a path or name), if set and executable. Back-compat with the
|
||||
pre-Crucible skeletons.
|
||||
2. **Standalone Crucible binary** — `crucible-module` / `crucible-topdata` /
|
||||
`crucible-hak` / `crucible-depot` on `PATH`.
|
||||
3. **Legacy name** — `sow-module-build` / `sow-topdata-build` on `PATH`
|
||||
(pre-D11; kept so nothing breaks during cutover).
|
||||
|
||||
If none resolve, the wrapper exits non-zero with a Phase 5 message. The
|
||||
`crucible` dispatcher (`crucible module …`) is for humans and CI introspection;
|
||||
wrappers prefer the single-token `crucible-<name>` shim so `"$builder" args`
|
||||
quoting stays correct.
|
||||
|
||||
## In CI (preferred)
|
||||
|
||||
Run the consumer job inside the pinned image and the binaries are on `PATH`:
|
||||
|
||||
```yaml
|
||||
container:
|
||||
image: registry.westgate.pw/sow/crucible:<sha> # pinned, immutable
|
||||
```
|
||||
|
||||
No host install, no `$HOME` layout, no developer machine assumptions.
|
||||
|
||||
## `NWN_ROOT` rule
|
||||
|
||||
Crucible **never guesses `NWN_ROOT` from `$HOME`** (this was a deploy-notes
|
||||
pain point). The NWN install/data root is passed explicitly:
|
||||
|
||||
- env `NWN_ROOT=/path/to/nwn`, or
|
||||
- flag `--nwn-root /path/to/nwn`.
|
||||
|
||||
A builder that needs `NWN_ROOT` and receives neither must fail closed with a
|
||||
clear message, not fall back to a home-directory default.
|
||||
|
||||
## Determinism
|
||||
|
||||
Same inputs → identical output bytes. Consumers may checksum artifacts across
|
||||
runs; non-determinism is a builder bug.
|
||||
@@ -0,0 +1,48 @@
|
||||
# Migrating from `nwn-tool` to Crucible (operator cutover)
|
||||
|
||||
The legacy toolchain lives at `gitea/sow-tools` as a single `nwn-tool` binary
|
||||
(entry `cmd/nwn-tool`, logic under `internal/`). This repo is the Crucible
|
||||
rewrite. Per the migration hard rules, the `internal/` source is **not**
|
||||
transplanted automatically — the operator migrates it. This is the checklist.
|
||||
|
||||
## Why a clean scaffold first
|
||||
|
||||
- The hard rules forbid copying old-repo source into the migration tree
|
||||
(`build from scratch; reference, don't transplant`).
|
||||
- The downstream skeletons (`sow-module`, `sow-topdata`) already fail closed
|
||||
until they can resolve a Crucible binary; the scaffold satisfies that contract
|
||||
without faking artifacts.
|
||||
- The scaffold pins the **shape** (module path, dispatcher, multi-binary layout,
|
||||
container, CI) so the migrated logic drops into a stable frame.
|
||||
|
||||
## Cutover steps
|
||||
|
||||
1. **Move the internal packages.** Bring `internal/{app,pipeline,project,erf,
|
||||
gff,topdata,music,changelog,validator}` from `gitea/sow-tools` into this repo.
|
||||
Re-home `internal/app` command wiring behind `internal/dispatch` instead of
|
||||
the old flag parser in `cmd/nwn-tool`.
|
||||
2. **Restore dependencies.** Add `golang.org/x/text` and `gopkg.in/yaml.v3` back
|
||||
to `go.mod`; commit the resulting `go.sum`. (`docker/Dockerfile` already
|
||||
tolerates an absent `go.sum` for the scaffold; it will be present after this.)
|
||||
3. **Wire each builder.** In `internal/dispatch`, register a handler per builder
|
||||
and remove it from the unwired fail-closed path. Map the legacy commands per
|
||||
[`command-surface.md`](command-surface.md).
|
||||
4. **Kill the home-dir / `NWN_ROOT` defaulting.** Honor only explicit
|
||||
env/flag roots per [`consumer-contract.md`](consumer-contract.md).
|
||||
5. **Drop the committed binary.** Delete `nwn-tool` / `tools/sow-toolkit` from
|
||||
the source tree; `.gitignore` already excludes them. Binaries become CI
|
||||
artifacts and image layers only.
|
||||
6. **Carry the determinism test.** Port
|
||||
`TestBuildNativeKeepsGeneratedIDsStableAcrossRepeatedBuilds` and keep it green.
|
||||
7. **Runtime base for music.** If music conversion is wired, switch the final
|
||||
Docker stage from `distroless/static` to a debian-slim base that includes
|
||||
`ffmpeg`.
|
||||
8. **Update `make smoke`.** Once a builder is wired, change the smoke
|
||||
expectation for it from exit `70` to a real success path.
|
||||
|
||||
## Wrapper contract carried over
|
||||
|
||||
Consumer repos keep the `SCRIPT_WRAPPER_CONTRACT` resolution behavior, now
|
||||
pointed at Crucible binaries — see [`consumer-contract.md`](consumer-contract.md).
|
||||
The old `SOW_TOOLS_DEV_BINARY` dev override maps to `$CRUCIBLE` /
|
||||
`$SOW_MODULE_BUILD` / `$SOW_TOPDATA_BUILD`.
|
||||
@@ -1,812 +0,0 @@
|
||||
# Generated Wiki Stale Purge Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add an opt-in `purge` stale policy that permanently removes only manifest-tracked generated NodeBB wiki topics that no longer exist in current generator output.
|
||||
|
||||
**Architecture:** Keep stale ownership and candidate selection inside the existing `deploy-wiki` manifest planner. Extend the deploy result/plan model with purge counts and purge actions, add a thin NodeBB topic purge client call, remove successfully purged entries from the next manifest, and surface the policy through existing toolkit validation and module release/docs paths.
|
||||
|
||||
**Tech Stack:** Go toolkit code and `httptest` deploy tests, YAML-backed toolkit project config validation, POSIX shell module release wrapper, Markdown docs.
|
||||
|
||||
---
|
||||
|
||||
## File Map
|
||||
|
||||
Toolkit files:
|
||||
|
||||
- Modify `internal/project/project.go`: validate `purge` as a supported stale policy and remove the unsupported `unpublish` config assumption.
|
||||
- Modify `internal/project/project_test.go`: prove project config accepts `purge` and rejects unsupported stale policies.
|
||||
- Modify `internal/topdata/wiki_deploy.go`: add purge deploy state, stale purge planning, manifest removal, and NodeBB topic purge requests.
|
||||
- Modify `internal/topdata/wiki_deploy_test.go`: cover dry-run/live purge and missing-topic-id refusal without changing archive/report behavior.
|
||||
- Modify `internal/app/app.go`: expose `purge` in deploy CLI help and print `purged` deploy counts.
|
||||
- Modify `internal/app/app_test.go`: cover CLI output/help parsing for the new count and policy text.
|
||||
- Modify `README.md`: document the destructive generated-manifest-scoped purge mode in the toolkit CLI docs.
|
||||
|
||||
Module files:
|
||||
|
||||
- Modify `../module/scripts/release-all.sh`: allow `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY=purge`.
|
||||
- Modify `../module/README.md`: document release usage and the destructive boundary.
|
||||
- Modify `../module/topdata/README.md`: document `deploy-wiki --stale-policy purge`.
|
||||
- Modify `../module/tests/release-all-cache-cleanup.sh`: prove release automation forwards `purge` into both wiki deploy phases.
|
||||
|
||||
## Task 1: Align Stale Policy Validation
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/project/project.go`
|
||||
- Modify: `internal/project/project_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing project-config test**
|
||||
|
||||
Add a project validation case in `internal/project/project_test.go` that loads stale policy config with `purge` and expects validation to pass:
|
||||
|
||||
```go
|
||||
func TestProjectValidationAcceptsWikiStalePurgePolicy(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeProjectFile(t, filepath.Join(root, ConfigFile), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
topdata:
|
||||
wiki:
|
||||
stale_pages:
|
||||
default: report
|
||||
live_cleanup: purge
|
||||
`)
|
||||
|
||||
proj, err := Load(root)
|
||||
if err != nil {
|
||||
t.Fatalf("load project: %v", err)
|
||||
}
|
||||
if err := proj.ValidateLayout(); err != nil {
|
||||
t.Fatalf("validate purge stale policy: %v", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the focused test to verify it fails**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/project -run TestProjectValidationAcceptsWikiStalePurgePolicy -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL because `topdata.wiki.stale_pages.live_cleanup "purge" is not supported`.
|
||||
|
||||
- [ ] **Step 3: Replace unsupported project validation vocabulary**
|
||||
|
||||
In `internal/project/project.go`, update the stale-policy switch from:
|
||||
|
||||
```go
|
||||
switch policy {
|
||||
case "report", "archive", "unpublish":
|
||||
default:
|
||||
failures = append(failures, fmt.Errorf("%s %q is not supported", key, policy))
|
||||
}
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```go
|
||||
switch policy {
|
||||
case "report", "archive", "purge":
|
||||
default:
|
||||
failures = append(failures, fmt.Errorf("%s %q is not supported", key, policy))
|
||||
}
|
||||
```
|
||||
|
||||
This removes the config-only `unpublish` value that the deploy command cannot execute.
|
||||
|
||||
- [ ] **Step 4: Add the unsupported-policy regression**
|
||||
|
||||
Add or extend a project validation test so `unpublish` and an arbitrary bad value fail:
|
||||
|
||||
```go
|
||||
func TestProjectValidationRejectsUnsupportedWikiStalePolicies(t *testing.T) {
|
||||
for _, policy := range []string{"unpublish", "remove-everything"} {
|
||||
t.Run(policy, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeProjectFile(t, filepath.Join(root, ConfigFile), fmt.Sprintf(`
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
topdata:
|
||||
wiki:
|
||||
stale_pages:
|
||||
live_cleanup: %s
|
||||
`, policy))
|
||||
|
||||
proj, err := Load(root)
|
||||
if err != nil {
|
||||
t.Fatalf("load project: %v", err)
|
||||
}
|
||||
err = proj.ValidateLayout()
|
||||
if err == nil || !strings.Contains(err.Error(), policy) {
|
||||
t.Fatalf("expected %q stale policy validation failure, got %v", policy, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Add `fmt` to the existing `internal/project/project_test.go` imports for the `Sprintf` YAML fixture.
|
||||
|
||||
- [ ] **Step 5: Run project tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/project -count=1
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 6: Commit the validation change**
|
||||
|
||||
```bash
|
||||
git add internal/project/project.go internal/project/project_test.go
|
||||
git commit -m "feat: validate wiki stale purge policy"
|
||||
```
|
||||
|
||||
## Task 2: Plan Manifest-Scoped Purge Actions
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/topdata/wiki_deploy.go`
|
||||
- Modify: `internal/topdata/wiki_deploy_test.go`
|
||||
|
||||
- [ ] **Step 1: Write failing deploy tests for dry-run purge planning**
|
||||
|
||||
In `internal/topdata/wiki_deploy_test.go`, add:
|
||||
|
||||
```go
|
||||
func TestDeployWikiDryRunPlansTrackedStalePagePurge(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sourceDir := filepath.Join(root, "pages")
|
||||
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
|
||||
t.Fatalf("create source dir: %v", err)
|
||||
}
|
||||
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
||||
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
|
||||
Version: "nodebb-v1",
|
||||
Pages: map[string]wikiDeployManifestPage{
|
||||
"skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3},
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write deploy manifest: %v", err)
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Fatalf("dry-run purge must not call NodeBB, got %s %s", r.Method, r.URL.Path)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
||||
SourceDir: sourceDir,
|
||||
Endpoint: server.URL,
|
||||
Token: "nodebb-token",
|
||||
ManifestPath: manifestPath,
|
||||
DryRun: true,
|
||||
StalePolicy: "purge",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("DeployWikiWithOptions stale purge dry run failed: %v", err)
|
||||
}
|
||||
if result.Stale != 1 || result.Purged != 1 {
|
||||
t.Fatalf("expected one planned stale purge, got %#v", result)
|
||||
}
|
||||
if _, ok := loadDeployManifest(manifestPath).Pages["skills:retired"]; !ok {
|
||||
t.Fatalf("dry-run purge must not change the deploy manifest")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Write the failing missing-topic-id test**
|
||||
|
||||
Add:
|
||||
|
||||
```go
|
||||
func TestDeployWikiPurgeRefusesStaleManifestEntryWithoutTopicID(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sourceDir := filepath.Join(root, "pages")
|
||||
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
|
||||
t.Fatalf("create source dir: %v", err)
|
||||
}
|
||||
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
||||
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
|
||||
Version: "nodebb-v1",
|
||||
Pages: map[string]wikiDeployManifestPage{
|
||||
"skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", PID: 42, CID: 3},
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write deploy manifest: %v", err)
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Fatalf("missing-topic-id purge must not call NodeBB, got %s %s", r.Method, r.URL.Path)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
||||
SourceDir: sourceDir,
|
||||
Endpoint: server.URL,
|
||||
Token: "nodebb-token",
|
||||
ManifestPath: manifestPath,
|
||||
DryRun: true,
|
||||
StalePolicy: "purge",
|
||||
}, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "topic id") {
|
||||
t.Fatalf("expected stale purge to require tracked topic id, got %v", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Run the focused tests to verify they fail**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/topdata -run 'TestDeployWiki.*Purge' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL because `purge` is not yet accepted and `deployResult` has no `Purged` field.
|
||||
|
||||
- [ ] **Step 4: Add purge plan/result model fields**
|
||||
|
||||
In `internal/topdata/wiki_deploy.go`, extend `deployResult`:
|
||||
|
||||
```go
|
||||
type deployResult struct {
|
||||
LocalPages int
|
||||
Created int
|
||||
Updated int
|
||||
Stale int
|
||||
Archived int
|
||||
Purged int
|
||||
Skipped int
|
||||
Drifted int
|
||||
Renamed int
|
||||
Manifest string
|
||||
}
|
||||
```
|
||||
|
||||
Allow `purge` during deploy option validation:
|
||||
|
||||
```go
|
||||
if opts.StalePolicy != "" &&
|
||||
opts.StalePolicy != "report" &&
|
||||
opts.StalePolicy != "archive" &&
|
||||
opts.StalePolicy != "purge" {
|
||||
return deployResult{}, fmt.Errorf("wiki stale policy %q is not supported", opts.StalePolicy)
|
||||
}
|
||||
```
|
||||
|
||||
Add the purge count to the deploy plan progress line so dry-run logs show the destructive action explicitly:
|
||||
|
||||
```go
|
||||
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, rename %d, skip %d, stale %d, archive %d, purge %d, drift %d",
|
||||
result.Created,
|
||||
result.Updated,
|
||||
result.Renamed,
|
||||
result.Skipped,
|
||||
result.Stale,
|
||||
result.Archived,
|
||||
result.Purged,
|
||||
result.Drifted,
|
||||
))
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Add stale purge planning**
|
||||
|
||||
In the stale manifest loop inside `planNodeBBDeploy`, keep the current report/archive branches and add the purge branch:
|
||||
|
||||
```go
|
||||
case "purge":
|
||||
if entry.TID == 0 {
|
||||
return nil, result, next, fmt.Errorf("stale generated wiki page %q cannot be purged without tracked NodeBB topic id", pageID)
|
||||
}
|
||||
result.Purged++
|
||||
delete(next.Pages, pageID)
|
||||
plans = append(plans, wikiDeployPlan{
|
||||
Page: wikiDeployPage{PageID: pageID, Title: entry.Title, Namespace: entry.Namespace},
|
||||
Entry: entry,
|
||||
Action: "purge",
|
||||
})
|
||||
```
|
||||
|
||||
Use a `switch policy` or equivalent structure so `report`, `archive`, and `purge` remain explicit. Do not add namespace search or title matching to find purge targets.
|
||||
|
||||
- [ ] **Step 6: Run the focused purge planning tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/topdata -run 'TestDeployWiki.*Purge' -count=1
|
||||
```
|
||||
|
||||
Expected: the dry-run and missing-topic-id purge tests pass once they compile.
|
||||
|
||||
- [ ] **Step 7: Commit purge planning**
|
||||
|
||||
```bash
|
||||
git add internal/topdata/wiki_deploy.go internal/topdata/wiki_deploy_test.go
|
||||
git commit -m "feat: plan stale generated wiki purges"
|
||||
```
|
||||
|
||||
## Task 3: Execute NodeBB Topic Purges
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/topdata/wiki_deploy.go`
|
||||
- Modify: `internal/topdata/wiki_deploy_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing live purge test**
|
||||
|
||||
Add a live stale purge test in `internal/topdata/wiki_deploy_test.go`:
|
||||
|
||||
```go
|
||||
func TestDeployWikiPurgesTrackedStaleGeneratedTopic(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sourceDir := filepath.Join(root, "pages")
|
||||
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
|
||||
t.Fatalf("create source dir: %v", err)
|
||||
}
|
||||
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
||||
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
|
||||
Version: "nodebb-v1",
|
||||
Pages: map[string]wikiDeployManifestPage{
|
||||
"skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3},
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write deploy manifest: %v", err)
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodDelete || r.URL.Path != "/api/v3/topics/7" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]any{"code": "ok"}})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
||||
SourceDir: sourceDir,
|
||||
Endpoint: server.URL,
|
||||
Token: "nodebb-token",
|
||||
ManifestPath: manifestPath,
|
||||
StalePolicy: "purge",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("DeployWikiWithOptions stale purge failed: %v", err)
|
||||
}
|
||||
if result.Stale != 1 || result.Purged != 1 {
|
||||
t.Fatalf("expected one stale purge, got %#v", result)
|
||||
}
|
||||
if _, ok := loadDeployManifest(manifestPath).Pages["skills:retired"]; ok {
|
||||
t.Fatalf("expected purged stale manifest entry to be removed")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This route is the NodeBB core purge route. `DELETE /api/v3/topics/:tid/state` is the separate soft-delete route and must not be used here.
|
||||
|
||||
- [ ] **Step 2: Run the live purge test to verify it fails**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/topdata -run 'TestDeployWiki.*Live.*Purge|TestDeployWiki.*Purges.*Stale' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL because purge plans are not yet executed by the NodeBB client.
|
||||
|
||||
- [ ] **Step 3: Add the NodeBB purge client method**
|
||||
|
||||
Place a thin method near `updatePost` and `renameWikiPage` in `internal/topdata/wiki_deploy.go`:
|
||||
|
||||
```go
|
||||
func (c *nodeBBClient) purgeTopic(tid int) error {
|
||||
if tid == 0 {
|
||||
return fmt.Errorf("NodeBB topic purge requires topic id")
|
||||
}
|
||||
return c.request(http.MethodDelete, fmt.Sprintf("/api/v3/topics/%d", tid), nil, nil)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Execute purge plans**
|
||||
|
||||
In the live plan execution switch, add:
|
||||
|
||||
```go
|
||||
case "purge":
|
||||
if err := client.purgeTopic(plan.Entry.TID); err != nil {
|
||||
return result, fmt.Errorf("deploy wiki page %q: purge NodeBB topic %d: %w", plan.Page.PageID, plan.Entry.TID, err)
|
||||
}
|
||||
```
|
||||
|
||||
Do not acquire wiki edit locks for purge: the NodeBB topic purge endpoint owns its own privilege checks.
|
||||
|
||||
- [ ] **Step 5: Keep current generated pages out of purge coverage**
|
||||
|
||||
Add this test so a current generated page with the same manifest page id is skipped instead of purged:
|
||||
|
||||
```go
|
||||
func TestDeployWikiPurgeDoesNotTargetCurrentGeneratedPages(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sourceDir := filepath.Join(root, "pages")
|
||||
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
|
||||
t.Fatalf("create source dir: %v", err)
|
||||
}
|
||||
generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Athletics\n\nGenerated athletics page\n[//]: # (sow-topdata-wiki:managed:end)\n"
|
||||
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.md"), []byte(generated), 0644); err != nil {
|
||||
t.Fatalf("write source page: %v", err)
|
||||
}
|
||||
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
||||
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
|
||||
Version: "nodebb-v1",
|
||||
Pages: map[string]wikiDeployManifestPage{
|
||||
"skills:athletics": {
|
||||
Hash: computeManagedHash(generated),
|
||||
Title: "Athletics",
|
||||
Namespace: "skills",
|
||||
TID: 7,
|
||||
PID: 42,
|
||||
CID: 3,
|
||||
},
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write deploy manifest: %v", err)
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Fatalf("current generated page must not call NodeBB during matching-hash deploy, got %s %s", r.Method, r.URL.Path)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
||||
SourceDir: sourceDir,
|
||||
Endpoint: server.URL,
|
||||
Token: "nodebb-token",
|
||||
ManifestPath: manifestPath,
|
||||
StalePolicy: "purge",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("DeployWikiWithOptions current page purge deploy failed: %v", err)
|
||||
}
|
||||
if result.Stale != 0 || result.Purged != 0 || result.Skipped != 1 {
|
||||
t.Fatalf("expected current generated page to skip purge, got %#v", result)
|
||||
}
|
||||
if _, ok := loadDeployManifest(manifestPath).Pages["skills:athletics"]; !ok {
|
||||
t.Fatalf("expected current generated page to remain in deploy manifest")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Run deploy tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/topdata -run 'TestDeployWiki' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 7: Commit purge execution**
|
||||
|
||||
```bash
|
||||
git add internal/topdata/wiki_deploy.go internal/topdata/wiki_deploy_test.go
|
||||
git commit -m "feat: purge stale generated wiki topics"
|
||||
```
|
||||
|
||||
## Task 4: Surface Purge In Toolkit CLI And Docs
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/app/app.go`
|
||||
- Modify: `internal/app/app_test.go`
|
||||
- Modify: `README.md`
|
||||
|
||||
- [ ] **Step 1: Write failing CLI-output coverage**
|
||||
|
||||
In `internal/app/app_test.go`, extend `TestTopdataConsoleDebugProgressAndRelativePaths` so the sample progress and emit call include a purge count:
|
||||
|
||||
```go
|
||||
console.progress("NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, purge 6, drift 0")
|
||||
console.emitWikiDeployResult(10, 1, 2, 3, 4, 5, 6, 0, "/workspace/project/build/wiki/deploy-manifest.json")
|
||||
```
|
||||
|
||||
Assert:
|
||||
|
||||
```go
|
||||
if !strings.Contains(output, "purged: 6") {
|
||||
t.Fatalf("expected purged deploy count, got %q", output)
|
||||
}
|
||||
```
|
||||
|
||||
Update the debug progress expectation to include `purge 6`.
|
||||
|
||||
- [ ] **Step 2: Run focused app tests to verify failure**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/app -run TestTopdataConsoleDebugProgressAndRelativePaths -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL because `emitWikiDeployResult` does not yet take a purge count.
|
||||
|
||||
- [ ] **Step 3: Print purge counts**
|
||||
|
||||
Update the emit signature and output in `internal/app/app.go`:
|
||||
|
||||
```go
|
||||
func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skipped, stale, archived, purged, drifted int, manifest string) {
|
||||
spin.linebreak()
|
||||
fmt.Fprintln(c.stdout, "Deploy Wiki ----------")
|
||||
fmt.Fprintf(c.stdout, "project: %s\n", c.projectName)
|
||||
fmt.Fprintf(c.stdout, "local pages: %d\n", localPages)
|
||||
fmt.Fprintf(c.stdout, "created: %d\n", created)
|
||||
fmt.Fprintf(c.stdout, "updated: %d\n", updated)
|
||||
fmt.Fprintf(c.stdout, "skipped: %d\n", skipped)
|
||||
fmt.Fprintf(c.stdout, "stale: %d\n", stale)
|
||||
fmt.Fprintf(c.stdout, "archived: %d\n", archived)
|
||||
fmt.Fprintf(c.stdout, "purged: %d\n", purged)
|
||||
fmt.Fprintf(c.stdout, "drifted: %d\n", drifted)
|
||||
fmt.Fprintf(c.stdout, "manifest: %s\n", c.relPath(manifest))
|
||||
}
|
||||
```
|
||||
|
||||
Pass `result.Purged` from the deploy command call site.
|
||||
|
||||
- [ ] **Step 4: Update CLI usage text**
|
||||
|
||||
Change deploy-wiki help text in `internal/app/app.go` from:
|
||||
|
||||
```go
|
||||
[--stale-policy <report|archive>]
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```go
|
||||
[--stale-policy <report|archive|purge>]
|
||||
```
|
||||
|
||||
Add this focused help assertion in `internal/app/app_test.go`:
|
||||
|
||||
```go
|
||||
func TestParseDeployWikiHelpListsPurgeStalePolicy(t *testing.T) {
|
||||
_, err := parseDeployWikiArgs("deploy-wiki", []string{"--help"})
|
||||
if err == nil || !strings.Contains(err.Error(), "--stale-policy <report|archive|purge>") {
|
||||
t.Fatalf("expected deploy-wiki help to list purge stale policy, got %v", err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Update toolkit README**
|
||||
|
||||
In `README.md`, update the `deploy-wiki` usage and stale-page explanation so it states:
|
||||
|
||||
```markdown
|
||||
`--stale-policy purge` permanently removes only stale generated topics already
|
||||
tracked in the wiki deploy manifest. Dry-run that policy before the live run;
|
||||
it never discovers manual wiki pages from NodeBB for purge.
|
||||
```
|
||||
|
||||
Keep `report` as the documented default and `archive` as the nondestructive cleanup option.
|
||||
|
||||
- [ ] **Step 6: Run app tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/app -count=1
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 7: Commit toolkit CLI/docs**
|
||||
|
||||
```bash
|
||||
git add internal/app/app.go internal/app/app_test.go README.md
|
||||
git commit -m "docs: expose wiki stale purge policy"
|
||||
```
|
||||
|
||||
## Task 5: Expose Purge Through Module Release Workflow
|
||||
|
||||
**Files:**
|
||||
- Modify: `../module/scripts/release-all.sh`
|
||||
- Modify: `../module/README.md`
|
||||
- Modify: `../module/topdata/README.md`
|
||||
- Modify: `../module/tests/release-all-cache-cleanup.sh`
|
||||
|
||||
- [ ] **Step 1: Write failing release wrapper coverage**
|
||||
|
||||
In `../module/tests/release-all-cache-cleanup.sh`, set the stale policy in the existing release invocation:
|
||||
|
||||
```bash
|
||||
SOW_MODULE_WIKI_DEPLOY_STALE_POLICY="purge" \
|
||||
TAG_NAME="v-test" \
|
||||
"${CHECKOUT}/scripts/release-all.sh" >"${TEST_TMP}/stdout.log" 2>"${TEST_TMP}/stderr.log"
|
||||
```
|
||||
|
||||
Update the wiki log expectations from:
|
||||
|
||||
```bash
|
||||
grep -Fxq "deploy-wiki --dry-run --create" "${TEST_TMP}/wiki.log" \
|
||||
|| fail "wiki dry run was not invoked"
|
||||
grep -Fxq "deploy-wiki --create" "${TEST_TMP}/wiki.log" \
|
||||
|| fail "wiki deploy was not invoked"
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```bash
|
||||
grep -Fxq "deploy-wiki --dry-run --create --stale-policy purge" "${TEST_TMP}/wiki.log" \
|
||||
|| fail "wiki stale purge dry run was not invoked"
|
||||
grep -Fxq "deploy-wiki --create --stale-policy purge" "${TEST_TMP}/wiki.log" \
|
||||
|| fail "wiki stale purge deploy was not invoked"
|
||||
```
|
||||
|
||||
Update the API ordering assertion to match the new dry-run log line:
|
||||
|
||||
```awk
|
||||
$0 == "WIKI deploy-wiki --dry-run --create --stale-policy purge" { wiki_dry_run = NR }
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the release wrapper test to verify it fails**
|
||||
|
||||
Run from `../module`:
|
||||
|
||||
```bash
|
||||
./tests/release-all-cache-cleanup.sh
|
||||
```
|
||||
|
||||
Expected: FAIL because `scripts/release-all.sh` rejects `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY=purge`.
|
||||
|
||||
- [ ] **Step 3: Update release wrapper validation**
|
||||
|
||||
In `../module/scripts/release-all.sh`, change:
|
||||
|
||||
```bash
|
||||
if [[ "${SOW_MODULE_WIKI_DEPLOY_STALE_POLICY}" != "report" && "${SOW_MODULE_WIKI_DEPLOY_STALE_POLICY}" != "archive" ]]; then
|
||||
echo "Unsupported SOW_MODULE_WIKI_DEPLOY_STALE_POLICY=${SOW_MODULE_WIKI_DEPLOY_STALE_POLICY}; expected report or archive" >&2
|
||||
return 1
|
||||
fi
|
||||
```
|
||||
|
||||
to:
|
||||
|
||||
```bash
|
||||
if [[ "${SOW_MODULE_WIKI_DEPLOY_STALE_POLICY}" != "report" && "${SOW_MODULE_WIKI_DEPLOY_STALE_POLICY}" != "archive" && "${SOW_MODULE_WIKI_DEPLOY_STALE_POLICY}" != "purge" ]]; then
|
||||
echo "Unsupported SOW_MODULE_WIKI_DEPLOY_STALE_POLICY=${SOW_MODULE_WIKI_DEPLOY_STALE_POLICY}; expected report, archive, or purge" >&2
|
||||
return 1
|
||||
fi
|
||||
```
|
||||
|
||||
The existing dry-run/live deploy argument plumbing already forwards every non-`report` stale policy through `--stale-policy`.
|
||||
|
||||
- [ ] **Step 4: Run the release wrapper test**
|
||||
|
||||
Run from `../module`:
|
||||
|
||||
```bash
|
||||
./tests/release-all-cache-cleanup.sh
|
||||
```
|
||||
|
||||
Expected: PASS and both release wiki deploy phases include `--stale-policy purge`.
|
||||
|
||||
- [ ] **Step 5: Document release controls**
|
||||
|
||||
In `../module/README.md`, revise the environment variable bullet to name both destructive modes explicitly:
|
||||
|
||||
```markdown
|
||||
- `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY` (defaults to `report`; use `archive`
|
||||
after a controlled dry run to retire stale generated pages in place, or
|
||||
`purge` after a controlled dry run to permanently remove stale generated
|
||||
topics already tracked in the wiki deploy manifest)
|
||||
```
|
||||
|
||||
In the wiki deployment paragraph, replace the archive-only sentence with a sentence that keeps these boundaries:
|
||||
|
||||
- `report` is default
|
||||
- `archive` rewrites stale generated pages in place
|
||||
- `purge` permanently removes only manifest-tracked stale generated topics
|
||||
- manual NodeBB wiki pages are not purge candidates
|
||||
|
||||
- [ ] **Step 6: Document local deploy controls**
|
||||
|
||||
In `../module/topdata/README.md`, replace:
|
||||
|
||||
```markdown
|
||||
reports stale manifest entries by default and can archive them in place with `--stale-policy archive`; it does not delete topics
|
||||
```
|
||||
|
||||
with:
|
||||
|
||||
```markdown
|
||||
reports stale manifest entries by default, can archive them in place with
|
||||
`--stale-policy archive`, and can permanently purge only manifest-tracked stale
|
||||
generated topics with `--stale-policy purge` after a dry run
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Check module diffs**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
git -C ../module diff --check
|
||||
git -C ../module diff -- scripts/release-all.sh tests/release-all-cache-cleanup.sh README.md topdata/README.md
|
||||
```
|
||||
|
||||
Expected: no whitespace failures; diff shows only wrapper policy vocabulary, wrapper coverage, and docs.
|
||||
|
||||
- [ ] **Step 8: Commit module wrapper/docs**
|
||||
|
||||
Run from `../module`:
|
||||
|
||||
```bash
|
||||
git add scripts/release-all.sh tests/release-all-cache-cleanup.sh README.md topdata/README.md
|
||||
git commit -m "docs: allow generated wiki stale purges"
|
||||
```
|
||||
|
||||
## Task 6: Full Verification And Operator Notes
|
||||
|
||||
**Files:**
|
||||
- Verify: toolkit and module working trees
|
||||
|
||||
- [ ] **Step 1: Format Go changes**
|
||||
|
||||
Run from toolkit:
|
||||
|
||||
```bash
|
||||
gofmt -w internal/project/project.go internal/project/project_test.go internal/topdata/wiki_deploy.go internal/topdata/wiki_deploy_test.go internal/app/app.go internal/app/app_test.go
|
||||
```
|
||||
|
||||
Expected: command exits 0.
|
||||
|
||||
- [ ] **Step 2: Run toolkit verification**
|
||||
|
||||
Run from toolkit:
|
||||
|
||||
```bash
|
||||
go test ./internal/project ./internal/topdata ./internal/app
|
||||
go test ./...
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Expected: all tests pass and diff check emits no whitespace errors.
|
||||
|
||||
- [ ] **Step 3: Build the toolkit development binary**
|
||||
|
||||
Run from toolkit:
|
||||
|
||||
```bash
|
||||
./build-tool.sh
|
||||
```
|
||||
|
||||
Expected: `tools/sow-toolkit` is rebuilt successfully for module wrapper verification.
|
||||
|
||||
- [ ] **Step 4: Run module verification with the development binary**
|
||||
|
||||
Run from module:
|
||||
|
||||
```bash
|
||||
SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./validate-topdata.sh
|
||||
SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Expected: topdata validation and wiki build pass; module diff check emits no whitespace errors.
|
||||
|
||||
- [ ] **Step 5: Record the cleanup command in the final handoff**
|
||||
|
||||
The final implementation response must state that destructive cleanup is opt-in and show the operator sequence:
|
||||
|
||||
```bash
|
||||
./deploy-wiki.sh --dry-run --stale-policy purge
|
||||
./deploy-wiki.sh --stale-policy purge
|
||||
```
|
||||
|
||||
For tag-triggered module release automation, state the equivalent variable:
|
||||
|
||||
```bash
|
||||
SOW_MODULE_WIKI_DEPLOY_STALE_POLICY=purge
|
||||
```
|
||||
|
||||
The handoff must mention that purge targets only stale generated pages already tracked in the deploy manifest and permanently removes their NodeBB topics/posts.
|
||||
@@ -1,200 +0,0 @@
|
||||
# Class Progression Feat Projections Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make class progression feat grouping configurable from wiki YAML so the module template can render granted feats and level-gated available feats separately.
|
||||
|
||||
**Architecture:** Extend the `class_progression` data provider config with named class-feat projection fields. The toolkit evaluates each configured projection against class feat rows and groups resolved feat links onto progression rows by level, while the module wiki config declares the project-specific `List` rules and the class template owns output wording.
|
||||
|
||||
**Tech Stack:** Go `sow-toolkit`, native topdata wiki YAML, HTML wiki templates, Go tests.
|
||||
|
||||
---
|
||||
|
||||
## File Map
|
||||
|
||||
- `internal/topdata/wiki_data_providers.go`: parse and validate data-provider projection configuration and pass it into class progression row assembly.
|
||||
- `internal/topdata/wiki_tables.go`: share class progression row assembly with configured class feat projections and keep default compatibility behavior.
|
||||
- `internal/topdata/wiki_native_test.go`: regression coverage for provider projection parsing, filtering, grouping, ordering, invalid config, and rendered template fields.
|
||||
- `../module/topdata/wiki/data.yaml`: declare module-owned `GrantedFeats` and `AvailableFeats` projections for class progression.
|
||||
- `../module/topdata/wiki/templates/pages/classes.html`: render bonus feats inline and use projected available feats in the Notes cell.
|
||||
- `../module/topdata/wiki/README.md`: document configurable data-provider projections for template authors.
|
||||
|
||||
### Task 1: Add Configured Class Feat Projection Tests
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/topdata/wiki_native_test.go`
|
||||
|
||||
- [ ] **Step 1: Write failing provider projection tests**
|
||||
|
||||
Add tests that build `wikiDataProviderDefinition` from YAML such as:
|
||||
|
||||
```yaml
|
||||
providers:
|
||||
ClassProgression:
|
||||
source: class_progression
|
||||
levels: 1-4
|
||||
class_feats:
|
||||
fields:
|
||||
GrantedFeats:
|
||||
when: "{{GrantedOnLevel > 0 && List == 3}}"
|
||||
AvailableFeats:
|
||||
when: "{{GrantedOnLevel > 0 && List != 3}}"
|
||||
```
|
||||
|
||||
Assert the parsed provider can render a progression row where `GrantedFeats`
|
||||
contains `List == 3` feat links, `AvailableFeats` contains positive-level
|
||||
non-`3` feat links, matched link order follows fixture source row order, and
|
||||
`BonusFeat` is still present.
|
||||
|
||||
- [ ] **Step 2: Add failing invalid configuration coverage**
|
||||
|
||||
Add a test that loads a provider field with an empty projection name or empty
|
||||
`when` expression and expects an error containing `data.yaml`, the provider
|
||||
name, and the projection field context.
|
||||
|
||||
- [ ] **Step 3: Run focused tests to verify RED**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/topdata -run 'TestWiki.*ClassProgression.*Projection|TestWikiData.*Provider.*Projection' -count=1
|
||||
```
|
||||
|
||||
Expected: FAIL because `wikiDataProviderDefinition` does not yet accept
|
||||
`class_feats.fields` and progression rows do not expose `AvailableFeats`.
|
||||
|
||||
### Task 2: Implement Generic Provider Projections
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/topdata/wiki_data_providers.go`
|
||||
- Modify: `internal/topdata/wiki_tables.go`
|
||||
- Test: `internal/topdata/wiki_native_test.go`
|
||||
|
||||
- [ ] **Step 1: Extend provider config types and validation**
|
||||
|
||||
Add typed YAML support for:
|
||||
|
||||
```go
|
||||
type wikiDataProviderClassFeatConfig struct {
|
||||
Fields map[string]wikiDataProviderProjectionField `json:"fields" yaml:"fields"`
|
||||
}
|
||||
|
||||
type wikiDataProviderProjectionField struct {
|
||||
When string `json:"when" yaml:"when"`
|
||||
}
|
||||
```
|
||||
|
||||
Validate each configured projection field name and `when` expression before
|
||||
provider execution.
|
||||
|
||||
- [ ] **Step 2: Carry projections into progression row assembly**
|
||||
|
||||
Change the provider path so `ClassProgression` passes provider-owned class feat
|
||||
projection fields into progression row assembly. Preserve table-based
|
||||
`class_progression` callers by giving them the current default `GrantedFeats`
|
||||
behavior.
|
||||
|
||||
- [ ] **Step 3: Evaluate and group projections**
|
||||
|
||||
In progression row assembly, resolve class feat links once per matching row,
|
||||
evaluate configured `when` expressions against class feat rows, and append
|
||||
matched links to the configured progression field for each positive
|
||||
`GrantedOnLevel` value in source row order.
|
||||
|
||||
- [ ] **Step 4: Run focused tests to verify GREEN**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/topdata -run 'TestWiki.*ClassProgression.*Projection|TestWikiData.*Provider.*Projection' -count=1
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
### Task 3: Configure Module Wiki Output
|
||||
|
||||
**Files:**
|
||||
- Modify: `../module/topdata/wiki/data.yaml`
|
||||
- Modify: `../module/topdata/wiki/templates/pages/classes.html`
|
||||
- Modify: `../module/topdata/wiki/README.md`
|
||||
|
||||
- [ ] **Step 1: Declare module projections**
|
||||
|
||||
Update `ClassProgression` in `../module/topdata/wiki/data.yaml` so module YAML
|
||||
owns `GrantedFeats` and `AvailableFeats` selection:
|
||||
|
||||
```yaml
|
||||
ClassProgression:
|
||||
source: class_progression
|
||||
levels: 1-12
|
||||
class_feats:
|
||||
fields:
|
||||
GrantedFeats:
|
||||
when: "{{GrantedOnLevel > 0 && List == 3}}"
|
||||
AvailableFeats:
|
||||
when: "{{GrantedOnLevel > 0 && List != 3}}"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Render projected fields in the class template**
|
||||
|
||||
Keep bonus feats in the Feats cell with a conditional comma. Render
|
||||
`AvailableFeats` in the Notes cell with module-owned “becomes available”
|
||||
wording using supported template loop/filter syntax.
|
||||
|
||||
- [ ] **Step 3: Document the projection surface**
|
||||
|
||||
Add a concise `data.yaml` note to the wiki README explaining that data-provider
|
||||
projection fields can expose configured per-level class feat lists to page
|
||||
templates and that wording stays in templates.
|
||||
|
||||
### Task 4: Verify The Cross-Repo Wiki Path
|
||||
|
||||
**Files:**
|
||||
- Verify: `internal/topdata/...`
|
||||
- Verify: `../module/topdata/wiki/...`
|
||||
|
||||
- [ ] **Step 1: Run toolkit regression tests**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
go test ./internal/topdata -count=1
|
||||
```
|
||||
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 2: Build the local toolkit**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
./build-tool.sh
|
||||
```
|
||||
|
||||
Expected: exit `0` and refresh `tools/sow-toolkit` when source changes require
|
||||
it.
|
||||
|
||||
- [ ] **Step 3: Build module wiki with the local toolkit**
|
||||
|
||||
Run from `../module`:
|
||||
|
||||
```bash
|
||||
SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force
|
||||
```
|
||||
|
||||
Expected: exit `0` and regenerate class wiki pages under `.cache/wiki/pages/`.
|
||||
|
||||
- [ ] **Step 4: Inspect fighter output**
|
||||
|
||||
Check generated fighter progression output for inline bonus feat text in Feats
|
||||
cells and `becomes available` notes for level-gated selectable feat links.
|
||||
|
||||
- [ ] **Step 5: Run diff checks**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
git diff --check
|
||||
```
|
||||
|
||||
in both toolkit and module repos.
|
||||
@@ -1,47 +0,0 @@
|
||||
# Wiki List Columns Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add a template-driven topdata wiki option for splitting provider-backed lists into configured columns with a fixed item count and optional forced empty columns.
|
||||
|
||||
**Architecture:** The toolkit wiki renderer gains a generic `columns` data-provider source that wraps another provider and exposes one row per rendered column. Each column row exposes `Items`, `Index`, and count metadata so page templates own markup while YAML owns row selection and chunking behavior. The module class page uses this provider for `ClassSkills`.
|
||||
|
||||
**Tech Stack:** Go `sow-toolkit`, YAML topdata wiki config, local HTML templates, Go tests.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Renderer Provider
|
||||
|
||||
**Files:**
|
||||
- Modify: `toolkit/internal/topdata/wiki_data_providers.go`
|
||||
- Modify: `toolkit/internal/topdata/wiki_template.go`
|
||||
- Test: `toolkit/internal/topdata/wiki_native_test.go`
|
||||
|
||||
- [ ] Write failing tests for a `columns` provider that chunks `ClassSkills` into rows with nested `Items`, including forced empty columns.
|
||||
- [ ] Run `go test ./internal/topdata -run 'TestWikiTemplate.*Column|TestBuildNativeWikiRendersClassSkillColumns'` and confirm the new assertions fail because `columns` is not supported.
|
||||
- [ ] Add `provider`, `columns`, `items_per_column`, and `force_columns` fields to `wikiDataProviderDefinition`.
|
||||
- [ ] Validate that `columns` providers name an existing provider, use positive integer `columns` and `items_per_column`, and do not point at themselves.
|
||||
- [ ] Allow nested `{{#each Items}}` loops by teaching template `#each` to iterate an array field from the current row before falling back to named providers.
|
||||
- [ ] Implement deterministic chunking: each column receives up to `items_per_column`; extra source items continue into additional overflow columns; `force_columns: true` emits empty configured columns when there are fewer items.
|
||||
|
||||
### Task 2: Module Template
|
||||
|
||||
**Files:**
|
||||
- Modify: `module/topdata/wiki/data.yaml`
|
||||
- Modify: `module/topdata/wiki/templates/pages/classes.html`
|
||||
- Modify: `module/topdata/wiki/README.md`
|
||||
|
||||
- [ ] Add `ClassSkillColumns` using `source: columns`, `provider: ClassSkills`, `columns: 3`, `items_per_column: 12`, and `force_columns: true`.
|
||||
- [ ] Replace the flat class skill `<ul>` with a `.wiki-list-columns .wiki-list-columns--3` wrapper and nested `{{#each Items}}` loops.
|
||||
- [ ] Document the generic `columns` provider fields for template authors.
|
||||
|
||||
### Task 3: Verification
|
||||
|
||||
**Files:**
|
||||
- Verify: `toolkit/internal/topdata/...`
|
||||
- Verify: `module/topdata/wiki/...`
|
||||
|
||||
- [ ] Run `go test ./internal/topdata -run 'TestWikiTemplate|TestBuildNativeWiki'`.
|
||||
- [ ] Run `./build-tool.sh` from `toolkit/`.
|
||||
- [ ] Run `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force` from `module/`.
|
||||
- [ ] Inspect generated class HTML for `wiki-list-columns--3`, three configured class-skill lists for shorter classes, and overflow columns for long lists.
|
||||
@@ -1,153 +0,0 @@
|
||||
# Generated Wiki Stale Purge Design
|
||||
|
||||
## Goal
|
||||
|
||||
Add an explicit bulk purge path for generated NodeBB wiki pages that have become
|
||||
stale after generator output changes, such as topdata visibility rules hiding
|
||||
previously generated pages.
|
||||
|
||||
## Context
|
||||
|
||||
The topdata wiki deployer already owns generated page state through
|
||||
`.cache/wiki/.wiki_deploy_manifest.json`. It currently recognizes stale
|
||||
generated pages that remain in the deploy manifest but no longer exist in the
|
||||
current generated wiki output. Existing stale policies can either report those
|
||||
pages or archive them by rewriting their managed content.
|
||||
|
||||
That is not sufficient when the stale generated wiki topics should be removed
|
||||
from NodeBB entirely. The Westgate Wiki plugin already treats a normal wiki topic
|
||||
delete as an irreversible purge, so the deployer should use NodeBB's explicit
|
||||
topic purge API when the operator asks for destructive generated-page cleanup.
|
||||
|
||||
## Scope
|
||||
|
||||
### In Scope
|
||||
|
||||
- Add `purge` as an explicit stale policy for topdata wiki deployment.
|
||||
- Plan purge candidates only from generated page entries already tracked in the
|
||||
wiki deploy manifest.
|
||||
- Purge the exact tracked NodeBB topic for each stale generated page.
|
||||
- Report purge counts in dry-run and live deploy output.
|
||||
- Remove successfully purged page entries from the deploy manifest so later
|
||||
deploys do not repeatedly target removed topics.
|
||||
- Allow release automation to opt into purge through the existing stale-policy
|
||||
environment variable path.
|
||||
- Update toolkit and module documentation for the destructive cleanup mode.
|
||||
|
||||
### Out Of Scope
|
||||
|
||||
- Purging manually-authored NodeBB wiki pages.
|
||||
- Discovering purge candidates by namespace browsing, title, slug, or wiki
|
||||
search.
|
||||
- Adding a soft-delete or restore workflow for stale generated wiki pages.
|
||||
- Changing the Westgate Wiki plugin delete behavior.
|
||||
- Adding a separate stale cleanup command outside `deploy-wiki`.
|
||||
|
||||
## Operator Interface
|
||||
|
||||
The existing `deploy-wiki` stale-policy surface gains a third explicit policy:
|
||||
|
||||
```bash
|
||||
./deploy-wiki.sh --dry-run --stale-policy purge
|
||||
./deploy-wiki.sh --stale-policy purge
|
||||
```
|
||||
|
||||
Release automation may use the same policy through:
|
||||
|
||||
```bash
|
||||
SOW_MODULE_WIKI_DEPLOY_STALE_POLICY=purge
|
||||
```
|
||||
|
||||
`purge` remains opt-in. Existing `report` and `archive` behavior remain
|
||||
unchanged.
|
||||
|
||||
## Purge Eligibility
|
||||
|
||||
A page may be purged only when all of these are true:
|
||||
|
||||
1. The page exists in `.cache/wiki/.wiki_deploy_manifest.json`.
|
||||
2. The page is absent from the current generated wiki page set in the deploy
|
||||
scope.
|
||||
3. The deploy manifest entry has the tracked NodeBB topic id required to call
|
||||
the topic purge API.
|
||||
|
||||
The deployer must not search NodeBB for candidate pages when purging stale
|
||||
generated output. It must not purge pages that are currently generated. It must
|
||||
not purge pages that only exist as manually-authored NodeBB wiki content.
|
||||
|
||||
If a stale manifest entry has no topic id, purge planning fails clearly instead
|
||||
of guessing a remote target.
|
||||
|
||||
## Deploy Flow
|
||||
|
||||
Stale planning stays inside the existing `deploy-wiki` deploy plan:
|
||||
|
||||
- `report` increments the stale count only.
|
||||
- `archive` produces the existing archive post-update action.
|
||||
- `purge` produces a destructive topic-purge action for the manifest entry's
|
||||
tracked topic id.
|
||||
|
||||
Dry runs calculate the same stale and purge counts as live runs but do not call
|
||||
NodeBB and do not mutate the deploy manifest.
|
||||
|
||||
Live purge execution calls NodeBB's topic purge API for each planned stale
|
||||
generated page. NodeBB auth, permission, missing-topic, or transport failures
|
||||
abort the deploy with the failing generated page id and remote topic context.
|
||||
|
||||
## Manifest State
|
||||
|
||||
Archive retains stale entries in the manifest because the archived post remains
|
||||
managed remotely.
|
||||
|
||||
Purge removes each successfully purged page entry from the next deploy manifest.
|
||||
This records that the generated page no longer has a managed remote NodeBB topic
|
||||
and prevents later deploys from attempting to purge an already removed topic.
|
||||
|
||||
If a purge deployment fails partway through, only state written after the
|
||||
existing deploy execution succeeds should reach the manifest. A later rerun may
|
||||
retry the stale manifest entries still present, matching the deployer's current
|
||||
all-or-error manifest persistence behavior.
|
||||
|
||||
## Reporting And Validation
|
||||
|
||||
Toolkit deploy output gains a `purged` count alongside the existing `stale` and
|
||||
`archived` counts. Dry-run summaries must make the destructive plan visible
|
||||
before a live purge run.
|
||||
|
||||
`purge` must be accepted wherever stale policy is validated:
|
||||
|
||||
- project config stale-page settings
|
||||
- `deploy-wiki --stale-policy`
|
||||
- module release wrapper validation for
|
||||
`SOW_MODULE_WIKI_DEPLOY_STALE_POLICY`
|
||||
|
||||
Unsupported stale policy values must keep failing early with clear error text.
|
||||
|
||||
## Testing
|
||||
|
||||
Toolkit tests should cover:
|
||||
|
||||
- stale policy validation accepts `purge`
|
||||
- unsupported stale policy values still fail
|
||||
- dry-run purge reports stale and purged counts without calling NodeBB or
|
||||
changing the manifest
|
||||
- live purge calls the NodeBB topic purge endpoint for a stale tracked generated
|
||||
topic
|
||||
- live purge removes the manifest entry after success
|
||||
- purge refuses a stale manifest entry without a tracked topic id
|
||||
- current generated pages are never purge candidates
|
||||
- archive/report behavior remains unchanged
|
||||
|
||||
Module wrapper/docs coverage should verify:
|
||||
|
||||
- release stale-policy validation accepts `purge`
|
||||
- docs describe purge as destructive, generated-manifest-scoped cleanup
|
||||
|
||||
## Compatibility
|
||||
|
||||
Default behavior remains `report`. Existing archive cleanup remains available and
|
||||
unchanged. The destructive purge path activates only when the operator selects
|
||||
`purge`.
|
||||
|
||||
The purge boundary is intentionally stricter than normal page adoption logic:
|
||||
generated manifest state authorizes the target; NodeBB discovery never does.
|
||||
@@ -1,105 +0,0 @@
|
||||
# Class Progression Feat Availability Design
|
||||
|
||||
## Goal
|
||||
|
||||
Class progression wiki providers should let module-owned wiki configuration
|
||||
distinguish automatically granted class feats from level-gated selectable feats
|
||||
while also exposing bonus feat gains declared by class `bfeat` tables in the
|
||||
progression table.
|
||||
|
||||
## Ownership
|
||||
|
||||
The authored wiki page layout and progression projection policy stay in
|
||||
`sow-module/topdata/wiki/`. The toolkit owns generic progression row data
|
||||
assembly from configured class feat and bonus-feat sources. The change should
|
||||
stay inside those boundaries:
|
||||
|
||||
- module wiki YAML decides which class feat rows project into progression
|
||||
fields;
|
||||
- module page templates decide how progression row fields are rendered;
|
||||
- toolkit progression row assembly groups configured projections by level
|
||||
without hardcoding the module's class feat list semantics or note wording.
|
||||
|
||||
## Behavior
|
||||
|
||||
Class feat rows with positive `GrantedOnLevel` values are not all equivalent.
|
||||
This module uses rows in class feat tables with `List` value `3` as
|
||||
automatically granted feats and positive-level selectable rows as feats that
|
||||
become available at the declared level. That distinction is repository policy
|
||||
and must be declared in wiki YAML rather than hidden in toolkit code.
|
||||
|
||||
The `ClassProgression` data provider should support configured class feat
|
||||
projection fields. This module can declare a shape such as:
|
||||
|
||||
```yaml
|
||||
providers:
|
||||
ClassProgression:
|
||||
source: class_progression
|
||||
levels: 1-12
|
||||
class_feats:
|
||||
fields:
|
||||
GrantedFeats:
|
||||
when: "{{GrantedOnLevel > 0 && List == 3}}"
|
||||
AvailableFeats:
|
||||
when: "{{GrantedOnLevel > 0 && List != 3}}"
|
||||
```
|
||||
|
||||
For each progression row, the toolkit should:
|
||||
|
||||
- continue exposing the per-level bonus feat count from the class `bfeat` table
|
||||
as `BonusFeat`;
|
||||
- expose each configured class feat field as a deterministic list of resolved
|
||||
wiki feat links for class feat rows whose configured expression matches the
|
||||
progression level;
|
||||
- leave non-matching class feat rows out of that configured field.
|
||||
|
||||
The default `class_progression` provider behavior should preserve the current
|
||||
`GrantedFeats` projection when no custom field configuration is declared.
|
||||
|
||||
## Rendering
|
||||
|
||||
The class progression template can render `BonusFeat` inline in its existing
|
||||
Feats cell rather than adding another progression column. It should emit the
|
||||
separator only when `GrantedFeats` already rendered content so bonus-only rows
|
||||
do not start with a comma.
|
||||
|
||||
The existing Notes column should render `AvailableFeats` in the module template
|
||||
with module-owned wording such as `<feat link> becomes available.`. The lower
|
||||
class feat table remains useful for listing class feat membership and list
|
||||
metadata; its level label should not be interpreted as an automatic grant for
|
||||
selectable rows.
|
||||
|
||||
## Data Flow
|
||||
|
||||
The toolkit resolves the class `FeatsTable` while building progression rows.
|
||||
Each configured class feat field evaluates its `when` expression against class
|
||||
feat rows, groups matched rows by positive `GrantedOnLevel`, resolves feat
|
||||
references to wiki link text, and writes the grouped list onto the progression
|
||||
row under the configured field name. Output order follows source table row
|
||||
order so repeated builds are deterministic.
|
||||
|
||||
The class `BonusFeatsTable` lookup continues to populate `BonusFeat` from the
|
||||
row whose id matches the progression level.
|
||||
|
||||
## Errors And Compatibility
|
||||
|
||||
The change should preserve current behavior for missing class feat tables,
|
||||
missing bonus feat tables, unresolved feat references, and classes without
|
||||
custom class feat projections. Invalid projection configuration should fail
|
||||
early with the provider file path and field name. Existing module templates
|
||||
that only read `GrantedFeats`, `BonusFeat`, and `Notes` continue to use the same
|
||||
progression row fields.
|
||||
|
||||
## Tests
|
||||
|
||||
Toolkit coverage should verify:
|
||||
|
||||
- default progression behavior still exposes `GrantedFeats`;
|
||||
- configured class feat fields filter by expression and group by level;
|
||||
- configured output order follows source table row order;
|
||||
- invalid configured class feat fields fail clearly;
|
||||
- bonus feat counts remain available to the progression template.
|
||||
|
||||
Module validation should rebuild the generated wiki output with the local
|
||||
toolkit binary after the toolkit change and inspect a class that has both
|
||||
granted feats and level-gated selectable feats, such as fighter.
|
||||
Generated
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1780749050,
|
||||
"narHash": "sha256-3av0pIjlOWQ6rDbNOmpUSvbNnJkGORQKKjb4LtCZsIY=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "a799d3e3886da994fa307f817a6bc705ae538eeb",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
description = "sow-tools / Crucible — NWN build/conversion/sync toolchain (self-contained Go devshell)";
|
||||
|
||||
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||
|
||||
outputs = { self, nixpkgs }:
|
||||
let
|
||||
systems = [ "x86_64-linux" "aarch64-linux" ];
|
||||
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system});
|
||||
in
|
||||
{
|
||||
# Self-contained (D8): a host with ONLY nix can enter this shell and run
|
||||
# `make check`. ffmpeg is present because the migrated music pipeline needs
|
||||
# it; the scaffold itself is pure stdlib Go.
|
||||
devShells = forAllSystems (pkgs: {
|
||||
default = pkgs.mkShell {
|
||||
name = "sow-tools";
|
||||
packages = with pkgs; [
|
||||
go
|
||||
gopls
|
||||
gotools
|
||||
ffmpeg
|
||||
git
|
||||
gnumake
|
||||
bashInteractive
|
||||
shellcheck
|
||||
yamllint
|
||||
jq
|
||||
];
|
||||
shellHook = ''
|
||||
export GOCACHE="$PWD/.cache/go-build"
|
||||
echo "sow-tools / Crucible devshell (self-contained). $(go version 2>/dev/null)"
|
||||
echo " make check go vet + go test + shellcheck + yamllint"
|
||||
echo " make build build every cmd/* into ./bin"
|
||||
echo " make smoke build + run the binary smoke test"
|
||||
'';
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -1,8 +1,3 @@
|
||||
module gitea.westgate.pw/ShadowsOverWestgate/sow-tools
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
golang.org/x/text v0.35.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
-2113
File diff suppressed because it is too large
Load Diff
@@ -1,575 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline"
|
||||
)
|
||||
|
||||
func TestRunBuildTopPackageUsesCachedCompiledOutputs(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
mkdirAll(t, filepath.Join(root, ".cache", "2da"))
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui"))
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust"))
|
||||
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: test_module
|
||||
topdata:
|
||||
source: topdata
|
||||
build: .cache
|
||||
package_hak: sow_top.hak
|
||||
package_tlk: sow_tlk.tlk
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, ".cache", "2da", "repadjust.2da"), "2DA V2.0\n\n Label\n0 TEST_LABEL\n")
|
||||
writeFile(t, filepath.Join(root, "build", "sow_tlk.tlk"), "compiled tlk")
|
||||
writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.png"), "icon-data")
|
||||
writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{
|
||||
"output": "repadjust.2da",
|
||||
"columns": ["Label"],
|
||||
"rows": [{"id": 0, "Label": "TEST_LABEL"}]
|
||||
}`+"\n")
|
||||
|
||||
sourceTime := time.Now().Add(-2 * time.Hour)
|
||||
outputTime := time.Now().Add(-1 * time.Hour)
|
||||
setTreeTime(t, filepath.Join(root, "topdata"), sourceTime)
|
||||
setFileTime(t, filepath.Join(root, ".cache", "2da", "repadjust.2da"), outputTime)
|
||||
setFileTime(t, filepath.Join(root, "build", "sow_tlk.tlk"), outputTime)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"build-top-package"},
|
||||
}
|
||||
|
||||
if err := runBuildTopPackage(ctx); err != nil {
|
||||
t.Fatalf("runBuildTopPackage failed: %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "top package hak: build/sow_top.hak") {
|
||||
t.Fatalf("expected build-top-package output, got %q", output)
|
||||
}
|
||||
if strings.Contains(output, "[build-top-package]") {
|
||||
t.Fatalf("did not expect raw progress lines in normal output, got %q", output)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "build", "sow_top.hak")); err != nil {
|
||||
t.Fatalf("expected packaged hak output: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBuildHAKsEmitsCompactSummary(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "assets", "envi", "music", "westgate"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
haks:
|
||||
- name: envi
|
||||
priority: 1
|
||||
max_bytes: 1048576
|
||||
split: false
|
||||
include:
|
||||
- envi/**
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "AleandAnecdotes.mp3"), "source-mp3")
|
||||
writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "CREDITS.md"), "# NWN Music Pack Credits\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Darren Curtis | Ale and Anecdotes | `mus_wg_andnc.bmu` | `AleandAnecdotes.mp3` | Darren's Commercially Free Music | | Darren Curtis Music | |\n")
|
||||
|
||||
ffprobePath := filepath.Join(root, "ffprobe")
|
||||
writeFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Unknown\",\"title\":\"Broken Metadata\"}}}'\n")
|
||||
if err := os.Chmod(ffprobePath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffprobe: %v", err)
|
||||
}
|
||||
|
||||
ffmpegPath := filepath.Join(root, "ffmpeg")
|
||||
writeFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'bmu-data' > \"$last\"\n")
|
||||
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffmpeg: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("SOW_FFMPEG", ffmpegPath)
|
||||
t.Setenv("SOW_FFPROBE", ffprobePath)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"build-haks"},
|
||||
}
|
||||
|
||||
if err := runBuildHAKs(ctx); err != nil {
|
||||
t.Fatalf("runBuildHAKs failed: %v", err)
|
||||
}
|
||||
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "Build HAKs ----------") {
|
||||
t.Fatalf("expected build header, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "mapped: 1 music file(s); use --verbose to list mappings") {
|
||||
t.Fatalf("expected compact mapping summary, got %q", output)
|
||||
}
|
||||
if strings.Contains(output, "AleandAnecdotes.mp3 -> mus_wg_andnc.bmu") {
|
||||
t.Fatalf("did not expect verbose mapping in normal mode, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "manifest: build/haks.json") {
|
||||
t.Fatalf("expected relative manifest path, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func setTreeTime(t *testing.T, root string, modTime time.Time) {
|
||||
t.Helper()
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
return os.Chtimes(path, modTime, modTime)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("set tree time under %s: %v", root, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBuildHAKsVerboseListsMappings(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "assets", "envi", "music", "westgate"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
mkdirAll(t, filepath.Join(root, "src"))
|
||||
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
haks:
|
||||
- name: envi
|
||||
priority: 1
|
||||
max_bytes: 1048576
|
||||
split: false
|
||||
include:
|
||||
- envi/**
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "AleandAnecdotes.mp3"), "source-mp3")
|
||||
writeFile(t, filepath.Join(root, "assets", "envi", "music", "westgate", "CREDITS.md"), "# NWN Music Pack Credits\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Darren Curtis | Ale and Anecdotes | `mus_wg_andnc.bmu` | `AleandAnecdotes.mp3` | Darren's Commercially Free Music | | Darren Curtis Music | |\n")
|
||||
|
||||
ffprobePath := filepath.Join(root, "ffprobe")
|
||||
writeFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Unknown\",\"title\":\"Broken Metadata\"}}}'\n")
|
||||
if err := os.Chmod(ffprobePath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffprobe: %v", err)
|
||||
}
|
||||
|
||||
ffmpegPath := filepath.Join(root, "ffmpeg")
|
||||
writeFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'bmu-data' > \"$last\"\n")
|
||||
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
|
||||
t.Fatalf("chmod ffmpeg: %v", err)
|
||||
}
|
||||
|
||||
t.Setenv("SOW_FFMPEG", ffmpegPath)
|
||||
t.Setenv("SOW_FFPROBE", ffprobePath)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"build-haks", "--verbose"},
|
||||
}
|
||||
|
||||
if err := runBuildHAKs(ctx); err != nil {
|
||||
t.Fatalf("runBuildHAKs failed: %v", err)
|
||||
}
|
||||
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "mappings:") {
|
||||
t.Fatalf("expected verbose mappings header, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "AleandAnecdotes.mp3 -> mus_wg_andnc.bmu") {
|
||||
t.Fatalf("expected verbose mapping output, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "wrote: envi (1 assets)") {
|
||||
t.Fatalf("expected verbose archive action, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopdataConsoleSuppressesProgressInNormalMode(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
console := &topdataConsole{
|
||||
stdout: &stdout,
|
||||
projectRoot: "/workspace/project",
|
||||
projectName: "Test Module",
|
||||
commandName: "build-topdata",
|
||||
commandLabel: "Build Topdata",
|
||||
level: logLevelNormal,
|
||||
}
|
||||
|
||||
console.progress("Packaging compiled topdata resources into sow_top.hak...")
|
||||
|
||||
if stdout.String() != "" {
|
||||
t.Fatalf("expected no normal-mode progress output, got %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpinnerEnabledForHonorsPlainTTYMode(t *testing.T) {
|
||||
t.Setenv("SOW_TOOLS_TTY_MODE", "plain")
|
||||
if spinnerEnabledFor(&bytes.Buffer{}, logLevelNormal) {
|
||||
t.Fatal("expected plain tty mode to disable spinner output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopdataConsoleDebugProgressAndRelativePaths(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
console := &topdataConsole{
|
||||
stdout: &stdout,
|
||||
projectRoot: "/workspace/project",
|
||||
projectName: "Test Module",
|
||||
commandName: "deploy-wiki",
|
||||
commandLabel: "Deploy Wiki",
|
||||
level: logLevelDebug,
|
||||
}
|
||||
|
||||
console.progress("NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, purge 6, drift 0")
|
||||
console.emitWikiDeployResult(10, 1, 2, 3, 4, 5, 6, 0, "/workspace/project/build/wiki/deploy-manifest.json")
|
||||
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, purge 6, drift 0") {
|
||||
t.Fatalf("expected debug progress line, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "archived: 5") {
|
||||
t.Fatalf("expected archived deploy count, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "purged: 6") {
|
||||
t.Fatalf("expected purged deploy count, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "manifest: build/wiki/deploy-manifest.json") {
|
||||
t.Fatalf("expected relative deploy manifest path, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDeployWikiHelpListsPurgeStalePolicy(t *testing.T) {
|
||||
_, err := parseDeployWikiArgs("deploy-wiki", []string{"--help"})
|
||||
if err == nil || !strings.Contains(err.Error(), "--stale-policy <report|archive|purge>") {
|
||||
t.Fatalf("expected deploy-wiki help to list purge stale policy, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDeployWikiResetManagedNamespacesFlag(t *testing.T) {
|
||||
opts, err := parseDeployWikiArgs("deploy-wiki", []string{"--reset-managed-namespaces"})
|
||||
if err != nil {
|
||||
t.Fatalf("parse deploy wiki reset flag: %v", err)
|
||||
}
|
||||
if !opts.ResetManagedNamespaces {
|
||||
t.Fatalf("expected --reset-managed-namespaces to enable namespace reset")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectConsoleSuppressesBuildModuleProgressInNormalMode(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
console := &projectConsole{
|
||||
stdout: &stdout,
|
||||
projectRoot: "/workspace/project",
|
||||
projectName: "Test Module",
|
||||
commandName: "build-module",
|
||||
commandLabel: "Build Module",
|
||||
level: logLevelNormal,
|
||||
}
|
||||
|
||||
console.progress("Writing module archive...")
|
||||
|
||||
if stdout.String() != "" {
|
||||
t.Fatalf("expected no normal-mode progress output, got %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectConsoleEmitsRelativePaths(t *testing.T) {
|
||||
var stdout bytes.Buffer
|
||||
console := &projectConsole{
|
||||
stdout: &stdout,
|
||||
projectRoot: "/workspace/project",
|
||||
projectName: "Test Module",
|
||||
commandName: "compare",
|
||||
commandLabel: "Compare",
|
||||
level: logLevelNormal,
|
||||
}
|
||||
|
||||
console.emitCompareResult(pipeline.CompareResult{
|
||||
ModulePath: "/workspace/project/build/test.mod",
|
||||
HAKPaths: []string{"/workspace/project/build/core.hak"},
|
||||
Checked: 42,
|
||||
})
|
||||
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "module: build/test.mod") {
|
||||
t.Fatalf("expected relative module path, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMusicScanUsesConfiguredDatasetsWithoutWriting(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "assets", "audio", "westgate"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
assets: assets
|
||||
build: build
|
||||
music:
|
||||
datasets:
|
||||
westgate_audio:
|
||||
source: audio/westgate
|
||||
output: generated/music
|
||||
prefix: wg_
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"music", "scan", "--dataset", "westgate_audio"},
|
||||
}
|
||||
|
||||
if err := runMusic(ctx); err != nil {
|
||||
t.Fatalf("runMusic failed: %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "music scan") || !strings.Contains(output, "tracks: 1") {
|
||||
t.Fatalf("unexpected music scan output: %q", output)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, ".cache", "credits")); !os.IsNotExist(err) {
|
||||
t.Fatalf("music scan should not write credits artifacts, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMusicListDatasets(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "assets"))
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
assets: assets
|
||||
music:
|
||||
datasets:
|
||||
westgate_audio:
|
||||
source: audio/westgate
|
||||
output: generated/music
|
||||
prefix: wg_
|
||||
`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"music", "list-datasets"},
|
||||
}
|
||||
|
||||
if err := runMusic(ctx); err != nil {
|
||||
t.Fatalf("runMusic failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "westgate_audio: source=audio/westgate output=generated/music prefix=wg_") {
|
||||
t.Fatalf("unexpected dataset list: %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigEffectiveJSONShowsDefaultsAndProvenance(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
source: src
|
||||
`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"config", "effective", "--json"},
|
||||
}
|
||||
|
||||
if err := runConfig(ctx); err != nil {
|
||||
t.Fatalf("runConfig failed: %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, `"hak_manifest": "haks.json"`) {
|
||||
t.Fatalf("expected HAK manifest default in effective config, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, `"paths.build":`) || !strings.Contains(output, `"toolkit default"`) {
|
||||
t.Fatalf("expected default provenance in effective config, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigExplainReportsYAMLSource(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
build: output
|
||||
`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"config", "explain", "paths.build"},
|
||||
}
|
||||
|
||||
if err := runConfig(ctx); err != nil {
|
||||
t.Fatalf("runConfig failed: %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "value: \"output\"") {
|
||||
t.Fatalf("expected configured build value, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "source: yaml") {
|
||||
t.Fatalf("expected YAML source, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigValidateDoesNotScanSourceInventory(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
source: missing-src
|
||||
build: build
|
||||
`)
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"config", "validate"},
|
||||
}
|
||||
|
||||
if err := runConfig(ctx); err != nil {
|
||||
t.Fatalf("runConfig failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "config: ok") {
|
||||
t.Fatalf("expected config validation output, got %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigSourcesListsActiveOverrides(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test Module
|
||||
resref: testmod
|
||||
paths:
|
||||
source: src
|
||||
`)
|
||||
t.Setenv("SOW_BUILD_HAKS_KEEP_EXISTING", "1")
|
||||
|
||||
var stdout bytes.Buffer
|
||||
ctx := context{
|
||||
stdout: &stdout,
|
||||
stderr: &bytes.Buffer{},
|
||||
cwd: root,
|
||||
args: []string{"config", "sources"},
|
||||
}
|
||||
|
||||
if err := runConfig(ctx); err != nil {
|
||||
t.Fatalf("runConfig failed: %v", err)
|
||||
}
|
||||
output := stdout.String()
|
||||
if !strings.Contains(output, "active overrides:") {
|
||||
t.Fatalf("expected active overrides section, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "build.keep_existing_haks=1") {
|
||||
t.Fatalf("expected keep existing override, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInlineFlagParsersRejectEmptyValues(t *testing.T) {
|
||||
if _, err := parseBuildHAKArgs([]string{"--hak="}); err == nil || !strings.Contains(err.Error(), "--hak requires a value") {
|
||||
t.Fatalf("expected empty --hak inline value error, got %v", err)
|
||||
}
|
||||
if _, err := parseMusicCommandArgs([]string{"--dataset="}); err == nil || !strings.Contains(err.Error(), "--dataset requires a value") {
|
||||
t.Fatalf("expected empty --dataset inline value error, got %v", err)
|
||||
}
|
||||
if _, err := parseDeployWikiArgs("deploy-wiki", []string{"--endpoint="}); err == nil || !strings.Contains(err.Error(), "--endpoint requires a value") {
|
||||
t.Fatalf("expected empty --endpoint inline value error, got %v", err)
|
||||
}
|
||||
if _, err := parseBuildChangelogArgs("build-changelog", []string{"--output="}); err == nil || !strings.Contains(err.Error(), "--output requires a value") {
|
||||
t.Fatalf("expected empty --output inline value error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBuildChangelogArgsAcceptsInlineValues(t *testing.T) {
|
||||
opts, err := parseBuildChangelogArgs("build-changelog", []string{
|
||||
"--config=scripts/changelog.json",
|
||||
"--output=CHANGELOG.md",
|
||||
"--current-tag=v2",
|
||||
"--previous-tag=v1",
|
||||
"--api-base-url=https://gitea.example/api/v1",
|
||||
"--token=secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("parseBuildChangelogArgs failed: %v", err)
|
||||
}
|
||||
if opts.configPath != "scripts/changelog.json" ||
|
||||
opts.outputPath != "CHANGELOG.md" ||
|
||||
opts.currentTag != "v2" ||
|
||||
opts.previousTag != "v1" ||
|
||||
opts.apiBaseURL != "https://gitea.example/api/v1" ||
|
||||
opts.token != "secret" {
|
||||
t.Fatalf("unexpected changelog opts: %#v", opts)
|
||||
}
|
||||
}
|
||||
|
||||
func mkdirAll(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(path, 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func setFileTime(t *testing.T, path string, modTime time.Time) {
|
||||
t.Helper()
|
||||
if err := os.Chtimes(path, modTime, modTime); err != nil {
|
||||
t.Fatalf("set file time %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Package buildinfo reports the Crucible build version.
|
||||
//
|
||||
// Version is empty in plain `go build`; the build scripts and Dockerfile inject
|
||||
// the short git sha via -ldflags. As a fallback we read the VCS revision that
|
||||
// the Go toolchain stamps into the binary, so a binary built without -ldflags
|
||||
// still self-identifies.
|
||||
package buildinfo
|
||||
|
||||
import "runtime/debug"
|
||||
|
||||
// Version is overridable at link time:
|
||||
//
|
||||
// -ldflags "-X gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo.Version=<sha>"
|
||||
var Version = ""
|
||||
|
||||
// String returns a human-readable version line, e.g. "crucible 8f31c2a".
|
||||
func String() string {
|
||||
return "crucible " + revision()
|
||||
}
|
||||
|
||||
func revision() string {
|
||||
if Version != "" {
|
||||
return Version
|
||||
}
|
||||
if bi, ok := debug.ReadBuildInfo(); ok {
|
||||
for _, s := range bi.Settings {
|
||||
if s.Key == "vcs.revision" && s.Value != "" {
|
||||
return s.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
return "devel"
|
||||
}
|
||||
@@ -1,455 +0,0 @@
|
||||
package changelog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
RepoURL string `json:"repo_url"`
|
||||
Categories []Category `json:"categories"`
|
||||
}
|
||||
|
||||
type Category struct {
|
||||
Title string `json:"title"`
|
||||
Sections []Section `json:"sections"`
|
||||
}
|
||||
|
||||
type Section struct {
|
||||
Title string `json:"title"`
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
RepoRoot string
|
||||
ConfigPath string
|
||||
OutputPath string
|
||||
CurrentTag string
|
||||
PreviousTag string
|
||||
APIBaseURL string
|
||||
Token string
|
||||
Stdout io.Writer
|
||||
}
|
||||
|
||||
type pullInfo struct {
|
||||
AuthorName string
|
||||
}
|
||||
|
||||
type generator struct {
|
||||
repoRoot string
|
||||
config Config
|
||||
currentTag string
|
||||
previousTag string
|
||||
apiBaseURL string
|
||||
repoURL string
|
||||
owner string
|
||||
name string
|
||||
token string
|
||||
httpClient *http.Client
|
||||
pullCache map[string]pullInfo
|
||||
}
|
||||
|
||||
type changelogEntry struct {
|
||||
Title string
|
||||
ReferenceLabel string
|
||||
ReferenceURL string
|
||||
ReferenceSort string
|
||||
AuthorName string
|
||||
}
|
||||
|
||||
var pullNumberPattern = regexp.MustCompile(`\(#([0-9]+)\)`)
|
||||
|
||||
func Generate(opts Options) error {
|
||||
repoRoot := strings.TrimSpace(opts.RepoRoot)
|
||||
if repoRoot == "" {
|
||||
detectedRepoRoot, err := gitOutput("", "rev-parse", "--show-toplevel")
|
||||
if err != nil {
|
||||
return fmt.Errorf("detect repo root: %w", err)
|
||||
}
|
||||
repoRoot = strings.TrimSpace(detectedRepoRoot)
|
||||
}
|
||||
|
||||
configPath := opts.ConfigPath
|
||||
if configPath == "" {
|
||||
configPath = filepath.Join(repoRoot, "scripts", "changelog.json")
|
||||
}
|
||||
config, err := loadConfig(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
currentTag := strings.TrimSpace(opts.CurrentTag)
|
||||
if currentTag == "" {
|
||||
currentTag, err = gitOutput(repoRoot, "describe", "--tags", "--exact-match")
|
||||
if err != nil {
|
||||
return fmt.Errorf("detect current tag: %w", err)
|
||||
}
|
||||
currentTag = strings.TrimSpace(currentTag)
|
||||
}
|
||||
if currentTag == "" {
|
||||
return fmt.Errorf("could not determine current tag")
|
||||
}
|
||||
|
||||
previousTag := strings.TrimSpace(opts.PreviousTag)
|
||||
if previousTag == "" {
|
||||
previousTag, err = previousTagFor(repoRoot, currentTag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
repoURL := strings.TrimSpace(config.RepoURL)
|
||||
if repoURL == "" {
|
||||
return fmt.Errorf("config %s is missing repo_url", configPath)
|
||||
}
|
||||
apiBaseURL := strings.TrimSpace(opts.APIBaseURL)
|
||||
if apiBaseURL == "" {
|
||||
if apiBaseURL = strings.TrimSpace(os.Getenv("GITEA_SERVER_URL")); apiBaseURL == "" {
|
||||
apiBaseURL = deriveAPIBaseURL(repoURL)
|
||||
}
|
||||
}
|
||||
apiBaseURL = normalizeAPIBaseURL(apiBaseURL)
|
||||
if apiBaseURL == "" {
|
||||
return fmt.Errorf("could not determine Gitea API base URL")
|
||||
}
|
||||
|
||||
owner, name, err := ownerAndRepoFromURL(repoURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
g := generator{
|
||||
repoRoot: repoRoot,
|
||||
config: config,
|
||||
currentTag: currentTag,
|
||||
previousTag: previousTag,
|
||||
apiBaseURL: strings.TrimRight(apiBaseURL, "/"),
|
||||
repoURL: strings.TrimRight(repoURL, "/"),
|
||||
owner: owner,
|
||||
name: name,
|
||||
token: firstNonEmpty(opts.Token, os.Getenv("GITEA_API_TOKEN"), os.Getenv("GITEA_TOKEN"), os.Getenv("SOW_TOOLS_TOKEN")),
|
||||
httpClient: http.DefaultClient,
|
||||
pullCache: map[string]pullInfo{},
|
||||
}
|
||||
|
||||
rendered, err := g.render()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if opts.OutputPath != "" {
|
||||
if err := os.MkdirAll(filepath.Dir(opts.OutputPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create output directory: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(opts.OutputPath, rendered, 0o644); err != nil {
|
||||
return fmt.Errorf("write changelog: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
stdout := opts.Stdout
|
||||
if stdout == nil {
|
||||
stdout = os.Stdout
|
||||
}
|
||||
_, err = stdout.Write(rendered)
|
||||
return err
|
||||
}
|
||||
|
||||
func loadConfig(path string) (Config, error) {
|
||||
var config Config
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("read changelog config %s: %w", path, err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &config); err != nil {
|
||||
return Config{}, fmt.Errorf("parse changelog config %s: %w", path, err)
|
||||
}
|
||||
if len(config.Categories) == 0 {
|
||||
return Config{}, fmt.Errorf("changelog config %s does not define any categories", path)
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func previousTagFor(repoRoot string, currentTag string) (string, error) {
|
||||
output, err := gitOutput(repoRoot, "for-each-ref", "--sort=-creatordate", "--format=%(refname:strip=2)", "refs/tags")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("list tags: %w", err)
|
||||
}
|
||||
|
||||
tags := strings.Fields(output)
|
||||
for idx, tag := range tags {
|
||||
if tag != currentTag {
|
||||
continue
|
||||
}
|
||||
if idx+1 < len(tags) {
|
||||
return tags[idx+1], nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("current tag %s was not found in repository tag list", currentTag)
|
||||
}
|
||||
|
||||
func deriveAPIBaseURL(repoURL string) string {
|
||||
parsed, err := url.Parse(repoURL)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%s://%s/api/v1", parsed.Scheme, parsed.Host)
|
||||
}
|
||||
|
||||
func normalizeAPIBaseURL(raw string) string {
|
||||
raw = strings.TrimRight(strings.TrimSpace(raw), "/")
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasSuffix(raw, "/api/v1") {
|
||||
return raw
|
||||
}
|
||||
return raw + "/api/v1"
|
||||
}
|
||||
|
||||
func ownerAndRepoFromURL(repoURL string) (string, string, error) {
|
||||
parsed, err := url.Parse(repoURL)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("parse repo_url %s: %w", repoURL, err)
|
||||
}
|
||||
|
||||
parts := strings.Split(strings.Trim(parsed.Path, "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return "", "", fmt.Errorf("repo_url %s does not include owner/repo", repoURL)
|
||||
}
|
||||
|
||||
owner := strings.TrimSpace(parts[0])
|
||||
name := strings.TrimSuffix(strings.TrimSpace(parts[1]), ".git")
|
||||
if owner == "" || name == "" {
|
||||
return "", "", fmt.Errorf("repo_url %s does not include owner/repo", repoURL)
|
||||
}
|
||||
return owner, name, nil
|
||||
}
|
||||
|
||||
func (g generator) render() ([]byte, error) {
|
||||
publishDate, err := gitOutput(g.repoRoot, "log", "-1", "--format=%cs", g.currentTag)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve publish date: %w", err)
|
||||
}
|
||||
publishDate = strings.TrimSpace(publishDate)
|
||||
|
||||
var buf bytes.Buffer
|
||||
fmt.Fprintf(&buf, "# %s - %s\n\n", g.currentTag, publishDate)
|
||||
if g.previousTag != "" {
|
||||
fmt.Fprintf(&buf, "_Compared against %s_\n\n", g.previousTag)
|
||||
} else {
|
||||
buf.WriteString("_Initial tagged release_\n\n")
|
||||
}
|
||||
|
||||
for _, category := range g.config.Categories {
|
||||
categoryText, err := g.renderCategory(category)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if categoryText == "" {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&buf, "## %s\n\n", category.Title)
|
||||
buf.WriteString(categoryText)
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func (g generator) renderCategory(category Category) (string, error) {
|
||||
var buf bytes.Buffer
|
||||
for _, section := range category.Sections {
|
||||
entries, err := g.sectionEntries(section)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&buf, "### %s\n", section.Title)
|
||||
for _, entry := range entries {
|
||||
fmt.Fprintf(&buf, "- %s ([%s](%s)) - From %s\n", entry.Title, entry.ReferenceLabel, entry.ReferenceURL, entry.AuthorName)
|
||||
}
|
||||
buf.WriteString("\n")
|
||||
}
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
func (g generator) sectionEntries(section Section) ([]changelogEntry, error) {
|
||||
if len(section.Paths) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
args := []string{"log", "--first-parent", "--pretty=format:%H%x1f%s%x1f%an"}
|
||||
if g.previousTag != "" {
|
||||
args = append(args, fmt.Sprintf("%s..%s", g.previousTag, g.currentTag))
|
||||
} else {
|
||||
args = append(args, g.currentTag)
|
||||
}
|
||||
args = append(args, "--")
|
||||
args = append(args, section.Paths...)
|
||||
|
||||
output, err := gitOutput(g.repoRoot, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list git log for section %s: %w", section.Title, err)
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
var entries []changelogEntry
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, "\x1f", 3)
|
||||
commitHash := strings.TrimSpace(parts[0])
|
||||
subject := ""
|
||||
if len(parts) > 1 {
|
||||
subject = strings.TrimSpace(parts[1])
|
||||
}
|
||||
fallbackAuthor := ""
|
||||
if len(parts) > 2 {
|
||||
fallbackAuthor = strings.TrimSpace(parts[2])
|
||||
}
|
||||
matches := pullNumberPattern.FindStringSubmatch(subject)
|
||||
entryKey := commitHash
|
||||
if len(matches) == 2 {
|
||||
entryKey = "pr:" + matches[1]
|
||||
}
|
||||
if _, exists := seen[entryKey]; exists {
|
||||
continue
|
||||
}
|
||||
seen[entryKey] = struct{}{}
|
||||
|
||||
entry := changelogEntry{
|
||||
Title: sanitizeSubject(subject),
|
||||
AuthorName: fallbackAuthor,
|
||||
}
|
||||
|
||||
if len(matches) == 2 {
|
||||
pullNumber := matches[1]
|
||||
pullInfo, err := g.pullDetails(pullNumber)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
authorName := strings.TrimSpace(pullInfo.AuthorName)
|
||||
if authorName == "" {
|
||||
authorName = fallbackAuthor
|
||||
}
|
||||
entry.ReferenceLabel = "#" + pullNumber
|
||||
entry.ReferenceURL = fmt.Sprintf("%s/pulls/%s", g.repoURL, pullNumber)
|
||||
entry.ReferenceSort = "pr:" + pullNumber
|
||||
entry.AuthorName = authorName
|
||||
} else {
|
||||
entry.ReferenceLabel = shortCommitHash(commitHash)
|
||||
entry.ReferenceURL = fmt.Sprintf("%s/commit/%s", g.repoURL, commitHash)
|
||||
entry.ReferenceSort = "commit:" + commitHash
|
||||
}
|
||||
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
if entries[i].Title == entries[j].Title {
|
||||
return entries[i].ReferenceSort < entries[j].ReferenceSort
|
||||
}
|
||||
return entries[i].Title < entries[j].Title
|
||||
})
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func sanitizeSubject(subject string) string {
|
||||
title := strings.TrimSpace(subject)
|
||||
title = pullNumberPattern.ReplaceAllString(title, "")
|
||||
title = strings.TrimSpace(title)
|
||||
title = strings.TrimPrefix(title, "Merge pull request")
|
||||
title = strings.TrimSpace(title)
|
||||
title = strings.Trim(title, `"'`)
|
||||
title = strings.TrimSpace(title)
|
||||
if title == "" {
|
||||
return strings.TrimSpace(subject)
|
||||
}
|
||||
return title
|
||||
}
|
||||
|
||||
func shortCommitHash(commitHash string) string {
|
||||
commitHash = strings.TrimSpace(commitHash)
|
||||
if len(commitHash) <= 7 {
|
||||
return commitHash
|
||||
}
|
||||
return commitHash[:7]
|
||||
}
|
||||
|
||||
func (g generator) pullDetails(pullNumber string) (pullInfo, error) {
|
||||
if cached, ok := g.pullCache[pullNumber]; ok {
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
endpoint := fmt.Sprintf("%s/repos/%s/%s/pulls/%s", g.apiBaseURL, url.PathEscape(g.owner), url.PathEscape(g.name), url.PathEscape(pullNumber))
|
||||
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return pullInfo{}, fmt.Errorf("build Gitea request for pull #%s: %w", pullNumber, err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if g.token != "" {
|
||||
req.Header.Set("Authorization", "token "+g.token)
|
||||
}
|
||||
|
||||
resp, err := g.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return pullInfo{}, fmt.Errorf("request Gitea pull #%s: %w", pullNumber, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return pullInfo{}, fmt.Errorf("request Gitea pull #%s: unexpected HTTP %d: %s", pullNumber, resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
User struct {
|
||||
FullName string `json:"full_name"`
|
||||
Login string `json:"login"`
|
||||
} `json:"user"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
|
||||
return pullInfo{}, fmt.Errorf("decode Gitea pull #%s: %w", pullNumber, err)
|
||||
}
|
||||
|
||||
info := pullInfo{AuthorName: firstNonEmpty(strings.TrimSpace(payload.User.FullName), strings.TrimSpace(payload.User.Login))}
|
||||
g.pullCache[pullNumber] = info
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func gitOutput(repoRoot string, args ...string) (string, error) {
|
||||
cmd := exec.Command("git", args...)
|
||||
if repoRoot != "" {
|
||||
cmd.Dir = repoRoot
|
||||
}
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %s", strings.Join(cmd.Args, " "), strings.TrimSpace(string(output)))
|
||||
}
|
||||
return string(output), nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,213 +0,0 @@
|
||||
package changelog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSanitizeSubject(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
subject string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "merge prefix with quotes",
|
||||
subject: "Merge pull request 'Add classes overhaul (#123)'",
|
||||
want: "Add classes overhaul",
|
||||
},
|
||||
{
|
||||
name: "plain subject",
|
||||
subject: "Add classes overhaul (#123)",
|
||||
want: "Add classes overhaul",
|
||||
},
|
||||
{
|
||||
name: "double quotes",
|
||||
subject: `Merge pull request "Add classes overhaul (#123)"`,
|
||||
want: "Add classes overhaul",
|
||||
},
|
||||
{
|
||||
name: "direct push",
|
||||
subject: "Fix direct push handling",
|
||||
want: "Fix direct push handling",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := sanitizeSubject(tt.subject); got != tt.want {
|
||||
t.Fatalf("sanitizeSubject(%q) = %q, want %q", tt.subject, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShortCommitHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := shortCommitHash("1234567890abcdef"); got != "1234567" {
|
||||
t.Fatalf("shortCommitHash() = %q, want %q", got, "1234567")
|
||||
}
|
||||
if got := shortCommitHash("1234567"); got != "1234567" {
|
||||
t.Fatalf("shortCommitHash() preserved %q unexpectedly as %q", "1234567", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwnerAndRepoFromURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
owner, repo, err := ownerAndRepoFromURL("https://gitea.example.test/org/repo.git")
|
||||
if err != nil {
|
||||
t.Fatalf("ownerAndRepoFromURL returned error: %v", err)
|
||||
}
|
||||
if owner != "org" || repo != "repo" {
|
||||
t.Fatalf("ownerAndRepoFromURL returned %q/%q", owner, repo)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveAPIBaseURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := deriveAPIBaseURL("https://gitea.example.test/org/repo")
|
||||
want := "https://gitea.example.test/api/v1"
|
||||
if got != want {
|
||||
t.Fatalf("deriveAPIBaseURL() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAPIBaseURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{input: "https://gitea.example.test", want: "https://gitea.example.test/api/v1"},
|
||||
{input: "https://gitea.example.test/api/v1", want: "https://gitea.example.test/api/v1"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
if got := normalizeAPIBaseURL(tt.input); got != tt.want {
|
||||
t.Fatalf("normalizeAPIBaseURL(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateIncludesPullRequestsAndDirectPushes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
repoRoot := t.TempDir()
|
||||
runGit(t, repoRoot, "init")
|
||||
runGit(t, repoRoot, "config", "user.name", "Test User")
|
||||
runGit(t, repoRoot, "config", "user.email", "test@example.com")
|
||||
|
||||
writeFile(t, filepath.Join(repoRoot, "content", "entry.txt"), "initial\n")
|
||||
runGit(t, repoRoot, "add", ".")
|
||||
runGit(t, repoRoot, "commit", "-m", "Initial import")
|
||||
runGit(t, repoRoot, "tag", "v1.0.0")
|
||||
|
||||
writeFile(t, filepath.Join(repoRoot, "content", "entry.txt"), "direct\n")
|
||||
runGit(t, repoRoot, "add", ".")
|
||||
runGit(t, repoRoot, "commit", "-m", "Fix direct push handling")
|
||||
directHash := strings.TrimSpace(runGit(t, repoRoot, "rev-parse", "HEAD"))
|
||||
|
||||
writeFile(t, filepath.Join(repoRoot, "content", "entry.txt"), "pull\n")
|
||||
runGit(t, repoRoot, "add", ".")
|
||||
runGit(t, repoRoot, "commit", "-m", "Add release summary (#12)")
|
||||
runGit(t, repoRoot, "tag", "v1.1.0")
|
||||
|
||||
configPath := filepath.Join(repoRoot, "scripts", "changelog.json")
|
||||
writeFile(t, configPath, `{
|
||||
"repo_url": "REPO_URL",
|
||||
"categories": [
|
||||
{
|
||||
"title": "Content",
|
||||
"sections": [
|
||||
{
|
||||
"title": "Entries",
|
||||
"paths": ["content/"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}`)
|
||||
|
||||
var pullRequests int
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/v1/repos/org/repo/pulls/12" {
|
||||
t.Fatalf("unexpected API path %q", r.URL.Path)
|
||||
}
|
||||
pullRequests++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"user":{"full_name":"Patch Author","login":"patch-author"}}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
repoURL := server.URL + "/org/repo"
|
||||
configData, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read changelog config: %v", err)
|
||||
}
|
||||
configData = bytes.ReplaceAll(configData, []byte("REPO_URL"), []byte(repoURL))
|
||||
if err := os.WriteFile(configPath, configData, 0o644); err != nil {
|
||||
t.Fatalf("write changelog config: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
err = Generate(Options{
|
||||
RepoRoot: repoRoot,
|
||||
ConfigPath: configPath,
|
||||
CurrentTag: "v1.1.0",
|
||||
PreviousTag: "v1.0.0",
|
||||
APIBaseURL: server.URL + "/api/v1",
|
||||
Stdout: &stdout,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate() returned error: %v", err)
|
||||
}
|
||||
|
||||
rendered := stdout.String()
|
||||
if !strings.Contains(rendered, "- Add release summary ([#12]("+repoURL+"/pulls/12)) - From Patch Author") {
|
||||
t.Fatalf("rendered changelog missing pull entry:\n%s", rendered)
|
||||
}
|
||||
if !strings.Contains(rendered, "- Fix direct push handling (["+shortCommitHash(directHash)+"]("+repoURL+"/commit/"+directHash+")) - From Test User") {
|
||||
t.Fatalf("rendered changelog missing direct-push entry:\n%s", rendered)
|
||||
}
|
||||
if pullRequests != 1 {
|
||||
t.Fatalf("expected 1 pull lookup, got %d", pullRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func runGit(t *testing.T, repoRoot string, args ...string) string {
|
||||
t.Helper()
|
||||
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = repoRoot
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, output)
|
||||
}
|
||||
return string(output)
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, path string, content string) {
|
||||
t.Helper()
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll(%q): %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%q): %v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
// Package dispatch is the Crucible command surface: a single registry of
|
||||
// builders shared by the `crucible` dispatcher and the standalone
|
||||
// `crucible-<name>` shims.
|
||||
//
|
||||
// Scaffold contract (Phase 5): every builder is registered but UNWIRED. Running
|
||||
// a builder fails closed (exit 70) with an operator-cutover message; it never
|
||||
// fakes an artifact. The internal pipeline/topdata/erf/wiki/music packages from
|
||||
// gitea/sow-tools are migrated by the operator at cutover (the workspace hard
|
||||
// rules forbid transplanting source here). Once a builder's handler is wired,
|
||||
// flip its Wired flag and register the handler.
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"text/tabwriter"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||
)
|
||||
|
||||
// Exit codes follow the sysexits(3) convention so CI can distinguish
|
||||
// "you called it wrong" from "the tool is not wired yet".
|
||||
const (
|
||||
exitOK = 0
|
||||
exitUsage = 64 // EX_USAGE — bad invocation / unknown builder
|
||||
exitUnwired = 70 // EX_SOFTWARE — builder registered but not yet implemented
|
||||
)
|
||||
|
||||
// Builder is one tool surface in the Crucible suite.
|
||||
type Builder struct {
|
||||
Name string // canonical dispatcher subcommand, e.g. "module"
|
||||
Bin string // standalone binary name, e.g. "crucible-module"
|
||||
Summary string // one-line description
|
||||
Legacy []string // nwn-tool commands this subsumes (migration aid; see docs/command-surface.md)
|
||||
}
|
||||
|
||||
// Registry is the single source of truth for the Crucible command surface.
|
||||
// Keep it in sync with docs/command-surface.md and the cmd/ directory.
|
||||
var Registry = []Builder{
|
||||
{
|
||||
Name: "depot",
|
||||
Bin: "crucible-depot",
|
||||
Summary: "verify/move content-addressed depot blobs (SeaweedFS)",
|
||||
Legacy: nil,
|
||||
},
|
||||
{
|
||||
Name: "hak",
|
||||
Bin: "crucible-hak",
|
||||
Summary: "pack/unpack ERF/HAK archives + hak manifests",
|
||||
Legacy: []string{"build-haks", "apply-hak-manifest"},
|
||||
},
|
||||
{
|
||||
Name: "module",
|
||||
Bin: "crucible-module",
|
||||
Summary: "build/extract/validate/compare the .mod",
|
||||
Legacy: []string{"build", "build-module", "extract", "validate", "compare"},
|
||||
},
|
||||
{
|
||||
Name: "topdata",
|
||||
Bin: "crucible-topdata",
|
||||
Summary: "compile 2da/tlk topdata + packages",
|
||||
Legacy: []string{"validate-topdata", "build-topdata", "build-top-package", "compare-topdata", "convert-topdata"},
|
||||
},
|
||||
{
|
||||
Name: "wiki",
|
||||
Bin: "crucible-wiki",
|
||||
Summary: "render + deploy mechanical wiki pages",
|
||||
Legacy: []string{"build-wiki", "deploy-wiki"},
|
||||
},
|
||||
}
|
||||
|
||||
func find(name string) (Builder, bool) {
|
||||
for _, b := range Registry {
|
||||
if b.Name == name {
|
||||
return b, true
|
||||
}
|
||||
}
|
||||
return Builder{}, false
|
||||
}
|
||||
|
||||
// Main is the `crucible` dispatcher entrypoint.
|
||||
func Main(args []string) int { return run(args, os.Stdout, os.Stderr) }
|
||||
|
||||
// RunBuilder is the standalone `crucible-<name>` entrypoint.
|
||||
func RunBuilder(name string, args []string) int {
|
||||
return runBuilder(name, args, os.Stdout, os.Stderr)
|
||||
}
|
||||
|
||||
func run(args []string, out, errw io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
usage(out)
|
||||
return exitUsage
|
||||
}
|
||||
switch args[0] {
|
||||
case "-h", "--help", "help":
|
||||
usage(out)
|
||||
return exitOK
|
||||
case "-V", "--version", "version":
|
||||
fmt.Fprintln(out, buildinfo.String())
|
||||
return exitOK
|
||||
case "list":
|
||||
list(out)
|
||||
return exitOK
|
||||
}
|
||||
return runBuilder(args[0], args[1:], out, errw)
|
||||
}
|
||||
|
||||
func runBuilder(name string, args []string, out, errw io.Writer) int {
|
||||
b, ok := find(name)
|
||||
if !ok {
|
||||
fmt.Fprintf(errw, "crucible: unknown builder %q\n\n", name)
|
||||
usage(errw)
|
||||
return exitUsage
|
||||
}
|
||||
if len(args) > 0 {
|
||||
switch args[0] {
|
||||
case "-h", "--help", "help":
|
||||
builderHelp(out, b)
|
||||
return exitOK
|
||||
}
|
||||
}
|
||||
// Every builder is unwired in the Phase 5 scaffold: fail closed, never fake.
|
||||
fmt.Fprintf(errw, unwiredMsg, b.Name, b.Bin)
|
||||
return exitUnwired
|
||||
}
|
||||
|
||||
const unwiredMsg = `crucible %[1]s: not wired yet.
|
||||
|
||||
The Crucible suite is scaffolded (Phase 5). The %[2]s implementation is migrated
|
||||
from gitea/sow-tools internal packages at operator cutover; the workspace hard
|
||||
rules forbid transplanting that source into this tree automatically.
|
||||
|
||||
This command fails closed (exit 70) rather than producing a fake artifact.
|
||||
See docs/migration-from-nwn-tool.md.
|
||||
`
|
||||
|
||||
func usage(w io.Writer) {
|
||||
fmt.Fprintf(w, "crucible — Shadows Over Westgate build/conversion/sync toolchain\n")
|
||||
fmt.Fprintf(w, "%s\n\n", buildinfo.String())
|
||||
fmt.Fprintf(w, "usage:\n")
|
||||
fmt.Fprintf(w, " crucible <builder> [args] dispatch to a builder\n")
|
||||
fmt.Fprintf(w, " crucible-<builder> [args] equivalent standalone binary\n\n")
|
||||
fmt.Fprintf(w, "builders:\n")
|
||||
list(w)
|
||||
fmt.Fprintf(w, "\nglobal commands:\n")
|
||||
fmt.Fprintf(w, " help | -h show this help\n")
|
||||
fmt.Fprintf(w, " version | -V show the build version\n")
|
||||
fmt.Fprintf(w, " list list builders (one per line: name<TAB>bin<TAB>summary)\n")
|
||||
fmt.Fprintf(w, "\nNWN_ROOT must be passed explicitly (env or flag); crucible never guesses\n")
|
||||
fmt.Fprintf(w, "from $HOME. See docs/consumer-contract.md.\n")
|
||||
}
|
||||
|
||||
func list(w io.Writer) {
|
||||
tw := tabwriter.NewWriter(w, 0, 2, 2, ' ', 0)
|
||||
for _, b := range Registry {
|
||||
fmt.Fprintf(tw, " %s\t%s\t%s\n", b.Name, b.Bin, b.Summary)
|
||||
}
|
||||
tw.Flush()
|
||||
}
|
||||
|
||||
func builderHelp(w io.Writer, b Builder) {
|
||||
fmt.Fprintf(w, "crucible %s (%s) — %s\n\n", b.Name, b.Bin, b.Summary)
|
||||
if len(b.Legacy) > 0 {
|
||||
fmt.Fprintf(w, "subsumes nwn-tool commands: ")
|
||||
for i, c := range b.Legacy {
|
||||
if i > 0 {
|
||||
fmt.Fprintf(w, ", ")
|
||||
}
|
||||
fmt.Fprintf(w, "%s", c)
|
||||
}
|
||||
fmt.Fprintf(w, "\n\n")
|
||||
}
|
||||
fmt.Fprintf(w, "status: scaffolded, not wired (Phase 5). See docs/command-surface.md.\n")
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package dispatch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestVersion(t *testing.T) {
|
||||
var out, errw bytes.Buffer
|
||||
for _, arg := range []string{"version", "-V", "--version"} {
|
||||
out.Reset()
|
||||
errw.Reset()
|
||||
if code := run([]string{arg}, &out, &errw); code != exitOK {
|
||||
t.Fatalf("%s: exit=%d want %d", arg, code, exitOK)
|
||||
}
|
||||
if !strings.HasPrefix(out.String(), "crucible ") {
|
||||
t.Fatalf("%s: version line %q missing prefix", arg, out.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelpAndNoArgs(t *testing.T) {
|
||||
var out, errw bytes.Buffer
|
||||
if code := run([]string{"help"}, &out, &errw); code != exitOK {
|
||||
t.Fatalf("help exit=%d want %d", code, exitOK)
|
||||
}
|
||||
if !strings.Contains(out.String(), "builders:") {
|
||||
t.Fatalf("help output missing builders section:\n%s", out.String())
|
||||
}
|
||||
// No args is a usage error (exit 64) but still prints help.
|
||||
out.Reset()
|
||||
if code := run(nil, &out, &errw); code != exitUsage {
|
||||
t.Fatalf("no-args exit=%d want %d", code, exitUsage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListCoversRegistry(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
list(&out)
|
||||
for _, b := range Registry {
|
||||
if !strings.Contains(out.String(), b.Name) || !strings.Contains(out.String(), b.Bin) {
|
||||
t.Fatalf("list missing %s/%s:\n%s", b.Name, b.Bin, out.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownBuilderFailsUsage(t *testing.T) {
|
||||
var out, errw bytes.Buffer
|
||||
if code := run([]string{"frobnicate"}, &out, &errw); code != exitUsage {
|
||||
t.Fatalf("unknown builder exit=%d want %d", code, exitUsage)
|
||||
}
|
||||
if !strings.Contains(errw.String(), "unknown builder") {
|
||||
t.Fatalf("unknown builder stderr=%q", errw.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryBuilderFailsClosed(t *testing.T) {
|
||||
for _, b := range Registry {
|
||||
var out, errw bytes.Buffer
|
||||
// Via dispatcher.
|
||||
if code := run([]string{b.Name}, &out, &errw); code != exitUnwired {
|
||||
t.Errorf("crucible %s: exit=%d want %d (must fail closed)", b.Name, code, exitUnwired)
|
||||
}
|
||||
if !strings.Contains(errw.String(), "not wired") {
|
||||
t.Errorf("crucible %s: stderr missing fail-closed message: %q", b.Name, errw.String())
|
||||
}
|
||||
// Via standalone shim path.
|
||||
out.Reset()
|
||||
errw.Reset()
|
||||
if code := runBuilder(b.Name, nil, &out, &errw); code != exitUnwired {
|
||||
t.Errorf("%s: exit=%d want %d (must fail closed)", b.Bin, code, exitUnwired)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuilderHelpIsOK(t *testing.T) {
|
||||
for _, b := range Registry {
|
||||
var out, errw bytes.Buffer
|
||||
if code := runBuilder(b.Name, []string{"--help"}, &out, &errw); code != exitOK {
|
||||
t.Errorf("%s --help: exit=%d want %d", b.Name, code, exitOK)
|
||||
}
|
||||
if !strings.Contains(out.String(), b.Summary) {
|
||||
t.Errorf("%s --help: missing summary", b.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,424 +0,0 @@
|
||||
package erf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
headerSize = 160
|
||||
versionV10 = "V1.0"
|
||||
strRefNone = 0xFFFFFFFF
|
||||
typeMOD = "MOD "
|
||||
typeHAK = "HAK "
|
||||
)
|
||||
|
||||
type Archive struct {
|
||||
FileType string
|
||||
Version string
|
||||
Resources []Resource
|
||||
}
|
||||
|
||||
type Resource struct {
|
||||
Name string
|
||||
Type uint16
|
||||
Data []byte
|
||||
SourcePath string
|
||||
Size int64
|
||||
}
|
||||
|
||||
type header struct {
|
||||
FileType [4]byte
|
||||
Version [4]byte
|
||||
LanguageCount uint32
|
||||
LocalizedStringSize uint32
|
||||
EntryCount uint32
|
||||
LocalizedStringOffset uint32
|
||||
KeyListOffset uint32
|
||||
ResourceListOffset uint32
|
||||
BuildYear uint32
|
||||
BuildDay uint32
|
||||
DescriptionStrRef uint32
|
||||
Reserved [116]byte
|
||||
}
|
||||
|
||||
type keyEntry struct {
|
||||
ResRef [16]byte
|
||||
ResourceID uint32
|
||||
ResourceType uint16
|
||||
Unused uint16
|
||||
}
|
||||
|
||||
type resourceEntry struct {
|
||||
Offset uint32
|
||||
Size uint32
|
||||
}
|
||||
|
||||
var extensionTypes = map[string]uint16{
|
||||
"res": 0x0000,
|
||||
"bmp": 0x0001,
|
||||
"mve": 0x0002,
|
||||
"tga": 0x0003,
|
||||
"wav": 0x0004,
|
||||
"plt": 0x0006,
|
||||
"ini": 0x0007,
|
||||
"bmu": 0x0008,
|
||||
"txt": 0x000A,
|
||||
"mdl": 0x07D2,
|
||||
"nss": 0x07D9,
|
||||
"ncs": 0x07DA,
|
||||
"are": 0x07DC,
|
||||
"set": 0x07DD,
|
||||
"ifo": 0x07DE,
|
||||
"bic": 0x07DF,
|
||||
"wok": 0x07E0,
|
||||
"2da": 0x07E1,
|
||||
"tlk": 0x07E2,
|
||||
"txi": 0x07E6,
|
||||
"git": 0x07E7,
|
||||
"uti": 0x07E9,
|
||||
"utc": 0x07EB,
|
||||
"dlg": 0x07ED,
|
||||
"itp": 0x07EE,
|
||||
"utt": 0x07F0,
|
||||
"dds": 0x07F1,
|
||||
"uts": 0x07F3,
|
||||
"fac": 0x07F6,
|
||||
"gff": 0x07F7,
|
||||
"ute": 0x07F8,
|
||||
"utd": 0x07FA,
|
||||
"utp": 0x07FC,
|
||||
"dfa": 0x07FD,
|
||||
"gic": 0x07FE,
|
||||
"gui": 0x07FF,
|
||||
"utm": 0x0803,
|
||||
"dwk": 0x0804,
|
||||
"pwk": 0x0805,
|
||||
"utg": 0x0807,
|
||||
"jrl": 0x0808,
|
||||
"utw": 0x080A,
|
||||
"ssf": 0x080C,
|
||||
"hak": 0x080D,
|
||||
"nwm": 0x080E,
|
||||
"bik": 0x080F,
|
||||
"ndb": 0x0810,
|
||||
"ptm": 0x0811,
|
||||
"ptt": 0x0812,
|
||||
"ltr": 0x0813,
|
||||
"shd": 0x0815,
|
||||
"mdb": 0x0816,
|
||||
"mtr": 0x0818,
|
||||
"jpg": 0x081C,
|
||||
"lod": 0x081E,
|
||||
"png": 0x0820,
|
||||
"lyt": 0x0BB8,
|
||||
"vis": 0x0BB9,
|
||||
"mdx": 0x0BC0,
|
||||
"wlk": 0x0BCC,
|
||||
"xml": 0x0BCD,
|
||||
"gr2": 0x0FA3,
|
||||
}
|
||||
|
||||
var typeExtensions = map[uint16]string{
|
||||
0x0000: "res",
|
||||
0x0001: "bmp",
|
||||
0x0002: "mve",
|
||||
0x0003: "tga",
|
||||
0x0004: "wav",
|
||||
0x0006: "plt",
|
||||
0x0007: "ini",
|
||||
0x0008: "bmu",
|
||||
0x000A: "txt",
|
||||
0x07D2: "mdl",
|
||||
0x07D9: "nss",
|
||||
0x07DA: "ncs",
|
||||
0x07DC: "are",
|
||||
0x07DD: "set",
|
||||
0x07DE: "ifo",
|
||||
0x07DF: "bic",
|
||||
0x07E0: "wok",
|
||||
0x07E1: "2da",
|
||||
0x07E2: "tlk",
|
||||
0x07E6: "txi",
|
||||
0x07E7: "git",
|
||||
0x07E9: "uti",
|
||||
0x07EB: "utc",
|
||||
0x07ED: "dlg",
|
||||
0x07EE: "itp",
|
||||
0x07F0: "utt",
|
||||
0x07F1: "dds",
|
||||
0x07F3: "uts",
|
||||
0x07F6: "fac",
|
||||
0x07F7: "gff",
|
||||
0x07F8: "ute",
|
||||
0x07FA: "utd",
|
||||
0x07FC: "utp",
|
||||
0x07FD: "dfa",
|
||||
0x07FE: "gic",
|
||||
0x07FF: "gui",
|
||||
0x0803: "utm",
|
||||
0x0804: "dwk",
|
||||
0x0805: "pwk",
|
||||
0x0807: "utg",
|
||||
0x0808: "jrl",
|
||||
0x080A: "utw",
|
||||
0x080C: "ssf",
|
||||
0x080D: "hak",
|
||||
0x080E: "nwm",
|
||||
0x080F: "bik",
|
||||
0x0810: "ndb",
|
||||
0x0811: "ptm",
|
||||
0x0812: "ptt",
|
||||
0x0813: "ltr",
|
||||
0x0815: "shd",
|
||||
0x0816: "mdb",
|
||||
0x0818: "mtr",
|
||||
0x081C: "jpg",
|
||||
0x081E: "lod",
|
||||
0x0820: "png",
|
||||
0x0BB8: "lyt",
|
||||
0x0BB9: "vis",
|
||||
0x0BC0: "mdx",
|
||||
0x0BCC: "wlk",
|
||||
0x0BCD: "xml",
|
||||
0x0FA3: "gr2",
|
||||
}
|
||||
|
||||
func init() {
|
||||
for ext, resourceType := range extensionTypes {
|
||||
canonicalExt, ok := typeExtensions[resourceType]
|
||||
if !ok {
|
||||
panic(fmt.Sprintf("missing canonical extension for resource type 0x%04X", resourceType))
|
||||
}
|
||||
if canonicalExt == ext {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func New(fileType string, resources []Resource) Archive {
|
||||
normalized := make([]Resource, len(resources))
|
||||
copy(normalized, resources)
|
||||
sort.Slice(normalized, func(i, j int) bool {
|
||||
if normalized[i].Name == normalized[j].Name {
|
||||
return normalized[i].Type < normalized[j].Type
|
||||
}
|
||||
return normalized[i].Name < normalized[j].Name
|
||||
})
|
||||
|
||||
return Archive{
|
||||
FileType: padFour(fileType),
|
||||
Version: versionV10,
|
||||
Resources: normalized,
|
||||
}
|
||||
}
|
||||
|
||||
func ArchiveSize(resources []Resource) int64 {
|
||||
return int64(headerSize+len(resources)*24+len(resources)*8) + totalResourceBytes(resources)
|
||||
}
|
||||
|
||||
func Write(w io.Writer, archive Archive) error {
|
||||
if archive.Version == "" {
|
||||
archive.Version = versionV10
|
||||
}
|
||||
if archive.FileType == "" {
|
||||
archive.FileType = typeMOD
|
||||
}
|
||||
|
||||
keys := make([]keyEntry, 0, len(archive.Resources))
|
||||
entries := make([]resourceEntry, 0, len(archive.Resources))
|
||||
|
||||
dataOffset := uint32(headerSize + len(archive.Resources)*24 + len(archive.Resources)*8)
|
||||
for index, resource := range archive.Resources {
|
||||
if len(resource.Name) > 16 {
|
||||
return fmt.Errorf("resource %q exceeds 16-byte resref limit", resource.Name)
|
||||
}
|
||||
size, err := payloadSize(resource)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size > int64(^uint32(0)) {
|
||||
return fmt.Errorf("resource %q exceeds 4 GiB resource size limit", resource.Name)
|
||||
}
|
||||
|
||||
entry := resourceEntry{
|
||||
Offset: dataOffset,
|
||||
Size: uint32(size),
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
dataOffset += uint32(size)
|
||||
|
||||
var key keyEntry
|
||||
copy(key.ResRef[:], []byte(resource.Name))
|
||||
key.ResourceID = uint32(index)
|
||||
key.ResourceType = resource.Type
|
||||
keys = append(keys, key)
|
||||
}
|
||||
|
||||
hdr := header{
|
||||
LanguageCount: 0,
|
||||
LocalizedStringSize: 0,
|
||||
EntryCount: uint32(len(archive.Resources)),
|
||||
LocalizedStringOffset: headerSize,
|
||||
KeyListOffset: headerSize,
|
||||
ResourceListOffset: uint32(headerSize + len(keys)*24),
|
||||
BuildYear: 0,
|
||||
BuildDay: 0,
|
||||
DescriptionStrRef: strRefNone,
|
||||
}
|
||||
copy(hdr.FileType[:], []byte(padFour(archive.FileType)))
|
||||
copy(hdr.Version[:], []byte(padFour(archive.Version)))
|
||||
|
||||
if err := binary.Write(w, binary.LittleEndian, hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := binary.Write(w, binary.LittleEndian, keys); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := binary.Write(w, binary.LittleEndian, entries); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, resource := range archive.Resources {
|
||||
if err := writeResourceData(w, resource); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Read(r io.Reader) (Archive, error) {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return Archive{}, fmt.Errorf("read erf: %w", err)
|
||||
}
|
||||
if len(data) < headerSize {
|
||||
return Archive{}, fmt.Errorf("erf file too small: %d bytes", len(data))
|
||||
}
|
||||
|
||||
var hdr header
|
||||
if err := binary.Read(bytes.NewReader(data[:headerSize]), binary.LittleEndian, &hdr); err != nil {
|
||||
return Archive{}, fmt.Errorf("decode erf header: %w", err)
|
||||
}
|
||||
|
||||
keyStart := int(hdr.KeyListOffset)
|
||||
keyEnd := keyStart + int(hdr.EntryCount)*24
|
||||
if keyEnd > len(data) {
|
||||
return Archive{}, fmt.Errorf("erf key list exceeds file bounds")
|
||||
}
|
||||
keys := make([]keyEntry, hdr.EntryCount)
|
||||
if err := binary.Read(bytes.NewReader(data[keyStart:keyEnd]), binary.LittleEndian, &keys); err != nil {
|
||||
return Archive{}, fmt.Errorf("decode key list: %w", err)
|
||||
}
|
||||
|
||||
resourceStart := int(hdr.ResourceListOffset)
|
||||
resourceEnd := resourceStart + int(hdr.EntryCount)*8
|
||||
if resourceEnd > len(data) {
|
||||
return Archive{}, fmt.Errorf("erf resource list exceeds file bounds")
|
||||
}
|
||||
entries := make([]resourceEntry, hdr.EntryCount)
|
||||
if err := binary.Read(bytes.NewReader(data[resourceStart:resourceEnd]), binary.LittleEndian, &entries); err != nil {
|
||||
return Archive{}, fmt.Errorf("decode resource list: %w", err)
|
||||
}
|
||||
|
||||
resources := make([]Resource, 0, hdr.EntryCount)
|
||||
for index, key := range keys {
|
||||
entry := entries[index]
|
||||
start := int(entry.Offset)
|
||||
end := start + int(entry.Size)
|
||||
if end > len(data) {
|
||||
return Archive{}, fmt.Errorf("resource %d exceeds file bounds", index)
|
||||
}
|
||||
|
||||
resref := string(bytes.TrimRight(key.ResRef[:], "\x00"))
|
||||
payload := make([]byte, entry.Size)
|
||||
copy(payload, data[start:end])
|
||||
resources = append(resources, Resource{
|
||||
Name: resref,
|
||||
Type: key.ResourceType,
|
||||
Data: payload,
|
||||
Size: int64(entry.Size),
|
||||
})
|
||||
}
|
||||
|
||||
return Archive{
|
||||
FileType: string(hdr.FileType[:]),
|
||||
Version: string(hdr.Version[:]),
|
||||
Resources: resources,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ResourceTypeForExtension(extension string) (uint16, bool) {
|
||||
resourceType, ok := extensionTypes[strings.TrimPrefix(strings.ToLower(extension), ".")]
|
||||
return resourceType, ok
|
||||
}
|
||||
|
||||
func HAKResourceTypeForExtension(extension string) (uint16, bool) {
|
||||
return ResourceTypeForExtension(extension)
|
||||
}
|
||||
|
||||
func ExtensionForResourceType(resourceType uint16) (string, bool) {
|
||||
extension, ok := typeExtensions[resourceType]
|
||||
return extension, ok
|
||||
}
|
||||
|
||||
func padFour(value string) string {
|
||||
value = strings.ToUpper(value)
|
||||
if len(value) >= 4 {
|
||||
return value[:4]
|
||||
}
|
||||
return value + strings.Repeat(" ", 4-len(value))
|
||||
}
|
||||
|
||||
func totalResourceBytes(resources []Resource) int64 {
|
||||
var total int64
|
||||
for _, resource := range resources {
|
||||
switch {
|
||||
case resource.Size > 0:
|
||||
total += resource.Size
|
||||
default:
|
||||
total += int64(len(resource.Data))
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func payloadSize(resource Resource) (int64, error) {
|
||||
if resource.Size > 0 {
|
||||
return resource.Size, nil
|
||||
}
|
||||
if resource.SourcePath != "" && len(resource.Data) == 0 {
|
||||
info, err := os.Stat(resource.SourcePath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stat resource %q: %w", resource.SourcePath, err)
|
||||
}
|
||||
return info.Size(), nil
|
||||
}
|
||||
return int64(len(resource.Data)), nil
|
||||
}
|
||||
|
||||
func writeResourceData(w io.Writer, resource Resource) error {
|
||||
if len(resource.Data) > 0 || resource.SourcePath == "" {
|
||||
_, err := w.Write(resource.Data)
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(resource.SourcePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open resource %q: %w", resource.SourcePath, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
written, err := io.Copy(w, file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copy resource %q: %w", resource.SourcePath, err)
|
||||
}
|
||||
if resource.Size > 0 && written != resource.Size {
|
||||
return fmt.Errorf("copy resource %q: expected %d bytes, wrote %d", resource.SourcePath, resource.Size, written)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
package erf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestArchiveRoundTrip(t *testing.T) {
|
||||
archive := New("MOD ", []Resource{
|
||||
{Name: "module", Type: 0x07DE, Data: []byte("ifo")},
|
||||
{Name: "start", Type: 0x07DC, Data: []byte("are")},
|
||||
})
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := Write(&buf, archive); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
decoded, err := Read(bytes.NewReader(buf.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
|
||||
if len(decoded.Resources) != 2 {
|
||||
t.Fatalf("expected 2 resources, got %d", len(decoded.Resources))
|
||||
}
|
||||
if decoded.Resources[0].Name != "module" || string(decoded.Resources[0].Data) != "ifo" {
|
||||
t.Fatalf("unexpected first resource: %#v", decoded.Resources[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionMappingsKeep2DAAndTLKDistinct(t *testing.T) {
|
||||
resourceType, ok := ResourceTypeForExtension("2da")
|
||||
if !ok {
|
||||
t.Fatal("expected 2da resource type")
|
||||
}
|
||||
if resourceType != 0x07E1 {
|
||||
t.Fatalf("expected 2da type 0x07E1, got 0x%04X", resourceType)
|
||||
}
|
||||
extension, ok := ExtensionForResourceType(resourceType)
|
||||
if !ok {
|
||||
t.Fatal("expected canonical extension for 2da type")
|
||||
}
|
||||
if extension != "2da" {
|
||||
t.Fatalf("expected 2da extension, got %q", extension)
|
||||
}
|
||||
|
||||
tlkType, ok := ResourceTypeForExtension("tlk")
|
||||
if !ok {
|
||||
t.Fatal("expected tlk resource type")
|
||||
}
|
||||
if tlkType != 0x07E2 {
|
||||
t.Fatalf("expected tlk type 0x07E2, got 0x%04X", tlkType)
|
||||
}
|
||||
if tlkType == resourceType {
|
||||
t.Fatal("expected 2da and tlk resource types to stay distinct")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHAKResourceTypeForExtensionSupportsPNG(t *testing.T) {
|
||||
if resourceType, ok := HAKResourceTypeForExtension("png"); !ok || resourceType != 0x0820 {
|
||||
t.Fatalf("expected png restype 0x0820 (2080) for HAK generation, got ok=%v type=0x%04X", ok, resourceType)
|
||||
}
|
||||
if resourceType, ok := HAKResourceTypeForExtension("2da"); !ok || resourceType != 0x07E1 {
|
||||
t.Fatalf("expected 2da to stay valid for HAK generation, got ok=%v type=0x%04X", ok, resourceType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionMappingsSupportModernAssetTypes(t *testing.T) {
|
||||
cases := map[string]uint16{
|
||||
"mtr": 0x0818,
|
||||
"shd": 0x0815,
|
||||
"txi": 0x07E6,
|
||||
"jpg": 0x081C,
|
||||
"mdb": 0x0816,
|
||||
"lyt": 0x0BB8,
|
||||
"vis": 0x0BB9,
|
||||
"mdx": 0x0BC0,
|
||||
"xml": 0x0BCD,
|
||||
"wlk": 0x0BCC,
|
||||
"gr2": 0x0FA3,
|
||||
}
|
||||
for ext, wantType := range cases {
|
||||
gotType, ok := ResourceTypeForExtension(ext)
|
||||
if !ok {
|
||||
t.Fatalf("expected %s to resolve to a resource type", ext)
|
||||
}
|
||||
if gotType != wantType {
|
||||
t.Fatalf("expected %s type 0x%04X, got 0x%04X", ext, wantType, gotType)
|
||||
}
|
||||
gotExt, ok := ExtensionForResourceType(wantType)
|
||||
if !ok {
|
||||
t.Fatalf("expected type 0x%04X to resolve back to an extension", wantType)
|
||||
}
|
||||
if gotExt != ext {
|
||||
t.Fatalf("expected type 0x%04X to resolve to %s, got %s", wantType, ext, gotExt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionMappingsSupportAllUTBlueprintTypes(t *testing.T) {
|
||||
cases := map[string]uint16{
|
||||
"uti": 0x07E9,
|
||||
"utc": 0x07EB,
|
||||
"utt": 0x07F0,
|
||||
"uts": 0x07F3,
|
||||
"ute": 0x07F8,
|
||||
"utd": 0x07FA,
|
||||
"utp": 0x07FC,
|
||||
"utm": 0x0803,
|
||||
"utg": 0x0807,
|
||||
"utw": 0x080A,
|
||||
}
|
||||
for ext, wantType := range cases {
|
||||
gotType, ok := ResourceTypeForExtension(ext)
|
||||
if !ok {
|
||||
t.Fatalf("expected %s to resolve to a resource type", ext)
|
||||
}
|
||||
if gotType != wantType {
|
||||
t.Fatalf("expected %s type 0x%04X, got 0x%04X", ext, wantType, gotType)
|
||||
}
|
||||
gotExt, ok := ExtensionForResourceType(wantType)
|
||||
if !ok {
|
||||
t.Fatalf("expected type 0x%04X to resolve back to an extension", wantType)
|
||||
}
|
||||
if gotExt != ext {
|
||||
t.Fatalf("expected type 0x%04X to resolve to %s, got %s", wantType, ext, gotExt)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,675 +0,0 @@
|
||||
package gff
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
)
|
||||
|
||||
const (
|
||||
headerSize = 56
|
||||
)
|
||||
|
||||
type header struct {
|
||||
FileType [4]byte
|
||||
FileVersion [4]byte
|
||||
StructOffset uint32
|
||||
StructCount uint32
|
||||
FieldOffset uint32
|
||||
FieldCount uint32
|
||||
LabelOffset uint32
|
||||
LabelCount uint32
|
||||
FieldDataOffset uint32
|
||||
FieldDataCount uint32
|
||||
FieldIndicesOffset uint32
|
||||
FieldIndicesCount uint32
|
||||
ListIndicesOffset uint32
|
||||
ListIndicesCount uint32
|
||||
}
|
||||
|
||||
type rawStruct struct {
|
||||
Type uint32
|
||||
DataOrOffset uint32
|
||||
FieldCount uint32
|
||||
}
|
||||
|
||||
type rawField struct {
|
||||
Type uint32
|
||||
LabelIndex uint32
|
||||
DataOrOffset uint32
|
||||
}
|
||||
|
||||
func Read(r io.Reader) (Document, error) {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return Document{}, fmt.Errorf("read gff: %w", err)
|
||||
}
|
||||
if len(data) < headerSize {
|
||||
return Document{}, fmt.Errorf("gff file too small: %d bytes", len(data))
|
||||
}
|
||||
|
||||
var hdr header
|
||||
if err := binary.Read(bytes.NewReader(data[:headerSize]), binary.LittleEndian, &hdr); err != nil {
|
||||
return Document{}, fmt.Errorf("decode header: %w", err)
|
||||
}
|
||||
|
||||
reader := binaryReader{data: data, hdr: hdr}
|
||||
rawStructs, err := reader.readStructs()
|
||||
if err != nil {
|
||||
return Document{}, err
|
||||
}
|
||||
rawFields, err := reader.readFields()
|
||||
if err != nil {
|
||||
return Document{}, err
|
||||
}
|
||||
labels, err := reader.readLabels()
|
||||
if err != nil {
|
||||
return Document{}, err
|
||||
}
|
||||
|
||||
root, err := reader.decodeStruct(0, rawStructs, rawFields, labels)
|
||||
if err != nil {
|
||||
return Document{}, err
|
||||
}
|
||||
|
||||
return Document{
|
||||
FileType: string(hdr.FileType[:]),
|
||||
FileVersion: string(hdr.FileVersion[:]),
|
||||
Root: root,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func Write(w io.Writer, doc Document) error {
|
||||
encoder := newBinaryEncoder(doc)
|
||||
data, err := encoder.encode()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = w.Write(data)
|
||||
return err
|
||||
}
|
||||
|
||||
type binaryReader struct {
|
||||
data []byte
|
||||
hdr header
|
||||
}
|
||||
|
||||
func (r binaryReader) readStructs() ([]rawStruct, error) {
|
||||
return readTable[rawStruct](r.data, r.hdr.StructOffset, r.hdr.StructCount)
|
||||
}
|
||||
|
||||
func (r binaryReader) readFields() ([]rawField, error) {
|
||||
return readTable[rawField](r.data, r.hdr.FieldOffset, r.hdr.FieldCount)
|
||||
}
|
||||
|
||||
func (r binaryReader) readLabels() ([]string, error) {
|
||||
start := int(r.hdr.LabelOffset)
|
||||
end := start + int(r.hdr.LabelCount)*16
|
||||
if end > len(r.data) {
|
||||
return nil, fmt.Errorf("label table exceeds file bounds")
|
||||
}
|
||||
|
||||
labels := make([]string, 0, r.hdr.LabelCount)
|
||||
for offset := start; offset < end; offset += 16 {
|
||||
chunk := r.data[offset : offset+16]
|
||||
n := bytes.IndexByte(chunk, 0)
|
||||
if n == -1 {
|
||||
n = len(chunk)
|
||||
}
|
||||
labels = append(labels, string(chunk[:n]))
|
||||
}
|
||||
return labels, nil
|
||||
}
|
||||
|
||||
func (r binaryReader) decodeStruct(index uint32, structs []rawStruct, fields []rawField, labels []string) (Struct, error) {
|
||||
if index >= uint32(len(structs)) {
|
||||
return Struct{}, fmt.Errorf("struct index %d out of range", index)
|
||||
}
|
||||
|
||||
raw := structs[index]
|
||||
fieldIndices, err := r.fieldIndices(raw)
|
||||
if err != nil {
|
||||
return Struct{}, err
|
||||
}
|
||||
|
||||
out := Struct{
|
||||
Type: raw.Type,
|
||||
Fields: make([]Field, 0, len(fieldIndices)),
|
||||
}
|
||||
for _, fieldIndex := range fieldIndices {
|
||||
if fieldIndex >= uint32(len(fields)) {
|
||||
return Struct{}, fmt.Errorf("field index %d out of range", fieldIndex)
|
||||
}
|
||||
field, err := r.decodeField(fields[fieldIndex], structs, fields, labels)
|
||||
if err != nil {
|
||||
return Struct{}, err
|
||||
}
|
||||
out.Fields = append(out.Fields, field)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r binaryReader) fieldIndices(s rawStruct) ([]uint32, error) {
|
||||
switch s.FieldCount {
|
||||
case 0:
|
||||
return nil, nil
|
||||
case 1:
|
||||
return []uint32{s.DataOrOffset}, nil
|
||||
default:
|
||||
start := int(r.hdr.FieldIndicesOffset + s.DataOrOffset)
|
||||
end := start + int(s.FieldCount)*4
|
||||
if end > len(r.data) {
|
||||
return nil, fmt.Errorf("field indices exceed file bounds")
|
||||
}
|
||||
|
||||
out := make([]uint32, s.FieldCount)
|
||||
reader := bytes.NewReader(r.data[start:end])
|
||||
if err := binary.Read(reader, binary.LittleEndian, &out); err != nil {
|
||||
return nil, fmt.Errorf("decode field indices: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (r binaryReader) decodeField(raw rawField, structs []rawStruct, fields []rawField, labels []string) (Field, error) {
|
||||
if raw.LabelIndex >= uint32(len(labels)) {
|
||||
return Field{}, fmt.Errorf("label index %d out of range", raw.LabelIndex)
|
||||
}
|
||||
fieldType := FieldType(raw.Type)
|
||||
value, err := r.decodeValue(fieldType, raw.DataOrOffset, structs, fields, labels)
|
||||
if err != nil {
|
||||
return Field{}, fmt.Errorf("decode field %q: %w", labels[raw.LabelIndex], err)
|
||||
}
|
||||
|
||||
return Field{
|
||||
Label: labels[raw.LabelIndex],
|
||||
Type: fieldType,
|
||||
Value: value,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r binaryReader) decodeValue(fieldType FieldType, data uint32, structs []rawStruct, fields []rawField, labels []string) (Value, error) {
|
||||
switch fieldType {
|
||||
case TypeByte:
|
||||
return ByteValue(uint8(data)), nil
|
||||
case TypeChar:
|
||||
return CharValue(int8(data)), nil
|
||||
case TypeWord:
|
||||
return WordValue(uint16(data)), nil
|
||||
case TypeShort:
|
||||
return ShortValue(int16(data)), nil
|
||||
case TypeDWord:
|
||||
return DWordValue(data), nil
|
||||
case TypeInt:
|
||||
return IntValue(int32(data)), nil
|
||||
case TypeFloat:
|
||||
return FloatValue(math.Float32frombits(data)), nil
|
||||
case TypeDWord64:
|
||||
raw, err := r.sliceFieldData(data, 8)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return DWord64Value(binary.LittleEndian.Uint64(raw)), nil
|
||||
case TypeInt64:
|
||||
raw, err := r.sliceFieldData(data, 8)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return Int64Value(int64(binary.LittleEndian.Uint64(raw))), nil
|
||||
case TypeDouble:
|
||||
raw, err := r.sliceFieldData(data, 8)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return DoubleValue(math.Float64frombits(binary.LittleEndian.Uint64(raw))), nil
|
||||
case TypeCExoString:
|
||||
raw, err := r.prefixedFieldData(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return StringValue(string(raw)), nil
|
||||
case TypeResRef:
|
||||
raw, err := r.prefixedByteFieldData(data, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ResRefValue(string(raw)), nil
|
||||
case TypeCExoLocString:
|
||||
raw, err := r.locStringFieldData(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeLocString(raw)
|
||||
case TypeVoid:
|
||||
raw, err := r.prefixedFieldData(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return VoidValue(raw), nil
|
||||
case TypeStruct:
|
||||
return r.decodeStruct(data, structs, fields, labels)
|
||||
case TypeList:
|
||||
return r.decodeList(data, structs, fields, labels)
|
||||
case TypeOrientation:
|
||||
raw, err := r.sliceFieldData(data, 16)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return Orientation{
|
||||
X: math.Float32frombits(binary.LittleEndian.Uint32(raw[0:4])),
|
||||
Y: math.Float32frombits(binary.LittleEndian.Uint32(raw[4:8])),
|
||||
Z: math.Float32frombits(binary.LittleEndian.Uint32(raw[8:12])),
|
||||
W: math.Float32frombits(binary.LittleEndian.Uint32(raw[12:16])),
|
||||
}, nil
|
||||
case TypeVector:
|
||||
raw, err := r.sliceFieldData(data, 12)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return Vector{
|
||||
X: math.Float32frombits(binary.LittleEndian.Uint32(raw[0:4])),
|
||||
Y: math.Float32frombits(binary.LittleEndian.Uint32(raw[4:8])),
|
||||
Z: math.Float32frombits(binary.LittleEndian.Uint32(raw[8:12])),
|
||||
}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported field type %d", fieldType)
|
||||
}
|
||||
}
|
||||
|
||||
func (r binaryReader) decodeList(offset uint32, structs []rawStruct, fields []rawField, labels []string) (ListValue, error) {
|
||||
start := int(r.hdr.ListIndicesOffset + offset)
|
||||
if start+4 > len(r.data) {
|
||||
return nil, fmt.Errorf("list data exceeds file bounds")
|
||||
}
|
||||
count := binary.LittleEndian.Uint32(r.data[start : start+4])
|
||||
start += 4
|
||||
end := start + int(count)*4
|
||||
if end > len(r.data) {
|
||||
return nil, fmt.Errorf("list indices exceed file bounds")
|
||||
}
|
||||
|
||||
list := make(ListValue, 0, count)
|
||||
for pos := start; pos < end; pos += 4 {
|
||||
index := binary.LittleEndian.Uint32(r.data[pos : pos+4])
|
||||
child, err := r.decodeStruct(index, structs, fields, labels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
list = append(list, child)
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func (r binaryReader) sliceFieldData(offset uint32, size int) ([]byte, error) {
|
||||
start := int(r.hdr.FieldDataOffset + offset)
|
||||
end := start + size
|
||||
if end > len(r.data) {
|
||||
return nil, fmt.Errorf("field data exceeds file bounds")
|
||||
}
|
||||
return r.data[start:end], nil
|
||||
}
|
||||
|
||||
func (r binaryReader) prefixedFieldData(offset uint32) ([]byte, error) {
|
||||
sizeRaw, err := r.sliceFieldData(offset, 4)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := int(binary.LittleEndian.Uint32(sizeRaw))
|
||||
return r.sliceFieldData(offset+4, size)
|
||||
}
|
||||
|
||||
func (r binaryReader) prefixedByteFieldData(offset uint32, prefixBytes uint32) ([]byte, error) {
|
||||
header, err := r.sliceFieldData(offset, int(prefixBytes))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := int(header[0])
|
||||
return r.sliceFieldData(offset+prefixBytes, size)
|
||||
}
|
||||
|
||||
func (r binaryReader) locStringFieldData(offset uint32) ([]byte, error) {
|
||||
sizeRaw, err := r.sliceFieldData(offset, 4)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := int(binary.LittleEndian.Uint32(sizeRaw))
|
||||
return r.sliceFieldData(offset, size+4)
|
||||
}
|
||||
|
||||
func readTable[T any](data []byte, offset uint32, count uint32) ([]T, error) {
|
||||
if count == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var row T
|
||||
size := binary.Size(row)
|
||||
start := int(offset)
|
||||
end := start + int(count)*size
|
||||
if end > len(data) {
|
||||
return nil, fmt.Errorf("table exceeds file bounds")
|
||||
}
|
||||
|
||||
out := make([]T, count)
|
||||
reader := bytes.NewReader(data[start:end])
|
||||
if err := binary.Read(reader, binary.LittleEndian, &out); err != nil {
|
||||
return nil, fmt.Errorf("decode table: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func decodeLocString(data []byte) (LocString, error) {
|
||||
if len(data) < 12 {
|
||||
return LocString{}, fmt.Errorf("locstring too small")
|
||||
}
|
||||
totalSize := binary.LittleEndian.Uint32(data[0:4])
|
||||
if totalSize+4 != uint32(len(data)) {
|
||||
return LocString{}, fmt.Errorf("locstring size mismatch")
|
||||
}
|
||||
stringRef := binary.LittleEndian.Uint32(data[4:8])
|
||||
count := binary.LittleEndian.Uint32(data[8:12])
|
||||
pos := 12
|
||||
entries := make([]LocStringEntry, 0, count)
|
||||
for i := uint32(0); i < count; i++ {
|
||||
if pos+8 > len(data) {
|
||||
return LocString{}, fmt.Errorf("locstring entry header truncated")
|
||||
}
|
||||
id := binary.LittleEndian.Uint32(data[pos : pos+4])
|
||||
length := binary.LittleEndian.Uint32(data[pos+4 : pos+8])
|
||||
pos += 8
|
||||
if pos+int(length) > len(data) {
|
||||
return LocString{}, fmt.Errorf("locstring entry %d truncated", i)
|
||||
}
|
||||
entries = append(entries, LocStringEntry{
|
||||
ID: id,
|
||||
Value: string(data[pos : pos+int(length)]),
|
||||
})
|
||||
pos += int(length)
|
||||
}
|
||||
|
||||
return LocString{
|
||||
StringRef: stringRef,
|
||||
Entries: entries,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type binaryEncoder struct {
|
||||
document Document
|
||||
structs []rawStruct
|
||||
fields []rawField
|
||||
labels []string
|
||||
labelIndex map[string]uint32
|
||||
fieldData bytes.Buffer
|
||||
fieldIndices []uint32
|
||||
listIndices []uint32
|
||||
}
|
||||
|
||||
func newBinaryEncoder(doc Document) *binaryEncoder {
|
||||
return &binaryEncoder{
|
||||
document: doc,
|
||||
labelIndex: map[string]uint32{},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *binaryEncoder) encode() ([]byte, error) {
|
||||
if err := e.addStruct(e.document.Root); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
hdr := header{}
|
||||
copy(hdr.FileType[:], []byte(padFour(e.document.FileType)))
|
||||
copy(hdr.FileVersion[:], []byte(padFour(defaultString(e.document.FileVersion, "V3.2"))))
|
||||
|
||||
hdr.StructOffset = headerSize
|
||||
hdr.StructCount = uint32(len(e.structs))
|
||||
hdr.FieldOffset = hdr.StructOffset + uint32(len(e.structs))*12
|
||||
hdr.FieldCount = uint32(len(e.fields))
|
||||
hdr.LabelOffset = hdr.FieldOffset + uint32(len(e.fields))*12
|
||||
hdr.LabelCount = uint32(len(e.labels))
|
||||
hdr.FieldDataOffset = hdr.LabelOffset + uint32(len(e.labels))*16
|
||||
hdr.FieldDataCount = uint32(e.fieldData.Len())
|
||||
hdr.FieldIndicesOffset = hdr.FieldDataOffset + uint32(e.fieldData.Len())
|
||||
hdr.FieldIndicesCount = uint32(len(e.fieldIndices) * 4)
|
||||
hdr.ListIndicesOffset = hdr.FieldIndicesOffset + uint32(len(e.fieldIndices))*4
|
||||
hdr.ListIndicesCount = uint32(len(e.listIndices) * 4)
|
||||
|
||||
if err := binary.Write(&buf, binary.LittleEndian, hdr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := binary.Write(&buf, binary.LittleEndian, e.structs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := binary.Write(&buf, binary.LittleEndian, e.fields); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, label := range e.labels {
|
||||
var raw [16]byte
|
||||
copy(raw[:], []byte(label))
|
||||
if _, err := buf.Write(raw[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if _, err := buf.Write(e.fieldData.Bytes()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := binary.Write(&buf, binary.LittleEndian, e.fieldIndices); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := binary.Write(&buf, binary.LittleEndian, e.listIndices); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func (e *binaryEncoder) addStruct(s Struct) error {
|
||||
index := uint32(len(e.structs))
|
||||
e.structs = append(e.structs, rawStruct{Type: s.Type})
|
||||
|
||||
fieldIndices := make([]uint32, 0, len(s.Fields))
|
||||
for _, field := range s.Fields {
|
||||
fieldIndex, err := e.addField(field)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fieldIndices = append(fieldIndices, fieldIndex)
|
||||
}
|
||||
|
||||
entry := &e.structs[index]
|
||||
entry.FieldCount = uint32(len(fieldIndices))
|
||||
switch len(fieldIndices) {
|
||||
case 0:
|
||||
entry.DataOrOffset = 0
|
||||
case 1:
|
||||
entry.DataOrOffset = fieldIndices[0]
|
||||
default:
|
||||
entry.DataOrOffset = uint32(len(e.fieldIndices) * 4)
|
||||
e.fieldIndices = append(e.fieldIndices, fieldIndices...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *binaryEncoder) addField(field Field) (uint32, error) {
|
||||
labelIndex, err := e.indexLabel(field.Label)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
data, err := e.encodeValue(field.Value)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("encode field %q: %w", field.Label, err)
|
||||
}
|
||||
|
||||
entry := rawField{
|
||||
Type: uint32(field.Type),
|
||||
LabelIndex: labelIndex,
|
||||
DataOrOffset: data,
|
||||
}
|
||||
index := uint32(len(e.fields))
|
||||
e.fields = append(e.fields, entry)
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func (e *binaryEncoder) encodeValue(value Value) (uint32, error) {
|
||||
switch typed := value.(type) {
|
||||
case ByteValue:
|
||||
return uint32(uint8(typed)), nil
|
||||
case CharValue:
|
||||
return uint32(uint8(typed)), nil
|
||||
case WordValue:
|
||||
return uint32(uint16(typed)), nil
|
||||
case ShortValue:
|
||||
return uint32(uint16(typed)), nil
|
||||
case DWordValue:
|
||||
return uint32(typed), nil
|
||||
case IntValue:
|
||||
return uint32(int32(typed)), nil
|
||||
case FloatValue:
|
||||
return math.Float32bits(float32(typed)), nil
|
||||
case DWord64Value:
|
||||
return e.writeFieldData(func(buf *bytes.Buffer) error {
|
||||
return binary.Write(buf, binary.LittleEndian, uint64(typed))
|
||||
}), nil
|
||||
case Int64Value:
|
||||
return e.writeFieldData(func(buf *bytes.Buffer) error {
|
||||
return binary.Write(buf, binary.LittleEndian, int64(typed))
|
||||
}), nil
|
||||
case DoubleValue:
|
||||
return e.writeFieldData(func(buf *bytes.Buffer) error {
|
||||
return binary.Write(buf, binary.LittleEndian, math.Float64bits(float64(typed)))
|
||||
}), nil
|
||||
case StringValue:
|
||||
return e.writeLengthPrefixed([]byte(typed)), nil
|
||||
case ResRefValue:
|
||||
if len(typed) > 255 {
|
||||
return 0, fmt.Errorf("resref exceeds 255 bytes")
|
||||
}
|
||||
return e.writeFieldData(func(buf *bytes.Buffer) error {
|
||||
if err := buf.WriteByte(byte(len(typed))); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := buf.Write([]byte(typed))
|
||||
return err
|
||||
}), nil
|
||||
case LocString:
|
||||
return e.writeFieldData(func(buf *bytes.Buffer) error {
|
||||
payload := bytes.Buffer{}
|
||||
if err := binary.Write(&payload, binary.LittleEndian, typed.StringRef); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := binary.Write(&payload, binary.LittleEndian, uint32(len(typed.Entries))); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range typed.Entries {
|
||||
if err := binary.Write(&payload, binary.LittleEndian, entry.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := binary.Write(&payload, binary.LittleEndian, uint32(len(entry.Value))); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := payload.Write([]byte(entry.Value)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := binary.Write(buf, binary.LittleEndian, uint32(payload.Len())); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := buf.Write(payload.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}), nil
|
||||
case VoidValue:
|
||||
return e.writeLengthPrefixed([]byte(typed)), nil
|
||||
case Struct:
|
||||
index := uint32(len(e.structs))
|
||||
if err := e.addStruct(typed); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return index, nil
|
||||
case ListValue:
|
||||
return e.writeList(typed)
|
||||
case Orientation:
|
||||
return e.writeFieldData(func(buf *bytes.Buffer) error {
|
||||
for _, component := range []float32{typed.X, typed.Y, typed.Z, typed.W} {
|
||||
if err := binary.Write(buf, binary.LittleEndian, math.Float32bits(component)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}), nil
|
||||
case Vector:
|
||||
return e.writeFieldData(func(buf *bytes.Buffer) error {
|
||||
for _, component := range []float32{typed.X, typed.Y, typed.Z} {
|
||||
if err := binary.Write(buf, binary.LittleEndian, math.Float32bits(component)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}), nil
|
||||
default:
|
||||
return 0, fmt.Errorf("unsupported value type %T", value)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *binaryEncoder) writeList(list ListValue) (uint32, error) {
|
||||
indices := make([]uint32, 0, len(list))
|
||||
for _, item := range list {
|
||||
index := uint32(len(e.structs))
|
||||
if err := e.addStruct(item); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
indices = append(indices, index)
|
||||
}
|
||||
|
||||
offset := uint32(len(e.listIndices) * 4)
|
||||
e.listIndices = append(e.listIndices, uint32(len(list)))
|
||||
e.listIndices = append(e.listIndices, indices...)
|
||||
return offset, nil
|
||||
}
|
||||
|
||||
func (e *binaryEncoder) writeFieldData(write func(*bytes.Buffer) error) uint32 {
|
||||
offset := uint32(e.fieldData.Len())
|
||||
if err := write(&e.fieldData); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return offset
|
||||
}
|
||||
|
||||
func (e *binaryEncoder) writeLengthPrefixed(data []byte) uint32 {
|
||||
return e.writeFieldData(func(buf *bytes.Buffer) error {
|
||||
if err := binary.Write(buf, binary.LittleEndian, uint32(len(data))); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := buf.Write(data)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (e *binaryEncoder) indexLabel(label string) (uint32, error) {
|
||||
if len(label) > 16 {
|
||||
return 0, fmt.Errorf("label %q exceeds 16 bytes", label)
|
||||
}
|
||||
if index, ok := e.labelIndex[label]; ok {
|
||||
return index, nil
|
||||
}
|
||||
index := uint32(len(e.labels))
|
||||
e.labels = append(e.labels, label)
|
||||
e.labelIndex[label] = index
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func padFour(value string) string {
|
||||
if len(value) >= 4 {
|
||||
return value[:4]
|
||||
}
|
||||
return value + string(bytes.Repeat([]byte(" "), 4-len(value)))
|
||||
}
|
||||
|
||||
func defaultString(value, fallback string) string {
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package gff
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRoundTripBinaryAndJSON(t *testing.T) {
|
||||
original := Document{
|
||||
FileType: "UTC ",
|
||||
FileVersion: "V3.2",
|
||||
Root: Struct{
|
||||
Type: 0,
|
||||
Fields: []Field{
|
||||
NewField("Tag", StringValue("test_creature")),
|
||||
NewField("TemplateResRef", ResRefValue("nw_test")),
|
||||
NewField("HP", IntValue(12)),
|
||||
NewField("Position", Vector{X: 1.25, Y: 2.5, Z: 3.75}),
|
||||
NewField("Inventory", ListValue{
|
||||
{
|
||||
Type: 1,
|
||||
Fields: []Field{
|
||||
NewField("Slot", DWordValue(0)),
|
||||
NewField("ResRef", ResRefValue("itm_sword")),
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var binaryBuf bytes.Buffer
|
||||
if err := Write(&binaryBuf, original); err != nil {
|
||||
t.Fatalf("write binary: %v", err)
|
||||
}
|
||||
|
||||
decoded, err := Read(bytes.NewReader(binaryBuf.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("read binary: %v", err)
|
||||
}
|
||||
|
||||
left, err := json.Marshal(original)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal original json: %v", err)
|
||||
}
|
||||
right, err := json.Marshal(decoded)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal decoded json: %v", err)
|
||||
}
|
||||
|
||||
if string(left) != string(right) {
|
||||
t.Fatalf("roundtrip mismatch\noriginal: %s\ndecoded: %s", left, right)
|
||||
}
|
||||
}
|
||||
@@ -1,317 +0,0 @@
|
||||
package gff
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type jsonDocument struct {
|
||||
FileType string `json:"file_type"`
|
||||
FileVersion string `json:"file_version"`
|
||||
Root jsonStruct `json:"root"`
|
||||
}
|
||||
|
||||
type jsonStruct struct {
|
||||
StructType uint32 `json:"struct_type"`
|
||||
Fields []jsonField `json:"fields"`
|
||||
}
|
||||
|
||||
type jsonField struct {
|
||||
Label string `json:"label"`
|
||||
Type string `json:"type"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
|
||||
type jsonLocString struct {
|
||||
StringRef uint32 `json:"string_ref"`
|
||||
Entries []LocStringEntry `json:"entries"`
|
||||
}
|
||||
|
||||
func (s Struct) MarshalJSON() ([]byte, error) {
|
||||
payload, err := marshalStruct(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(payload)
|
||||
}
|
||||
|
||||
func (s *Struct) UnmarshalJSON(data []byte) error {
|
||||
var payload jsonStruct
|
||||
if err := json.Unmarshal(data, &payload); err != nil {
|
||||
return err
|
||||
}
|
||||
decoded, err := unmarshalStruct(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*s = decoded
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d Document) MarshalJSON() ([]byte, error) {
|
||||
root, err := marshalStruct(d.Root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return json.Marshal(jsonDocument{
|
||||
FileType: d.FileType,
|
||||
FileVersion: d.FileVersion,
|
||||
Root: root,
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Document) UnmarshalJSON(data []byte) error {
|
||||
var payload jsonDocument
|
||||
if err := json.Unmarshal(data, &payload); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
root, err := unmarshalStruct(payload.Root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.FileType = payload.FileType
|
||||
d.FileVersion = payload.FileVersion
|
||||
d.Root = root
|
||||
return nil
|
||||
}
|
||||
|
||||
func marshalStruct(in Struct) (jsonStruct, error) {
|
||||
fields := make([]jsonField, 0, len(in.Fields))
|
||||
for _, field := range in.Fields {
|
||||
payload, err := marshalValue(field.Value)
|
||||
if err != nil {
|
||||
return jsonStruct{}, fmt.Errorf("marshal field %q: %w", field.Label, err)
|
||||
}
|
||||
fields = append(fields, jsonField{
|
||||
Label: field.Label,
|
||||
Type: field.Type.String(),
|
||||
Value: payload,
|
||||
})
|
||||
}
|
||||
|
||||
return jsonStruct{
|
||||
StructType: in.Type,
|
||||
Fields: fields,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func unmarshalStruct(in jsonStruct) (Struct, error) {
|
||||
fields := make([]Field, 0, len(in.Fields))
|
||||
for _, field := range in.Fields {
|
||||
ft, err := parseFieldType(field.Type)
|
||||
if err != nil {
|
||||
return Struct{}, fmt.Errorf("field %q: %w", field.Label, err)
|
||||
}
|
||||
|
||||
value, err := unmarshalValue(ft, field.Value)
|
||||
if err != nil {
|
||||
return Struct{}, fmt.Errorf("field %q: %w", field.Label, err)
|
||||
}
|
||||
|
||||
fields = append(fields, Field{
|
||||
Label: field.Label,
|
||||
Type: ft,
|
||||
Value: value,
|
||||
})
|
||||
}
|
||||
|
||||
return Struct{
|
||||
Type: in.StructType,
|
||||
Fields: fields,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func marshalValue(value Value) (json.RawMessage, error) {
|
||||
switch typed := value.(type) {
|
||||
case ByteValue:
|
||||
return json.Marshal(uint8(typed))
|
||||
case CharValue:
|
||||
return json.Marshal(int8(typed))
|
||||
case WordValue:
|
||||
return json.Marshal(uint16(typed))
|
||||
case ShortValue:
|
||||
return json.Marshal(int16(typed))
|
||||
case DWordValue:
|
||||
return json.Marshal(uint32(typed))
|
||||
case IntValue:
|
||||
return json.Marshal(int32(typed))
|
||||
case DWord64Value:
|
||||
return json.Marshal(uint64(typed))
|
||||
case Int64Value:
|
||||
return json.Marshal(int64(typed))
|
||||
case FloatValue:
|
||||
return json.Marshal(float32(typed))
|
||||
case DoubleValue:
|
||||
return json.Marshal(float64(typed))
|
||||
case StringValue:
|
||||
return json.Marshal(string(typed))
|
||||
case ResRefValue:
|
||||
return json.Marshal(string(typed))
|
||||
case LocString:
|
||||
return json.Marshal(jsonLocString(typed))
|
||||
case VoidValue:
|
||||
return json.Marshal(base64.StdEncoding.EncodeToString([]byte(typed)))
|
||||
case Struct:
|
||||
payload, err := marshalStruct(typed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(payload)
|
||||
case ListValue:
|
||||
payload := make([]jsonStruct, 0, len(typed))
|
||||
for _, item := range typed {
|
||||
entry, err := marshalStruct(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
payload = append(payload, entry)
|
||||
}
|
||||
return json.Marshal(payload)
|
||||
case Orientation:
|
||||
return json.Marshal(typed)
|
||||
case Vector:
|
||||
return json.Marshal(typed)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported value type %T", value)
|
||||
}
|
||||
}
|
||||
|
||||
func unmarshalValue(ft FieldType, data []byte) (Value, error) {
|
||||
switch ft {
|
||||
case TypeByte:
|
||||
var out uint8
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ByteValue(out), nil
|
||||
case TypeChar:
|
||||
var out int8
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return CharValue(out), nil
|
||||
case TypeWord:
|
||||
var out uint16
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return WordValue(out), nil
|
||||
case TypeShort:
|
||||
var out int16
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ShortValue(out), nil
|
||||
case TypeDWord:
|
||||
var out uint32
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return DWordValue(out), nil
|
||||
case TypeInt:
|
||||
var out int32
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return IntValue(out), nil
|
||||
case TypeDWord64:
|
||||
var out uint64
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return DWord64Value(out), nil
|
||||
case TypeInt64:
|
||||
var out int64
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return Int64Value(out), nil
|
||||
case TypeFloat:
|
||||
var out float32
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return FloatValue(out), nil
|
||||
case TypeDouble:
|
||||
var out float64
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return DoubleValue(out), nil
|
||||
case TypeCExoString:
|
||||
var out string
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return StringValue(out), nil
|
||||
case TypeResRef:
|
||||
var out string
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ResRefValue(out), nil
|
||||
case TypeCExoLocString:
|
||||
var out jsonLocString
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return LocString(out), nil
|
||||
case TypeVoid:
|
||||
var out string
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(out)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode base64 void: %w", err)
|
||||
}
|
||||
return VoidValue(decoded), nil
|
||||
case TypeStruct:
|
||||
var payload jsonStruct
|
||||
if err := json.Unmarshal(data, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return unmarshalStruct(payload)
|
||||
case TypeList:
|
||||
var payload []jsonStruct
|
||||
if err := json.Unmarshal(data, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(ListValue, 0, len(payload))
|
||||
for _, item := range payload {
|
||||
decoded, err := unmarshalStruct(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, decoded)
|
||||
}
|
||||
return out, nil
|
||||
case TypeOrientation:
|
||||
var out Orientation
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
case TypeVector:
|
||||
var out Vector
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported field type %d", ft)
|
||||
}
|
||||
}
|
||||
|
||||
func parseFieldType(name string) (FieldType, error) {
|
||||
for kind, candidate := range fieldTypeNames {
|
||||
if candidate == name {
|
||||
return kind, nil
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("unknown field type %q", name)
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
package gff
|
||||
|
||||
import "fmt"
|
||||
|
||||
type FieldType uint32
|
||||
|
||||
const (
|
||||
TypeByte FieldType = iota
|
||||
TypeChar
|
||||
TypeWord
|
||||
TypeShort
|
||||
TypeDWord
|
||||
TypeInt
|
||||
TypeDWord64
|
||||
TypeInt64
|
||||
TypeFloat
|
||||
TypeDouble
|
||||
TypeCExoString
|
||||
TypeResRef
|
||||
TypeCExoLocString
|
||||
TypeVoid
|
||||
TypeStruct
|
||||
TypeList
|
||||
TypeOrientation
|
||||
TypeVector
|
||||
)
|
||||
|
||||
var fieldTypeNames = map[FieldType]string{
|
||||
TypeByte: "Byte",
|
||||
TypeChar: "Char",
|
||||
TypeWord: "Word",
|
||||
TypeShort: "Short",
|
||||
TypeDWord: "DWord",
|
||||
TypeInt: "Int",
|
||||
TypeDWord64: "DWord64",
|
||||
TypeInt64: "Int64",
|
||||
TypeFloat: "Float",
|
||||
TypeDouble: "Double",
|
||||
TypeCExoString: "CExoString",
|
||||
TypeResRef: "ResRef",
|
||||
TypeCExoLocString: "CExoLocString",
|
||||
TypeVoid: "Void",
|
||||
TypeStruct: "Struct",
|
||||
TypeList: "List",
|
||||
TypeOrientation: "Orientation",
|
||||
TypeVector: "Vector",
|
||||
}
|
||||
|
||||
type Document struct {
|
||||
FileType string `json:"file_type"`
|
||||
FileVersion string `json:"file_version"`
|
||||
Root Struct `json:"root"`
|
||||
}
|
||||
|
||||
type Struct struct {
|
||||
Type uint32 `json:"struct_type"`
|
||||
Fields []Field `json:"fields"`
|
||||
}
|
||||
|
||||
type Field struct {
|
||||
Label string `json:"label"`
|
||||
Type FieldType `json:"-"`
|
||||
Value Value `json:"-"`
|
||||
}
|
||||
|
||||
type Value interface {
|
||||
fieldType() FieldType
|
||||
}
|
||||
|
||||
type LocString struct {
|
||||
StringRef uint32 `json:"string_ref"`
|
||||
Entries []LocStringEntry `json:"entries"`
|
||||
}
|
||||
|
||||
type LocStringEntry struct {
|
||||
ID uint32 `json:"id"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type Vector struct {
|
||||
X float32 `json:"x"`
|
||||
Y float32 `json:"y"`
|
||||
Z float32 `json:"z"`
|
||||
}
|
||||
|
||||
type Orientation struct {
|
||||
X float32 `json:"x"`
|
||||
Y float32 `json:"y"`
|
||||
Z float32 `json:"z"`
|
||||
W float32 `json:"w"`
|
||||
}
|
||||
|
||||
type ByteValue uint8
|
||||
type CharValue int8
|
||||
type WordValue uint16
|
||||
type ShortValue int16
|
||||
type DWordValue uint32
|
||||
type IntValue int32
|
||||
type DWord64Value uint64
|
||||
type Int64Value int64
|
||||
type FloatValue float32
|
||||
type DoubleValue float64
|
||||
type StringValue string
|
||||
type ResRefValue string
|
||||
type VoidValue []byte
|
||||
type ListValue []Struct
|
||||
|
||||
func (ByteValue) fieldType() FieldType { return TypeByte }
|
||||
func (CharValue) fieldType() FieldType { return TypeChar }
|
||||
func (WordValue) fieldType() FieldType { return TypeWord }
|
||||
func (ShortValue) fieldType() FieldType { return TypeShort }
|
||||
func (DWordValue) fieldType() FieldType { return TypeDWord }
|
||||
func (IntValue) fieldType() FieldType { return TypeInt }
|
||||
func (DWord64Value) fieldType() FieldType { return TypeDWord64 }
|
||||
func (Int64Value) fieldType() FieldType { return TypeInt64 }
|
||||
func (FloatValue) fieldType() FieldType { return TypeFloat }
|
||||
func (DoubleValue) fieldType() FieldType { return TypeDouble }
|
||||
func (StringValue) fieldType() FieldType { return TypeCExoString }
|
||||
func (ResRefValue) fieldType() FieldType { return TypeResRef }
|
||||
func (LocString) fieldType() FieldType { return TypeCExoLocString }
|
||||
func (VoidValue) fieldType() FieldType { return TypeVoid }
|
||||
func (Struct) fieldType() FieldType { return TypeStruct }
|
||||
func (ListValue) fieldType() FieldType { return TypeList }
|
||||
func (Orientation) fieldType() FieldType { return TypeOrientation }
|
||||
func (Vector) fieldType() FieldType { return TypeVector }
|
||||
|
||||
func (t FieldType) String() string {
|
||||
if name, ok := fieldTypeNames[t]; ok {
|
||||
return name
|
||||
}
|
||||
return fmt.Sprintf("FieldType(%d)", t)
|
||||
}
|
||||
|
||||
func NewField(label string, value Value) Field {
|
||||
return Field{
|
||||
Label: label,
|
||||
Type: value.fieldType(),
|
||||
Value: value,
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type DatasetConfig struct {
|
||||
Source string `json:"source" yaml:"source"`
|
||||
Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
|
||||
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
|
||||
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
|
||||
}
|
||||
|
||||
type DefaultsConfig struct {
|
||||
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
|
||||
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
|
||||
CreditsOverlay string `json:"credits_overlay,omitempty" yaml:"credits_overlay,omitempty"`
|
||||
MaxStemLength *int `json:"max_stem_length,omitempty" yaml:"max_stem_length,omitempty"`
|
||||
}
|
||||
|
||||
func ValidateDatasetSource(field, source string) error {
|
||||
trimmed := strings.TrimSpace(source)
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("%s.source is required", field)
|
||||
}
|
||||
if strings.Contains(trimmed, "\x00") {
|
||||
return fmt.Errorf("%s.source must not contain NUL bytes", field)
|
||||
}
|
||||
clean := filepath.Clean(filepath.FromSlash(trimmed))
|
||||
if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||
return fmt.Errorf("%s.source must not escape project root", field)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateDatasetID(id string) error {
|
||||
trimmed := strings.TrimSpace(id)
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("dataset id must not be empty")
|
||||
}
|
||||
if strings.Contains(trimmed, "/") || strings.Contains(trimmed, "\\") {
|
||||
return fmt.Errorf("dataset id %q must not contain path separators", trimmed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,292 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func ParseCreditsOverlay(path string) (CreditsOverlay, error) {
|
||||
entries, err := ParseCreditsMarkdown(path)
|
||||
if err != nil {
|
||||
return CreditsOverlay{}, err
|
||||
}
|
||||
overlay := CreditsOverlay{
|
||||
ByOriginal: make(map[string]CreditsEntry),
|
||||
ByOutput: make(map[string]CreditsEntry),
|
||||
Entries: entries,
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.OriginalFile != "" {
|
||||
key := strings.ToLower(strings.TrimSpace(entry.OriginalFile))
|
||||
if _, exists := overlay.ByOriginal[key]; exists {
|
||||
return CreditsOverlay{}, fmt.Errorf("duplicate manual credits row for original file %s", entry.OriginalFile)
|
||||
}
|
||||
overlay.ByOriginal[key] = entry
|
||||
}
|
||||
if entry.OutputFile != "" {
|
||||
key := strings.ToLower(strings.TrimSpace(entry.OutputFile))
|
||||
if _, exists := overlay.ByOutput[key]; exists {
|
||||
return CreditsOverlay{}, fmt.Errorf("duplicate manual credits row for output file %s", entry.OutputFile)
|
||||
}
|
||||
overlay.ByOutput[key] = entry
|
||||
}
|
||||
}
|
||||
return overlay, nil
|
||||
}
|
||||
|
||||
func (o CreditsOverlay) Match(originalFile, outputFile string) *CreditsEntry {
|
||||
if entry, ok := o.ByOutput[strings.ToLower(strings.TrimSpace(outputFile))]; ok {
|
||||
entryCopy := entry
|
||||
return &entryCopy
|
||||
}
|
||||
if entry, ok := o.ByOriginal[strings.ToLower(strings.TrimSpace(originalFile))]; ok {
|
||||
entryCopy := entry
|
||||
return &entryCopy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateOverlayEntries(dir string, overlay CreditsOverlay, generated []CreditsEntry) error {
|
||||
if len(overlay.Entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
validOriginal := make(map[string]struct{}, len(generated))
|
||||
validOutput := make(map[string]struct{}, len(generated))
|
||||
for _, entry := range generated {
|
||||
validOriginal[strings.ToLower(entry.OriginalFile)] = struct{}{}
|
||||
validOutput[strings.ToLower(entry.OutputFile)] = struct{}{}
|
||||
}
|
||||
for _, entry := range overlay.Entries {
|
||||
if entry.OriginalFile != "" {
|
||||
if _, ok := validOriginal[strings.ToLower(entry.OriginalFile)]; !ok {
|
||||
return fmt.Errorf("manual credits entry in %s references unknown original file %s", dir, entry.OriginalFile)
|
||||
}
|
||||
}
|
||||
if entry.OutputFile != "" {
|
||||
if _, ok := validOutput[strings.ToLower(entry.OutputFile)]; !ok {
|
||||
return fmt.Errorf("manual credits entry in %s references unknown output file %s", dir, entry.OutputFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ParseCreditsMarkdown(path string) ([]CreditsEntry, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
lines := strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n")
|
||||
header := []string(nil)
|
||||
entries := make([]CreditsEntry, 0)
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "|") {
|
||||
continue
|
||||
}
|
||||
cells := ParseMarkdownTableRow(line)
|
||||
if len(cells) == 0 {
|
||||
continue
|
||||
}
|
||||
if header == nil {
|
||||
header = make([]string, len(cells))
|
||||
for i, cell := range cells {
|
||||
header[i] = NormalizeCreditsHeader(cell)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if IsMarkdownSeparatorRow(cells) {
|
||||
continue
|
||||
}
|
||||
entry := CreditsEntry{}
|
||||
for i, key := range header {
|
||||
if i >= len(cells) {
|
||||
continue
|
||||
}
|
||||
value := CleanCreditsCell(cells[i])
|
||||
switch key {
|
||||
case "artist":
|
||||
entry.Artist = value
|
||||
case "title":
|
||||
entry.Title = value
|
||||
case "output_file":
|
||||
entry.OutputFile = value
|
||||
case "original_file":
|
||||
entry.OriginalFile = value
|
||||
case "album":
|
||||
entry.Album = value
|
||||
case "date":
|
||||
entry.Date = value
|
||||
case "rights":
|
||||
entry.Rights = value
|
||||
case "notes":
|
||||
entry.Notes = value
|
||||
}
|
||||
}
|
||||
if entry.Artist == "" && entry.Title == "" && entry.OutputFile == "" && entry.OriginalFile == "" {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func ParseMarkdownTableRow(line string) []string {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
trimmed = strings.TrimPrefix(trimmed, "|")
|
||||
trimmed = strings.TrimSuffix(trimmed, "|")
|
||||
parts := strings.Split(trimmed, "|")
|
||||
cells := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
cells = append(cells, strings.TrimSpace(strings.ReplaceAll(part, `\|`, "|")))
|
||||
}
|
||||
return cells
|
||||
}
|
||||
|
||||
func IsMarkdownSeparatorRow(cells []string) bool {
|
||||
if len(cells) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, cell := range cells {
|
||||
cell = strings.TrimSpace(cell)
|
||||
cell = strings.ReplaceAll(cell, "-", "")
|
||||
cell = strings.ReplaceAll(cell, ":", "")
|
||||
if cell != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func NormalizeCreditsHeader(value string) string {
|
||||
switch strings.ToLower(CleanCreditsCell(value)) {
|
||||
case "artist":
|
||||
return "artist"
|
||||
case "title":
|
||||
return "title"
|
||||
case "output file":
|
||||
return "output_file"
|
||||
case "original file":
|
||||
return "original_file"
|
||||
case "album":
|
||||
return "album"
|
||||
case "date":
|
||||
return "date"
|
||||
case "license / copyright":
|
||||
return "rights"
|
||||
case "notes":
|
||||
return "notes"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func CleanCreditsCell(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
value = strings.Trim(value, "`")
|
||||
value = strings.ReplaceAll(value, "<br>", "\n")
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func RenderCreditsMarkdown(entries []CreditsEntry) string {
|
||||
var builder strings.Builder
|
||||
builder.WriteString(CreditsHeader)
|
||||
for _, entry := range entries {
|
||||
builder.WriteString("| ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Artist))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Title))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(WrapCodeCell(entry.OutputFile)))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(WrapCodeCell(entry.OriginalFile)))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Album))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Date))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Rights))
|
||||
builder.WriteString(" | ")
|
||||
builder.WriteString(EscapeMarkdownCell(entry.Notes))
|
||||
builder.WriteString(" |\n")
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func WrapCodeCell(value string) string {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return ""
|
||||
}
|
||||
return "`" + value + "`"
|
||||
}
|
||||
|
||||
func EscapeMarkdownCell(value string) string {
|
||||
value = strings.ReplaceAll(value, `\`, `\\`)
|
||||
value = strings.ReplaceAll(value, "|", `\|`)
|
||||
value = strings.ReplaceAll(value, "\n", "<br>")
|
||||
return value
|
||||
}
|
||||
|
||||
func ApplyCreditsOverlay(entry *CreditsEntry, overlay *CreditsEntry) {
|
||||
if overlay == nil {
|
||||
return
|
||||
}
|
||||
entry.Artist = FirstNonEmpty(overlay.Artist, entry.Artist)
|
||||
entry.Title = FirstNonEmpty(overlay.Title, entry.Title)
|
||||
entry.Album = FirstNonEmpty(overlay.Album, entry.Album)
|
||||
entry.Date = FirstNonEmpty(overlay.Date, entry.Date)
|
||||
entry.Rights = FirstNonEmpty(overlay.Rights, entry.Rights)
|
||||
entry.Notes = FirstNonEmpty(overlay.Notes, entry.Notes)
|
||||
}
|
||||
|
||||
func SyncGeneratedCreditsArtifacts(root string, desiredFiles map[string][]byte) (int, error) {
|
||||
changedFiles := 0
|
||||
existingFiles := make(map[string]struct{})
|
||||
if err := filepath.WalkDir(root, func(path string, d os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
if os.IsNotExist(walkErr) {
|
||||
return nil
|
||||
}
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
existingFiles[path] = struct{}{}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return 0, fmt.Errorf("scan credits dir: %w", err)
|
||||
}
|
||||
|
||||
for path, content := range desiredFiles {
|
||||
current, err := os.ReadFile(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return 0, fmt.Errorf("read credits artifact %s: %w", path, err)
|
||||
}
|
||||
if err == nil && string(current) == string(content) {
|
||||
delete(existingFiles, path)
|
||||
continue
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return 0, fmt.Errorf("create generated credits dir: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(path, content, 0o644); err != nil {
|
||||
return 0, fmt.Errorf("write credits artifact %s: %w", path, err)
|
||||
}
|
||||
changedFiles++
|
||||
delete(existingFiles, path)
|
||||
}
|
||||
|
||||
for path := range existingFiles {
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return 0, fmt.Errorf("remove stale credits artifact %s: %w", path, err)
|
||||
}
|
||||
changedFiles++
|
||||
}
|
||||
return changedFiles, nil
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func ProbeMetadata(ffprobePath, sourcePath string) (Metadata, error) {
|
||||
cmd := exec.Command(ffprobePath, "-v", "quiet", "-print_format", "json", "-show_format", sourcePath)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return Metadata{}, err
|
||||
}
|
||||
var payload struct {
|
||||
Format struct {
|
||||
Tags map[string]string `json:"tags"`
|
||||
} `json:"format"`
|
||||
}
|
||||
if err := json.Unmarshal(output, &payload); err != nil {
|
||||
return Metadata{}, err
|
||||
}
|
||||
tags := make(map[string]string, len(payload.Format.Tags))
|
||||
for key, value := range payload.Format.Tags {
|
||||
tags[strings.ToLower(strings.TrimSpace(key))] = strings.TrimSpace(value)
|
||||
}
|
||||
return Metadata{
|
||||
Title: FirstNonEmpty(tags["title"]),
|
||||
Artist: FirstNonEmpty(tags["artist"], tags["album_artist"], tags["composer"]),
|
||||
Album: FirstNonEmpty(tags["album"]),
|
||||
Date: FirstNonEmpty(tags["date"], tags["year"]),
|
||||
Comment: FirstNonEmpty(tags["comment"], tags["description"]),
|
||||
Copyright: FirstNonEmpty(tags["copyright"]),
|
||||
License: FirstNonEmpty(tags["license"], tags["license_url"]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ConvertSource(ffmpegPath, sourcePath, outputPath string) error {
|
||||
cmd := exec.Command(
|
||||
ffmpegPath, "-y",
|
||||
"-i", sourcePath,
|
||||
"-vn",
|
||||
"-map_metadata", "-1",
|
||||
"-ar", "44100",
|
||||
"-ac", "2",
|
||||
"-b:a", "192k",
|
||||
"-codec:a", "libmp3lame",
|
||||
"-write_xing", "0",
|
||||
"-f", "mp3",
|
||||
outputPath,
|
||||
)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
message := strings.TrimSpace(string(output))
|
||||
if message == "" {
|
||||
message = err.Error()
|
||||
}
|
||||
return fmt.Errorf("%s", message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxStemLen = 16
|
||||
CreditsHeader = "# NWN Music Pack Credits\n\nOriginal rights remain with the respective composers and licensors.\n\n## Processed Files\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n"
|
||||
)
|
||||
|
||||
var (
|
||||
defaultConvertExtensions = []string{
|
||||
".flac",
|
||||
".m4a",
|
||||
".mp3",
|
||||
".ogg",
|
||||
".wav",
|
||||
}
|
||||
ConvertExtensions = map[string]struct{}{
|
||||
".flac": {},
|
||||
".m4a": {},
|
||||
".mp3": {},
|
||||
".ogg": {},
|
||||
".wav": {},
|
||||
}
|
||||
PassthroughExtensions = map[string]struct{}{
|
||||
".bmu": {},
|
||||
".wav": {},
|
||||
}
|
||||
DropWords = map[string]struct{}{
|
||||
"mp3": {}, "wav": {}, "flac": {}, "ogg": {}, "m4a": {},
|
||||
"music": {}, "track": {}, "audio": {}, "song": {}, "soundtrack": {},
|
||||
"final": {}, "version": {}, "ver": {}, "v": {}, "mix": {}, "master": {}, "remaster": {},
|
||||
"free": {}, "royalty": {}, "copyright": {}, "background": {}, "bgm": {},
|
||||
"loop": {}, "loops": {}, "loopable": {}, "seamless": {},
|
||||
"official": {}, "download": {}, "preview": {},
|
||||
"the": {}, "a": {}, "an": {}, "and": {}, "of": {}, "to": {}, "in": {}, "for": {}, "with": {}, "by": {},
|
||||
}
|
||||
BracketRE = regexp.MustCompile(`\[[^\]]*\]|\([^)]*\)|\{[^}]*\}`)
|
||||
SplitRE = regexp.MustCompile(`[^A-Za-z0-9]+`)
|
||||
YearRE = regexp.MustCompile(`^\d{4,}$`)
|
||||
VowelRE = regexp.MustCompile(`[aeiou]`)
|
||||
)
|
||||
|
||||
func DefaultConvertExtensions() []string {
|
||||
return append([]string(nil), defaultConvertExtensions...)
|
||||
}
|
||||
|
||||
type Metadata struct {
|
||||
Title string
|
||||
Artist string
|
||||
Album string
|
||||
Date string
|
||||
Rights string
|
||||
Notes string
|
||||
Comment string
|
||||
Copyright string
|
||||
License string
|
||||
}
|
||||
|
||||
type CreditsEntry struct {
|
||||
Artist string `json:"artist"`
|
||||
Title string `json:"title"`
|
||||
OutputFile string `json:"output_file"`
|
||||
OriginalFile string `json:"original_file"`
|
||||
Album string `json:"album"`
|
||||
Date string `json:"date"`
|
||||
Rights string `json:"rights"`
|
||||
Notes string `json:"notes"`
|
||||
}
|
||||
|
||||
type CreditsSource struct {
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
Entries []CreditsEntry `json:"entries"`
|
||||
}
|
||||
|
||||
type CreditsInventory struct {
|
||||
Sources []CreditsSource `json:"sources"`
|
||||
}
|
||||
|
||||
type CreditsOverlay struct {
|
||||
ByOriginal map[string]CreditsEntry
|
||||
ByOutput map[string]CreditsEntry
|
||||
Entries []CreditsEntry
|
||||
}
|
||||
|
||||
func ResolveFFmpeg() (string, error) {
|
||||
return ResolveFFmpegWithConfig("")
|
||||
}
|
||||
|
||||
func ResolveFFmpegWithConfig(configuredPath string) (string, error) {
|
||||
if configured := strings.TrimSpace(configuredPath); configured != "" {
|
||||
return configured, nil
|
||||
}
|
||||
if configured := strings.TrimSpace(os.Getenv("SOW_FFMPEG")); configured != "" {
|
||||
return configured, nil
|
||||
}
|
||||
path, err := exec.LookPath("ffmpeg")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func ResolveFFprobe() (string, error) {
|
||||
return ResolveFFprobeWithConfig("")
|
||||
}
|
||||
|
||||
func ResolveFFprobeWithConfig(configuredPath string) (string, error) {
|
||||
if configured := strings.TrimSpace(configuredPath); configured != "" {
|
||||
return configured, nil
|
||||
}
|
||||
if configured := strings.TrimSpace(os.Getenv("SOW_FFPROBE")); configured != "" {
|
||||
return configured, nil
|
||||
}
|
||||
path, err := exec.LookPath("ffprobe")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func IsMusicAssetPath(rel string) bool {
|
||||
rel = filepath.ToSlash(strings.TrimSpace(rel))
|
||||
return rel == "envi/music" || strings.HasPrefix(rel, "envi/music/")
|
||||
}
|
||||
|
||||
func PathIsUnder(root, rel string) bool {
|
||||
root = TrimSlashes(filepath.ToSlash(root))
|
||||
rel = TrimSlashes(filepath.ToSlash(rel))
|
||||
if root == "" || rel == "" {
|
||||
return false
|
||||
}
|
||||
return rel == root || strings.HasPrefix(rel, root+"/")
|
||||
}
|
||||
|
||||
func PrefixForDir(prefixes map[string]string, dir string) string {
|
||||
dir = TrimSlashes(filepath.ToSlash(dir))
|
||||
bestPrefix := ""
|
||||
bestLen := -1
|
||||
keys := make([]string, 0, len(prefixes))
|
||||
for key := range prefixes {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
value := prefixes[key]
|
||||
normalizedKey := TrimSlashes(filepath.ToSlash(key))
|
||||
if normalizedKey == "" {
|
||||
continue
|
||||
}
|
||||
if dir != normalizedKey && !strings.HasPrefix(dir, normalizedKey+"/") {
|
||||
continue
|
||||
}
|
||||
if len(normalizedKey) > bestLen {
|
||||
bestLen = len(normalizedKey)
|
||||
bestPrefix = SanitizePrefix(value)
|
||||
}
|
||||
}
|
||||
return bestPrefix
|
||||
}
|
||||
|
||||
func SanitizePrefix(prefix string) string {
|
||||
prefix = strings.ToLower(strings.TrimSpace(prefix))
|
||||
var builder strings.Builder
|
||||
for _, r := range prefix {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
builder.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
builder.WriteRune(r)
|
||||
case r == '_':
|
||||
builder.WriteRune(r)
|
||||
}
|
||||
}
|
||||
result := builder.String()
|
||||
if strings.TrimSpace(prefix) != "" && strings.HasSuffix(strings.TrimSpace(prefix), "_") && !strings.HasSuffix(result, "_") {
|
||||
result += "_"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func TrimSlashes(value string) string {
|
||||
return strings.Trim(strings.TrimSpace(value), "/")
|
||||
}
|
||||
|
||||
func Min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func Max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func FirstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func JoinNonEmpty(sep string, values ...string) string {
|
||||
parts := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value != "" {
|
||||
parts = append(parts, value)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, sep)
|
||||
}
|
||||
@@ -1,404 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUniqueNameHandlesShortStemCollisions(t *testing.T) {
|
||||
used := map[string]struct{}{
|
||||
"mus_wg_mystc": {},
|
||||
}
|
||||
|
||||
got := UniqueName("mus_wg_mystc", used)
|
||||
if got != "mus_wg_mystc_1" {
|
||||
t.Fatalf("unexpected collision result: got %q want %q", got, "mus_wg_mystc_1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUniqueNameNoCollision(t *testing.T) {
|
||||
used := map[string]struct{}{}
|
||||
got := UniqueName("new_stem", used)
|
||||
if got != "new_stem" {
|
||||
t.Fatalf("expected 'new_stem', got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUniqueNameMultipleCollisions(t *testing.T) {
|
||||
used := map[string]struct{}{
|
||||
"stem": {},
|
||||
"stem_1": {},
|
||||
"stem_2": {},
|
||||
}
|
||||
got := UniqueName("stem", used)
|
||||
if got != "stem_3" {
|
||||
t.Fatalf("expected 'stem_3', got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateStem(t *testing.T) {
|
||||
used := map[string]struct{}{
|
||||
"existing": {},
|
||||
}
|
||||
stem, err := GenerateStem("My Cool Track.mp3", "mus_", used)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if stem == "" {
|
||||
t.Fatal("expected non-empty stem")
|
||||
}
|
||||
if len(stem) > MaxStemLen {
|
||||
t.Fatalf("stem %q exceeds max length %d", stem, MaxStemLen)
|
||||
}
|
||||
if _, exists := used[stem]; !exists {
|
||||
t.Fatal("expected stem to be registered in used set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateStemPrefixTooLong(t *testing.T) {
|
||||
_, err := GenerateStem("test.mp3", "this_prefix_is_way_too_long_for_stem_", nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for too-long prefix")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlugWords(t *testing.T) {
|
||||
words := SlugWords("My Cool Music Track (Official) [HD]")
|
||||
if len(words) == 0 {
|
||||
t.Fatal("expected non-empty words")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlugWordsStripsBrackets(t *testing.T) {
|
||||
words := SlugWords("Song [Explicit] (Remix)")
|
||||
for _, w := range words {
|
||||
if w == "explicit" || w == "remix" {
|
||||
t.Fatalf("bracket content should be removed, got word %q", w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizePrefix(t *testing.T) {
|
||||
if got := SanitizePrefix("Mus_WG_"); got != "mus_wg_" {
|
||||
t.Fatalf("unexpected prefix: %q", got)
|
||||
}
|
||||
if got := SanitizePrefix(" Test "); got != "test" {
|
||||
t.Fatalf("unexpected prefix: %q", got)
|
||||
}
|
||||
if got := SanitizePrefix(""); got != "" {
|
||||
t.Fatalf("expected empty prefix, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReserveName(t *testing.T) {
|
||||
used := map[string]struct{}{}
|
||||
if err := ReserveName("testname", used); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if _, exists := used["testname"]; !exists {
|
||||
t.Fatal("expected name to be reserved")
|
||||
}
|
||||
if err := ReserveName("testname", used); err == nil {
|
||||
t.Fatal("expected error for duplicate name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateManualOutputFile(t *testing.T) {
|
||||
if err := ValidateManualOutputFile("test.bmu"); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if err := ValidateManualOutputFile("test.txt"); err == nil {
|
||||
t.Fatal("expected error for non-bmu extension")
|
||||
}
|
||||
if err := ValidateManualOutputFile(".bmu"); err == nil {
|
||||
t.Fatal("expected error for empty stem")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCreditsMarkdownMissingFile(t *testing.T) {
|
||||
entries, err := ParseCreditsMarkdown("nonexistent.md")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error for missing file: %v", err)
|
||||
}
|
||||
if entries != nil {
|
||||
t.Fatalf("expected nil entries for missing file, got %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCreditsMarkdownRealContent(t *testing.T) {
|
||||
content := "# Header\n\n| Artist | Title | Output File | Original File | Album | Date | License / Copyright | Notes |\n|---|---|---|---|---|---|---|---|\n| Test Artist | Test Title | `output.bmu` | `original.mp3` | Test Album | 2024 | MIT | test |\n"
|
||||
path := filepath.Join(t.TempDir(), "CREDITS.md")
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write test credits: %v", err)
|
||||
}
|
||||
|
||||
entries, err := ParseCreditsMarkdown(path)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", len(entries))
|
||||
}
|
||||
if entries[0].Artist != "Test Artist" {
|
||||
t.Fatalf("expected 'Test Artist', got %q", entries[0].Artist)
|
||||
}
|
||||
if entries[0].Title != "Test Title" {
|
||||
t.Fatalf("expected 'Test Title', got %q", entries[0].Title)
|
||||
}
|
||||
if entries[0].OutputFile != "output.bmu" {
|
||||
t.Fatalf("expected 'output.bmu', got %q", entries[0].OutputFile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderCreditsMarkdown(t *testing.T) {
|
||||
entries := []CreditsEntry{
|
||||
{
|
||||
Artist: "Test Artist",
|
||||
Title: "Test Title",
|
||||
OutputFile: "output.bmu",
|
||||
OriginalFile: "original.mp3",
|
||||
Album: "Test Album",
|
||||
Date: "2024",
|
||||
Rights: "MIT",
|
||||
Notes: "test note",
|
||||
},
|
||||
}
|
||||
|
||||
result := RenderCreditsMarkdown(entries)
|
||||
if !strings.Contains(result, "Test Artist") {
|
||||
t.Fatal("expected rendered output to contain artist")
|
||||
}
|
||||
if !strings.Contains(result, "output.bmu") {
|
||||
t.Fatal("expected rendered output to contain output file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCreditsOverlay(t *testing.T) {
|
||||
content := "# Header\n\n| Artist | Title | Output File | Original File |\n|---|---|---|---|\n| Overlay Artist | Ovr Title | `override.bmu` | `original.mp3` |\n"
|
||||
path := filepath.Join(t.TempDir(), "CREDITS.md")
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatalf("write test overlay: %v", err)
|
||||
}
|
||||
|
||||
overlay, err := ParseCreditsOverlay(path)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlay.Entries) != 1 {
|
||||
t.Fatalf("expected 1 overlay entry, got %d", len(overlay.Entries))
|
||||
}
|
||||
|
||||
matched := overlay.Match("original.mp3", "")
|
||||
if matched == nil {
|
||||
t.Fatal("expected overlay match by original file")
|
||||
}
|
||||
if matched.Artist != "Overlay Artist" {
|
||||
t.Fatalf("expected 'Overlay Artist', got %q", matched.Artist)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCreditsOverlay(t *testing.T) {
|
||||
entry := &CreditsEntry{
|
||||
Artist: "Original Artist",
|
||||
Title: "Original Title",
|
||||
}
|
||||
overlay := &CreditsEntry{
|
||||
Artist: "Overlay Artist",
|
||||
}
|
||||
|
||||
ApplyCreditsOverlay(entry, overlay)
|
||||
if entry.Artist != "Overlay Artist" {
|
||||
t.Fatalf("expected 'Overlay Artist', got %q", entry.Artist)
|
||||
}
|
||||
if entry.Title != "Original Title" {
|
||||
t.Fatalf("expected 'Original Title' preserved, got %q", entry.Title)
|
||||
}
|
||||
|
||||
ApplyCreditsOverlay(entry, nil)
|
||||
if entry.Artist != "Overlay Artist" {
|
||||
t.Fatalf("expected nil overlay to be no-op, got %q", entry.Artist)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateOverlayEntries(t *testing.T) {
|
||||
generated := []CreditsEntry{
|
||||
{OriginalFile: "track1.mp3", OutputFile: "track1.bmu"},
|
||||
}
|
||||
overlay := CreditsOverlay{
|
||||
Entries: []CreditsEntry{
|
||||
{OriginalFile: "track1.mp3"},
|
||||
},
|
||||
}
|
||||
if err := ValidateOverlayEntries("test", overlay, generated); err != nil {
|
||||
t.Fatalf("unexpected validation error: %v", err)
|
||||
}
|
||||
|
||||
badOverlay := CreditsOverlay{
|
||||
Entries: []CreditsEntry{
|
||||
{OriginalFile: "nonexistent.mp3"},
|
||||
},
|
||||
}
|
||||
if err := ValidateOverlayEntries("test", badOverlay, generated); err == nil {
|
||||
t.Fatal("expected validation error for unknown original file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMusicAssetPath(t *testing.T) {
|
||||
if !IsMusicAssetPath("envi/music") {
|
||||
t.Fatal("expected 'envi/music' to be music path")
|
||||
}
|
||||
if !IsMusicAssetPath("envi/music/westgate") {
|
||||
t.Fatal("expected 'envi/music/westgate' to be music path")
|
||||
}
|
||||
if IsMusicAssetPath("envi/textures") {
|
||||
t.Fatal("expected 'envi/textures' not to be music path")
|
||||
}
|
||||
if IsMusicAssetPath("") {
|
||||
t.Fatal("expected empty string not to be music path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstNonEmpty(t *testing.T) {
|
||||
if got := FirstNonEmpty("", "hello", "world"); got != "hello" {
|
||||
t.Fatalf("expected 'hello', got %q", got)
|
||||
}
|
||||
if got := FirstNonEmpty("", "", ""); got != "" {
|
||||
t.Fatalf("expected empty, got %q", got)
|
||||
}
|
||||
if got := FirstNonEmpty("first"); got != "first" {
|
||||
t.Fatalf("expected 'first', got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJoinNonEmpty(t *testing.T) {
|
||||
if got := JoinNonEmpty(", ", "a", "", "b"); got != "a, b" {
|
||||
t.Fatalf("expected 'a, b', got %q", got)
|
||||
}
|
||||
if got := JoinNonEmpty(", "); got != "" {
|
||||
t.Fatalf("expected empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimSlashes(t *testing.T) {
|
||||
if got := TrimSlashes("/foo/bar/"); got != "foo/bar" {
|
||||
t.Fatalf("expected 'foo/bar', got %q", got)
|
||||
}
|
||||
if got := TrimSlashes("///"); got != "" {
|
||||
t.Fatalf("expected empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscapeMarkdownCell(t *testing.T) {
|
||||
result := EscapeMarkdownCell("a|b\nc")
|
||||
if !strings.Contains(result, `\|`) {
|
||||
t.Fatal("expected pipe to be escaped")
|
||||
}
|
||||
if !strings.Contains(result, "<br>") {
|
||||
t.Fatal("expected newline to be replaced with <br>")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapCodeCell(t *testing.T) {
|
||||
if got := WrapCodeCell("test.bmu"); got != "`test.bmu`" {
|
||||
t.Fatalf("expected '`test.bmu`', got %q", got)
|
||||
}
|
||||
if got := WrapCodeCell(""); got != "" {
|
||||
t.Fatalf("expected empty string, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncGeneratedCreditsArtifactsWritesNewFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
desired := map[string][]byte{
|
||||
filepath.Join(root, "test.md"): []byte("content"),
|
||||
}
|
||||
changed, err := SyncGeneratedCreditsArtifacts(root, desired)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if changed != 1 {
|
||||
t.Fatalf("expected 1 changed file, got %d", changed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncGeneratedCreditsArtifactsNoChanges(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "test.md")
|
||||
if err := os.WriteFile(path, []byte("content"), 0o644); err != nil {
|
||||
t.Fatalf("write file: %v", err)
|
||||
}
|
||||
desired := map[string][]byte{
|
||||
path: []byte("content"),
|
||||
}
|
||||
changed, err := SyncGeneratedCreditsArtifacts(root, desired)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if changed != 0 {
|
||||
t.Fatalf("expected 0 changed files, got %d", changed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncGeneratedCreditsArtifactsRemovesStale(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
stalePath := filepath.Join(root, "stale.md")
|
||||
if err := os.WriteFile(stalePath, []byte("stale"), 0o644); err != nil {
|
||||
t.Fatalf("write stale file: %v", err)
|
||||
}
|
||||
desired := map[string][]byte{}
|
||||
changed, err := SyncGeneratedCreditsArtifacts(root, desired)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if changed != 1 {
|
||||
t.Fatalf("expected 1 removed file, got %d", changed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefixForDir(t *testing.T) {
|
||||
prefixes := map[string]string{
|
||||
"envi/music/westgate": "mus_wg_",
|
||||
}
|
||||
if got := PrefixForDir(prefixes, "envi/music/westgate"); got != "mus_wg_" {
|
||||
t.Fatalf("expected 'mus_wg_', got %q", got)
|
||||
}
|
||||
if got := PrefixForDir(prefixes, "envi/music/other"); got != "" {
|
||||
t.Fatalf("expected empty prefix, got %q", got)
|
||||
}
|
||||
if got := PrefixForDir(nil, "envi/music"); got != "" {
|
||||
t.Fatalf("expected empty prefix for nil map, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxMin(t *testing.T) {
|
||||
if got := Min(3, 5); got != 3 {
|
||||
t.Fatalf("Min(3,5) = %d, want 3", got)
|
||||
}
|
||||
if got := Max(3, 5); got != 5 {
|
||||
t.Fatalf("Max(3,5) = %d, want 5", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFFmpegEnv(t *testing.T) {
|
||||
t.Setenv("SOW_FFMPEG", "/custom/ffmpeg")
|
||||
path, err := ResolveFFmpeg()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if path != "/custom/ffmpeg" {
|
||||
t.Fatalf("expected '/custom/ffmpeg', got %q", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveFFprobeEnv(t *testing.T) {
|
||||
t.Setenv("SOW_FFPROBE", "/custom/ffprobe")
|
||||
path, err := ResolveFFprobe()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if path != "/custom/ffprobe" {
|
||||
t.Fatalf("expected '/custom/ffprobe', got %q", path)
|
||||
}
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
package music
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func GenerateStem(fileName, prefix string, used map[string]struct{}) (string, error) {
|
||||
return GenerateStemWithMax(fileName, prefix, MaxStemLen, used)
|
||||
}
|
||||
|
||||
func GenerateStemWithMax(fileName, prefix string, maxStemLen int, used map[string]struct{}) (string, error) {
|
||||
if maxStemLen <= 0 {
|
||||
maxStemLen = MaxStemLen
|
||||
}
|
||||
maxCore := maxStemLen - len(prefix)
|
||||
if maxCore < 3 {
|
||||
return "", fmt.Errorf("prefix too long: %q", prefix)
|
||||
}
|
||||
base := strings.TrimSuffix(fileName, filepath.Ext(fileName))
|
||||
words := SlugWords(base)
|
||||
core := MakeMusicCore(words, maxCore)
|
||||
if core == "" {
|
||||
sum := sha1.Sum([]byte(fileName))
|
||||
core = fmt.Sprintf("%x", sum[:])[:maxCore]
|
||||
}
|
||||
stem := strings.TrimRight((prefix + core)[:Min(maxStemLen, len(prefix)+len(core))], "_")
|
||||
if stem == "" {
|
||||
return "", fmt.Errorf("could not derive music stem for %s", fileName)
|
||||
}
|
||||
return UniqueNameWithMax(stem, maxStemLen, used), nil
|
||||
}
|
||||
|
||||
func SlugWords(value string) []string {
|
||||
value = BracketRE.ReplaceAllString(value, " ")
|
||||
value = strings.ReplaceAll(value, "&", " and ")
|
||||
value = strings.ReplaceAll(value, "'", "")
|
||||
value = strings.ReplaceAll(value, "\u2019", "")
|
||||
value = strings.ToLower(value)
|
||||
var ascii strings.Builder
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
ascii.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
ascii.WriteRune(r)
|
||||
default:
|
||||
ascii.WriteRune(' ')
|
||||
}
|
||||
}
|
||||
rawWords := SplitRE.Split(ascii.String(), -1)
|
||||
words := make([]string, 0, len(rawWords))
|
||||
for _, word := range rawWords {
|
||||
if word == "" {
|
||||
continue
|
||||
}
|
||||
if _, drop := DropWords[word]; drop {
|
||||
continue
|
||||
}
|
||||
if YearRE.MatchString(word) {
|
||||
continue
|
||||
}
|
||||
words = append(words, word)
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
func MakeMusicCore(words []string, maxCore int) string {
|
||||
core := ""
|
||||
for _, word := range words {
|
||||
candidate := word
|
||||
if core != "" {
|
||||
candidate = core + "_" + word
|
||||
}
|
||||
if len(candidate) <= maxCore {
|
||||
core = candidate
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if core != "" {
|
||||
return core
|
||||
}
|
||||
abbrev := make([]string, 0, Min(4, len(words)))
|
||||
for _, word := range words {
|
||||
if len(abbrev) == 4 {
|
||||
break
|
||||
}
|
||||
abbrev = append(abbrev, AbbreviateWord(word, 5))
|
||||
}
|
||||
for _, word := range abbrev {
|
||||
candidate := word
|
||||
if core != "" {
|
||||
candidate = core + "_" + word
|
||||
}
|
||||
if len(candidate) <= maxCore {
|
||||
core = candidate
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if core != "" {
|
||||
return core
|
||||
}
|
||||
if len(words) > 0 {
|
||||
return strings.Trim(words[0][:Min(maxCore, len(words[0]))], "_")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func AbbreviateWord(word string, maxLen int) string {
|
||||
if len(word) <= maxLen {
|
||||
return word
|
||||
}
|
||||
consonants := VowelRE.ReplaceAllString(word, "")
|
||||
candidate := word[:1]
|
||||
if len(consonants) > 1 {
|
||||
candidate += consonants[1:]
|
||||
}
|
||||
if len(candidate) >= 3 {
|
||||
return candidate[:Min(maxLen, len(candidate))]
|
||||
}
|
||||
return word[:Min(maxLen, len(word))]
|
||||
}
|
||||
|
||||
func UniqueName(stem string, used map[string]struct{}) string {
|
||||
return UniqueNameWithMax(stem, MaxStemLen, used)
|
||||
}
|
||||
|
||||
func UniqueNameWithMax(stem string, maxStemLen int, used map[string]struct{}) string {
|
||||
if maxStemLen <= 0 {
|
||||
maxStemLen = MaxStemLen
|
||||
}
|
||||
if _, exists := used[stem]; !exists {
|
||||
used[stem] = struct{}{}
|
||||
return stem
|
||||
}
|
||||
for index := 1; ; index++ {
|
||||
suffix := fmt.Sprintf("_%d", index)
|
||||
prefixLen := Min(len(stem), Max(0, maxStemLen-len(suffix)))
|
||||
candidate := strings.TrimRight(stem[:prefixLen], "_") + suffix
|
||||
if _, exists := used[candidate]; exists {
|
||||
continue
|
||||
}
|
||||
used[candidate] = struct{}{}
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
|
||||
func ReserveName(stem string, used map[string]struct{}) error {
|
||||
stem = strings.ToLower(strings.TrimSpace(stem))
|
||||
if stem == "" {
|
||||
return fmt.Errorf("output filename is empty")
|
||||
}
|
||||
if _, exists := used[stem]; exists {
|
||||
return fmt.Errorf("output filename %s collides with an existing asset", stem)
|
||||
}
|
||||
used[stem] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateManualOutputFile(name string) error {
|
||||
return ValidateManualOutputFileWithRules(name, ".bmu", MaxStemLen)
|
||||
}
|
||||
|
||||
func ValidateManualOutputFileWithRules(name, extension string, maxStemLen int) error {
|
||||
name = strings.ToLower(strings.TrimSpace(name))
|
||||
extension = strings.ToLower(strings.TrimSpace(extension))
|
||||
if extension == "" {
|
||||
extension = ".bmu"
|
||||
}
|
||||
if maxStemLen <= 0 {
|
||||
maxStemLen = MaxStemLen
|
||||
}
|
||||
if !strings.HasSuffix(name, extension) {
|
||||
return fmt.Errorf("output file must end with %s", extension)
|
||||
}
|
||||
stem := strings.TrimSuffix(name, extension)
|
||||
if stem == "" {
|
||||
return fmt.Errorf("output file stem is empty")
|
||||
}
|
||||
if len(stem) > maxStemLen {
|
||||
return fmt.Errorf("output file stem %q exceeds %d characters", stem, maxStemLen)
|
||||
}
|
||||
for _, r := range stem {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
case r >= '0' && r <= '9':
|
||||
case r == '_':
|
||||
default:
|
||||
return fmt.Errorf("output file stem %q contains unsupported character %q", stem, string(r))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
type ApplyManifestResult struct {
|
||||
ManifestPath string
|
||||
ModuleSource string
|
||||
HAKCount int
|
||||
}
|
||||
|
||||
func ApplyHAKManifest(p *project.Project, manifestPath string) (ApplyManifestResult, error) {
|
||||
if manifestPath == "" {
|
||||
manifestPath = p.HAKManifestPath()
|
||||
}
|
||||
if strings.TrimSpace(p.EffectiveConfig().Paths.Source) == "" {
|
||||
return ApplyManifestResult{}, fmt.Errorf("cannot apply hak manifest: paths.source is not configured")
|
||||
}
|
||||
sourceRoot := filepath.Clean(p.SourceDir())
|
||||
if sourceRoot == "." || sourceRoot == string(filepath.Separator) || sourceRoot == filepath.Clean(p.Root) {
|
||||
return ApplyManifestResult{}, fmt.Errorf("cannot apply hak manifest: paths.source resolves to unsafe source root %s", sourceRoot)
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("read hak manifest: %w", err)
|
||||
}
|
||||
|
||||
var manifest BuildManifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("parse hak manifest: %w", err)
|
||||
}
|
||||
|
||||
moduleSource := filepath.Join(sourceRoot, "module", "module.ifo.json")
|
||||
sourceRaw, err := os.ReadFile(moduleSource)
|
||||
if err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("read module ifo source: %w", err)
|
||||
}
|
||||
|
||||
var document gff.Document
|
||||
if err := json.Unmarshal(sourceRaw, &document); err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("parse module ifo source: %w", err)
|
||||
}
|
||||
|
||||
setModuleHAKList(&document, manifest.ModuleHAKs)
|
||||
|
||||
formatted, err := json.MarshalIndent(document, "", " ")
|
||||
if err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("marshal updated module ifo: %w", err)
|
||||
}
|
||||
formatted = append(formatted, '\n')
|
||||
|
||||
if err := os.WriteFile(moduleSource, formatted, 0o644); err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("write module ifo source: %w", err)
|
||||
}
|
||||
|
||||
return ApplyManifestResult{
|
||||
ManifestPath: manifestPath,
|
||||
ModuleSource: moduleSource,
|
||||
HAKCount: len(manifest.ModuleHAKs),
|
||||
}, nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,27 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
)
|
||||
|
||||
func ensureUniqueChunkResources(chunk hakChunk) error {
|
||||
seen := map[string]string{}
|
||||
for _, asset := range chunk.Assets {
|
||||
key := fmt.Sprintf("%s:%04x", asset.Resource.Name, asset.Resource.Type)
|
||||
if previous, exists := seen[key]; exists {
|
||||
return fmt.Errorf("resource %s from %s conflicts with %s inside generated hak %s", chunkResourceLabel(asset), asset.Rel, previous, chunk.Name)
|
||||
}
|
||||
seen[key] = asset.Rel
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func chunkResourceLabel(asset assetResource) string {
|
||||
extension, ok := erf.ExtensionForResourceType(asset.Resource.Type)
|
||||
if !ok {
|
||||
return asset.Resource.Name
|
||||
}
|
||||
return asset.Resource.Name + "." + extension
|
||||
}
|
||||
@@ -1,359 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
type CompareResult struct {
|
||||
ModulePath string
|
||||
HAKPaths []string
|
||||
Checked int
|
||||
}
|
||||
|
||||
type resourceExpectation struct {
|
||||
Key string
|
||||
Kind string
|
||||
Bytes []byte
|
||||
}
|
||||
|
||||
func Compare(p *project.Project) (CompareResult, error) {
|
||||
modulePath := p.ModuleArchivePath()
|
||||
|
||||
var hakPaths []string
|
||||
if len(p.Inventory.AssetFiles) > 0 {
|
||||
var err error
|
||||
hakPaths, err = manifestHAKPaths(p)
|
||||
if err != nil {
|
||||
return CompareResult{}, err
|
||||
}
|
||||
}
|
||||
if err := ensureBuildIsCurrent(p, modulePath, hakPaths); err != nil {
|
||||
return CompareResult{ModulePath: modulePath, HAKPaths: hakPaths}, err
|
||||
}
|
||||
|
||||
moduleArchive, err := readArchive(modulePath)
|
||||
if err != nil {
|
||||
return CompareResult{}, err
|
||||
}
|
||||
|
||||
expected, err := expectedResources(p)
|
||||
if err != nil {
|
||||
return CompareResult{}, err
|
||||
}
|
||||
|
||||
actual := archiveIndex(moduleArchive)
|
||||
for _, hakPath := range hakPaths {
|
||||
hakArchive, err := readArchive(hakPath)
|
||||
if err != nil {
|
||||
return CompareResult{}, err
|
||||
}
|
||||
for key, value := range archiveIndex(hakArchive) {
|
||||
actual[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
var diagnostics []error
|
||||
checked := 0
|
||||
|
||||
expectedKeys := make([]string, 0, len(expected))
|
||||
for key := range expected {
|
||||
expectedKeys = append(expectedKeys, key)
|
||||
}
|
||||
slices.Sort(expectedKeys)
|
||||
|
||||
for _, key := range expectedKeys {
|
||||
want := expected[key]
|
||||
got, ok := actual[key]
|
||||
if !ok {
|
||||
diagnostics = append(diagnostics, fmt.Errorf("missing resource in built archives: %s", key))
|
||||
continue
|
||||
}
|
||||
if !bytes.Equal(want.Bytes, got.Data) {
|
||||
diagnostics = append(diagnostics, fmt.Errorf("resource content mismatch: %s", key))
|
||||
continue
|
||||
}
|
||||
checked++
|
||||
delete(actual, key)
|
||||
}
|
||||
|
||||
if len(actual) > 0 {
|
||||
extraKeys := make([]string, 0, len(actual))
|
||||
for key := range actual {
|
||||
extraKeys = append(extraKeys, key)
|
||||
}
|
||||
slices.Sort(extraKeys)
|
||||
for _, key := range extraKeys {
|
||||
diagnostics = append(diagnostics, fmt.Errorf("unexpected extra resource in built archives: %s", key))
|
||||
}
|
||||
}
|
||||
|
||||
result := CompareResult{
|
||||
ModulePath: modulePath,
|
||||
HAKPaths: hakPaths,
|
||||
Checked: checked,
|
||||
}
|
||||
if len(diagnostics) > 0 {
|
||||
return result, errors.Join(diagnostics...)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func expectedResources(p *project.Project) (map[string]resourceExpectation, error) {
|
||||
out := map[string]resourceExpectation{}
|
||||
moduleHakOrder, err := plannedModuleHAKOrder(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, rel := range p.Inventory.SourceFiles {
|
||||
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
|
||||
name, extension, err := splitSourceName(abs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s: %w", abs, err)
|
||||
}
|
||||
var document gff.Document
|
||||
if err := json.Unmarshal(raw, &document); err != nil {
|
||||
return nil, fmt.Errorf("parse gff json %s: %w", abs, err)
|
||||
}
|
||||
if extension == ".ifo" && strings.EqualFold(name, "module") && len(moduleHakOrder) > 0 {
|
||||
setModuleHAKList(&document, moduleHakOrder)
|
||||
}
|
||||
|
||||
canonical, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal canonical gff json %s: %w", abs, err)
|
||||
}
|
||||
|
||||
out[resourceKey(name, extension)] = resourceExpectation{
|
||||
Key: resourceKey(name, extension),
|
||||
Kind: "gff-json",
|
||||
Bytes: canonical,
|
||||
}
|
||||
}
|
||||
|
||||
for _, rel := range p.Inventory.ScriptFiles {
|
||||
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
|
||||
data, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s: %w", abs, err)
|
||||
}
|
||||
name := strings.ToLower(strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs)))
|
||||
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(abs)), ".")
|
||||
out[resourceKey(name, extension)] = resourceExpectation{
|
||||
Key: resourceKey(name, extension),
|
||||
Kind: "raw",
|
||||
Bytes: data,
|
||||
}
|
||||
}
|
||||
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
|
||||
data, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s: %w", abs, err)
|
||||
}
|
||||
name := strings.ToLower(strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs)))
|
||||
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(abs)), ".")
|
||||
out[resourceKey(name, extension)] = resourceExpectation{
|
||||
Key: resourceKey(name, extension),
|
||||
Kind: "raw",
|
||||
Bytes: data,
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func ensureBuildIsCurrent(p *project.Project, modulePath string, hakPaths []string) error {
|
||||
archivePaths := make([]string, 0, 1+len(hakPaths))
|
||||
archivePaths = append(archivePaths, modulePath)
|
||||
archivePaths = append(archivePaths, hakPaths...)
|
||||
|
||||
oldestArchive := time.Time{}
|
||||
for _, path := range archivePaths {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat archive %s: %w", path, err)
|
||||
}
|
||||
if oldestArchive.IsZero() || info.ModTime().Before(oldestArchive) {
|
||||
oldestArchive = info.ModTime()
|
||||
}
|
||||
}
|
||||
|
||||
sourcePaths := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
|
||||
for _, rel := range p.Inventory.SourceFiles {
|
||||
sourcePaths = append(sourcePaths, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
for _, rel := range p.Inventory.ScriptFiles {
|
||||
sourcePaths = append(sourcePaths, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
sourcePaths = append(sourcePaths, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
|
||||
newestSource := time.Time{}
|
||||
newestPath := ""
|
||||
for _, path := range sourcePaths {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat source %s: %w", path, err)
|
||||
}
|
||||
if newestSource.IsZero() || info.ModTime().After(newestSource) {
|
||||
newestSource = info.ModTime()
|
||||
newestPath = path
|
||||
}
|
||||
}
|
||||
|
||||
if !newestSource.IsZero() && newestSource.After(oldestArchive) {
|
||||
return fmt.Errorf("built archives are older than source file %s; run build-module or build-haks before compare", newestPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readArchive(path string) (erf.Archive, error) {
|
||||
input, err := os.Open(path)
|
||||
if err != nil {
|
||||
return erf.Archive{}, fmt.Errorf("open archive %s: %w", path, err)
|
||||
}
|
||||
defer input.Close()
|
||||
|
||||
archive, err := erf.Read(input)
|
||||
if err != nil {
|
||||
return erf.Archive{}, fmt.Errorf("read archive %s: %w", path, err)
|
||||
}
|
||||
return archive, nil
|
||||
}
|
||||
|
||||
func archiveIndex(archive erf.Archive) map[string]erf.Resource {
|
||||
out := map[string]erf.Resource{}
|
||||
for _, resource := range archive.Resources {
|
||||
extension, ok := erf.ExtensionForResourceType(resource.Type)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
key := resourceKey(strings.ToLower(resource.Name), extension)
|
||||
switch extension {
|
||||
case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
|
||||
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl":
|
||||
document, err := gff.Read(bytes.NewReader(resource.Data))
|
||||
if err != nil {
|
||||
out[key] = resource
|
||||
continue
|
||||
}
|
||||
canonical, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
out[key] = resource
|
||||
continue
|
||||
}
|
||||
resource.Data = canonical
|
||||
}
|
||||
|
||||
out[key] = resource
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resourceKey(name, extension string) string {
|
||||
return strings.ToLower(name) + "." + strings.TrimPrefix(strings.ToLower(extension), ".")
|
||||
}
|
||||
|
||||
func manifestHAKPaths(p *project.Project) ([]string, error) {
|
||||
manifestPath := p.HAKManifestPath()
|
||||
raw, err := os.ReadFile(manifestPath)
|
||||
if err == nil {
|
||||
var manifest BuildManifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("parse hak manifest: %w", err)
|
||||
}
|
||||
paths := make([]string, 0, len(manifest.HAKs))
|
||||
for _, hak := range manifest.HAKs {
|
||||
paths = append(paths, p.HAKArchivePath(hak.Name))
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("read hak manifest: %w", err)
|
||||
}
|
||||
|
||||
legacy := filepath.Join(p.BuildDir(), "*.hak")
|
||||
paths, err := filepath.Glob(legacy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan hak archives: %w", err)
|
||||
}
|
||||
filtered := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
if filepath.Base(path) == p.TopDataPackageHAKName() {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, path)
|
||||
}
|
||||
slices.Sort(filtered)
|
||||
return filtered, nil
|
||||
}
|
||||
|
||||
func topPackageSourcePaths(p *project.Project) ([]string, error) {
|
||||
sourceDir := p.TopDataSourceDir()
|
||||
generated2DADir := filepath.Join(sourceDir, "assets", "2da")
|
||||
paths := make([]string, 0)
|
||||
for _, candidate := range []string{
|
||||
filepath.Join(sourceDir, ".tlk_state.json"),
|
||||
filepath.Join(sourceDir, "base_dialog.json"),
|
||||
} {
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
paths = append(paths, candidate)
|
||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("stat topdata source %s: %w", candidate, err)
|
||||
}
|
||||
}
|
||||
for _, dir := range []string{
|
||||
filepath.Join(sourceDir, "data"),
|
||||
filepath.Join(sourceDir, "assets"),
|
||||
} {
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("stat topdata path %s: %w", dir, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
continue
|
||||
}
|
||||
err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() && path == generated2DADir {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
paths = append(paths, path)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan topdata path %s: %w", dir, err)
|
||||
}
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
@@ -1,621 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
type ExtractResult struct {
|
||||
ModulePath string
|
||||
HAKPaths []string
|
||||
DeletedArchivePaths []string
|
||||
Written int
|
||||
Overwritten int
|
||||
Removed int
|
||||
Skipped int
|
||||
}
|
||||
|
||||
type extractionArchive struct {
|
||||
Path string
|
||||
Rel string
|
||||
Extension string
|
||||
}
|
||||
|
||||
type extractionScope struct {
|
||||
Source bool
|
||||
Assets bool
|
||||
}
|
||||
|
||||
func Extract(p *project.Project, files ...string) (ExtractResult, error) {
|
||||
var result ExtractResult
|
||||
var failures []error
|
||||
desired := map[string]struct{}{}
|
||||
scope := extractionScope{}
|
||||
|
||||
archives, err := resolveExtractionArchives(p, files)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
for _, archivePath := range archives {
|
||||
input, err := os.Open(archivePath.Path)
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Errorf("open archive %s: %w", archivePath.Path, err))
|
||||
continue
|
||||
}
|
||||
archive, err := erf.Read(input)
|
||||
input.Close()
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Errorf("read archive %s: %w", archivePath.Path, err))
|
||||
continue
|
||||
}
|
||||
|
||||
switch archivePath.Extension {
|
||||
case ".mod":
|
||||
if result.ModulePath == "" {
|
||||
result.ModulePath = archivePath.Path
|
||||
}
|
||||
case ".hak":
|
||||
result.HAKPaths = append(result.HAKPaths, archivePath.Path)
|
||||
}
|
||||
|
||||
written, overwritten, skipped, extractedScope, errs := extractArchiveResources(p, archive, desired)
|
||||
result.Written += written
|
||||
result.Overwritten += overwritten
|
||||
result.Skipped += skipped
|
||||
scope.Source = scope.Source || extractedScope.Source
|
||||
scope.Assets = scope.Assets || extractedScope.Assets
|
||||
failures = append(failures, errs...)
|
||||
}
|
||||
|
||||
if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale {
|
||||
removed, errs := cleanupStaleFiles(p, desired, scope)
|
||||
result.Removed = removed
|
||||
failures = append(failures, errs...)
|
||||
}
|
||||
|
||||
if len(failures) > 0 {
|
||||
return result, errors.Join(failures...)
|
||||
}
|
||||
consumeAll, consumeModules := extractionConsumePolicy(p)
|
||||
if consumeAll || consumeModules {
|
||||
for _, archivePath := range archives {
|
||||
if !consumeAll && archivePath.Extension != ".mod" {
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(archivePath.Path); err != nil {
|
||||
return result, fmt.Errorf("delete consumed archive %s: %w", archivePath.Path, err)
|
||||
}
|
||||
result.DeletedArchivePaths = append(result.DeletedArchivePaths, archivePath.Path)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, extractionScope, []error) {
|
||||
var failures []error
|
||||
writtenCount := 0
|
||||
overwrittenCount := 0
|
||||
skippedCount := 0
|
||||
scope := extractionScope{}
|
||||
|
||||
ignored := make(map[string]bool)
|
||||
for _, ext := range p.Config.Extract.IgnoreExtensions {
|
||||
ext := strings.TrimPrefix(ext, ".")
|
||||
ignored[ext] = true
|
||||
}
|
||||
|
||||
for _, resource := range archive.Resources {
|
||||
ext, ok := erf.ExtensionForResourceType(resource.Type)
|
||||
if !ok {
|
||||
failures = append(failures, fmt.Errorf("unsupported resource type 0x%04X for %s", resource.Type, resource.Name))
|
||||
continue
|
||||
}
|
||||
if ignored[ext] {
|
||||
skippedCount++
|
||||
continue
|
||||
}
|
||||
|
||||
target, data, err := extractedFile(p, resource, ext)
|
||||
if err != nil {
|
||||
failures = append(failures, err)
|
||||
continue
|
||||
}
|
||||
desired[target] = struct{}{}
|
||||
scope.Source = scope.Source || pathWithinRoot(target, p.SourceDir())
|
||||
scope.Assets = scope.Assets || pathWithinRoot(target, p.AssetsDir())
|
||||
|
||||
state, err := writeManagedFile(target, data)
|
||||
if err != nil {
|
||||
failures = append(failures, err)
|
||||
continue
|
||||
}
|
||||
switch state {
|
||||
case writeNew:
|
||||
writtenCount++
|
||||
case writeOverwritten:
|
||||
overwrittenCount++
|
||||
case writeSkipped:
|
||||
skippedCount++
|
||||
}
|
||||
}
|
||||
|
||||
return writtenCount, overwrittenCount, skippedCount, scope, failures
|
||||
}
|
||||
|
||||
func resolveExtractionArchives(p *project.Project, overrides []string) ([]extractionArchive, error) {
|
||||
patterns := p.EffectiveConfig().Extract.Archives
|
||||
if len(overrides) > 0 {
|
||||
patterns = overrides
|
||||
}
|
||||
if len(patterns) == 0 {
|
||||
return nil, fmt.Errorf("extract.archives must contain at least one build-relative archive pattern")
|
||||
}
|
||||
|
||||
buildRoot := filepath.Clean(p.BuildDir())
|
||||
byPath := map[string]extractionArchive{}
|
||||
for _, rawPattern := range patterns {
|
||||
pattern, err := normalizeExtractionArchivePattern(p, rawPattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
matched, err := matchExtractionArchives(buildRoot, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(matched) == 0 {
|
||||
return nil, fmt.Errorf("extract archive pattern %q matched no files under %s", rawPattern, buildRoot)
|
||||
}
|
||||
for _, archive := range matched {
|
||||
byPath[archive.Path] = archive
|
||||
}
|
||||
}
|
||||
|
||||
archives := make([]extractionArchive, 0, len(byPath))
|
||||
for _, archive := range byPath {
|
||||
archives = append(archives, archive)
|
||||
}
|
||||
slices.SortFunc(archives, func(a, b extractionArchive) int {
|
||||
return strings.Compare(a.Rel, b.Rel)
|
||||
})
|
||||
return archives, nil
|
||||
}
|
||||
|
||||
func normalizeExtractionArchivePattern(p *project.Project, raw string) (string, error) {
|
||||
pattern := strings.TrimSpace(raw)
|
||||
if pattern == "" {
|
||||
return "", fmt.Errorf("extract archive pattern must not be empty")
|
||||
}
|
||||
pattern = strings.NewReplacer(
|
||||
"{module.resref}", strings.TrimSpace(p.Config.Module.ResRef),
|
||||
).Replace(pattern)
|
||||
if strings.Contains(pattern, "\x00") {
|
||||
return "", fmt.Errorf("extract archive pattern %q must not contain NUL bytes", raw)
|
||||
}
|
||||
if filepath.IsAbs(pattern) {
|
||||
return "", fmt.Errorf("extract archive pattern %q must be relative to paths.build", raw)
|
||||
}
|
||||
clean := filepath.Clean(filepath.FromSlash(pattern))
|
||||
if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("extract archive pattern %q must not escape paths.build", raw)
|
||||
}
|
||||
return filepath.ToSlash(clean), nil
|
||||
}
|
||||
|
||||
func matchExtractionArchives(buildRoot, pattern string) ([]extractionArchive, error) {
|
||||
var archives []extractionArchive
|
||||
err := filepath.WalkDir(buildRoot, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(buildRoot, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if !matchPathPattern(rel, pattern) {
|
||||
return nil
|
||||
}
|
||||
archive, err := validateExtractionArchive(buildRoot, rel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
archives = append(archives, archive)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("scan build directory %s: %w", buildRoot, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return archives, nil
|
||||
}
|
||||
|
||||
func validateExtractionArchive(buildRoot, rel string) (extractionArchive, error) {
|
||||
cleanRel := filepath.Clean(filepath.FromSlash(rel))
|
||||
if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(filepath.Separator)) {
|
||||
return extractionArchive{}, fmt.Errorf("extract archive %q must not escape paths.build", rel)
|
||||
}
|
||||
path := filepath.Join(buildRoot, cleanRel)
|
||||
if !pathWithinRoot(path, buildRoot) {
|
||||
return extractionArchive{}, fmt.Errorf("extract archive %q resolves outside paths.build", rel)
|
||||
}
|
||||
extension := strings.ToLower(filepath.Ext(cleanRel))
|
||||
switch extension {
|
||||
case ".mod", ".hak":
|
||||
return extractionArchive{Path: path, Rel: filepath.ToSlash(cleanRel), Extension: extension}, nil
|
||||
case ".erf":
|
||||
return extractionArchive{}, fmt.Errorf("extract archive %q is an .erf; ERF extraction is unsafe because ERFs lack module.ifo context", rel)
|
||||
default:
|
||||
return extractionArchive{}, fmt.Errorf("extract archive %q uses unsupported extension %q", rel, extension)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionConsumePolicy(p *project.Project) (consumeAll bool, consumeModules bool) {
|
||||
if configured := p.Config.Extract.ConsumeArchives; configured != nil {
|
||||
return *configured, false
|
||||
}
|
||||
return false, p.Config.Extract.DeleteModuleArchiveAfterSuccess
|
||||
}
|
||||
|
||||
func pathWithinRoot(path, root string) bool {
|
||||
root = filepath.Clean(root)
|
||||
if root == "" || root == "." {
|
||||
return false
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel)
|
||||
}
|
||||
|
||||
func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) {
|
||||
resref := strings.ToLower(resource.Name)
|
||||
effective := p.EffectiveConfig()
|
||||
|
||||
switch extension {
|
||||
case "nss":
|
||||
target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), "scripts", resref+".nss")
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return target, resource.Data, nil
|
||||
case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
|
||||
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl":
|
||||
document, err := gff.Read(bytes.NewReader(resource.Data))
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err)
|
||||
}
|
||||
target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json")
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if err := mergeExtractedGFFJSON(p, target, &document); err != nil {
|
||||
return "", nil, fmt.Errorf("merge extracted gff json %s.%s: %w", resource.Name, extension, err)
|
||||
}
|
||||
formatted, err := json.MarshalIndent(document, "", " ")
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err)
|
||||
}
|
||||
formatted = append(formatted, '\n')
|
||||
return target, formatted, nil
|
||||
default:
|
||||
target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return target, resource.Data, nil
|
||||
}
|
||||
}
|
||||
|
||||
func extractionTarget(p *project.Project, field, configured, root string, parts ...string) (string, error) {
|
||||
if strings.TrimSpace(configured) == "" {
|
||||
return "", fmt.Errorf("cannot extract resource: %s is not configured", field)
|
||||
}
|
||||
|
||||
cleanRoot := filepath.Clean(root)
|
||||
if cleanRoot == "." || cleanRoot == string(filepath.Separator) || cleanRoot == filepath.Clean(p.Root) {
|
||||
return "", fmt.Errorf("cannot extract resource: %s resolves to unsafe extraction root %s", field, cleanRoot)
|
||||
}
|
||||
|
||||
target := filepath.Join(append([]string{cleanRoot}, parts...)...)
|
||||
rel, err := filepath.Rel(cleanRoot, target)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve extraction target %s: %w", target, err)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
|
||||
return "", fmt.Errorf("refusing to extract resource outside %s: %s", cleanRoot, target)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
func mergeExtractedGFFJSON(p *project.Project, target string, extracted *gff.Document) error {
|
||||
rule, ok, err := extractGFFJSONMergeRule(p, target)
|
||||
if err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("read existing source %s: %w", target, err)
|
||||
}
|
||||
var existing gff.Document
|
||||
if err := json.Unmarshal(raw, &existing); err != nil {
|
||||
return fmt.Errorf("parse existing source %s: %w", target, err)
|
||||
}
|
||||
|
||||
for _, label := range rule.PreserveFields {
|
||||
if field, ok := gffField(existing.Root, label); ok {
|
||||
setGFFField(&extracted.Root, field)
|
||||
}
|
||||
}
|
||||
for _, listRule := range rule.MergeLists {
|
||||
if err := mergeGFFListByKey(&extracted.Root, existing.Root, listRule); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractGFFJSONMergeRule(p *project.Project, target string) (project.ExtractGFFJSONMergeRule, bool, error) {
|
||||
rel, err := filepath.Rel(filepath.Clean(p.SourceDir()), filepath.Clean(target))
|
||||
if err != nil {
|
||||
return project.ExtractGFFJSONMergeRule{}, false, fmt.Errorf("resolve source-relative target %s: %w", target, err)
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if rel == ".." || strings.HasPrefix(rel, "../") {
|
||||
return project.ExtractGFFJSONMergeRule{}, false, nil
|
||||
}
|
||||
for _, rule := range p.EffectiveConfig().Extract.Merge.GFFJSON {
|
||||
if rule.Target == rel {
|
||||
return rule, true, nil
|
||||
}
|
||||
}
|
||||
return project.ExtractGFFJSONMergeRule{}, false, nil
|
||||
}
|
||||
|
||||
func gffField(s gff.Struct, label string) (gff.Field, bool) {
|
||||
for _, field := range s.Fields {
|
||||
if field.Label == label {
|
||||
return field, true
|
||||
}
|
||||
}
|
||||
return gff.Field{}, false
|
||||
}
|
||||
|
||||
func setGFFField(s *gff.Struct, replacement gff.Field) {
|
||||
for index, field := range s.Fields {
|
||||
if field.Label == replacement.Label {
|
||||
s.Fields[index] = replacement
|
||||
return
|
||||
}
|
||||
}
|
||||
s.Fields = append(s.Fields, replacement)
|
||||
}
|
||||
|
||||
func mergeGFFListByKey(extracted *gff.Struct, existing gff.Struct, rule project.ExtractListMergeRule) error {
|
||||
extractedField, ok := gffField(*extracted, rule.Field)
|
||||
if !ok {
|
||||
return fmt.Errorf("extracted field %q not found", rule.Field)
|
||||
}
|
||||
extractedList, ok := extractedField.Value.(gff.ListValue)
|
||||
if !ok {
|
||||
return fmt.Errorf("extracted field %q is %s, not List", rule.Field, extractedField.Type)
|
||||
}
|
||||
existingField, ok := gffField(existing, rule.Field)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
existingList, ok := existingField.Value.(gff.ListValue)
|
||||
if !ok {
|
||||
return fmt.Errorf("existing field %q is %s, not List", rule.Field, existingField.Type)
|
||||
}
|
||||
|
||||
extractedByKey := map[string]gff.Struct{}
|
||||
extractedOrder := make([]string, 0, len(extractedList))
|
||||
for _, item := range extractedList {
|
||||
key, err := gffStructKey(item, rule.KeyField)
|
||||
if err != nil {
|
||||
return fmt.Errorf("extracted field %q: %w", rule.Field, err)
|
||||
}
|
||||
if _, exists := extractedByKey[key]; exists {
|
||||
return fmt.Errorf("extracted field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
|
||||
}
|
||||
extractedByKey[key] = item
|
||||
extractedOrder = append(extractedOrder, key)
|
||||
}
|
||||
|
||||
merged := make(gff.ListValue, 0, len(extractedList))
|
||||
seen := map[string]struct{}{}
|
||||
for _, item := range existingList {
|
||||
key, err := gffStructKey(item, rule.KeyField)
|
||||
if err != nil {
|
||||
return fmt.Errorf("existing field %q: %w", rule.Field, err)
|
||||
}
|
||||
if _, duplicate := seen[key]; duplicate {
|
||||
return fmt.Errorf("existing field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
|
||||
}
|
||||
if extractedItem, exists := extractedByKey[key]; exists {
|
||||
merged = append(merged, extractedItem)
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, key := range extractedOrder {
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
merged = append(merged, extractedByKey[key])
|
||||
}
|
||||
|
||||
setGFFField(extracted, gff.NewField(rule.Field, merged))
|
||||
return nil
|
||||
}
|
||||
|
||||
func gffStructKey(s gff.Struct, keyField string) (string, error) {
|
||||
field, ok := gffField(s, keyField)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("key field %q not found", keyField)
|
||||
}
|
||||
switch value := field.Value.(type) {
|
||||
case gff.ResRefValue:
|
||||
return string(value), nil
|
||||
case gff.StringValue:
|
||||
return string(value), nil
|
||||
default:
|
||||
return "", fmt.Errorf("key field %q is %s, not ResRef or CExoString", keyField, field.Type)
|
||||
}
|
||||
}
|
||||
|
||||
type writeState int
|
||||
|
||||
const (
|
||||
writeSkipped writeState = iota
|
||||
writeNew
|
||||
writeOverwritten
|
||||
)
|
||||
|
||||
func writeManagedFile(path string, data []byte) (writeState, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return writeSkipped, fmt.Errorf("create parent directory for %s: %w", path, err)
|
||||
}
|
||||
|
||||
existing, err := os.ReadFile(path)
|
||||
if err == nil {
|
||||
if bytes.Equal(existing, data) {
|
||||
return writeSkipped, nil
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
return writeSkipped, fmt.Errorf("overwrite %s: %w", path, err)
|
||||
}
|
||||
return writeOverwritten, nil
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return writeSkipped, fmt.Errorf("check existing file %s: %w", path, err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
return writeSkipped, fmt.Errorf("write %s: %w", path, err)
|
||||
}
|
||||
return writeNew, nil
|
||||
}
|
||||
|
||||
func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) {
|
||||
candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
|
||||
if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) {
|
||||
for _, rel := range p.Inventory.SourceFiles {
|
||||
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
for _, rel := range p.Inventory.ScriptFiles {
|
||||
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
}
|
||||
if scope.Assets && safeCleanupRoot(p.AssetsDir(), p.Root) {
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
}
|
||||
|
||||
removed := 0
|
||||
var failures []error
|
||||
for _, path := range candidates {
|
||||
if _, keep := desired[path]; keep {
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
failures = append(failures, fmt.Errorf("remove stale file %s: %w", path, err))
|
||||
continue
|
||||
}
|
||||
removed++
|
||||
cleanupEmptyParents(filepath.Dir(path), p.SourceDir(), p.AssetsDir())
|
||||
}
|
||||
|
||||
return removed, failures
|
||||
}
|
||||
|
||||
func safeCleanupRoot(root, projectRoot string) bool {
|
||||
cleanRoot := filepath.Clean(root)
|
||||
return cleanRoot != "." && cleanRoot != string(filepath.Separator) && cleanRoot != filepath.Clean(projectRoot)
|
||||
}
|
||||
|
||||
func cleanupEmptyParents(dir string, roots ...string) {
|
||||
for {
|
||||
if dir == "." || dir == string(filepath.Separator) {
|
||||
return
|
||||
}
|
||||
stop := false
|
||||
for _, root := range roots {
|
||||
if dir == root {
|
||||
stop = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if stop {
|
||||
return
|
||||
}
|
||||
if err := os.Remove(dir); err != nil {
|
||||
return
|
||||
}
|
||||
dir = filepath.Dir(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func sourceSubdir(extension string) string {
|
||||
switch strings.ToLower(extension) {
|
||||
case "are":
|
||||
return "areas"
|
||||
case "dlg":
|
||||
return "dialogs"
|
||||
case "fac":
|
||||
return "factions"
|
||||
case "gic":
|
||||
return "instance"
|
||||
case "git":
|
||||
return "instance"
|
||||
case "ifo":
|
||||
return "module"
|
||||
case "itp":
|
||||
return "palettes"
|
||||
case "jrl":
|
||||
return "journal"
|
||||
case "utc":
|
||||
return filepath.Join("blueprints", "creatures")
|
||||
case "utd":
|
||||
return filepath.Join("blueprints", "doors")
|
||||
case "ute":
|
||||
return filepath.Join("blueprints", "encounters")
|
||||
case "uti":
|
||||
return filepath.Join("blueprints", "items")
|
||||
case "utm":
|
||||
return filepath.Join("blueprints", "merchants")
|
||||
case "utp":
|
||||
return filepath.Join("blueprints", "placeables")
|
||||
case "uts":
|
||||
return filepath.Join("blueprints", "sounds")
|
||||
case "utt":
|
||||
return filepath.Join("blueprints", "triggers")
|
||||
case "utw":
|
||||
return filepath.Join("blueprints", "waypoints")
|
||||
default:
|
||||
return extension
|
||||
}
|
||||
}
|
||||
@@ -1,524 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
type preparedMusicAssets struct {
|
||||
Generated []assetResource
|
||||
SkipSourceRel map[string]struct{}
|
||||
Artifacts []string
|
||||
TempRoots []string
|
||||
Summary CreditsRefreshSummary
|
||||
}
|
||||
|
||||
type MusicBuildOptions struct {
|
||||
DatasetIDs []string
|
||||
Write bool
|
||||
}
|
||||
|
||||
type musicPrepareOptions struct {
|
||||
datasetIDs []string
|
||||
write bool
|
||||
}
|
||||
|
||||
type musicSourceGroup struct {
|
||||
DatasetID string
|
||||
Dataset project.EffectiveMusicDataset
|
||||
SourceDir string
|
||||
OutputDir string
|
||||
Sources []string
|
||||
}
|
||||
|
||||
type musicCreditGroup struct {
|
||||
CreditsRoot string
|
||||
SourceDir string
|
||||
Entries []music.CreditsEntry
|
||||
}
|
||||
|
||||
func musicRootPath(p *project.Project, path string) string {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return p.Root
|
||||
}
|
||||
if filepath.IsAbs(path) {
|
||||
return path
|
||||
}
|
||||
return filepath.Join(p.Root, filepath.FromSlash(path))
|
||||
}
|
||||
|
||||
func effectiveMusicDatasets(p *project.Project, onlyIDs []string) (map[string]project.EffectiveMusicDataset, error) {
|
||||
effective := p.EffectiveConfig()
|
||||
wanted := map[string]struct{}{}
|
||||
for _, id := range onlyIDs {
|
||||
id = strings.TrimSpace(id)
|
||||
if id != "" {
|
||||
wanted[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
datasets := map[string]project.EffectiveMusicDataset{}
|
||||
for id, dataset := range effective.Music.Datasets {
|
||||
if len(wanted) > 0 {
|
||||
if _, ok := wanted[id]; !ok {
|
||||
continue
|
||||
}
|
||||
delete(wanted, id)
|
||||
}
|
||||
datasets[id] = dataset
|
||||
}
|
||||
if len(wanted) > 0 {
|
||||
unknown := make([]string, 0, len(wanted))
|
||||
for id := range wanted {
|
||||
unknown = append(unknown, id)
|
||||
}
|
||||
sort.Strings(unknown)
|
||||
return nil, fmt.Errorf("unknown music dataset(s): %s", strings.Join(unknown, ", "))
|
||||
}
|
||||
if len(datasets) == 0 && len(wanted) == 0 {
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
ext := strings.ToLower(filepath.Ext(rel))
|
||||
if !music.PathIsUnder("envi/music", rel) {
|
||||
continue
|
||||
}
|
||||
if _, ok := extensionSet(effective.Music.ConvertExtensions)[ext]; ok {
|
||||
datasets["legacy_envi_music"] = project.EffectiveMusicDataset{
|
||||
Source: "envi/music",
|
||||
Output: "envi/music",
|
||||
StageRoot: effective.Music.StageRoot,
|
||||
CreditsRoot: effective.Music.CreditsRoot,
|
||||
NamingScheme: effective.Music.NamingScheme,
|
||||
OutputExtension: effective.Music.OutputExtension,
|
||||
MaxStemLength: effective.Music.MaxStemLength,
|
||||
PackageMode: "hak_asset",
|
||||
ConvertExtensions: effective.Music.ConvertExtensions,
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return datasets, nil
|
||||
}
|
||||
|
||||
func extensionSet(values []string) map[string]struct{} {
|
||||
out := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value != "" {
|
||||
out[value] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func datasetForMusicSource(datasets map[string]project.EffectiveMusicDataset, rel, ext string) (string, project.EffectiveMusicDataset, bool) {
|
||||
bestID := ""
|
||||
var best project.EffectiveMusicDataset
|
||||
bestLen := -1
|
||||
for id, dataset := range datasets {
|
||||
if _, ok := extensionSet(dataset.ConvertExtensions)[ext]; !ok {
|
||||
continue
|
||||
}
|
||||
if !music.PathIsUnder(dataset.Source, rel) {
|
||||
continue
|
||||
}
|
||||
if len(dataset.Source) > bestLen {
|
||||
bestID = id
|
||||
best = dataset
|
||||
bestLen = len(dataset.Source)
|
||||
}
|
||||
}
|
||||
return bestID, best, bestID != ""
|
||||
}
|
||||
|
||||
func plannedMusicAsset(outputRel, sourceRel string) (assetResource, error) {
|
||||
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(outputRel)), ".")
|
||||
resourceType, ok := erf.HAKResourceTypeForExtension(extension)
|
||||
if !ok {
|
||||
return assetResource{}, fmt.Errorf("unsupported generated music resource extension %q", filepath.Ext(outputRel))
|
||||
}
|
||||
name := strings.ToLower(strings.TrimSuffix(filepath.Base(outputRel), filepath.Ext(outputRel)))
|
||||
return assetResource{
|
||||
Rel: outputRel,
|
||||
Resource: erf.Resource{
|
||||
Name: name,
|
||||
Type: resourceType,
|
||||
},
|
||||
ContentID: "music-plan:" + filepath.ToSlash(sourceRel),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func BuildMusic(p *project.Project, opts MusicBuildOptions) (BuildResult, error) {
|
||||
prepared, err := prepareMusicAssetsWithOptions(p, musicPrepareOptions{datasetIDs: opts.DatasetIDs, write: opts.Write})
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
if err := cleanupPreparedMusicAssets(prepared); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
return BuildResult{
|
||||
CreditsArtifactPaths: prepared.Artifacts,
|
||||
CreditsSummary: prepared.Summary,
|
||||
HAKAssets: len(prepared.Generated),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) {
|
||||
return prepareMusicAssetsWithOptions(p, musicPrepareOptions{write: true})
|
||||
}
|
||||
|
||||
func prepareMusicAssetsWithOptions(p *project.Project, opts musicPrepareOptions) (*preparedMusicAssets, error) {
|
||||
usedNames := make(map[string]struct{})
|
||||
groupsByKey := make(map[string]*musicSourceGroup)
|
||||
datasets, err := effectiveMusicDatasets(p, opts.datasetIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
rel = filepath.ToSlash(rel)
|
||||
ext := strings.ToLower(filepath.Ext(rel))
|
||||
name := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
|
||||
if datasetID, dataset, ok := datasetForMusicSource(datasets, rel, ext); ok {
|
||||
sourceDir := filepath.ToSlash(filepath.Dir(rel))
|
||||
relDir, err := filepath.Rel(filepath.FromSlash(dataset.Source), filepath.FromSlash(sourceDir))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve music source directory %s: %w", sourceDir, err)
|
||||
}
|
||||
relDir = filepath.ToSlash(relDir)
|
||||
outputDir := dataset.Output
|
||||
if relDir != "." && relDir != "" {
|
||||
outputDir = filepath.ToSlash(filepath.Join(outputDir, filepath.FromSlash(relDir)))
|
||||
}
|
||||
key := datasetID + "\x00" + sourceDir
|
||||
group := groupsByKey[key]
|
||||
if group == nil {
|
||||
group = &musicSourceGroup{
|
||||
DatasetID: datasetID,
|
||||
Dataset: dataset,
|
||||
SourceDir: sourceDir,
|
||||
OutputDir: outputDir,
|
||||
}
|
||||
groupsByKey[key] = group
|
||||
}
|
||||
group.Sources = append(group.Sources, rel)
|
||||
continue
|
||||
}
|
||||
if name != "" {
|
||||
usedNames[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
result := &preparedMusicAssets{
|
||||
SkipSourceRel: make(map[string]struct{}),
|
||||
}
|
||||
if len(groupsByKey) == 0 {
|
||||
if !opts.write {
|
||||
return result, nil
|
||||
}
|
||||
writeResult, err := writeCreditsArtifacts(p, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Artifacts = writeResult.Artifacts
|
||||
result.Summary = writeResult.Summary
|
||||
return result, nil
|
||||
}
|
||||
|
||||
groups := make([]*musicSourceGroup, 0, len(groupsByKey))
|
||||
for _, group := range groupsByKey {
|
||||
sort.Strings(group.Sources)
|
||||
groups = append(groups, group)
|
||||
}
|
||||
sort.Slice(groups, func(i, j int) bool {
|
||||
if groups[i].SourceDir != groups[j].SourceDir {
|
||||
return groups[i].SourceDir < groups[j].SourceDir
|
||||
}
|
||||
return groups[i].DatasetID < groups[j].DatasetID
|
||||
})
|
||||
|
||||
creditGroups := make([]musicCreditGroup, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
result.Summary.ScannedDirs = append(result.Summary.ScannedDirs, group.SourceDir)
|
||||
result.Summary.TrackCount += len(group.Sources)
|
||||
}
|
||||
ffmpegPath := ""
|
||||
ffprobePath := ""
|
||||
if opts.write {
|
||||
var err error
|
||||
ffmpegPath, err = music.ResolveFFmpegWithConfig(p.MusicFFmpegPath())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ffprobePath, err = music.ResolveFFprobeWithConfig(p.MusicFFprobePath())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
for _, group := range groups {
|
||||
stageRoot := musicRootPath(p, group.Dataset.StageRoot)
|
||||
result.TempRoots = append(result.TempRoots, stageRoot)
|
||||
prefix := music.SanitizePrefix(group.Dataset.Prefix)
|
||||
overlayPath := filepath.Join(p.AssetsDir(), filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay)
|
||||
overlay, err := music.ParseCreditsOverlay(overlayPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse credits overlay %s: %w", overlayPath, err)
|
||||
}
|
||||
|
||||
entries := make([]music.CreditsEntry, 0, len(group.Sources))
|
||||
for _, rel := range group.Sources {
|
||||
base := filepath.Base(rel)
|
||||
manual := overlay.Match(base, "")
|
||||
outputFile := ""
|
||||
switch {
|
||||
case manual != nil && strings.TrimSpace(manual.OutputFile) != "":
|
||||
outputFile = strings.ToLower(strings.TrimSpace(manual.OutputFile))
|
||||
if err := music.ValidateManualOutputFileWithRules(outputFile, group.Dataset.OutputExtension, group.Dataset.MaxStemLength); err != nil {
|
||||
return nil, fmt.Errorf("manual credits output for %s: %w", rel, err)
|
||||
}
|
||||
if err := music.ReserveName(strings.TrimSuffix(outputFile, filepath.Ext(outputFile)), usedNames); err != nil {
|
||||
return nil, fmt.Errorf("manual credits output for %s: %w", rel, err)
|
||||
}
|
||||
default:
|
||||
stem, err := music.GenerateStemWithMax(base, prefix, group.Dataset.MaxStemLength, usedNames)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate music name for %s: %w", rel, err)
|
||||
}
|
||||
outputFile = stem + group.Dataset.OutputExtension
|
||||
}
|
||||
outputRel := filepath.ToSlash(filepath.Join(group.OutputDir, outputFile))
|
||||
result.Summary.Mappings = append(result.Summary.Mappings, MusicFileMapping{
|
||||
Source: rel,
|
||||
Output: outputRel,
|
||||
})
|
||||
result.SkipSourceRel[rel] = struct{}{}
|
||||
|
||||
if !opts.write {
|
||||
if group.Dataset.PackageMode == "hak_asset" {
|
||||
asset, err := plannedMusicAsset(outputRel, rel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Generated = append(result.Generated, asset)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
metadata, err := music.ProbeMetadata(ffprobePath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("probe music metadata for %s: %w", rel, err)
|
||||
}
|
||||
entry := music.CreditsEntry{
|
||||
Artist: metadata.Artist,
|
||||
Title: music.FirstNonEmpty(metadata.Title, strings.TrimSuffix(base, filepath.Ext(base))),
|
||||
OutputFile: outputFile,
|
||||
OriginalFile: base,
|
||||
Album: metadata.Album,
|
||||
Date: metadata.Date,
|
||||
Rights: music.FirstNonEmpty(metadata.Rights, music.JoinNonEmpty("; ", metadata.License, metadata.Copyright)),
|
||||
Notes: music.FirstNonEmpty(metadata.Notes, metadata.Comment),
|
||||
}
|
||||
music.ApplyCreditsOverlay(&entry, manual)
|
||||
|
||||
stagePath := filepath.Join(stageRoot, filepath.FromSlash(outputRel))
|
||||
if err := os.MkdirAll(filepath.Dir(stagePath), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create music stage dir: %w", err)
|
||||
}
|
||||
if err := music.ConvertSource(ffmpegPath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)), stagePath); err != nil {
|
||||
return nil, fmt.Errorf("convert music %s: %w", rel, err)
|
||||
}
|
||||
if group.Dataset.PackageMode == "hak_asset" {
|
||||
resourceInfo, err := assetResourceFromPath(stagePath, outputRel, lfsAssetInfo{}, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load converted music %s: %w", outputRel, err)
|
||||
}
|
||||
info, err := os.Stat(stagePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat converted music %s: %w", stagePath, err)
|
||||
}
|
||||
result.Generated = append(result.Generated, assetResource{
|
||||
Rel: outputRel,
|
||||
Resource: resourceInfo.Resource,
|
||||
Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}),
|
||||
CreatedAt: info.ModTime(),
|
||||
ContentID: resourceInfo.ContentID,
|
||||
})
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if opts.write {
|
||||
if err := music.ValidateOverlayEntries(group.SourceDir, overlay, entries); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
creditGroups = append(creditGroups, musicCreditGroup{
|
||||
CreditsRoot: group.Dataset.CreditsRoot,
|
||||
SourceDir: group.SourceDir,
|
||||
Entries: entries,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.write {
|
||||
sort.Slice(result.Generated, func(i, j int) bool {
|
||||
return compareResourceKeys(result.Generated[i].Resource, result.Generated[j].Resource) < 0
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
writeResult, err := writeCreditsArtifacts(p, creditGroups)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writeResult.Summary.ScannedDirs = append(writeResult.Summary.ScannedDirs, result.Summary.ScannedDirs...)
|
||||
writeResult.Summary.TrackCount = result.Summary.TrackCount
|
||||
result.Artifacts = writeResult.Artifacts
|
||||
result.Summary = writeResult.Summary
|
||||
sort.Slice(result.Generated, func(i, j int) bool {
|
||||
return compareResourceKeys(result.Generated[i].Resource, result.Generated[j].Resource) < 0
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type creditsArtifactWriteResult struct {
|
||||
Artifacts []string
|
||||
Summary CreditsRefreshSummary
|
||||
}
|
||||
|
||||
func writeCreditsArtifacts(p *project.Project, generated []musicCreditGroup) (creditsArtifactWriteResult, error) {
|
||||
defaultCreditsRoot := p.CreditsDir()
|
||||
sources := make([]music.CreditsSource, 0)
|
||||
desiredByRoot := map[string]map[string][]byte{}
|
||||
summary := CreditsRefreshSummary{}
|
||||
if generated != nil {
|
||||
sort.Slice(generated, func(i, j int) bool {
|
||||
if generated[i].SourceDir != generated[j].SourceDir {
|
||||
return generated[i].SourceDir < generated[j].SourceDir
|
||||
}
|
||||
return generated[i].CreditsRoot < generated[j].CreditsRoot
|
||||
})
|
||||
for _, group := range generated {
|
||||
creditsRoot := musicRootPath(p, group.CreditsRoot)
|
||||
if desiredByRoot[creditsRoot] == nil {
|
||||
desiredByRoot[creditsRoot] = map[string][]byte{}
|
||||
}
|
||||
entries := append([]music.CreditsEntry(nil), group.Entries...)
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
if strings.ToLower(entries[i].Artist) != strings.ToLower(entries[j].Artist) {
|
||||
return strings.ToLower(entries[i].Artist) < strings.ToLower(entries[j].Artist)
|
||||
}
|
||||
if strings.ToLower(entries[i].Title) != strings.ToLower(entries[j].Title) {
|
||||
return strings.ToLower(entries[i].Title) < strings.ToLower(entries[j].Title)
|
||||
}
|
||||
return strings.ToLower(entries[i].OutputFile) < strings.ToLower(entries[j].OutputFile)
|
||||
})
|
||||
outputPath := filepath.Join(creditsRoot, filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay)
|
||||
desiredByRoot[creditsRoot][outputPath] = []byte(music.RenderCreditsMarkdown(entries))
|
||||
summary.GeneratedPaths = append(summary.GeneratedPaths, outputPath)
|
||||
for _, entry := range entries {
|
||||
summary.Mappings = append(summary.Mappings, MusicFileMapping{
|
||||
Source: entry.OriginalFile,
|
||||
Output: entry.OutputFile,
|
||||
})
|
||||
}
|
||||
sources = append(sources, music.CreditsSource{
|
||||
Path: filepath.ToSlash(outputPath),
|
||||
Kind: "generated_music",
|
||||
Entries: entries,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
err := filepath.WalkDir(p.AssetsDir(), func(path string, d os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if strings.ToLower(d.Name()) != "credits.md" {
|
||||
return nil
|
||||
}
|
||||
entries, err := music.ParseCreditsMarkdown(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse credits %s: %w", path, err)
|
||||
}
|
||||
rel, err := filepath.Rel(p.Root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sources = append(sources, music.CreditsSource{
|
||||
Path: filepath.ToSlash(rel),
|
||||
Kind: "authored",
|
||||
Entries: entries,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return creditsArtifactWriteResult{}, err
|
||||
}
|
||||
|
||||
sort.Slice(sources, func(i, j int) bool { return sources[i].Path < sources[j].Path })
|
||||
payload, err := json.MarshalIndent(music.CreditsInventory{Sources: sources}, "", " ")
|
||||
if err != nil {
|
||||
return creditsArtifactWriteResult{}, fmt.Errorf("marshal credits inventory: %w", err)
|
||||
}
|
||||
payload = append(payload, '\n')
|
||||
if len(desiredByRoot) == 0 {
|
||||
desiredByRoot[defaultCreditsRoot] = map[string][]byte{}
|
||||
}
|
||||
artifacts := make([]string, 0)
|
||||
changedFiles := 0
|
||||
roots := make([]string, 0, len(desiredByRoot))
|
||||
for root := range desiredByRoot {
|
||||
roots = append(roots, root)
|
||||
}
|
||||
sort.Strings(roots)
|
||||
for _, creditsRoot := range roots {
|
||||
desiredFiles := desiredByRoot[creditsRoot]
|
||||
inventoryPath := filepath.Join(creditsRoot, "credits.json")
|
||||
desiredFiles[inventoryPath] = payload
|
||||
if err := os.MkdirAll(creditsRoot, 0o755); err != nil {
|
||||
return creditsArtifactWriteResult{}, fmt.Errorf("create credits dir: %w", err)
|
||||
}
|
||||
changed, err := music.SyncGeneratedCreditsArtifacts(creditsRoot, desiredFiles)
|
||||
if err != nil {
|
||||
return creditsArtifactWriteResult{}, err
|
||||
}
|
||||
changedFiles += changed
|
||||
for path := range desiredFiles {
|
||||
artifacts = append(artifacts, path)
|
||||
}
|
||||
if summary.InventoryPath == "" {
|
||||
summary.InventoryPath = inventoryPath
|
||||
}
|
||||
}
|
||||
sort.Strings(artifacts)
|
||||
summary.ArtifactPaths = append(summary.ArtifactPaths, artifacts...)
|
||||
summary.ChangedFiles = changedFiles
|
||||
return creditsArtifactWriteResult{
|
||||
Artifacts: artifacts,
|
||||
Summary: summary,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func cleanupPreparedMusicAssets(prepared *preparedMusicAssets) error {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
}
|
||||
for _, root := range prepared.TempRoots {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(root); err != nil {
|
||||
return fmt.Errorf("remove temporary music artifact root %s: %w", root, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,738 +0,0 @@
|
||||
package project
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultBuildPath = "build"
|
||||
DefaultCachePath = ".cache"
|
||||
DefaultToolsPath = "tools"
|
||||
DefaultModuleArchiveTemplate = "{module.resref}.mod"
|
||||
DefaultHAKManifest = "haks.json"
|
||||
DefaultHAKArchiveTemplate = "{hak.name}.hak"
|
||||
DefaultSourceJSONPattern = "{resref}.{extension}.json"
|
||||
DefaultValidationProfile = "nwn_module"
|
||||
DefaultMusicStageRoot = "{paths.cache}/music"
|
||||
DefaultCreditsRoot = "{paths.cache}/credits"
|
||||
DefaultCreditsOverlayFile = "CREDITS.md"
|
||||
DefaultMusicNamingScheme = "nwn_bmu"
|
||||
DefaultMusicOutputExtension = ".bmu"
|
||||
DefaultTopDataBuild = "{paths.build}/topdata"
|
||||
DefaultTopDataCompiled2DADir = "2da"
|
||||
DefaultTopDataCompiledTLK = "sow_tlk.tlk"
|
||||
DefaultTopDataWikiOutputRoot = "wiki"
|
||||
DefaultTopDataWikiPagesDir = "pages"
|
||||
DefaultTopDataWikiStateFile = "state.json"
|
||||
DefaultTopDataWikiSource = "topdata/wiki"
|
||||
DefaultTopDataWikiRenderer = "nodebb_tiptap_html"
|
||||
DefaultTopDataWikiLinkStrategy = "preserve_westgate_wiki_links"
|
||||
DefaultTopDataWikiNamespacesFile = "namespaces.yaml"
|
||||
DefaultTopDataWikiTablesFile = "tables.yaml"
|
||||
DefaultTopDataWikiVisibilityFile = "visibility.yaml"
|
||||
DefaultTopDataWikiDataFile = "data.yaml"
|
||||
DefaultTopDataWikiPagePathsFile = "wiki.yaml"
|
||||
DefaultTopDataWikiTemplatesDir = "templates"
|
||||
DefaultTopDataWikiPageTemplatesDir = "templates/pages"
|
||||
DefaultTopDataWikiManualSectionsDir = "manual-sections"
|
||||
DefaultTopDataWikiManualSectionsFile = "manual-sections/default.yaml"
|
||||
DefaultTopDataWikiPageIndexFile = "page-index.json"
|
||||
DefaultTopDataWikiStaleDefault = "report"
|
||||
DefaultTopDataWikiStaleLiveCleanup = "archive"
|
||||
DefaultTopDataWikiMarkerFormat = "html_comments"
|
||||
DefaultTopDataWikiPageMarkerPrefix = "sow-topdata-wiki"
|
||||
DefaultTopDataWikiTitlePrefixMinLength = 3
|
||||
DefaultTopDataWikiStatusPages = true
|
||||
DefaultTopDataWikiStatusListingScope = "all"
|
||||
DefaultTopDataWikiAlignmentLinkFormat = "wiki_markup"
|
||||
DefaultWikiDeployManifest = ".wiki_deploy_manifest.json"
|
||||
DefaultWikiDeployEditSummary = "Auto-generated from native builder"
|
||||
DefaultTopDataPackageHAK = "sow_top.hak"
|
||||
DefaultTopDataPackageTLK = "sow_tlk.tlk"
|
||||
DefaultAutogenCacheRoot = "{paths.cache}"
|
||||
DefaultAutogenCacheMaxAge = time.Hour
|
||||
DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH"
|
||||
DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH"
|
||||
DefaultAutogenReleaseProvider = "gitea"
|
||||
DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL"
|
||||
DefaultAutogenRepoEnv = "SOW_ASSETS_REPO"
|
||||
DefaultExtractLayout = "nwn_canonical_json"
|
||||
DefaultExtractHAKDiscovery = "build_glob"
|
||||
)
|
||||
|
||||
type ConfigProvenance map[string]ConfigValueProvenance
|
||||
|
||||
type ConfigValueProvenance struct {
|
||||
Source string `json:"source" yaml:"source"`
|
||||
Detail string `json:"detail,omitempty" yaml:"detail,omitempty"`
|
||||
Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"`
|
||||
}
|
||||
|
||||
type EffectiveConfig struct {
|
||||
ConfigSource ConfigSource `json:"config_source" yaml:"config_source"`
|
||||
Module ModuleConfig `json:"module" yaml:"module"`
|
||||
Paths EffectivePathConfig `json:"paths" yaml:"paths"`
|
||||
Outputs EffectiveOutputConfig `json:"outputs" yaml:"outputs"`
|
||||
Build BuildConfig `json:"build" yaml:"build"`
|
||||
Generated GeneratedConfig `json:"generated_assets" yaml:"generated_assets"`
|
||||
Inventory InventoryConfig `json:"inventory" yaml:"inventory"`
|
||||
Validation ValidationConfig `json:"validation" yaml:"validation"`
|
||||
Music EffectiveMusicConfig `json:"music" yaml:"music"`
|
||||
TopData EffectiveTopDataConfig `json:"topdata" yaml:"topdata"`
|
||||
Extract ExtractConfig `json:"extract" yaml:"extract"`
|
||||
Autogen EffectiveAutogenConfig `json:"autogen" yaml:"autogen"`
|
||||
HAKs []HAKConfig `json:"haks" yaml:"haks"`
|
||||
Provenance ConfigProvenance `json:"provenance" yaml:"provenance"`
|
||||
Overrides []ConfigOverride `json:"overrides,omitempty" yaml:"overrides,omitempty"`
|
||||
}
|
||||
|
||||
type EffectivePathConfig struct {
|
||||
Source string `json:"source" yaml:"source"`
|
||||
Assets string `json:"assets" yaml:"assets"`
|
||||
Build string `json:"build" yaml:"build"`
|
||||
Cache string `json:"cache" yaml:"cache"`
|
||||
Tools string `json:"tools" yaml:"tools"`
|
||||
}
|
||||
|
||||
type EffectiveOutputConfig struct {
|
||||
ModuleArchive string `json:"module_archive" yaml:"module_archive"`
|
||||
HAKManifest string `json:"hak_manifest" yaml:"hak_manifest"`
|
||||
HAKArchive string `json:"hak_archive" yaml:"hak_archive"`
|
||||
}
|
||||
|
||||
type EffectiveMusicConfig struct {
|
||||
Prefixes map[string]string `json:"prefixes" yaml:"prefixes"`
|
||||
Tools MusicToolsConfig `json:"tools" yaml:"tools"`
|
||||
StageRoot string `json:"stage_root" yaml:"stage_root"`
|
||||
CreditsRoot string `json:"credits_root" yaml:"credits_root"`
|
||||
CreditsOverlay string `json:"credits_overlay" yaml:"credits_overlay"`
|
||||
MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"`
|
||||
NamingScheme string `json:"naming_scheme" yaml:"naming_scheme"`
|
||||
OutputExtension string `json:"output_extension" yaml:"output_extension"`
|
||||
ConvertExtensions []string `json:"convert_extensions" yaml:"convert_extensions"`
|
||||
Datasets map[string]EffectiveMusicDataset `json:"datasets,omitempty" yaml:"datasets,omitempty"`
|
||||
}
|
||||
|
||||
type EffectiveMusicDataset struct {
|
||||
Source string `json:"source" yaml:"source"`
|
||||
Output string `json:"output" yaml:"output"`
|
||||
Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
|
||||
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
|
||||
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
|
||||
NamingScheme string `json:"naming_scheme" yaml:"naming_scheme"`
|
||||
OutputExtension string `json:"output_extension" yaml:"output_extension"`
|
||||
MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"`
|
||||
PackageMode string `json:"package_mode" yaml:"package_mode"`
|
||||
ConvertExtensions []string `json:"convert_extensions" yaml:"convert_extensions"`
|
||||
}
|
||||
|
||||
type EffectiveTopDataConfig struct {
|
||||
Source string `json:"source" yaml:"source"`
|
||||
Build string `json:"build" yaml:"build"`
|
||||
ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"`
|
||||
Assets string `json:"assets" yaml:"assets"`
|
||||
Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"`
|
||||
CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"`
|
||||
PackageHAK string `json:"package_hak" yaml:"package_hak"`
|
||||
PackageTLK string `json:"package_tlk" yaml:"package_tlk"`
|
||||
ValueEncodings []TopDataValueEncodingConfig `json:"value_encodings" yaml:"value_encodings"`
|
||||
ValueDefaults []TopDataValueDefaultConfig `json:"value_defaults" yaml:"value_defaults"`
|
||||
RowGeneration []TopDataRowGenerationConfig `json:"row_generation" yaml:"row_generation"`
|
||||
RowExtensions []TopDataRowExtensionConfig `json:"row_extensions" yaml:"row_extensions"`
|
||||
ClassFeatInjections TopDataClassFeatInjectionConfig `json:"class_feat_injections" yaml:"class_feat_injections"`
|
||||
Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"`
|
||||
}
|
||||
|
||||
type EffectiveAutogenConfig struct {
|
||||
Producers []AutogenProducerConfig `json:"producers" yaml:"producers"`
|
||||
Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"`
|
||||
Cache EffectiveAutogenCacheConfig `json:"cache" yaml:"cache"`
|
||||
ReleaseSource EffectiveAutogenReleaseSourceConfig `json:"release_source" yaml:"release_source"`
|
||||
}
|
||||
|
||||
type EffectiveAutogenCacheConfig struct {
|
||||
Root string `json:"root" yaml:"root"`
|
||||
MaxAge string `json:"max_age" yaml:"max_age"`
|
||||
RefreshEnv string `json:"refresh_env" yaml:"refresh_env"`
|
||||
PartsManifestRefreshEnv string `json:"parts_manifest_refresh_env" yaml:"parts_manifest_refresh_env"`
|
||||
}
|
||||
|
||||
type EffectiveAutogenReleaseSourceConfig struct {
|
||||
Provider string `json:"provider" yaml:"provider"`
|
||||
ServerURLEnv string `json:"server_url_env" yaml:"server_url_env"`
|
||||
RepoEnv string `json:"repo_env" yaml:"repo_env"`
|
||||
}
|
||||
|
||||
type ConfigOverride struct {
|
||||
Key string `json:"key" yaml:"key"`
|
||||
Value string `json:"value" yaml:"value"`
|
||||
Source string `json:"source" yaml:"source"`
|
||||
}
|
||||
|
||||
func (p *Project) EffectiveConfig() EffectiveConfig {
|
||||
provenance := cloneProvenance(p.Provenance)
|
||||
markMissingDefaults(provenance)
|
||||
|
||||
paths := EffectivePathConfig{
|
||||
Source: strings.TrimSpace(p.Config.Paths.Source),
|
||||
Assets: strings.TrimSpace(p.Config.Paths.Assets),
|
||||
Build: defaultString(p.Config.Paths.Build, DefaultBuildPath),
|
||||
Cache: defaultString(p.Config.Paths.Cache, DefaultCachePath),
|
||||
Tools: defaultString(p.Config.Paths.Tools, DefaultToolsPath),
|
||||
}
|
||||
outputs := EffectiveOutputConfig{
|
||||
ModuleArchive: defaultString(p.Config.Outputs.ModuleArchive, DefaultModuleArchiveTemplate),
|
||||
HAKManifest: defaultString(p.Config.Outputs.HAKManifest, DefaultHAKManifest),
|
||||
HAKArchive: defaultString(p.Config.Outputs.HAKArchive, DefaultHAKArchiveTemplate),
|
||||
}
|
||||
inventory := InventoryConfig{
|
||||
SourceExtensions: defaultStringSlice(p.Config.Inventory.SourceExtensions, SourceExtensions),
|
||||
AssetExtensions: defaultStringSlice(p.Config.Inventory.AssetExtensions, AssetExtensions),
|
||||
SourceJSONPattern: defaultString(p.Config.Inventory.SourceJSONPattern, DefaultSourceJSONPattern),
|
||||
}
|
||||
validation := ValidationConfig{
|
||||
Profile: defaultString(p.Config.Validation.Profile, DefaultValidationProfile),
|
||||
BuiltinScriptPrefixes: normalizeLowerStringSlice(defaultStringSlice(p.Config.Validation.BuiltinScriptPrefixes, DefaultBuiltinScriptPrefixes())),
|
||||
RequiredFields: defaultRequiredFields(p.Config.Validation.RequiredFields),
|
||||
}
|
||||
topBuild := expandPathTemplate(defaultString(p.Config.TopData.Build, DefaultTopDataBuild), paths)
|
||||
top := EffectiveTopDataConfig{
|
||||
Source: strings.TrimSpace(p.Config.TopData.Source),
|
||||
Build: topBuild,
|
||||
ReferenceBuilder: strings.TrimSpace(p.Config.TopData.ReferenceBuilder),
|
||||
Assets: strings.TrimSpace(p.Config.TopData.Assets),
|
||||
Compiled2DADir: defaultString(p.Config.TopData.Compiled2DADir, DefaultTopDataCompiled2DADir),
|
||||
CompiledTLK: defaultString(p.Config.TopData.CompiledTLK, DefaultTopDataCompiledTLK),
|
||||
PackageHAK: defaultString(p.Config.TopData.PackageHAK, DefaultTopDataPackageHAK),
|
||||
PackageTLK: defaultString(p.Config.TopData.PackageTLK, DefaultTopDataPackageTLK),
|
||||
ValueEncodings: cloneTopDataValueEncodings(p.Config.TopData.ValueEncodings),
|
||||
ValueDefaults: cloneTopDataValueDefaults(p.Config.TopData.ValueDefaults),
|
||||
RowGeneration: cloneTopDataRowGeneration(p.Config.TopData.RowGeneration),
|
||||
RowExtensions: cloneTopDataRowExtensions(p.Config.TopData.RowExtensions),
|
||||
ClassFeatInjections: cloneTopDataClassFeatInjections(p.Config.TopData.ClassFeatInjections),
|
||||
Wiki: TopDataWikiConfig{
|
||||
OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot),
|
||||
PagesDir: defaultString(p.Config.TopData.Wiki.PagesDir, DefaultTopDataWikiPagesDir),
|
||||
StateFile: defaultString(p.Config.TopData.Wiki.StateFile, DefaultTopDataWikiStateFile),
|
||||
ManagedNamespaces: defaultStringSlice(p.Config.TopData.Wiki.ManagedNamespaces, []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}),
|
||||
DeployManifest: defaultString(p.Config.TopData.Wiki.DeployManifest, DefaultWikiDeployManifest),
|
||||
DeployEditSummary: defaultString(p.Config.TopData.Wiki.DeployEditSummary, DefaultWikiDeployEditSummary),
|
||||
Source: defaultString(p.Config.TopData.Wiki.Source, DefaultTopDataWikiSource),
|
||||
Renderer: defaultString(p.Config.TopData.Wiki.Renderer, DefaultTopDataWikiRenderer),
|
||||
LinkStrategy: defaultString(p.Config.TopData.Wiki.LinkStrategy, DefaultTopDataWikiLinkStrategy),
|
||||
NamespacesFile: defaultString(p.Config.TopData.Wiki.NamespacesFile, DefaultTopDataWikiNamespacesFile),
|
||||
TablesFile: defaultString(p.Config.TopData.Wiki.TablesFile, DefaultTopDataWikiTablesFile),
|
||||
VisibilityFile: defaultString(p.Config.TopData.Wiki.VisibilityFile, DefaultTopDataWikiVisibilityFile),
|
||||
DataFile: defaultString(p.Config.TopData.Wiki.DataFile, DefaultTopDataWikiDataFile),
|
||||
PagePathsFile: defaultString(p.Config.TopData.Wiki.PagePathsFile, DefaultTopDataWikiPagePathsFile),
|
||||
TemplatesDir: defaultString(p.Config.TopData.Wiki.TemplatesDir, DefaultTopDataWikiTemplatesDir),
|
||||
PageTemplatesDir: defaultWikiPageTemplatesDir(p.Config.TopData.Wiki),
|
||||
ManualSectionsDir: defaultString(p.Config.TopData.Wiki.ManualSectionsDir, DefaultTopDataWikiManualSectionsDir),
|
||||
ManualSectionsFile: defaultWikiManualSectionsFile(p.Config.TopData.Wiki),
|
||||
PageIndexFile: defaultString(p.Config.TopData.Wiki.PageIndexFile, DefaultTopDataWikiPageIndexFile),
|
||||
AlignmentLinks: TopDataWikiAlignmentLinks{
|
||||
TargetPattern: strings.TrimSpace(p.Config.TopData.Wiki.AlignmentLinks.TargetPattern),
|
||||
LinkFormat: defaultString(p.Config.TopData.Wiki.AlignmentLinks.LinkFormat, DefaultTopDataWikiAlignmentLinkFormat),
|
||||
},
|
||||
TitlePrefixMinLength: defaultInt(p.Config.TopData.Wiki.TitlePrefixMinLength, DefaultTopDataWikiTitlePrefixMinLength),
|
||||
StatusPages: defaultBoolPointer(p.Config.TopData.Wiki.StatusPages, DefaultTopDataWikiStatusPages),
|
||||
StatusListingScope: defaultString(p.Config.TopData.Wiki.StatusListingScope, DefaultTopDataWikiStatusListingScope),
|
||||
StalePages: TopDataWikiStalePagesConfig{
|
||||
Default: defaultString(p.Config.TopData.Wiki.StalePages.Default, DefaultTopDataWikiStaleDefault),
|
||||
LiveCleanup: defaultString(p.Config.TopData.Wiki.StalePages.LiveCleanup, DefaultTopDataWikiStaleLiveCleanup),
|
||||
},
|
||||
ManagedRegion: TopDataWikiManagedRegionConfig{
|
||||
MarkerFormat: defaultString(p.Config.TopData.Wiki.ManagedRegion.MarkerFormat, DefaultTopDataWikiMarkerFormat),
|
||||
PageMarkerPrefix: defaultString(p.Config.TopData.Wiki.ManagedRegion.PageMarkerPrefix, DefaultTopDataWikiPageMarkerPrefix),
|
||||
},
|
||||
},
|
||||
}
|
||||
generated := p.Config.Generated
|
||||
for index := range generated.TopData2DA {
|
||||
generated.TopData2DA[index].Output = expandPathTemplate(defaultString(generated.TopData2DA[index].Output, "{paths.cache}/generated-assets/"+generated.TopData2DA[index].ID), paths)
|
||||
}
|
||||
|
||||
extract := p.Config.Extract
|
||||
extract.Layout = defaultString(extract.Layout, DefaultExtractLayout)
|
||||
extract.HAKDiscovery = defaultString(extract.HAKDiscovery, DefaultExtractHAKDiscovery)
|
||||
if len(extract.Archives) == 0 {
|
||||
extract.Archives = []string{outputs.ModuleArchiveName(p.Config.Module)}
|
||||
}
|
||||
if extract.CleanupStale == nil {
|
||||
defaultCleanup := true
|
||||
extract.CleanupStale = &defaultCleanup
|
||||
}
|
||||
if extract.ConsumeArchives == nil {
|
||||
defaultConsume := false
|
||||
extract.ConsumeArchives = &defaultConsume
|
||||
}
|
||||
|
||||
musicStageRoot := expandPathTemplate(DefaultMusicStageRoot, paths)
|
||||
musicCreditsRoot := expandPathTemplate(DefaultCreditsRoot, paths)
|
||||
musicCreditsOverlay := DefaultCreditsOverlayFile
|
||||
musicMaxStemLength := 16
|
||||
musicNamingScheme := DefaultMusicNamingScheme
|
||||
musicOutputExtension := DefaultMusicOutputExtension
|
||||
musicConvertExtensions := music.DefaultConvertExtensions()
|
||||
if d := p.Config.Music.Defaults; d != nil {
|
||||
if d.StageRoot != "" {
|
||||
musicStageRoot = expandPathTemplate(d.StageRoot, paths)
|
||||
}
|
||||
if d.CreditsRoot != "" {
|
||||
musicCreditsRoot = expandPathTemplate(d.CreditsRoot, paths)
|
||||
}
|
||||
if d.CreditsOverlay != "" {
|
||||
musicCreditsOverlay = d.CreditsOverlay
|
||||
}
|
||||
if d.MaxStemLength != nil {
|
||||
musicMaxStemLength = *d.MaxStemLength
|
||||
}
|
||||
if d.NamingScheme != "" {
|
||||
musicNamingScheme = d.NamingScheme
|
||||
}
|
||||
if d.OutputExtension != "" {
|
||||
musicOutputExtension = normalizeExtension(d.OutputExtension)
|
||||
}
|
||||
if len(d.ConvertExtensions) > 0 {
|
||||
musicConvertExtensions = normalizeExtensionSlice(d.ConvertExtensions)
|
||||
}
|
||||
}
|
||||
musicPrefixes := cloneStringMap(p.Config.Music.Prefixes)
|
||||
if len(musicPrefixes) == 0 {
|
||||
musicPrefixes = make(map[string]string, len(p.Config.Music.Datasets))
|
||||
for _, ds := range p.Config.Music.Datasets {
|
||||
if ds.Source != "" && ds.Prefix != "" {
|
||||
musicPrefixes[ds.Source] = ds.Prefix
|
||||
}
|
||||
}
|
||||
}
|
||||
effectiveDatasets := make(map[string]EffectiveMusicDataset, len(p.Config.Music.Datasets))
|
||||
for id, ds := range p.Config.Music.Datasets {
|
||||
dsStageRoot := ds.StageRoot
|
||||
if dsStageRoot == "" {
|
||||
dsStageRoot = musicStageRoot
|
||||
}
|
||||
dsCreditsRoot := ds.CreditsRoot
|
||||
if dsCreditsRoot == "" {
|
||||
dsCreditsRoot = musicCreditsRoot
|
||||
}
|
||||
dsOutput := strings.TrimSpace(ds.Output)
|
||||
if dsOutput == "" {
|
||||
dsOutput = ds.Source
|
||||
}
|
||||
dsNamingScheme := defaultString(ds.NamingScheme, musicNamingScheme)
|
||||
dsOutputExtension := musicOutputExtension
|
||||
if ds.OutputExtension != "" {
|
||||
dsOutputExtension = normalizeExtension(ds.OutputExtension)
|
||||
}
|
||||
dsConvertExtensions := musicConvertExtensions
|
||||
if len(ds.ConvertExtensions) > 0 {
|
||||
dsConvertExtensions = normalizeExtensionSlice(ds.ConvertExtensions)
|
||||
}
|
||||
effectiveDatasets[id] = EffectiveMusicDataset{
|
||||
Source: ds.Source,
|
||||
Output: dsOutput,
|
||||
Prefix: ds.Prefix,
|
||||
StageRoot: dsStageRoot,
|
||||
CreditsRoot: dsCreditsRoot,
|
||||
NamingScheme: dsNamingScheme,
|
||||
OutputExtension: dsOutputExtension,
|
||||
MaxStemLength: musicMaxStemLength,
|
||||
PackageMode: defaultString(ds.PackageMode, "hak_asset"),
|
||||
ConvertExtensions: dsConvertExtensions,
|
||||
}
|
||||
}
|
||||
effective := EffectiveConfig{
|
||||
ConfigSource: p.ConfigSource,
|
||||
Module: p.Config.Module,
|
||||
Paths: paths,
|
||||
Outputs: outputs,
|
||||
Build: p.Config.Build,
|
||||
Generated: generated,
|
||||
Inventory: inventory,
|
||||
Validation: validation,
|
||||
Music: EffectiveMusicConfig{
|
||||
Prefixes: musicPrefixes,
|
||||
Tools: p.Config.Music.Tools,
|
||||
StageRoot: musicStageRoot,
|
||||
CreditsRoot: musicCreditsRoot,
|
||||
CreditsOverlay: musicCreditsOverlay,
|
||||
MaxStemLength: musicMaxStemLength,
|
||||
NamingScheme: musicNamingScheme,
|
||||
OutputExtension: musicOutputExtension,
|
||||
ConvertExtensions: musicConvertExtensions,
|
||||
Datasets: effectiveDatasets,
|
||||
},
|
||||
TopData: top,
|
||||
Extract: extract,
|
||||
Autogen: EffectiveAutogenConfig{
|
||||
Producers: slices.Clone(p.Config.Autogen.Producers),
|
||||
Consumers: slices.Clone(p.Config.Autogen.Consumers),
|
||||
Cache: EffectiveAutogenCacheConfig{
|
||||
Root: expandPathTemplate(defaultString(p.Config.Autogen.Cache.Root, DefaultAutogenCacheRoot), paths),
|
||||
MaxAge: defaultString(p.Config.Autogen.Cache.MaxAge, DefaultAutogenCacheMaxAge.String()),
|
||||
RefreshEnv: defaultString(p.Config.Autogen.Cache.RefreshEnv, DefaultAutogenRefreshEnv),
|
||||
PartsManifestRefreshEnv: DefaultPartsManifestRefreshEnv,
|
||||
},
|
||||
ReleaseSource: EffectiveAutogenReleaseSourceConfig{
|
||||
Provider: defaultString(p.Config.Autogen.ReleaseSource.Provider, DefaultAutogenReleaseProvider),
|
||||
ServerURLEnv: defaultString(p.Config.Autogen.ReleaseSource.ServerURLEnv, DefaultAutogenServerURLEnv),
|
||||
RepoEnv: defaultString(p.Config.Autogen.ReleaseSource.RepoEnv, DefaultAutogenRepoEnv),
|
||||
},
|
||||
},
|
||||
HAKs: slices.Clone(p.Config.HAKs),
|
||||
Provenance: provenance,
|
||||
}
|
||||
effective.Overrides = activeOverrides(effective)
|
||||
return effective
|
||||
}
|
||||
|
||||
func cloneTopDataValueEncodings(values []TopDataValueEncodingConfig) []TopDataValueEncodingConfig {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]TopDataValueEncodingConfig, len(values))
|
||||
copy(out, values)
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneTopDataValueDefaults(values []TopDataValueDefaultConfig) []TopDataValueDefaultConfig {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]TopDataValueDefaultConfig, len(values))
|
||||
copy(out, values)
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneTopDataRowGeneration(values []TopDataRowGenerationConfig) []TopDataRowGenerationConfig {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]TopDataRowGenerationConfig, len(values))
|
||||
copy(out, values)
|
||||
for index := range out {
|
||||
if strings.TrimSpace(out[index].Dataset) == "" {
|
||||
out[index].Dataset = strings.TrimSpace(out[index].Namespace)
|
||||
}
|
||||
if strings.TrimSpace(out[index].Namespace) == "" {
|
||||
out[index].Namespace = strings.TrimSpace(out[index].Dataset)
|
||||
}
|
||||
if strings.TrimSpace(out[index].Mode) == "" {
|
||||
out[index].Mode = "after_base"
|
||||
}
|
||||
out[index].Dataset = filepath.ToSlash(strings.TrimSpace(out[index].Dataset))
|
||||
out[index].Namespace = filepath.ToSlash(strings.TrimSpace(out[index].Namespace))
|
||||
out[index].Mode = strings.TrimSpace(out[index].Mode)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneTopDataRowExtensions(values []TopDataRowExtensionConfig) []TopDataRowExtensionConfig {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]TopDataRowExtensionConfig, len(values))
|
||||
copy(out, values)
|
||||
for index := range out {
|
||||
out[index].Dataset = filepath.ToSlash(strings.TrimSpace(out[index].Dataset))
|
||||
out[index].Mode = strings.TrimSpace(out[index].Mode)
|
||||
out[index].LevelColumn = strings.TrimSpace(out[index].LevelColumn)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneTopDataClassFeatInjections(value TopDataClassFeatInjectionConfig) TopDataClassFeatInjectionConfig {
|
||||
out := TopDataClassFeatInjectionConfig{
|
||||
GlobalFeats: slices.Clone(value.GlobalFeats),
|
||||
ClassSkillMasterfeats: slices.Clone(value.ClassSkillMasterfeats),
|
||||
}
|
||||
for index := range out.GlobalFeats {
|
||||
out.GlobalFeats[index].RequirePresent = slices.Clone(value.GlobalFeats[index].RequirePresent)
|
||||
out.GlobalFeats[index].UnlessPresent = slices.Clone(value.GlobalFeats[index].UnlessPresent)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *Project) EffectiveConfigJSON() ([]byte, error) {
|
||||
return json.MarshalIndent(p.EffectiveConfig(), "", " ")
|
||||
}
|
||||
|
||||
func (p *Project) ExplainConfig(key string) (any, ConfigValueProvenance, bool) {
|
||||
effective := p.EffectiveConfig()
|
||||
value, ok := lookupEffectiveValue(effective, key)
|
||||
if !ok {
|
||||
return nil, ConfigValueProvenance{}, false
|
||||
}
|
||||
prov := effective.Provenance[key]
|
||||
if prov.Source == "" {
|
||||
prov = ConfigValueProvenance{Source: "yaml", Detail: p.ConfigSource.Name}
|
||||
}
|
||||
return value, prov, true
|
||||
}
|
||||
|
||||
func lookupEffectiveValue(effective EffectiveConfig, key string) (any, bool) {
|
||||
raw, err := json.Marshal(effective)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
var data any
|
||||
if err := json.Unmarshal(raw, &data); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
current := data
|
||||
for _, part := range strings.Split(key, ".") {
|
||||
object, ok := current.(map[string]any)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
current, ok = object[part]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
return current, true
|
||||
}
|
||||
|
||||
func markMissingDefaults(provenance ConfigProvenance) {
|
||||
defaults := map[string]string{
|
||||
"paths.build": "build",
|
||||
"paths.cache": ".cache",
|
||||
"paths.tools": "tools",
|
||||
"outputs.module_archive": DefaultModuleArchiveTemplate,
|
||||
"outputs.hak_manifest": DefaultHAKManifest,
|
||||
"outputs.hak_archive": DefaultHAKArchiveTemplate,
|
||||
"generated_assets.topdata_2da": "no generated topdata 2DA assets",
|
||||
"inventory.source_extensions": "NWN source resource extensions",
|
||||
"inventory.asset_extensions": "NWN asset resource extensions",
|
||||
"inventory.source_json_pattern": DefaultSourceJSONPattern,
|
||||
"validation.profile": DefaultValidationProfile,
|
||||
"validation.builtin_script_prefixes": "NWN built-in script prefixes",
|
||||
"validation.required_fields": "NWN required GFF fields",
|
||||
"music.stage_root": DefaultMusicStageRoot,
|
||||
"music.credits_root": DefaultCreditsRoot,
|
||||
"music.credits_overlay": DefaultCreditsOverlayFile,
|
||||
"music.max_stem_length": "16",
|
||||
"music.naming_scheme": DefaultMusicNamingScheme,
|
||||
"music.output_extension": DefaultMusicOutputExtension,
|
||||
"music.convert_extensions": strings.Join(music.DefaultConvertExtensions(), ", "),
|
||||
"topdata.build": DefaultTopDataBuild,
|
||||
"topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir,
|
||||
"topdata.compiled_tlk": DefaultTopDataCompiledTLK,
|
||||
"topdata.package_hak": DefaultTopDataPackageHAK,
|
||||
"topdata.package_tlk": DefaultTopDataPackageTLK,
|
||||
"topdata.wiki.output_root": DefaultTopDataWikiOutputRoot,
|
||||
"topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir,
|
||||
"topdata.wiki.state_file": DefaultTopDataWikiStateFile,
|
||||
"topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces",
|
||||
"topdata.wiki.deploy_manifest": DefaultWikiDeployManifest,
|
||||
"topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary,
|
||||
"topdata.wiki.source": DefaultTopDataWikiSource,
|
||||
"topdata.wiki.renderer": DefaultTopDataWikiRenderer,
|
||||
"topdata.wiki.link_strategy": DefaultTopDataWikiLinkStrategy,
|
||||
"topdata.wiki.namespaces_file": DefaultTopDataWikiNamespacesFile,
|
||||
"topdata.wiki.tables_file": DefaultTopDataWikiTablesFile,
|
||||
"topdata.wiki.visibility_file": DefaultTopDataWikiVisibilityFile,
|
||||
"topdata.wiki.data_file": DefaultTopDataWikiDataFile,
|
||||
"topdata.wiki.page_paths_file": DefaultTopDataWikiPagePathsFile,
|
||||
"topdata.wiki.templates_dir": DefaultTopDataWikiTemplatesDir,
|
||||
"topdata.wiki.page_templates_dir": DefaultTopDataWikiPageTemplatesDir,
|
||||
"topdata.wiki.manual_sections_dir": DefaultTopDataWikiManualSectionsDir,
|
||||
"topdata.wiki.manual_sections_file": DefaultTopDataWikiManualSectionsFile,
|
||||
"topdata.wiki.page_index_file": DefaultTopDataWikiPageIndexFile,
|
||||
"topdata.wiki.title_prefix_min_length": "3",
|
||||
"topdata.wiki.status_pages": "true",
|
||||
"topdata.wiki.status_listing_scope": DefaultTopDataWikiStatusListingScope,
|
||||
"topdata.wiki.alignment_links.link_format": DefaultTopDataWikiAlignmentLinkFormat,
|
||||
"topdata.wiki.stale_pages.default": DefaultTopDataWikiStaleDefault,
|
||||
"topdata.wiki.stale_pages.live_cleanup": DefaultTopDataWikiStaleLiveCleanup,
|
||||
"topdata.wiki.managed_region.marker_format": DefaultTopDataWikiMarkerFormat,
|
||||
"topdata.wiki.managed_region.page_marker_prefix": DefaultTopDataWikiPageMarkerPrefix,
|
||||
"extract.layout": DefaultExtractLayout,
|
||||
"extract.hak_discovery": DefaultExtractHAKDiscovery,
|
||||
"extract.archives": DefaultModuleArchiveTemplate,
|
||||
"extract.cleanup_stale": "true",
|
||||
"extract.consume_archives": "false",
|
||||
"autogen.cache.root": DefaultAutogenCacheRoot,
|
||||
"autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(),
|
||||
"autogen.cache.refresh_env": DefaultAutogenRefreshEnv,
|
||||
"autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv,
|
||||
"autogen.release_source.provider": DefaultAutogenReleaseProvider,
|
||||
"autogen.release_source.server_url_env": DefaultAutogenServerURLEnv,
|
||||
"autogen.release_source.repo_env": DefaultAutogenRepoEnv,
|
||||
}
|
||||
for key, detail := range defaults {
|
||||
if _, ok := provenance[key]; !ok {
|
||||
provenance[key] = ConfigValueProvenance{Source: "toolkit default", Detail: detail}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cloneProvenance(input ConfigProvenance) ConfigProvenance {
|
||||
out := ConfigProvenance{}
|
||||
for key, value := range input {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func defaultString(value, fallback string) string {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return fallback
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
func defaultInt(value, fallback int) int {
|
||||
if value == 0 {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func defaultBoolPointer(value *bool, fallback bool) *bool {
|
||||
out := fallback
|
||||
if value != nil {
|
||||
out = *value
|
||||
}
|
||||
return &out
|
||||
}
|
||||
|
||||
func defaultStringSlice(value, fallback []string) []string {
|
||||
if len(value) == 0 {
|
||||
return slices.Clone(fallback)
|
||||
}
|
||||
return slices.Clone(value)
|
||||
}
|
||||
|
||||
func defaultWikiPageTemplatesDir(cfg TopDataWikiConfig) string {
|
||||
if strings.TrimSpace(cfg.PageTemplatesDir) != "" {
|
||||
return strings.TrimSpace(cfg.PageTemplatesDir)
|
||||
}
|
||||
if strings.TrimSpace(cfg.TemplatesDir) != "" {
|
||||
return filepath.ToSlash(filepath.Join(filepath.FromSlash(strings.TrimSpace(cfg.TemplatesDir)), "pages"))
|
||||
}
|
||||
return DefaultTopDataWikiPageTemplatesDir
|
||||
}
|
||||
|
||||
func defaultWikiManualSectionsFile(cfg TopDataWikiConfig) string {
|
||||
if strings.TrimSpace(cfg.ManualSectionsFile) != "" {
|
||||
return strings.TrimSpace(cfg.ManualSectionsFile)
|
||||
}
|
||||
if strings.TrimSpace(cfg.ManualSectionsDir) != "" {
|
||||
return filepath.ToSlash(filepath.Join(filepath.FromSlash(strings.TrimSpace(cfg.ManualSectionsDir)), "default.yaml"))
|
||||
}
|
||||
return DefaultTopDataWikiManualSectionsFile
|
||||
}
|
||||
|
||||
func defaultRequiredFields(configured map[string][]string) map[string][]string {
|
||||
if len(configured) > 0 {
|
||||
return cloneStringSliceMap(normalizeRequiredFields(configured))
|
||||
}
|
||||
return DefaultRequiredFields()
|
||||
}
|
||||
|
||||
func cloneStringSliceMap(input map[string][]string) map[string][]string {
|
||||
out := make(map[string][]string, len(input))
|
||||
for key, values := range input {
|
||||
out[key] = slices.Clone(values)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneStringMap(input map[string]string) map[string]string {
|
||||
if len(input) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(input))
|
||||
for key, value := range input {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func expandPathTemplate(template string, paths EffectivePathConfig) string {
|
||||
replacer := strings.NewReplacer(
|
||||
"{paths.source}", paths.Source,
|
||||
"{paths.assets}", paths.Assets,
|
||||
"{paths.build}", paths.Build,
|
||||
"{paths.cache}", paths.Cache,
|
||||
"{paths.tools}", paths.Tools,
|
||||
)
|
||||
return filepath.ToSlash(replacer.Replace(template))
|
||||
}
|
||||
|
||||
func expandPathTemplates(templates []string, paths EffectivePathConfig) []string {
|
||||
out := make([]string, 0, len(templates))
|
||||
for _, template := range templates {
|
||||
out = append(out, expandPathTemplate(template, paths))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *Project) ActiveOverrides() []ConfigOverride {
|
||||
return activeOverrides(p.EffectiveConfig())
|
||||
}
|
||||
|
||||
func activeOverrides(effective EffectiveConfig) []ConfigOverride {
|
||||
envs := map[string]string{
|
||||
"build.keep_existing_haks": effectiveBuildKeepExistingEnv,
|
||||
"autogen.cache.refresh": effective.Autogen.Cache.RefreshEnv,
|
||||
"autogen.cache.parts_manifest_refresh": effective.Autogen.Cache.PartsManifestRefreshEnv,
|
||||
"autogen.release_source.server_url": effective.Autogen.ReleaseSource.ServerURLEnv,
|
||||
"autogen.release_source.repo": effective.Autogen.ReleaseSource.RepoEnv,
|
||||
"music.ffmpeg": "SOW_FFMPEG",
|
||||
"music.ffprobe": "SOW_FFPROBE",
|
||||
"wiki.endpoint": "NODEBB_API_ENDPOINT",
|
||||
"wiki.token": "NODEBB_API_TOKEN",
|
||||
"wiki.categories": "NODEBB_WIKI_CATEGORIES",
|
||||
"release.version": "GITHUB_REF_NAME",
|
||||
}
|
||||
keys := make([]string, 0, len(envs))
|
||||
for key := range envs {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
overrides := make([]ConfigOverride, 0)
|
||||
for _, key := range keys {
|
||||
env := envs[key]
|
||||
if value := strings.TrimSpace(os.Getenv(env)); value != "" {
|
||||
if isSensitiveOverrideKey(key) || isSensitiveOverrideKey(env) {
|
||||
value = "<set>"
|
||||
}
|
||||
overrides = append(overrides, ConfigOverride{Key: key, Value: value, Source: "env:" + env})
|
||||
}
|
||||
}
|
||||
return overrides
|
||||
}
|
||||
|
||||
const effectiveBuildKeepExistingEnv = "SOW_BUILD_HAKS_KEEP_EXISTING"
|
||||
|
||||
func isSensitiveOverrideKey(key string) bool {
|
||||
lower := strings.ToLower(key)
|
||||
return strings.Contains(lower, "token") || strings.Contains(lower, "password")
|
||||
}
|
||||
|
||||
func (o EffectiveOutputConfig) ModuleArchiveName(module ModuleConfig) string {
|
||||
return strings.NewReplacer("{module.resref}", strings.TrimSpace(module.ResRef)).Replace(o.ModuleArchive)
|
||||
}
|
||||
|
||||
func (o EffectiveOutputConfig) HAKArchiveName(name string) string {
|
||||
return strings.NewReplacer("{hak.name}", strings.TrimSpace(name)).Replace(o.HAKArchive)
|
||||
}
|
||||
|
||||
func (p ConfigValueProvenance) String() string {
|
||||
if p.Detail == "" {
|
||||
return p.Source
|
||||
}
|
||||
return fmt.Sprintf("%s (%s)", p.Source, p.Detail)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,155 +0,0 @@
|
||||
# AI Agent Contract: Auto-Include Existing Part Models in 2DA Generation
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-17:
|
||||
|
||||
- Superseded for current Shadows Over Westgate builds.
|
||||
- The module no longer consumes a released parts manifest. Parts data is owned by
|
||||
`assets/`, generated as HAK-side 2DA assets from `assets/topdata/data/parts`
|
||||
plus a direct scan of `assets/content/part/**/*.mdl`, and packaged in
|
||||
`sow_part`.
|
||||
- This file remains as historical context for the legacy manifest-backed
|
||||
consumer path.
|
||||
|
||||
## Objective
|
||||
|
||||
The native 2DA generation pipeline must automatically include existing part models already present in the **`sow-assets` repository** when building final parts tables.
|
||||
|
||||
This behavior is required for parity with the current asset set and must not depend on manually authored entries for models that already exist in repository assets.
|
||||
|
||||
---
|
||||
|
||||
## Source of Truth
|
||||
|
||||
The superseded manifest-backed native builder obtained existing model
|
||||
information from a released artifact published by the **`sow-assets` repository**.
|
||||
|
||||
Resolution rules:
|
||||
|
||||
- if `topdata.assets` is configured to a local path, use that exact path
|
||||
- otherwise derive the **`sow-assets` repository** from the module repo's Gitea origin
|
||||
and fetch only the configured released manifest
|
||||
|
||||
Normal builds must not clone or scan the `sow-assets` repo tree. They may cache the
|
||||
downloaded manifest locally for reuse, but the source of truth remains the released
|
||||
manifest on Gitea.
|
||||
|
||||
Manifest-backed discovery is derived from:
|
||||
|
||||
```text
|
||||
sow-assets/assets/part/**/*.mdl
|
||||
```
|
||||
|
||||
The manifest generator may also accept the repo's legacy `assets/parts/` layout when
|
||||
resolving the physical paths, but the native builder itself must consume the manifest,
|
||||
not the repo tree.
|
||||
|
||||
---
|
||||
|
||||
## Required Scope
|
||||
|
||||
Process existing `.mdl` files for these parts table categories only:
|
||||
|
||||
- `belt`
|
||||
- `bicep`
|
||||
- `chest`
|
||||
- `foot`
|
||||
- `forearm`
|
||||
- `leg`
|
||||
- `neck`
|
||||
- `pelvis`
|
||||
- `robe`
|
||||
- `shin`
|
||||
- `shoulder`
|
||||
|
||||
Dataset mapping rules:
|
||||
|
||||
- `parts/legs` maps to asset category `leg`
|
||||
- `parts/hand` remains unsupported and is not auto-generated
|
||||
|
||||
The scan must recurse through subdirectories under each applicable category path.
|
||||
|
||||
---
|
||||
|
||||
## Required Behavior
|
||||
|
||||
### 1. Manifest-backed discovery
|
||||
|
||||
For each supported part category, read the released manifest and discover all existing
|
||||
numeric model IDs listed for that category.
|
||||
|
||||
### 2. Trailing numeric suffix extraction
|
||||
|
||||
The released manifest must be generated using trailing-numeric-suffix extraction from
|
||||
the source `.mdl` filenames. Native topdata consumes the resulting numeric IDs only.
|
||||
|
||||
### 3. Row generation
|
||||
|
||||
If a trailing numeric suffix is present:
|
||||
|
||||
- use that numeric suffix as the output row ID
|
||||
- emit a row for that part model in the final generated 2DA
|
||||
|
||||
### 4. Default emitted values
|
||||
|
||||
For rows generated from discovered existing models, set:
|
||||
|
||||
- `COSTMODIFIER = 0`
|
||||
- `ACBONUS = 0.00`
|
||||
|
||||
Rows without discovered existing models may be intentionally emitted as dense
|
||||
2DA null rows when configured. In that mode, missing rows must have
|
||||
`COSTMODIFIER = ****` and `ACBONUS = ****` in the final generated 2DA.
|
||||
|
||||
### 5. Override precedence
|
||||
|
||||
If file-based overrides exist, they must take precedence over discovered defaults.
|
||||
|
||||
That means:
|
||||
|
||||
- manifest discovery provides the baseline row presence and default values
|
||||
- override data may replace or augment those values
|
||||
- override data wins in any conflict
|
||||
|
||||
---
|
||||
|
||||
## Non-Negotiable Requirements
|
||||
|
||||
- The implementation must read model presence from the **released parts manifest**,
|
||||
not from assumption, hardcoded lists, or legacy output.
|
||||
- Existing repository models with trailing numeric suffixes must be reflected in generated output even if they are not otherwise explicitly authored.
|
||||
- Override application is mandatory and must win over auto-discovered defaults.
|
||||
- This is replacement behavior for the native pipeline, not an optional enhancement.
|
||||
|
||||
---
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
The superseded implementation was correct only if all of the following were true:
|
||||
|
||||
1. The builder fetches the configured released manifest artifact for part models.
|
||||
2. Only the supported part categories are considered.
|
||||
3. Manifest IDs derived from trailing numeric suffixes generate corresponding 2DA rows.
|
||||
4. The numeric suffix is used as the row ID.
|
||||
5. Auto-generated rows default to:
|
||||
- `COSTMODIFIER = 0`
|
||||
- `ACBONUS = 0.00`
|
||||
|
||||
6. Authored base rows remain exposed. Authored `ACBONUS = ****` rows remain
|
||||
null unless model discovery activates the row.
|
||||
7. Dense gaps without authored rows or discovered models are emitted as null
|
||||
rows with `COSTMODIFIER = ****` and `ACBONUS = ****`.
|
||||
8. File overrides are applied after discovery and take precedence.
|
||||
9. The final output includes all eligible existing models present in the manifest.
|
||||
10. Discovery does not require a sibling local `sow-assets` checkout or a Git clone.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Directive
|
||||
|
||||
When generating native parts 2DAs, first scan the project-resolved assets repo for
|
||||
supported part models, collect existing supported part models by category, extract
|
||||
trailing numeric suffixes from filenames, emit rows using those suffixes as row IDs
|
||||
with default `COSTMODIFIER = 0` and `ACBONUS = 0.00`, then apply overrides with
|
||||
override values taking precedence.
|
||||
@@ -1,76 +0,0 @@
|
||||
# Class Feat Global Injection Contract
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-06-05:
|
||||
|
||||
- Preferred class feat injection policy is authored in
|
||||
`topdata/data/classes/feats/global.json`.
|
||||
- `global.json` rows flow through the ordinary class feat expansion pipeline,
|
||||
including successor expansion, masterfeat expansion, class-skill filtering,
|
||||
reference resolution, deduplication, and final 2DA emission.
|
||||
- Legacy `topdata.class_feat_injections` YAML still loads only as a
|
||||
compatibility path when no applicable class feat `global.json` exists.
|
||||
- Toolkit hardcoded defaults are compatibility-only and must not be the normal
|
||||
source of project policy.
|
||||
|
||||
## Objective
|
||||
|
||||
Native class feat generation must produce deterministic, legacy-equivalent
|
||||
effective output while keeping project policy in topdata authoring files.
|
||||
|
||||
## Requirements
|
||||
|
||||
- For each class skill, inject the configured skill-focus masterfeat rows from
|
||||
`classes/feats/global.json` using `FeatIndex.filter = "classskills"`.
|
||||
- Inject `feat:literate` only when the class table does not already include
|
||||
`feat:illiterate`.
|
||||
- Inject the shared combat/menu feat rows declared in the manifest unless a
|
||||
class already authors the equivalent `FeatIndex`.
|
||||
- Explicit class feat rows take precedence over injected rows.
|
||||
- Injections must be deterministic for identical inputs.
|
||||
|
||||
## Manifest Shape
|
||||
|
||||
```json
|
||||
{
|
||||
"position": "prepend",
|
||||
"injections": [
|
||||
{
|
||||
"row": {
|
||||
"FeatIndex": { "id": "feat:literate" },
|
||||
"GrantedOnLevel": 1,
|
||||
"List": 3,
|
||||
"OnMenu": 0
|
||||
},
|
||||
"unless_present": [
|
||||
{ "field": "FeatIndex", "id": "feat:illiterate" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"row": {
|
||||
"FeatIndex": {
|
||||
"id": "masterfeats:skill_focus",
|
||||
"filter": "classskills"
|
||||
},
|
||||
"GrantedOnLevel": -1,
|
||||
"List": 1,
|
||||
"OnMenu": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`position` defaults to `append`. The module uses `prepend` for class feat
|
||||
injections to keep generated policy rows before per-class authored rows.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- Class feat output contains the same effective shared feats as the prior YAML
|
||||
policy.
|
||||
- Literacy injection is skipped for illiterate classes.
|
||||
- Skill-focus masterfeat rows expand only for class skills.
|
||||
- YAML compatibility continues to work for consumers that have not migrated.
|
||||
- A project with `classes/feats/global.json` does not also receive YAML or
|
||||
hardcoded default class feat injections.
|
||||
@@ -1,119 +0,0 @@
|
||||
# Generic Family Expansion Contract
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- Implemented and actively used by native feat generation.
|
||||
- `family_expansion.go`, `native.go`, and `topdata_test.go` cover the current
|
||||
underscore-family interpretation, generated family metadata, and reapplication
|
||||
behavior.
|
||||
- Remaining work is mostly consumer expansion: use the same primitive in more
|
||||
datasets only when the authored data actually benefits from it.
|
||||
|
||||
Topdata now treats underscore expansion as a structural rule:
|
||||
|
||||
- `parent_child` means `child` is an expansion of `parent`
|
||||
- underscores are for family structure, not simulated spaces
|
||||
- existing canonical keys still win when lock or TLK state already established them
|
||||
|
||||
This is a global interpretation rule, not a wiki-only convention.
|
||||
|
||||
## Identity
|
||||
|
||||
- `weaponspecialization_club`
|
||||
- parent: `weaponspecialization`
|
||||
- child: `club`
|
||||
- `gnome_rock`
|
||||
- parent: `gnome`
|
||||
- child: `rock`
|
||||
- `toughness_10`
|
||||
- parent: `toughness`
|
||||
- child: `10`
|
||||
|
||||
Standalone keys without an underscore are treated as a parent identity with no child.
|
||||
|
||||
## Generated Family Files
|
||||
|
||||
Canonical generated families declare:
|
||||
|
||||
```json
|
||||
{
|
||||
"family": "weapon_focus",
|
||||
"family_key": "weaponfocus",
|
||||
"template": "masterfeats:weaponfocus",
|
||||
"name_prefix": "Weapon Focus",
|
||||
"label_prefix": "FEAT_WEAPON_FOCUS",
|
||||
"constant_prefix": "FEAT_WEAPON_FOCUS",
|
||||
"legacy_family_keys": ["epic_weapon_focus"],
|
||||
"default_fields": {
|
||||
"DESCRIPTION": {
|
||||
"tlk": {
|
||||
"key": "masterfeats:weaponfocus.description",
|
||||
"text": "..."
|
||||
}
|
||||
},
|
||||
"MINATTACKBONUS": "1",
|
||||
"TOOLSCATEGORIES": "1"
|
||||
},
|
||||
"apply_after_modules": true,
|
||||
"identity_source": "child_source_value",
|
||||
"auto_prereq_fields": {
|
||||
"PREREQFEAT1": "weaponfocus"
|
||||
},
|
||||
"child_source": {
|
||||
"dataset": "baseitems",
|
||||
"column": "WeaponFocusFeat"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- `family` is an authored label and is not a hardcoded switch key
|
||||
- `family_key` is the structural parent identity used in generated child keys
|
||||
- `template` is optional for the primitive in general, but required by current
|
||||
masterfeat-backed `feat` families
|
||||
- `name_prefix`, `label_prefix`, and `constant_prefix` define how new child rows are
|
||||
named when they are not already authored
|
||||
- `template_fields` lists template-backed fields copied from the referenced template row
|
||||
when authored; it is optional if the family owns its shared values directly
|
||||
- `default_fields` applies authored shared/default values to every generated family row
|
||||
and can be used to move family-wide shared values out of the template row
|
||||
- `apply_after_modules: true` re-applies the generated family after normal module files,
|
||||
so authored family-shared values remain authoritative even for legacy explicitly
|
||||
authored child rows
|
||||
- `child_ref_field` binds the generated child row back to the source row key, e.g.
|
||||
`REQSKILL`
|
||||
- `identity_source: "child_source_value"` tells the builder to reuse IDs/keys from the
|
||||
source column when that column already carries feat identity
|
||||
- `legacy_family_keys` declares previous generated-family prefixes that should donate
|
||||
existing locked row IDs to the current family when a family is renamed
|
||||
- `auto_prereq_fields` binds prerequisite fields to other authored families by
|
||||
`family_key`, without code changes
|
||||
- `allow_existing_only` limits expansion to already-authored child rows when a family is
|
||||
intentionally partial
|
||||
- `child_source.dataset` identifies the canonical dataset driving expansion
|
||||
- `child_source.column` is used when expansion is gated by a non-null source field
|
||||
- `child_source.predicate` is used when expansion depends on a named rule such as
|
||||
accessibility
|
||||
|
||||
## Row Metadata
|
||||
|
||||
Generated rows can carry:
|
||||
|
||||
```json
|
||||
{
|
||||
"meta": {
|
||||
"family": {
|
||||
"parent": "weaponfocus",
|
||||
"child": "club",
|
||||
"source": "baseitems:club",
|
||||
"template": "masterfeats:weaponfocus"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This metadata is builder-owned and does not affect emitted 2DA columns. It preserves
|
||||
family structure for later phases without enabling wiki generation yet.
|
||||
@@ -1,174 +0,0 @@
|
||||
# `feat` Generated Families Contract
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- Implemented for the current native `feat` pipeline.
|
||||
- Native tests cover generated-family expansion, family-owned shared defaults,
|
||||
masterfeat-backed inheritance, and lock/TLK identity preservation.
|
||||
- The one explicitly deferred item in this contract is still current:
|
||||
`feat.2da` remains native-built but excluded from `compare-topdata`
|
||||
reference-catalog coverage through `compare_reference: false`.
|
||||
|
||||
Canonical authored paths:
|
||||
|
||||
- `topdata/data/feat/base.json`
|
||||
- `topdata/data/feat/lock.json`
|
||||
- `topdata/data/feat/generated/skill_focus.json`
|
||||
- `topdata/data/feat/generated/greater_skill_focus.json`
|
||||
- `topdata/data/feat/generated/weapon_focus.json`
|
||||
- `topdata/data/feat/generated/weapon_specialization.json`
|
||||
- `topdata/data/feat/generated/greater_weapon_focus.json`
|
||||
- `topdata/data/feat/generated/greater_weapon_specialization.json`
|
||||
- `topdata/data/feat/generated/improved_critical.json`
|
||||
- `topdata/data/feat/generated/overwhelming_critical.json`
|
||||
- `topdata/data/feat/modules/activecombat/core.json`
|
||||
- `topdata/data/feat/modules/activecombat/specialattacks.json`
|
||||
- `topdata/data/feat/modules/class/core.json`
|
||||
- `topdata/data/feat/modules/combat/core.json`
|
||||
- `topdata/data/feat/modules/defensive/core.json`
|
||||
- `topdata/data/feat/modules/magical/core.json`
|
||||
- `topdata/data/feat/modules/other/core.json`
|
||||
- `topdata/data/feat/modules/proficiency/core.json`
|
||||
- `topdata/data/feat/modules/racial/core.json`
|
||||
- `topdata/data/feat/modules/removedandhidden/core.json`
|
||||
- `topdata/data/feat/modules/skillfeat/affinity.json`
|
||||
- `topdata/data/feat/modules/skillfeat/focus.json`
|
||||
- `topdata/data/feat/modules/skillfeat/greaterfocus.json`
|
||||
- `topdata/data/feat/modules/skillfeat/other.json`
|
||||
|
||||
`skill_affinity` is no longer authored as a generated feat file. It is synthesized from
|
||||
canonical `racialtypes` race grants.
|
||||
|
||||
Compact family-expansion shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"family": "skill_focus",
|
||||
"family_key": "skillfocus",
|
||||
"template": "masterfeats:skillfocus",
|
||||
"name_prefix": "Skill Focus",
|
||||
"label_prefix": "FEAT_SKILL_FOCUS",
|
||||
"constant_prefix": "FEAT_SKILL_FOCUS",
|
||||
"default_fields": {
|
||||
"DESCRIPTION": {
|
||||
"tlk": {
|
||||
"key": "masterfeats:skillfocus.description",
|
||||
"text": "..."
|
||||
}
|
||||
},
|
||||
"CRValue": "0.5",
|
||||
"ReqSkillMinRanks": "1",
|
||||
"TOOLSCATEGORIES": "6"
|
||||
},
|
||||
"apply_after_modules": true,
|
||||
"child_ref_field": "REQSKILL",
|
||||
"child_source": {
|
||||
"dataset": "skills",
|
||||
"predicate": "accessible"
|
||||
},
|
||||
"title_style": {
|
||||
"child_case": "lower",
|
||||
"child_parenthetical": "comma"
|
||||
},
|
||||
"overrides": {
|
||||
"skills:craft_alchemy": {
|
||||
"ICON": "ife_foc_alchm"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Weapon family shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"family": "weapon_focus",
|
||||
"family_key": "weaponfocus",
|
||||
"template": "masterfeats:weaponfocus",
|
||||
"name_prefix": "Weapon Focus",
|
||||
"label_prefix": "FEAT_WEAPON_FOCUS",
|
||||
"constant_prefix": "FEAT_WEAPON_FOCUS",
|
||||
"legacy_family_keys": ["epic_weapon_focus"],
|
||||
"default_fields": {
|
||||
"DESCRIPTION": {
|
||||
"tlk": {
|
||||
"key": "masterfeats:weaponfocus.description",
|
||||
"text": "..."
|
||||
}
|
||||
},
|
||||
"MINATTACKBONUS": "1",
|
||||
"TOOLSCATEGORIES": "1"
|
||||
},
|
||||
"apply_after_modules": true,
|
||||
"identity_source": "child_source_value",
|
||||
"child_source": {
|
||||
"dataset": "baseitems",
|
||||
"column": "WeaponFocusFeat"
|
||||
},
|
||||
"title_style": {
|
||||
"child_case": "lower",
|
||||
"child_parenthetical": "preserve"
|
||||
},
|
||||
"overrides": {
|
||||
"baseitems:heavymace": {
|
||||
"ICON": "ife_wepfoc_Lma"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- any generated feat file with family-expansion fields is treated as an authored
|
||||
family-expansion definition
|
||||
- `family` is descriptive only; the builder does not hardcode family names
|
||||
- family behavior is driven by authored fields such as `template_fields`,
|
||||
`default_fields`, `apply_after_modules`, `identity_source`, `child_ref_field`,
|
||||
`auto_prereq_fields`, and optional `title_style`
|
||||
- `default_fields` is a family-wide shared layer; it applies to existing and newly
|
||||
generated child rows before per-child overrides are merged
|
||||
- `apply_after_modules: true` makes the generated family the final authoritative shared
|
||||
layer for legacy explicitly authored child rows, so shared baselines can live in the
|
||||
generated family file instead of `masterfeats` overrides
|
||||
- `legacy_family_keys` declares old generated-family prefixes whose existing row IDs
|
||||
should be inherited by the current family; this is how renamed families such as
|
||||
`greater_skill_focus` retaining vanilla `epic_skill_focus` rows preserve hardcoded
|
||||
engine behavior without hardcoding the rename in toolkit logic
|
||||
- family-expansion files must declare `template`, `family_key`, and `child_source.dataset`
|
||||
- `child_source.column` is used when expansion is gated by a non-null source field
|
||||
- `child_source.predicate: "accessible"` currently means `HideFromLevelUp != 1`
|
||||
- `title_style` only changes generated child display text while composing titles;
|
||||
generated feat keys, TLK keys, row IDs, family metadata, and source references
|
||||
remain tied to generated family identity
|
||||
- generated family title text is authoritative for family-managed existing rows
|
||||
as well as newly allocated rows, so in-game TLK names and generated wiki titles
|
||||
are derived from the same normalized `FEAT` text
|
||||
- `title_style.child_case` defaults to `preserve`; `lower` lowercases the resolved
|
||||
child display text after parenthetical formatting
|
||||
- `title_style.child_parenthetical` defaults to `preserve`; `comma` flattens one
|
||||
child parenthetical suffix such as `Knowledge (local)` into
|
||||
`Knowledge, local` before the child text is wrapped by the family title
|
||||
- unsupported `title_style` values fail generated-family validation instead of
|
||||
silently changing title behavior
|
||||
- skill focus families intentionally do not set `allow_existing_only`, so any new
|
||||
accessible skill automatically receives matching focus rows
|
||||
- compact `overrides` are keyed by source dataset key:
|
||||
- `skills:*` for skill families
|
||||
- `baseitems:*` for weapon families
|
||||
- generated rows carry `meta.family` so the underscore family rule is preserved even
|
||||
before wiki generation exists
|
||||
- emitted feat keys must prefer existing canonical lock/TLK identities over newly
|
||||
derived child spellings
|
||||
- manual and irregular feat content continues to live under `feat/modules/`
|
||||
- generated families and module-authored feats are merged into the same final `feat.2da`
|
||||
pipeline and share inheritance, override, TLK, and family-metadata behavior
|
||||
|
||||
Current rollout policy:
|
||||
|
||||
- `feat.2da` still builds natively
|
||||
- `feat.2da` is still excluded from `compare-topdata` output-catalog self-check coverage
|
||||
via `compare_reference: false`
|
||||
- manual and irregular feat families remain authored directly in modules rather than
|
||||
compact generated-family files
|
||||
@@ -1,78 +0,0 @@
|
||||
# Topdata Inheritance Contract
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- Implemented.
|
||||
- `native.go` supports both row-sugar inheritance and generic object
|
||||
inheritance, including cycle detection and missing-target failures.
|
||||
- Validation and build regression coverage live in `topdata_test.go`.
|
||||
|
||||
Native topdata now supports two inheritance forms:
|
||||
|
||||
## 1. Row Sugar Compatibility
|
||||
|
||||
This is the existing narrow row helper:
|
||||
|
||||
```json
|
||||
{
|
||||
"PARENT": { "id": "custom:parent" },
|
||||
"inherit": {
|
||||
"from": "PARENT",
|
||||
"fields": ["VALUE", "ICON"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
It remains supported for row-local field copying.
|
||||
|
||||
## 2. Generic Object Inheritance
|
||||
|
||||
This is the new reusable primitive:
|
||||
|
||||
```json
|
||||
{
|
||||
"inherit": {
|
||||
"ref": "masterfeats:skillfocus"
|
||||
},
|
||||
"LABEL": "FEAT_SKILL_FOCUS_ATHLETICS"
|
||||
}
|
||||
```
|
||||
|
||||
Or for a nested object:
|
||||
|
||||
```json
|
||||
{
|
||||
"DETAILS": {
|
||||
"inherit": {
|
||||
"ref": "custom:parent",
|
||||
"field": "DETAILS"
|
||||
},
|
||||
"meta": {
|
||||
"cost": "9"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Rules
|
||||
|
||||
- `inherit.ref` must use stable `dataset:key` identity.
|
||||
- `inherit.field` is optional and selects an object-valued field on the target row.
|
||||
- Generic inheritance can appear on any authored object, not only top-level rows.
|
||||
- Merge precedence is `local overrides inherited`.
|
||||
- Scalars replace inherited values.
|
||||
- Arrays replace inherited values.
|
||||
- Objects merge recursively unless the local object is an atomic topdata value object such as:
|
||||
- TLK payloads
|
||||
- row refs
|
||||
- table refs
|
||||
- Cycles fail the build.
|
||||
- Missing targets fail the build.
|
||||
|
||||
## Current Intended Consumer
|
||||
|
||||
`feat` is the first dataset expected to use this primitive heavily, especially for
|
||||
borrowing shared properties from `masterfeats` while keeping `feat`'s own dataset
|
||||
contract and row model.
|
||||
@@ -1,82 +0,0 @@
|
||||
# `masterfeats` Canonical Contract
|
||||
|
||||
## Status Snapshot
|
||||
|
||||
Current state as of 2026-05-13:
|
||||
|
||||
- Implemented as a first-class native dataset.
|
||||
- The repository contains `topdata/data/masterfeats/`, migration/import support,
|
||||
validator checks, and native output coverage for `masterfeats.2da`.
|
||||
- Remaining work is only downstream adoption and regression maintenance as other
|
||||
native datasets continue to consume `masterfeats` more heavily.
|
||||
|
||||
`masterfeats` is a stable native canonical family consumed directly by `feat`
|
||||
generation, class-feat expansion, and inheritance/ref resolution.
|
||||
|
||||
Current canonical scope:
|
||||
|
||||
- dataset root: `topdata/data/masterfeats/`
|
||||
- required files:
|
||||
- `base.json`
|
||||
- `lock.json`
|
||||
- generated output:
|
||||
- `masterfeats.2da`
|
||||
|
||||
## `base.json`
|
||||
|
||||
Canonical shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"output": "masterfeats.2da",
|
||||
"columns": ["LABEL", "STRREF", "DESCRIPTION", "..."],
|
||||
"rows": [
|
||||
{
|
||||
"id": 0,
|
||||
"key": "masterfeats:weaponfocus",
|
||||
"LABEL": "WeaponFocus",
|
||||
"STRREF": "6490",
|
||||
"DESCRIPTION": "436"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- `output` may be omitted; native dataset discovery deterministically derives
|
||||
`masterfeats.2da` from `topdata/data/masterfeats/base.json`
|
||||
- if `output` is present, it must be exactly `masterfeats.2da`
|
||||
- `columns` must include `LABEL`, `STRREF`, and `DESCRIPTION`
|
||||
- every canonical row must have a non-empty `key`
|
||||
- row keys must start with `masterfeats:`
|
||||
- inline TLK payloads are allowed in `STRREF` and `DESCRIPTION`
|
||||
- numeric/text references are both allowed where the native TLK compiler already supports them
|
||||
|
||||
## `lock.json`
|
||||
|
||||
Canonical shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"masterfeats:weaponfocus": 0
|
||||
}
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- every key must start with `masterfeats:`
|
||||
- every value must be numeric
|
||||
- the lockfile remains the canonical stable id source for authored keys
|
||||
|
||||
## Module contributions
|
||||
|
||||
This family continues to support the shared canonical module shapes already used by native
|
||||
datasets:
|
||||
|
||||
- `columns`
|
||||
- `entries`
|
||||
- `overrides`
|
||||
|
||||
Those shapes are validated generically in `topdata.go`; this contract only adds the
|
||||
family-specific guarantees for `masterfeats`.
|
||||
@@ -1,96 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func importLegacyAppearance(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
_ = legacyTLK
|
||||
|
||||
legacyDir := filepath.Join(referenceBuilderDir, "data", "appearance")
|
||||
if _, err := os.Stat(legacyDir); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
targetDir := filepath.Join(dataDir, "appearance")
|
||||
targetBasePath := filepath.Join(targetDir, "base.json")
|
||||
targetLockPath := filepath.Join(targetDir, "lock.json")
|
||||
targetModulesDir := filepath.Join(targetDir, "modules")
|
||||
moduleNames := []string{
|
||||
"cotblreaver.json",
|
||||
"crawlingclaw.json",
|
||||
"halfogre.json",
|
||||
"zombieknight.json",
|
||||
}
|
||||
targetModulePaths := make([]string, 0, len(moduleNames))
|
||||
for _, name := range moduleNames {
|
||||
targetModulePaths = append(targetModulePaths, filepath.Join(targetModulesDir, name))
|
||||
}
|
||||
if fileExists(targetBasePath) && fileExists(targetLockPath) {
|
||||
allModulesPresent := true
|
||||
for _, path := range targetModulePaths {
|
||||
if !fileExists(path) {
|
||||
allModulesPresent = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allModulesPresent {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
baseObj["output"] = "appearance.2da"
|
||||
|
||||
lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
moduleObjs := make([]map[string]any, 0, len(moduleNames))
|
||||
for _, name := range moduleNames {
|
||||
obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", name))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
moduleObjs = append(moduleObjs, obj)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(targetModulesDir, 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
writes := []struct {
|
||||
path string
|
||||
obj map[string]any
|
||||
}{
|
||||
{path: targetBasePath, obj: baseObj},
|
||||
{path: targetLockPath, obj: lockObj},
|
||||
}
|
||||
for i, path := range targetModulePaths {
|
||||
writes = append(writes, struct {
|
||||
path string
|
||||
obj map[string]any
|
||||
}{path: path, obj: moduleObjs[i]})
|
||||
}
|
||||
|
||||
for _, write := range writes {
|
||||
raw, err := json.MarshalIndent(write.obj, "", " ")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(write.path, raw, 0o644); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(writes), nil
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
func importLegacyArmor(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
_ = legacyTLK
|
||||
|
||||
legacyDir := filepath.Join(referenceBuilderDir, "data", "armor")
|
||||
if _, err := os.Stat(legacyDir); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
targetDir := filepath.Join(dataDir, "armor")
|
||||
targetPath := filepath.Join(targetDir, "armor.json")
|
||||
if _, err := os.Stat(targetPath); err == nil {
|
||||
obj, err := loadJSONObject(targetPath)
|
||||
if err == nil && countLegacyTLKRefsInValue(obj) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := mergeArmorRows(baseObj, filepath.Join(legacyDir, "modules"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := removePathIfExists(filepath.Join(targetDir, "base.json")); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := removePathIfExists(filepath.Join(targetDir, "lock.json")); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := removePathIfExists(filepath.Join(targetDir, "modules")); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
out := map[string]any{
|
||||
"output": "armor.2da",
|
||||
"columns": baseObj["columns"],
|
||||
"rows": rows,
|
||||
}
|
||||
raw, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(targetPath, raw, 0o644); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func mergeArmorRows(baseObj map[string]any, modulesDir string) ([]any, error) {
|
||||
rawRows, ok := baseObj["rows"].([]any)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rows := make([]map[string]any, 0, len(rawRows))
|
||||
byID := map[int]map[string]any{}
|
||||
for _, raw := range rawRows {
|
||||
row, ok := deepCopyValue(raw).(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
delete(row, "key")
|
||||
rawID, ok := row["id"]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
id, err := asInt(rawID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row["id"] = id
|
||||
rows = append(rows, row)
|
||||
byID[id] = row
|
||||
}
|
||||
|
||||
modulePaths, err := collectModulePaths(modulesDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, path := range modulePaths {
|
||||
obj, err := loadJSONObject(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if entries, ok := obj["entries"].(map[string]any); ok {
|
||||
for _, raw := range entries {
|
||||
row, ok := deepCopyValue(raw).(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
delete(row, "key")
|
||||
rawID, ok := row["id"]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
id, err := asInt(rawID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row["id"] = id
|
||||
rows = append(rows, row)
|
||||
byID[id] = row
|
||||
}
|
||||
}
|
||||
if overrides, ok := obj["overrides"].([]any); ok {
|
||||
for _, raw := range overrides {
|
||||
override, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rawID, ok := override["id"]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
id, err := asInt(rawID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row := byID[id]
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
for key, value := range override {
|
||||
if key == "id" || key == "key" || key == "_tlk" || isMetadataField(key) {
|
||||
continue
|
||||
}
|
||||
row[key] = deepCopyValue(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
return rows[i]["id"].(int) < rows[j]["id"].(int)
|
||||
})
|
||||
out := make([]any, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
func resolvePartsAssetsOverrideDir(p *project.Project) (string, error) {
|
||||
spec := strings.TrimSpace(p.Config.TopData.Assets)
|
||||
if spec == "" {
|
||||
return "", nil
|
||||
}
|
||||
if looksLikeGitRepoSpec(spec) {
|
||||
return "", fmt.Errorf("topdata.assets no longer supports git repo specs for parts discovery; use a local path override or the default released manifest")
|
||||
}
|
||||
path := p.TopDataAssetsDir()
|
||||
if path == "" {
|
||||
return "", nil
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return "", fmt.Errorf("configured topdata.assets path %s is not accessible: %w", path, err)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func looksLikeGitRepoSpec(value string) bool {
|
||||
return strings.Contains(value, "://") ||
|
||||
strings.HasPrefix(value, "git@") ||
|
||||
strings.HasSuffix(value, ".git")
|
||||
}
|
||||
|
||||
func gitOutput(dir string, args ...string) (string, error) {
|
||||
cmd := exec.Command("git", args...)
|
||||
if dir != "" {
|
||||
cmd.Dir = dir
|
||||
}
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%v: %s", args, strings.TrimSpace(string(output)))
|
||||
}
|
||||
return strings.TrimSpace(string(output)), nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,41 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func importLegacyBaseDialog(referenceBuilderDir, sourceDir string) (int, error) {
|
||||
legacyPath := filepath.Join(referenceBuilderDir, "tlk", "base.json")
|
||||
if _, err := os.Stat(legacyPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
targetPath := filepath.Join(sourceDir, "base_dialog.json")
|
||||
if fileExists(targetPath) {
|
||||
current, err := loadJSONObject(targetPath)
|
||||
if err == nil {
|
||||
if entries, ok := current["entries"].(map[string]any); ok && len(entries) > 0 {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
obj, err := loadJSONObject(legacyPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw, err := json.MarshalIndent(obj, "", " ")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(targetPath, raw, 0o644); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func importLegacyBaseitems(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
_ = legacyTLK
|
||||
|
||||
legacyDir := filepath.Join(referenceBuilderDir, "data", "baseitems")
|
||||
if _, err := os.Stat(legacyDir); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
targetDir := filepath.Join(dataDir, "baseitems")
|
||||
targetBasePath := filepath.Join(targetDir, "base.json")
|
||||
targetLockPath := filepath.Join(targetDir, "lock.json")
|
||||
targetModulesDir := filepath.Join(targetDir, "modules")
|
||||
moduleNames := []string{
|
||||
"blunderbuss.json",
|
||||
"coin.json",
|
||||
"estoc.json",
|
||||
"heavymace.json",
|
||||
"maulandfalchion.json",
|
||||
"ovr_baseitems.json",
|
||||
"ovr_cloak255.json",
|
||||
"ovr_helmet255.json",
|
||||
"shortspear.json",
|
||||
}
|
||||
targetModulePaths := make([]string, 0, len(moduleNames))
|
||||
for _, name := range moduleNames {
|
||||
targetModulePaths = append(targetModulePaths, filepath.Join(targetModulesDir, name))
|
||||
}
|
||||
if fileExists(targetBasePath) && fileExists(targetLockPath) {
|
||||
allModulesPresent := true
|
||||
for _, path := range targetModulePaths {
|
||||
if !fileExists(path) {
|
||||
allModulesPresent = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allModulesPresent {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
baseObj["output"] = "baseitems.2da"
|
||||
canonicalizeWikiMetadataDocument(baseObj)
|
||||
|
||||
lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
moduleObjs := make([]map[string]any, 0, len(moduleNames))
|
||||
for _, name := range moduleNames {
|
||||
obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", name))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
canonicalizeWikiMetadataDocument(obj)
|
||||
moduleObjs = append(moduleObjs, obj)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(targetModulesDir, 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
writes := []struct {
|
||||
path string
|
||||
obj map[string]any
|
||||
}{
|
||||
{path: targetBasePath, obj: baseObj},
|
||||
{path: targetLockPath, obj: lockObj},
|
||||
}
|
||||
for index, path := range targetModulePaths {
|
||||
writes = append(writes, struct {
|
||||
path string
|
||||
obj map[string]any
|
||||
}{
|
||||
path: path,
|
||||
obj: moduleObjs[index],
|
||||
})
|
||||
}
|
||||
for _, write := range writes {
|
||||
raw, err := json.MarshalIndent(write.obj, "", " ")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(write.path, raw, 0o644); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(writes), nil
|
||||
}
|
||||
@@ -1,373 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type classSpellbookSpec struct {
|
||||
Path string
|
||||
Key string
|
||||
Class string
|
||||
Levels map[int][]string
|
||||
}
|
||||
|
||||
type classSpellbookPlan struct {
|
||||
Spec classSpellbookSpec
|
||||
Column string
|
||||
SpellLevels map[string]int
|
||||
}
|
||||
|
||||
func isClassSpellbookPath(path string) bool {
|
||||
slashed := filepath.ToSlash(path)
|
||||
return strings.Contains(slashed, "/data/classes/spellbooks/") && strings.HasSuffix(strings.ToLower(slashed), ".json")
|
||||
}
|
||||
|
||||
func validateClassSpellbookShape(path string, obj map[string]any, report *ValidationReport) {
|
||||
if _, ok := obj["class"]; !ok {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: "class spellbook requires class"})
|
||||
}
|
||||
if _, ok := obj["levels"]; !ok {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: "class spellbook requires levels"})
|
||||
}
|
||||
}
|
||||
|
||||
func validateClassSpellbooks(dataDir string, report *ValidationReport) {
|
||||
collected, err := collectSpellbookValidationDatasets(dataDir)
|
||||
if err != nil {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityError,
|
||||
Path: dataDir,
|
||||
Message: fmt.Sprintf("validate class spellbooks: %v", err),
|
||||
})
|
||||
return
|
||||
}
|
||||
_, diagnostics := classSpellbookPlans(dataDir, collected)
|
||||
report.Diagnostics = append(report.Diagnostics, diagnostics...)
|
||||
if len(diagnostics) == 0 {
|
||||
warnSpellModulesForManagedSpellbookColumns(dataDir, collected, report)
|
||||
}
|
||||
}
|
||||
|
||||
func collectSpellbookValidationDatasets(dataDir string) ([]nativeCollectedDataset, error) {
|
||||
datasets, err := discoverNativeDatasets(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]nativeCollectedDataset, 0, 2)
|
||||
for _, dataset := range datasets {
|
||||
if dataset.Name != "classes/core" && dataset.Name != "spells" {
|
||||
continue
|
||||
}
|
||||
collected, err := collectNativeDataset(dataset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, collected)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func applyClassSpellbooks(sourceDir string, collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) {
|
||||
dataDir := filepath.Join(sourceDir, "data")
|
||||
plans, diagnostics := classSpellbookPlans(dataDir, collected)
|
||||
if len(diagnostics) > 0 {
|
||||
return nil, fmt.Errorf("class spellbook validation failed: %s", diagnosticsTextForError(diagnostics))
|
||||
}
|
||||
if len(plans) == 0 {
|
||||
return collected, nil
|
||||
}
|
||||
|
||||
out := make([]nativeCollectedDataset, len(collected))
|
||||
copy(out, collected)
|
||||
spellsIndex := -1
|
||||
for index := range out {
|
||||
if out[index].Dataset.Name == "spells" {
|
||||
spellsIndex = index
|
||||
break
|
||||
}
|
||||
}
|
||||
if spellsIndex == -1 {
|
||||
return nil, fmt.Errorf("class spellbooks require canonical dataset %q", "spells")
|
||||
}
|
||||
|
||||
spells := out[spellsIndex]
|
||||
columns := append([]string(nil), spells.Columns...)
|
||||
rows := make([]map[string]any, 0, len(spells.Rows))
|
||||
for _, row := range spells.Rows {
|
||||
rows = append(rows, cloneRowMap(row))
|
||||
}
|
||||
rowByKey := rowsByKey(rows)
|
||||
|
||||
for _, plan := range plans {
|
||||
column := plan.Column
|
||||
if _, ok := canonicalColumn(columns, column); !ok {
|
||||
columns = append(columns, column)
|
||||
ensureRowsExposeColumns(rows, columns)
|
||||
}
|
||||
for _, row := range rows {
|
||||
row[column] = nullValue
|
||||
}
|
||||
for spellKey, level := range plan.SpellLevels {
|
||||
row := rowByKey[spellKey]
|
||||
if row == nil {
|
||||
return nil, fmt.Errorf("%s: unknown spell %q", plan.Spec.Path, spellKey)
|
||||
}
|
||||
row[column] = level
|
||||
}
|
||||
}
|
||||
|
||||
spells.Columns = columns
|
||||
spells.Rows = rows
|
||||
out[spellsIndex] = spells
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func classSpellbookPlans(dataDir string, collected []nativeCollectedDataset) ([]classSpellbookPlan, []Diagnostic) {
|
||||
specs, diagnostics := loadClassSpellbookSpecs(filepath.Join(dataDir, "classes", "spellbooks"))
|
||||
if len(specs) == 0 {
|
||||
return nil, diagnostics
|
||||
}
|
||||
classes := collectedDatasetByName(collected, "classes/core")
|
||||
spells := collectedDatasetByName(collected, "spells")
|
||||
if classes == nil {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: dataDir, Message: `class spellbooks require canonical dataset "classes/core"`})
|
||||
return nil, diagnostics
|
||||
}
|
||||
if spells == nil {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: dataDir, Message: `class spellbooks require canonical dataset "spells"`})
|
||||
return nil, diagnostics
|
||||
}
|
||||
|
||||
classRows := rowsByKey(classes.Rows)
|
||||
spellRows := rowsByKey(spells.Rows)
|
||||
seenColumns := map[string]string{}
|
||||
plans := make([]classSpellbookPlan, 0, len(specs))
|
||||
for _, spec := range specs {
|
||||
classRow := classRows[spec.Class]
|
||||
if classRow == nil {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: spec.Path, Message: fmt.Sprintf("unknown class %q", spec.Class)})
|
||||
}
|
||||
column := ""
|
||||
if classRow != nil {
|
||||
column, _ = classRow["SpellTableColumn"].(string)
|
||||
column = strings.TrimSpace(column)
|
||||
if column == "" || column == nullValue {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: spec.Path, Message: fmt.Sprintf("class %q has no SpellTableColumn", spec.Class)})
|
||||
}
|
||||
}
|
||||
if column != "" && column != nullValue {
|
||||
folded := strings.ToLower(column)
|
||||
if previous, ok := seenColumns[folded]; ok {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: spec.Path, Message: fmt.Sprintf("SpellTableColumn %q is already managed by %s", column, previous)})
|
||||
} else {
|
||||
seenColumns[folded] = spec.Path
|
||||
}
|
||||
}
|
||||
|
||||
spellLevels := map[string]int{}
|
||||
for _, level := range sortedIntKeys(spec.Levels) {
|
||||
for _, spellKey := range spec.Levels[level] {
|
||||
if spellRows[spellKey] == nil {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: spec.Path, Message: fmt.Sprintf("unknown spell %q", spellKey)})
|
||||
continue
|
||||
}
|
||||
if previous, ok := spellLevels[spellKey]; ok {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: spec.Path, Message: fmt.Sprintf("spell %q appears at both level %d and level %d", spellKey, previous, level)})
|
||||
continue
|
||||
}
|
||||
spellLevels[spellKey] = level
|
||||
}
|
||||
}
|
||||
|
||||
if column != "" && column != nullValue {
|
||||
plans = append(plans, classSpellbookPlan{Spec: spec, Column: column, SpellLevels: spellLevels})
|
||||
}
|
||||
}
|
||||
return plans, diagnostics
|
||||
}
|
||||
|
||||
func warnSpellModulesForManagedSpellbookColumns(dataDir string, collected []nativeCollectedDataset, report *ValidationReport) {
|
||||
plans, diagnostics := classSpellbookPlans(dataDir, collected)
|
||||
if len(diagnostics) > 0 || len(plans) == 0 {
|
||||
return
|
||||
}
|
||||
managed := map[string]classSpellbookPlan{}
|
||||
for _, plan := range plans {
|
||||
managed[strings.ToLower(plan.Column)] = plan
|
||||
}
|
||||
paths, err := collectModulePaths(filepath.Join(dataDir, "spells", "modules"))
|
||||
if err != nil {
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{Severity: SeverityError, Path: dataDir, Message: fmt.Sprintf("collect spell modules for spellbook warnings: %v", err)})
|
||||
return
|
||||
}
|
||||
for _, path := range paths {
|
||||
obj, err := loadJSONObject(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
warnSpellbookManagedColumnsInValue(dataDir, path, obj["entries"], managed, report)
|
||||
warnSpellbookManagedColumnsInValue(dataDir, path, obj["overrides"], managed, report)
|
||||
warnSpellbookManagedColumnsInValue(dataDir, path, obj["rows"], managed, report)
|
||||
}
|
||||
}
|
||||
|
||||
func warnSpellbookManagedColumnsInValue(dataDir, path string, value any, managed map[string]classSpellbookPlan, report *ValidationReport) {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
for _, key := range sortedKeys(typed) {
|
||||
row, ok := typed[key].(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
warnSpellbookManagedColumnsInRow(dataDir, path, row, managed, report)
|
||||
}
|
||||
case []any:
|
||||
for _, raw := range typed {
|
||||
row, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
warnSpellbookManagedColumnsInRow(dataDir, path, row, managed, report)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func warnSpellbookManagedColumnsInRow(dataDir, path string, row map[string]any, managed map[string]classSpellbookPlan, report *ValidationReport) {
|
||||
for field, value := range row {
|
||||
plan, ok := managed[strings.ToLower(field)]
|
||||
if !ok || isNullLikeValue(value) {
|
||||
continue
|
||||
}
|
||||
spellbookPath := displayTopdataPath(dataDir, plan.Spec.Path)
|
||||
report.Diagnostics = append(report.Diagnostics, Diagnostic{
|
||||
Severity: SeverityWarning,
|
||||
Path: path,
|
||||
Message: fmt.Sprintf(
|
||||
"spell module authors class spell column %q managed by class spellbook %s; use %s instead",
|
||||
plan.Column,
|
||||
spellbookPath,
|
||||
spellbookPath,
|
||||
),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func displayTopdataPath(dataDir, path string) string {
|
||||
topdataDir := filepath.Dir(dataDir)
|
||||
rel, err := filepath.Rel(filepath.Dir(topdataDir), path)
|
||||
if err != nil {
|
||||
return filepath.ToSlash(path)
|
||||
}
|
||||
return filepath.ToSlash(rel)
|
||||
}
|
||||
|
||||
func loadClassSpellbookSpecs(dir string) ([]classSpellbookSpec, []Diagnostic) {
|
||||
paths, err := collectModulePaths(dir)
|
||||
if err != nil {
|
||||
return nil, []Diagnostic{{Severity: SeverityError, Path: dir, Message: err.Error()}}
|
||||
}
|
||||
specs := make([]classSpellbookSpec, 0, len(paths))
|
||||
diagnostics := []Diagnostic{}
|
||||
for _, path := range paths {
|
||||
obj, err := loadJSONObject(path)
|
||||
if err != nil {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: err.Error()})
|
||||
continue
|
||||
}
|
||||
spec, specDiagnostics := parseClassSpellbookSpec(path, obj)
|
||||
diagnostics = append(diagnostics, specDiagnostics...)
|
||||
if spec.Class != "" && spec.Levels != nil {
|
||||
specs = append(specs, spec)
|
||||
}
|
||||
}
|
||||
return specs, diagnostics
|
||||
}
|
||||
|
||||
func parseClassSpellbookSpec(path string, obj map[string]any) (classSpellbookSpec, []Diagnostic) {
|
||||
diagnostics := []Diagnostic{}
|
||||
spec := classSpellbookSpec{Path: path, Levels: map[int][]string{}}
|
||||
if key, ok := obj["key"].(string); ok {
|
||||
spec.Key = strings.TrimSpace(key)
|
||||
}
|
||||
classKey, ok := obj["class"].(string)
|
||||
if !ok || strings.TrimSpace(classKey) == "" {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: "class spellbook requires class"})
|
||||
} else {
|
||||
spec.Class = strings.TrimSpace(classKey)
|
||||
if !strings.HasPrefix(spec.Class, "classes:") {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: "class must use a classes: reference"})
|
||||
}
|
||||
}
|
||||
rawLevels, ok := obj["levels"].(map[string]any)
|
||||
if !ok {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: "class spellbook levels must be an object"})
|
||||
return spec, diagnostics
|
||||
}
|
||||
for _, rawLevel := range sortedKeys(rawLevels) {
|
||||
level, err := strconv.Atoi(rawLevel)
|
||||
if err != nil || level < 0 {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: fmt.Sprintf("level key %q must be a nonnegative integer", rawLevel)})
|
||||
continue
|
||||
}
|
||||
rawList, ok := rawLevels[rawLevel].([]any)
|
||||
if !ok {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: fmt.Sprintf("level %d must be an array", level)})
|
||||
continue
|
||||
}
|
||||
for index, rawSpell := range rawList {
|
||||
spellKey, ok := rawSpell.(string)
|
||||
if !ok || strings.TrimSpace(spellKey) == "" {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: fmt.Sprintf("level %d spell %d must be a spells: reference", level, index)})
|
||||
continue
|
||||
}
|
||||
spellKey = strings.TrimSpace(spellKey)
|
||||
if !strings.HasPrefix(spellKey, "spells:") {
|
||||
diagnostics = append(diagnostics, Diagnostic{Severity: SeverityError, Path: path, Message: fmt.Sprintf("level %d spell %d must use a spells: reference", level, index)})
|
||||
continue
|
||||
}
|
||||
spec.Levels[level] = append(spec.Levels[level], spellKey)
|
||||
}
|
||||
}
|
||||
return spec, diagnostics
|
||||
}
|
||||
|
||||
func collectedDatasetByName(collected []nativeCollectedDataset, name string) *nativeCollectedDataset {
|
||||
for index := range collected {
|
||||
if collected[index].Dataset.Name == name {
|
||||
return &collected[index]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rowsByKey(rows []map[string]any) map[string]map[string]any {
|
||||
out := make(map[string]map[string]any, len(rows))
|
||||
for _, row := range rows {
|
||||
key, _ := row["key"].(string)
|
||||
if key != "" {
|
||||
out[key] = row
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedIntKeys(values map[int][]string) []int {
|
||||
keys := make([]int, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func diagnosticsTextForError(diags []Diagnostic) string {
|
||||
messages := make([]string, 0, len(diags))
|
||||
for _, diag := range diags {
|
||||
messages = append(messages, diag.Message)
|
||||
}
|
||||
return strings.Join(messages, "; ")
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var legacyClassesPlainFamilies = []string{"feats", "skills", "savthr", "bfeat", "pres"}
|
||||
|
||||
func importLegacyClasses(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
legacyRoot := filepath.Join(referenceBuilderDir, "data", "classes")
|
||||
if _, err := os.Stat(legacyRoot); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
targetRoot := filepath.Join(dataDir, "classes")
|
||||
if canonicalClassesPresent(targetRoot) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
coreCollected, plainCollected, err := collectLegacyClassesDatasets(legacyRoot)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
tableKeyByOutput := map[string]string{}
|
||||
for _, dataset := range plainCollected {
|
||||
if dataset.TableKey == "" {
|
||||
continue
|
||||
}
|
||||
registerLegacyClassesTableKey(tableKeyByOutput, dataset.TableKey, dataset.Dataset.OutputName)
|
||||
}
|
||||
|
||||
coreRows := make([]map[string]any, 0, len(coreCollected.Rows))
|
||||
for _, rawRow := range coreCollected.Rows {
|
||||
row, ok := deepCopyValue(rawRow).(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if legacyTLK != nil {
|
||||
if _, _, err := inlineLegacyTLKValue(row, legacyTLK); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
rewriteLegacyClassesCoreRow(row, tableKeyByOutput)
|
||||
coreRows = append(coreRows, row)
|
||||
}
|
||||
|
||||
if err := removePathIfExists(targetRoot); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(targetRoot, "core"), 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
writes := []struct {
|
||||
path string
|
||||
obj map[string]any
|
||||
}{
|
||||
{
|
||||
path: filepath.Join(targetRoot, "core", "base.json"),
|
||||
obj: map[string]any{
|
||||
"output": coreCollected.Dataset.OutputName,
|
||||
"columns": stringSliceToAny(coreCollected.Columns),
|
||||
"rows": rowsToAny(coreRows),
|
||||
},
|
||||
},
|
||||
{
|
||||
path: filepath.Join(targetRoot, "core", "lock.json"),
|
||||
obj: anyMapInt(coreCollected.LockData),
|
||||
},
|
||||
}
|
||||
|
||||
for _, collected := range plainCollected {
|
||||
obj := map[string]any{
|
||||
"output": collected.Dataset.OutputName,
|
||||
"columns": stringSliceToAny(collected.Columns),
|
||||
"rows": rowsToAny(collected.Rows),
|
||||
}
|
||||
if strings.TrimSpace(collected.TableKey) != "" {
|
||||
obj["key"] = collected.TableKey
|
||||
}
|
||||
writes = append(writes, struct {
|
||||
path string
|
||||
obj map[string]any
|
||||
}{
|
||||
path: filepath.Join(dataDir, collected.Dataset.Name+".json"),
|
||||
obj: obj,
|
||||
})
|
||||
}
|
||||
|
||||
for _, write := range writes {
|
||||
if err := os.MkdirAll(filepath.Dir(write.path), 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw, err := json.MarshalIndent(write.obj, "", " ")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(write.path, raw, 0o644); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return len(writes), nil
|
||||
}
|
||||
|
||||
func canonicalClassesPresent(targetRoot string) bool {
|
||||
if !fileExists(filepath.Join(targetRoot, "core", "base.json")) || !fileExists(filepath.Join(targetRoot, "core", "lock.json")) {
|
||||
return false
|
||||
}
|
||||
for _, family := range legacyClassesPlainFamilies {
|
||||
ok, err := hasJSONFiles(filepath.Join(targetRoot, family))
|
||||
if err != nil || !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func collectLegacyClassesDatasets(legacyRoot string) (nativeCollectedDataset, []nativeCollectedDataset, error) {
|
||||
coreCollected, err := collectBaseDataset(nativeDataset{
|
||||
Name: "classes/core",
|
||||
BasePath: filepath.Join(legacyRoot, "core", "base.json"),
|
||||
LockPath: filepath.Join(legacyRoot, "core", "lock.json"),
|
||||
ModulesDir: filepath.Join(legacyRoot, "core", "modules"),
|
||||
OutputName: "classes.2da",
|
||||
Spec: specForDataset("classes"),
|
||||
})
|
||||
if err != nil {
|
||||
return nativeCollectedDataset{}, nil, err
|
||||
}
|
||||
coreCollected.Dataset.OutputName = "classes.2da"
|
||||
|
||||
plainCollected := make([]nativeCollectedDataset, 0)
|
||||
for _, family := range legacyClassesPlainFamilies {
|
||||
familyDir := filepath.Join(legacyRoot, family)
|
||||
entries, err := os.ReadDir(familyDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
return nativeCollectedDataset{}, nil, err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" || strings.HasPrefix(entry.Name(), ".") || entry.Name() == "lock.json" {
|
||||
continue
|
||||
}
|
||||
filePath := filepath.Join(familyDir, entry.Name())
|
||||
tableData, err := loadJSONObject(filePath)
|
||||
if err != nil {
|
||||
return nativeCollectedDataset{}, nil, err
|
||||
}
|
||||
outputName, _ := tableData["output"].(string)
|
||||
if strings.TrimSpace(outputName) == "" {
|
||||
outputName = strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())) + ".2da"
|
||||
}
|
||||
collected, err := collectPlainDataset(nativeDataset{
|
||||
Name: filepath.ToSlash(filepath.Join("classes", family, strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())))),
|
||||
BasePath: filePath,
|
||||
LockPath: filepath.Join(familyDir, "lock.json"),
|
||||
OutputName: outputName,
|
||||
Spec: specForDataset(filepath.ToSlash(filepath.Join("classes", family, strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name()))))),
|
||||
})
|
||||
if err != nil {
|
||||
return nativeCollectedDataset{}, nil, err
|
||||
}
|
||||
plainCollected = append(plainCollected, collected)
|
||||
}
|
||||
}
|
||||
|
||||
slices.SortFunc(plainCollected, func(a, b nativeCollectedDataset) int {
|
||||
return strings.Compare(a.Dataset.Name, b.Dataset.Name)
|
||||
})
|
||||
return coreCollected, plainCollected, nil
|
||||
}
|
||||
|
||||
func registerLegacyClassesTableKey(tableKeyByOutput map[string]string, tableKey, outputName string) {
|
||||
trimmedOutput := strings.TrimSpace(outputName)
|
||||
if trimmedOutput != "" {
|
||||
tableKeyByOutput[trimmedOutput] = tableKey
|
||||
}
|
||||
trimmedStem := strings.TrimSpace(outputStem(outputName))
|
||||
if trimmedStem != "" {
|
||||
tableKeyByOutput[trimmedStem] = tableKey
|
||||
}
|
||||
}
|
||||
|
||||
func rewriteLegacyClassesCoreRow(row map[string]any, tableKeyByOutput map[string]string) {
|
||||
for _, field := range []string{"FeatsTable", "SavingThrowTable", "SkillsTable", "BonusFeatsTable", "PreReqTable"} {
|
||||
tableKey := legacyClassesTableKeyForValue(row[field], tableKeyByOutput)
|
||||
if tableKey == "" {
|
||||
continue
|
||||
}
|
||||
row[field] = map[string]any{"table": tableKey}
|
||||
}
|
||||
}
|
||||
|
||||
func legacyClassesTableKeyForValue(value any, tableKeyByOutput map[string]string) string {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
tableKey, _ := typed["table"].(string)
|
||||
return strings.TrimSpace(tableKey)
|
||||
case string:
|
||||
trimmed := strings.TrimSpace(typed)
|
||||
if trimmed == "" || trimmed == nullValue {
|
||||
return ""
|
||||
}
|
||||
if trimmed != strings.ToLower(trimmed) {
|
||||
return ""
|
||||
}
|
||||
if tableKey, ok := tableKeyByOutput[trimmed]; ok {
|
||||
return tableKey
|
||||
}
|
||||
return ""
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
package topdata
|
||||
|
||||
func importLegacyCloakmodel(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
_ = legacyTLK
|
||||
return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "cloakmodel", "cloakmodel.2da", nil)
|
||||
}
|
||||
@@ -1,888 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type convertOptions struct {
|
||||
Namespace string
|
||||
KeyFields []string
|
||||
CollisionMode string
|
||||
BaseDialog string
|
||||
Type string
|
||||
Name string
|
||||
FilenamePrefixes map[string]string
|
||||
}
|
||||
|
||||
func RunConvertCommand(args []string, stdout io.Writer) error {
|
||||
if len(args) == 0 || args[0] == "--help" || args[0] == "-h" {
|
||||
printConvertUsage(stdout)
|
||||
return nil
|
||||
}
|
||||
switch args[0] {
|
||||
case "2da-to-json":
|
||||
if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
|
||||
_, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-json [flags] <input.2da> <output.json>")
|
||||
return nil
|
||||
}
|
||||
opts, input, output, err := parseConvertArgs(args[1:], false, "suffix")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := parse2DAFile(input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := convert2DAToJSON(data, output, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeJSONFile(output, result)
|
||||
case "2da-to-module":
|
||||
if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
|
||||
_, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-module [--namespace <dataset>] [--name <module-name>] [--type entries|override] [flags] <input.2da> [output.json]")
|
||||
return nil
|
||||
}
|
||||
opts, input, output, err := parseConvertArgs(args[1:], true, "error")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := parse2DAFile(input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Type == "override" {
|
||||
result := map[string]any{"overrides": toOverridesRows(data.Rows)}
|
||||
if err := writeJSONFile(output, result); err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout, "converted overrides: %d\n", len(result["overrides"].([]map[string]any)))
|
||||
return nil
|
||||
}
|
||||
result, err := convert2DAToModule(data, output, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeJSONFile(output, result); err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout, "converted entries: %d\n", len(result["entries"].(map[string]any)))
|
||||
return nil
|
||||
case "json-to-2da":
|
||||
if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
|
||||
_, _ = fmt.Fprintln(stdout, "usage: convert-topdata json-to-2da <input.json> <output.2da>")
|
||||
return nil
|
||||
}
|
||||
if len(args) != 3 {
|
||||
return errors.New("usage: convert-topdata json-to-2da <input.json> <output.2da>")
|
||||
}
|
||||
data, err := readCanonicalJSON(args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return write2DAFile(data, args[2])
|
||||
default:
|
||||
return fmt.Errorf("unknown convert-topdata subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func printConvertUsage(stdout io.Writer) {
|
||||
_, _ = fmt.Fprintln(stdout, "usage: convert-topdata <2da-to-json|2da-to-module|json-to-2da> ...")
|
||||
_, _ = fmt.Fprintln(stdout, "")
|
||||
_, _ = fmt.Fprintln(stdout, "subcommands:")
|
||||
_, _ = fmt.Fprintln(stdout, " 2da-to-json Convert a 2DA file into canonical JSON rows")
|
||||
_, _ = fmt.Fprintln(stdout, " 2da-to-module Convert a 2DA file into module entries or override JSON")
|
||||
_, _ = fmt.Fprintln(stdout, " json-to-2da Convert canonical JSON rows into a 2DA file")
|
||||
_, _ = fmt.Fprintln(stdout, "")
|
||||
_, _ = fmt.Fprintln(stdout, "2da-to-module notes:")
|
||||
_, _ = fmt.Fprintln(stdout, " - --namespace is optional when topdata/templates/config.yaml declares it for the input table")
|
||||
_, _ = fmt.Fprintln(stdout, " - --name controls inferred output naming: <prefix>_<namespace>_<name>.json")
|
||||
_, _ = fmt.Fprintln(stdout, " - topdata/templates/config.yaml module_output.filename_prefixes can set entries/override prefixes")
|
||||
_, _ = fmt.Fprintln(stdout, " - flags accept both --flag value and --flag=value forms")
|
||||
_, _ = fmt.Fprintln(stdout, " - unkeyed rows now fail conversion instead of being skipped")
|
||||
}
|
||||
|
||||
type parsed2DA struct {
|
||||
Columns []string `json:"columns"`
|
||||
Rows []map[string]any `json:"rows"`
|
||||
}
|
||||
|
||||
func parseConvertArgs(args []string, allowFormat bool, defaultCollision string) (convertOptions, string, string, error) {
|
||||
opts := convertOptions{
|
||||
CollisionMode: defaultCollision,
|
||||
Type: "entries",
|
||||
}
|
||||
positional := make([]string, 0, 2)
|
||||
for index := 0; index < len(args); index++ {
|
||||
arg := args[index]
|
||||
switch arg {
|
||||
case "--namespace":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", errors.New("--namespace requires a value")
|
||||
}
|
||||
opts.Namespace = args[index]
|
||||
case "--key-field":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", errors.New("--key-field requires a value")
|
||||
}
|
||||
opts.KeyFields = append(opts.KeyFields, args[index])
|
||||
case "--name":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", errors.New("--name requires a value")
|
||||
}
|
||||
opts.Name = args[index]
|
||||
case "--collision":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", errors.New("--collision requires a value")
|
||||
}
|
||||
opts.CollisionMode = args[index]
|
||||
case "--base-dialog":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", errors.New("--base-dialog requires a value")
|
||||
}
|
||||
opts.BaseDialog = args[index]
|
||||
case "--format", "--type":
|
||||
if !allowFormat {
|
||||
return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), arg)
|
||||
}
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", fmt.Errorf("%s requires a value", arg)
|
||||
}
|
||||
opts.Type = args[index]
|
||||
default:
|
||||
if value, ok := parseInlineFlagValue(arg, "--namespace"); ok {
|
||||
opts.Namespace = value
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--key-field"); ok {
|
||||
opts.KeyFields = append(opts.KeyFields, value)
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--name"); ok {
|
||||
opts.Name = value
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--collision"); ok {
|
||||
opts.CollisionMode = value
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--base-dialog"); ok {
|
||||
opts.BaseDialog = value
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--format"); ok {
|
||||
if !allowFormat {
|
||||
return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), "--format")
|
||||
}
|
||||
opts.Type = value
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--type"); ok {
|
||||
if !allowFormat {
|
||||
return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), "--type")
|
||||
}
|
||||
opts.Type = value
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(arg, "--") {
|
||||
return opts, "", "", fmt.Errorf("unknown flag %q", arg)
|
||||
}
|
||||
positional = append(positional, arg)
|
||||
}
|
||||
}
|
||||
if allowFormat {
|
||||
if len(positional) < 1 || len(positional) > 2 {
|
||||
return opts, "", "", errors.New("usage: convert-topdata 2da-to-module [--namespace <dataset>] [--name <module-name>] [--type entries|override] [flags] <input.2da> [output.json]")
|
||||
}
|
||||
} else if len(positional) != 2 {
|
||||
return opts, "", "", errors.New("usage: convert-topdata 2da-to-json [flags] <input.2da> <output.json>")
|
||||
}
|
||||
if opts.CollisionMode != "suffix" && opts.CollisionMode != "error" {
|
||||
return opts, "", "", fmt.Errorf("unsupported collision mode %q", opts.CollisionMode)
|
||||
}
|
||||
if allowFormat {
|
||||
output := ""
|
||||
if len(positional) == 2 {
|
||||
output = positional[1]
|
||||
}
|
||||
ctx, err := resolveConvertContext(positional[0])
|
||||
if err != nil {
|
||||
return opts, "", "", err
|
||||
}
|
||||
if ctx.Project != nil {
|
||||
if outputCtx, err := resolveConvertOutputContext(ctx.Project, output); err != nil {
|
||||
return opts, "", "", err
|
||||
} else if ctx.Template.Namespace == "" && len(ctx.Template.KeyFields) == 0 {
|
||||
ctx.Template = outputCtx
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(opts.Namespace) == "" {
|
||||
opts.Namespace = ctx.Template.Namespace
|
||||
}
|
||||
if len(opts.KeyFields) == 0 && len(ctx.Template.KeyFields) > 0 {
|
||||
opts.KeyFields = append(opts.KeyFields, ctx.Template.KeyFields...)
|
||||
}
|
||||
if len(opts.FilenamePrefixes) == 0 && len(ctx.Config.ModuleOutput.FilenamePrefixes) > 0 {
|
||||
opts.FilenamePrefixes = cloneStringMap(ctx.Config.ModuleOutput.FilenamePrefixes)
|
||||
}
|
||||
if strings.TrimSpace(opts.Namespace) == "" {
|
||||
return opts, "", "", errors.New("2da-to-module requires --namespace or a topdata/templates/config.yaml entry for the input table")
|
||||
}
|
||||
switch opts.Type {
|
||||
case "entry":
|
||||
opts.Type = "entries"
|
||||
case "overrides":
|
||||
opts.Type = "override"
|
||||
}
|
||||
if opts.Type != "entries" && opts.Type != "override" {
|
||||
return opts, "", "", fmt.Errorf("unsupported module type %q", opts.Type)
|
||||
}
|
||||
if output == "" {
|
||||
resolved, err := defaultModuleOutputPath(positional[0], opts, ctx.Project)
|
||||
if err != nil {
|
||||
return opts, "", "", err
|
||||
}
|
||||
output = resolved
|
||||
}
|
||||
return opts, positional[0], output, nil
|
||||
}
|
||||
ctx, err := resolveConvertContext(positional[0])
|
||||
if err != nil {
|
||||
return opts, "", "", err
|
||||
}
|
||||
if ctx.Project != nil {
|
||||
if outputCtx, err := resolveConvertOutputContext(ctx.Project, positional[1]); err != nil {
|
||||
return opts, "", "", err
|
||||
} else if ctx.Template.Namespace == "" && len(ctx.Template.KeyFields) == 0 {
|
||||
ctx.Template = outputCtx
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(opts.Namespace) == "" {
|
||||
opts.Namespace = ctx.Template.Namespace
|
||||
}
|
||||
if len(opts.KeyFields) == 0 && len(ctx.Template.KeyFields) > 0 {
|
||||
opts.KeyFields = append(opts.KeyFields, ctx.Template.KeyFields...)
|
||||
}
|
||||
return opts, positional[0], positional[1], nil
|
||||
}
|
||||
|
||||
func parseInlineFlagValue(arg, name string) (string, bool) {
|
||||
prefix := name + "="
|
||||
if !strings.HasPrefix(arg, prefix) {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(arg[len(prefix):]), true
|
||||
}
|
||||
|
||||
func positionalSafe(args []string) string {
|
||||
if len(args) == 0 {
|
||||
return "command"
|
||||
}
|
||||
return args[0]
|
||||
}
|
||||
|
||||
func parse2DAFile(path string) (parsed2DA, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return parsed2DA{}, fmt.Errorf("read %s: %w", path, err)
|
||||
}
|
||||
lines := strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n")
|
||||
trimmed := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" {
|
||||
trimmed = append(trimmed, line)
|
||||
}
|
||||
}
|
||||
if len(trimmed) < 2 || !strings.HasPrefix(trimmed[0], "2DA") {
|
||||
return parsed2DA{}, fmt.Errorf("%s: invalid 2DA file", path)
|
||||
}
|
||||
columns := split2DALine(trimmed[1])
|
||||
if len(columns) == 0 {
|
||||
return parsed2DA{}, fmt.Errorf("%s: missing 2DA columns", path)
|
||||
}
|
||||
rows := make([]map[string]any, 0, max(0, len(trimmed)-2))
|
||||
seenIDs := map[int]struct{}{}
|
||||
for _, line := range trimmed[2:] {
|
||||
fields := split2DALine(line)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
rowID, err := strconv.Atoi(fields[0])
|
||||
if err != nil {
|
||||
return parsed2DA{}, fmt.Errorf("%s: invalid 2DA row id %q", path, fields[0])
|
||||
}
|
||||
if len(fields)-1 > len(columns) {
|
||||
return parsed2DA{}, fmt.Errorf("%s: row %d has %d values but only %d columns are declared", path, rowID, len(fields)-1, len(columns))
|
||||
}
|
||||
if _, ok := seenIDs[rowID]; ok {
|
||||
return parsed2DA{}, fmt.Errorf("%s: duplicate 2DA row id %d", path, rowID)
|
||||
}
|
||||
seenIDs[rowID] = struct{}{}
|
||||
row := map[string]any{"id": rowID}
|
||||
for index, column := range columns {
|
||||
if index+1 < len(fields) {
|
||||
row[column] = smartConvertScalar(fields[index+1])
|
||||
continue
|
||||
}
|
||||
row[column] = nullValue
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return parsed2DA{Columns: columns, Rows: rows}, nil
|
||||
}
|
||||
|
||||
func split2DALine(line string) []string {
|
||||
fields := make([]string, 0)
|
||||
var current strings.Builder
|
||||
inQuotes := false
|
||||
for _, ch := range line {
|
||||
switch {
|
||||
case ch == '"':
|
||||
inQuotes = !inQuotes
|
||||
case !inQuotes && (ch == '\t' || ch == ' '):
|
||||
if current.Len() > 0 {
|
||||
fields = append(fields, current.String())
|
||||
current.Reset()
|
||||
}
|
||||
default:
|
||||
current.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
if current.Len() > 0 {
|
||||
fields = append(fields, current.String())
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func smartConvertScalar(value string) any {
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
if parsed, err := strconv.Atoi(value); err == nil {
|
||||
return parsed
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func convert2DAToJSON(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) {
|
||||
rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outRows := make([]map[string]any, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
ordered := map[string]any{"id": row["id"]}
|
||||
if key, ok := row["key"].(string); ok && key != "" {
|
||||
ordered["key"] = key
|
||||
}
|
||||
for _, column := range data.Columns {
|
||||
ordered[column] = row[column]
|
||||
}
|
||||
outRows = append(outRows, ordered)
|
||||
}
|
||||
return map[string]any{"columns": data.Columns, "rows": outRows}, nil
|
||||
}
|
||||
|
||||
func convert2DAToModule(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) {
|
||||
rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries := map[string]any{}
|
||||
missing := []string{}
|
||||
for _, row := range rows {
|
||||
key, _ := row["key"].(string)
|
||||
if key == "" {
|
||||
missing = append(missing, strconv.Itoa(row["id"].(int)))
|
||||
continue
|
||||
}
|
||||
entry := map[string]any{}
|
||||
for _, column := range data.Columns {
|
||||
value := row[column]
|
||||
if format2DAValue(value) == nullValue {
|
||||
continue
|
||||
}
|
||||
entry[column] = value
|
||||
}
|
||||
entries[key] = entry
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return nil, fmt.Errorf("unable to generate keys for row ids %s; declare --key-field or configure key_fields in topdata/templates/config.yaml", strings.Join(missing, ", "))
|
||||
}
|
||||
return map[string]any{"entries": entries}, nil
|
||||
}
|
||||
|
||||
func toOverridesRows(rows []map[string]any) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
override := map[string]any{"id": row["id"]}
|
||||
keys := make([]string, 0, len(row))
|
||||
for key := range row {
|
||||
if key == "id" || key == "key" {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
if format2DAValue(row[key]) == nullValue {
|
||||
continue
|
||||
}
|
||||
override[key] = row[key]
|
||||
}
|
||||
out = append(out, override)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assignConvertedRowKeys(rows []map[string]any, outputPath string, opts convertOptions) ([]map[string]any, error) {
|
||||
namespace := strings.TrimSpace(opts.Namespace)
|
||||
out := make([]map[string]any, 0, len(rows))
|
||||
used := map[string]struct{}{}
|
||||
rowIDsByKey := map[string]int{}
|
||||
for _, row := range rows {
|
||||
cloned := cloneRowMap(row)
|
||||
if namespace == "" {
|
||||
out = append(out, cloned)
|
||||
continue
|
||||
}
|
||||
candidates := convertedRowKeyCandidates(cloned, opts.KeyFields)
|
||||
if len(candidates) == 0 {
|
||||
out = append(out, cloned)
|
||||
continue
|
||||
}
|
||||
key := ""
|
||||
for _, candidate := range candidates {
|
||||
candidateKey := namespace + ":" + candidate
|
||||
if _, ok := used[candidateKey]; ok {
|
||||
continue
|
||||
}
|
||||
key = candidateKey
|
||||
break
|
||||
}
|
||||
if key == "" {
|
||||
key = namespace + ":" + candidates[0]
|
||||
if opts.CollisionMode == "error" {
|
||||
return nil, fmt.Errorf("duplicate generated key %q for row %v (already used by row %d)", key, cloned["id"], rowIDsByKey[key])
|
||||
}
|
||||
key = fmt.Sprintf("%s_%v", key, cloned["id"])
|
||||
}
|
||||
used[key] = struct{}{}
|
||||
rowID, _ := asInt(cloned["id"])
|
||||
rowIDsByKey[key] = rowID
|
||||
cloned["key"] = key
|
||||
out = append(out, cloned)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var convertKeyWhitespace = regexp.MustCompile(`\s+`)
|
||||
var convertKeyCleaner = regexp.MustCompile(`[^a-z0-9_]`)
|
||||
|
||||
func convertedRowKeyCandidates(row map[string]any, preferred []string) []string {
|
||||
fields := preferred
|
||||
if len(fields) == 0 {
|
||||
fields = []string{"LABEL", "Label", "Name"}
|
||||
} else {
|
||||
for _, field := range fields {
|
||||
value, ok := row[field]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
text := strings.TrimSpace(format2DAValue(value))
|
||||
if text == "" || text == nullValue {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
values := make([]string, 0, len(fields))
|
||||
seen := map[string]struct{}{}
|
||||
candidates := make([]string, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
value, ok := row[field]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
text := strings.TrimSpace(format2DAValue(value))
|
||||
if text == "" || text == nullValue {
|
||||
continue
|
||||
}
|
||||
normalized := normalizeConvertedKeyText(text)
|
||||
if normalized == "" {
|
||||
continue
|
||||
}
|
||||
values = append(values, normalized)
|
||||
joined := strings.Join(values, "_")
|
||||
if _, ok := seen[joined]; ok {
|
||||
continue
|
||||
}
|
||||
seen[joined] = struct{}{}
|
||||
candidates = append(candidates, joined)
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func normalizeConvertedKeyText(text string) string {
|
||||
text = strings.TrimSpace(strings.ToLower(text))
|
||||
text = convertKeyWhitespace.ReplaceAllString(text, "_")
|
||||
text = convertKeyCleaner.ReplaceAllString(text, "")
|
||||
text = strings.Trim(text, "_")
|
||||
return text
|
||||
}
|
||||
|
||||
type convertTemplateTable struct {
|
||||
Namespace string `json:"namespace" yaml:"namespace"`
|
||||
KeyFields []string `json:"key_fields" yaml:"key_fields"`
|
||||
}
|
||||
|
||||
type convertTemplateConfig struct {
|
||||
ModuleOutput convertModuleOutputConfig `json:"module_output" yaml:"module_output"`
|
||||
Tables map[string]convertTemplateTable `json:"tables" yaml:"tables"`
|
||||
}
|
||||
|
||||
type convertModuleOutputConfig struct {
|
||||
FilenamePrefixes map[string]string `json:"filename_prefixes" yaml:"filename_prefixes"`
|
||||
}
|
||||
|
||||
type convertContext struct {
|
||||
Project *project.Project
|
||||
Config convertTemplateConfig
|
||||
Template convertTemplateTable
|
||||
}
|
||||
|
||||
func resolveConvertContext(input string) (convertContext, error) {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return convertContext{}, err
|
||||
}
|
||||
root, err := project.FindRoot(cwd)
|
||||
if err != nil {
|
||||
return convertContext{}, nil
|
||||
}
|
||||
p, err := project.Load(root)
|
||||
if err != nil {
|
||||
return convertContext{}, err
|
||||
}
|
||||
if !p.HasTopData() {
|
||||
return convertContext{Project: p}, nil
|
||||
}
|
||||
config, err := loadConvertTemplateConfig(p)
|
||||
if err != nil {
|
||||
return convertContext{}, err
|
||||
}
|
||||
table, err := templateTableConfig(config, p, input)
|
||||
if err != nil {
|
||||
return convertContext{}, err
|
||||
}
|
||||
return convertContext{
|
||||
Project: p,
|
||||
Config: config,
|
||||
Template: table,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func loadTemplateTableConfig(p *project.Project, input string) (convertTemplateTable, error) {
|
||||
config, err := loadConvertTemplateConfig(p)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
return templateTableConfig(config, p, input)
|
||||
}
|
||||
|
||||
func templateTableConfig(config convertTemplateConfig, p *project.Project, input string) (convertTemplateTable, error) {
|
||||
if len(config.Tables) == 0 {
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
templatesDir := filepath.Join(p.TopDataSourceDir(), "templates")
|
||||
inputAbs, err := filepath.Abs(input)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
templatesAbs, err := filepath.Abs(templatesDir)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
rel, err := filepath.Rel(templatesAbs, inputAbs)
|
||||
if err != nil || strings.HasPrefix(rel, "..") || rel == "." {
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
|
||||
rel = filepath.ToSlash(rel)
|
||||
base := filepath.Base(rel)
|
||||
stem := strings.TrimSuffix(base, filepath.Ext(base))
|
||||
for _, key := range []string{rel, base, stem} {
|
||||
if table, ok := config.Tables[key]; ok {
|
||||
return table, nil
|
||||
}
|
||||
}
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
|
||||
func loadConvertTemplateConfig(p *project.Project) (convertTemplateConfig, error) {
|
||||
templatesDir := filepath.Join(p.TopDataSourceDir(), "templates")
|
||||
for _, candidate := range []string{"config.yaml", "config.yml", "config.json"} {
|
||||
configPath := filepath.Join(templatesDir, candidate)
|
||||
raw, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return convertTemplateConfig{}, fmt.Errorf("read %s: %w", configPath, err)
|
||||
}
|
||||
|
||||
var config convertTemplateConfig
|
||||
switch filepath.Ext(configPath) {
|
||||
case ".yaml", ".yml":
|
||||
if err := yaml.Unmarshal(raw, &config); err != nil {
|
||||
return convertTemplateConfig{}, fmt.Errorf("parse %s: %w", configPath, err)
|
||||
}
|
||||
default:
|
||||
if err := json.Unmarshal(raw, &config); err != nil {
|
||||
return convertTemplateConfig{}, fmt.Errorf("parse %s: %w", configPath, err)
|
||||
}
|
||||
}
|
||||
if err := validateConvertTemplateConfig(config); err != nil {
|
||||
return convertTemplateConfig{}, fmt.Errorf("validate %s: %w", configPath, err)
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
return convertTemplateConfig{}, nil
|
||||
}
|
||||
|
||||
func validateConvertTemplateConfig(config convertTemplateConfig) error {
|
||||
for moduleType, prefix := range config.ModuleOutput.FilenamePrefixes {
|
||||
switch moduleType {
|
||||
case "entries", "override":
|
||||
default:
|
||||
return fmt.Errorf("unsupported module_output.filename_prefixes key %q; expected entries or override", moduleType)
|
||||
}
|
||||
if normalizeModuleFilenamePrefix(prefix) == "" {
|
||||
return fmt.Errorf("module_output.filename_prefixes.%s must contain at least one filename-safe character", moduleType)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveConvertOutputContext(p *project.Project, output string) (convertTemplateTable, error) {
|
||||
if p == nil || !p.HasTopData() || strings.TrimSpace(output) == "" {
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
config, err := loadConvertTemplateConfig(p)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
if len(config.Tables) == 0 {
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
dataDir := filepath.Join(p.TopDataSourceDir(), "data")
|
||||
dataAbs, err := filepath.Abs(dataDir)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
outputAbs, err := filepath.Abs(output)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
rel, err := filepath.Rel(dataAbs, outputAbs)
|
||||
if err != nil || rel == "." || strings.HasPrefix(rel, "..") {
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
|
||||
bestMatch := ""
|
||||
bestTable := convertTemplateTable{}
|
||||
for _, table := range config.Tables {
|
||||
namespacePath := namespaceToPath(table.Namespace)
|
||||
if namespacePath == "" {
|
||||
continue
|
||||
}
|
||||
if rel != namespacePath && !strings.HasPrefix(rel, namespacePath+"/") {
|
||||
continue
|
||||
}
|
||||
if len(namespacePath) <= len(bestMatch) {
|
||||
continue
|
||||
}
|
||||
bestMatch = namespacePath
|
||||
bestTable = table
|
||||
}
|
||||
return bestTable, nil
|
||||
}
|
||||
|
||||
func namespaceToPath(namespace string) string {
|
||||
namespace = strings.TrimSpace(strings.ToLower(namespace))
|
||||
if namespace == "" {
|
||||
return ""
|
||||
}
|
||||
namespace = strings.ReplaceAll(namespace, "\\", "/")
|
||||
namespace = strings.ReplaceAll(namespace, ":", "/")
|
||||
namespace = convertKeyWhitespace.ReplaceAllString(namespace, "_")
|
||||
namespace = strings.Trim(namespace, "/")
|
||||
return namespace
|
||||
}
|
||||
|
||||
func defaultModuleOutputPath(input string, opts convertOptions, p *project.Project) (string, error) {
|
||||
if p == nil {
|
||||
return "", errors.New("2da-to-module requires an explicit output path when not run inside a project root")
|
||||
}
|
||||
if !p.HasTopData() {
|
||||
return "", errors.New("2da-to-module requires topdata to be configured when inferring an output path")
|
||||
}
|
||||
filename, err := defaultModuleFileName(input, opts, p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
modulesDir := filepath.Join(p.TopDataSourceDir(), "data", filepath.FromSlash(opts.Namespace), "modules")
|
||||
return filepath.Join(modulesDir, filename), nil
|
||||
}
|
||||
|
||||
func defaultModuleFileName(input string, opts convertOptions, p *project.Project) (string, error) {
|
||||
if strings.TrimSpace(opts.Name) != "" {
|
||||
name := normalizeConvertedKeyText(opts.Name)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("invalid --name %q", opts.Name)
|
||||
}
|
||||
namespace := normalizeModuleFilenameNamespace(opts.Namespace)
|
||||
prefix := moduleFilenamePrefix(opts)
|
||||
return fmt.Sprintf("%s_%s_%s.json", prefix, namespace, name), nil
|
||||
}
|
||||
if isTemplateInput(p, input) {
|
||||
return "", errors.New("2da-to-module requires --name when converting from topdata/templates without an explicit output path")
|
||||
}
|
||||
return strings.TrimSuffix(filepath.Base(input), filepath.Ext(input)) + ".json", nil
|
||||
}
|
||||
|
||||
func moduleFilenamePrefix(opts convertOptions) string {
|
||||
moduleType := strings.TrimSpace(opts.Type)
|
||||
if moduleType == "entry" {
|
||||
moduleType = "entries"
|
||||
}
|
||||
if moduleType == "overrides" {
|
||||
moduleType = "override"
|
||||
}
|
||||
if configured := strings.TrimSpace(opts.FilenamePrefixes[moduleType]); configured != "" {
|
||||
if prefix := normalizeModuleFilenamePrefix(configured); prefix != "" {
|
||||
return prefix
|
||||
}
|
||||
}
|
||||
return "add"
|
||||
}
|
||||
|
||||
func normalizeModuleFilenamePrefix(prefix string) string {
|
||||
prefix = strings.TrimSpace(strings.ToLower(prefix))
|
||||
prefix = strings.ReplaceAll(prefix, "/", "_")
|
||||
prefix = strings.ReplaceAll(prefix, "\\", "_")
|
||||
prefix = convertKeyWhitespace.ReplaceAllString(prefix, "_")
|
||||
prefix = convertKeyCleaner.ReplaceAllString(prefix, "_")
|
||||
prefix = strings.Trim(prefix, "_")
|
||||
return prefix
|
||||
}
|
||||
|
||||
func normalizeModuleFilenameNamespace(namespace string) string {
|
||||
namespace = strings.TrimSpace(strings.ToLower(namespace))
|
||||
namespace = strings.ReplaceAll(namespace, "/", "_")
|
||||
namespace = strings.ReplaceAll(namespace, "\\", "_")
|
||||
namespace = convertKeyWhitespace.ReplaceAllString(namespace, "_")
|
||||
namespace = convertKeyCleaner.ReplaceAllString(namespace, "_")
|
||||
namespace = strings.Trim(namespace, "_")
|
||||
if namespace == "" {
|
||||
return "module"
|
||||
}
|
||||
return namespace
|
||||
}
|
||||
|
||||
func isTemplateInput(p *project.Project, input string) bool {
|
||||
if p == nil || !p.HasTopData() {
|
||||
return false
|
||||
}
|
||||
templatesAbs, err := filepath.Abs(filepath.Join(p.TopDataSourceDir(), "templates"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
inputAbs, err := filepath.Abs(input)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
rel, err := filepath.Rel(templatesAbs, inputAbs)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel != "." && !strings.HasPrefix(rel, "..")
|
||||
}
|
||||
|
||||
func readCanonicalJSON(path string) (parsed2DA, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return parsed2DA{}, err
|
||||
}
|
||||
var payload struct {
|
||||
Columns []string `json:"columns"`
|
||||
Rows []map[string]any `json:"rows"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
return parsed2DA{}, err
|
||||
}
|
||||
return parsed2DA{Columns: payload.Columns, Rows: payload.Rows}, nil
|
||||
}
|
||||
|
||||
func writeJSONFile(path string, payload any) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." {
|
||||
return err
|
||||
}
|
||||
raw, err := json.MarshalIndent(payload, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
return os.WriteFile(path, raw, 0o644)
|
||||
}
|
||||
|
||||
func write2DAFile(data parsed2DA, path string) error {
|
||||
rows := make([]map[string]any, 0, len(data.Rows))
|
||||
for _, row := range data.Rows {
|
||||
cloned := cloneRowMap(row)
|
||||
if rowID, err := asInt(cloned["id"]); err == nil {
|
||||
cloned["id"] = rowID
|
||||
}
|
||||
rows = append(rows, cloned)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
left, _ := asInt(rows[i]["id"])
|
||||
right, _ := asInt(rows[j]["id"])
|
||||
return left < right
|
||||
})
|
||||
table := map[string]any{
|
||||
"columns": data.Columns,
|
||||
"rows": rows,
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." {
|
||||
return err
|
||||
}
|
||||
return write2DA(table, path, false, -1)
|
||||
}
|
||||
|
||||
func cloneStringMap(in map[string]string) map[string]string {
|
||||
out := make(map[string]string, len(in))
|
||||
for key, value := range in {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,611 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRunConvertCommandInfersTemplateNamespaceAndOutput(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test
|
||||
resref: test
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
topdata:
|
||||
source: topdata
|
||||
build: build/topdata
|
||||
`)
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "templates"))
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance", "modules"))
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), `
|
||||
tables:
|
||||
template_appearance:
|
||||
namespace: appearance
|
||||
key_fields:
|
||||
- LABEL
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "template_appearance.2da"), "2DA V2.0\n\nLABEL STRING_REF NAME\n960 \"Zombie, Ash, Hot\" **** Zombie_Ash_Hot\n961 \"Zombie, Ash, Done\" **** Zombie_Ash_Done\n")
|
||||
|
||||
oldCwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(oldCwd)
|
||||
})
|
||||
if err := os.Chdir(root); err != nil {
|
||||
t.Fatalf("chdir: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
if err := RunConvertCommand([]string{
|
||||
"2da-to-module",
|
||||
"--name", "ashzombies",
|
||||
"topdata/templates/template_appearance.2da",
|
||||
}, &stdout); err != nil {
|
||||
t.Fatalf("RunConvertCommand failed: %v", err)
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(root, "topdata", "data", "appearance", "modules", "add_appearance_ashzombies.json")
|
||||
raw, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
text := string(raw)
|
||||
if !strings.Contains(text, `"appearance:zombie_ash_hot"`) || !strings.Contains(text, `"appearance:zombie_ash_done"`) {
|
||||
t.Fatalf("unexpected output:\n%s", text)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "converted entries: 2") {
|
||||
t.Fatalf("unexpected stdout: %s", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConvertCommandInfersOverrideOutputPrefixFromTemplateConfig(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test
|
||||
resref: test
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
topdata:
|
||||
source: topdata
|
||||
build: build/topdata
|
||||
`)
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "templates"))
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), `
|
||||
module_output:
|
||||
filename_prefixes:
|
||||
entries: add
|
||||
override: ovr
|
||||
tables:
|
||||
template_appearance:
|
||||
namespace: appearance
|
||||
key_fields:
|
||||
- LABEL
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "template_appearance.2da"), "2DA V2.0\n\nLABEL STRING_REF NAME\n960 \"Zombie, Ash, Hot\" **** Zombie_Ash_Hot\n")
|
||||
|
||||
oldCwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(oldCwd)
|
||||
})
|
||||
if err := os.Chdir(root); err != nil {
|
||||
t.Fatalf("chdir: %v", err)
|
||||
}
|
||||
|
||||
var stdout bytes.Buffer
|
||||
if err := RunConvertCommand([]string{
|
||||
"2da-to-module",
|
||||
"--type", "override",
|
||||
"--name", "ashzombies",
|
||||
"topdata/templates/template_appearance.2da",
|
||||
}, &stdout); err != nil {
|
||||
t.Fatalf("RunConvertCommand failed: %v", err)
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(root, "topdata", "data", "appearance", "modules", "ovr_appearance_ashzombies.json")
|
||||
raw, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(raw), `"overrides"`) {
|
||||
t.Fatalf("expected override payload, got:\n%s", string(raw))
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "topdata", "data", "appearance", "modules", "add_appearance_ashzombies.json")); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected add-prefixed output to be absent, stat err: %v", err)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "converted overrides: 1") {
|
||||
t.Fatalf("unexpected stdout: %s", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConvertCommand2DAToJSONInfersTemplateKeys(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test
|
||||
resref: test
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
topdata:
|
||||
source: topdata
|
||||
build: build/topdata
|
||||
`)
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "templates"))
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), `
|
||||
tables:
|
||||
template_appearance:
|
||||
namespace: appearance
|
||||
key_fields:
|
||||
- LABEL
|
||||
`)
|
||||
inputPath := filepath.Join(root, "topdata", "templates", "template_appearance.2da")
|
||||
writeFile(t, inputPath, "2DA V2.0\n\nLABEL STRING_REF NAME\n960 \"Zombie, Ash, Hot\" **** Zombie_Ash_Hot\n961 \"Zombie, Ash, Done\" **** Zombie_Ash_Done\n")
|
||||
|
||||
oldCwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(oldCwd)
|
||||
})
|
||||
if err := os.Chdir(root); err != nil {
|
||||
t.Fatalf("chdir: %v", err)
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(root, "out", "appearance.json")
|
||||
if err := RunConvertCommand([]string{
|
||||
"2da-to-json",
|
||||
"topdata/templates/template_appearance.2da",
|
||||
outputPath,
|
||||
}, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("RunConvertCommand failed: %v", err)
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
var payload struct {
|
||||
Rows []map[string]any `json:"rows"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
t.Fatalf("unmarshal output: %v", err)
|
||||
}
|
||||
if got := payload.Rows[0]["key"]; got != "appearance:zombie_ash_hot" {
|
||||
t.Fatalf("expected first key to be inferred from template config, got %#v", got)
|
||||
}
|
||||
if got := payload.Rows[1]["key"]; got != "appearance:zombie_ash_done" {
|
||||
t.Fatalf("expected second key to be inferred from template config, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConvertCommand2DAToJSONInfersOutputDatasetKeysFromConfig(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test
|
||||
resref: test
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
topdata:
|
||||
source: topdata
|
||||
build: build/topdata
|
||||
`)
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "templates"))
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "data", "soundset"))
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), `
|
||||
tables:
|
||||
template_soundset:
|
||||
namespace: soundset
|
||||
key_fields:
|
||||
- RESREF
|
||||
- LABEL
|
||||
`)
|
||||
inputPath := filepath.Join(root, "scratch_soundset.2da")
|
||||
writeFile(t, inputPath, "2DA V2.0\n\nLABEL RESREF STRREF GENDER TYPE\n0 \"Bandit Male\" nw_bandit 100 1 4\n1 \"Bandit Female\" nw_bandit 101 2 4\n")
|
||||
|
||||
oldCwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(oldCwd)
|
||||
})
|
||||
if err := os.Chdir(root); err != nil {
|
||||
t.Fatalf("chdir: %v", err)
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(root, "topdata", "data", "soundset", "base.json")
|
||||
if err := RunConvertCommand([]string{
|
||||
"2da-to-json",
|
||||
"scratch_soundset.2da",
|
||||
outputPath,
|
||||
}, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("RunConvertCommand failed: %v", err)
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
var payload struct {
|
||||
Rows []map[string]any `json:"rows"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
t.Fatalf("unmarshal output: %v", err)
|
||||
}
|
||||
if got := payload.Rows[0]["key"]; got != "soundset:nw_bandit" {
|
||||
t.Fatalf("expected first soundset key, got %#v", got)
|
||||
}
|
||||
if got := payload.Rows[1]["key"]; got != "soundset:nw_bandit_bandit_female" {
|
||||
t.Fatalf("expected second soundset key to use config disambiguation, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConvertCommand2DAToJSONInfersAmbientTemplateKeysFromResource(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test
|
||||
resref: test
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
topdata:
|
||||
source: topdata
|
||||
build: build/topdata
|
||||
`)
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "templates"))
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), `
|
||||
tables:
|
||||
template_ambientmusic:
|
||||
namespace: ambientmusic
|
||||
key_fields:
|
||||
- Resource
|
||||
`)
|
||||
inputPath := filepath.Join(root, "topdata", "templates", "template_ambientmusic.2da")
|
||||
writeFile(t, inputPath, "2DA V2.0\n\nDescription DisplayName Resource Stinger1 Stinger2 Stinger3\n0 61901 **** **** **** **** ****\n1 61842 **** mus_ruralday1 **** **** ****\n2 61843 **** mus_ruralday2 **** **** ****\n")
|
||||
|
||||
oldCwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(oldCwd)
|
||||
})
|
||||
if err := os.Chdir(root); err != nil {
|
||||
t.Fatalf("chdir: %v", err)
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(root, "out", "ambientmusic.json")
|
||||
if err := RunConvertCommand([]string{
|
||||
"2da-to-json",
|
||||
"topdata/templates/template_ambientmusic.2da",
|
||||
outputPath,
|
||||
}, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("RunConvertCommand failed: %v", err)
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
var payload struct {
|
||||
Rows []map[string]any `json:"rows"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
t.Fatalf("unmarshal output: %v", err)
|
||||
}
|
||||
if _, ok := payload.Rows[0]["key"]; ok {
|
||||
t.Fatalf("expected null Resource row to remain unkeyed, got %#v", payload.Rows[0]["key"])
|
||||
}
|
||||
if got := payload.Rows[1]["key"]; got != "ambientmusic:mus_ruralday1" {
|
||||
t.Fatalf("expected first populated resource row key, got %#v", got)
|
||||
}
|
||||
if got := payload.Rows[2]["key"]; got != "ambientmusic:mus_ruralday2" {
|
||||
t.Fatalf("expected second populated resource row key, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConvertCommand2DAToJSONSuffixesDuplicateGeneratedKeys(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test
|
||||
resref: test
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
topdata:
|
||||
source: topdata
|
||||
build: build/topdata
|
||||
`)
|
||||
inputPath := filepath.Join(root, "dup.2da")
|
||||
writeFile(t, inputPath, "2DA V2.0\n\nResource\n1 cmp_reserved\n2 cmp_reserved\n")
|
||||
|
||||
oldCwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(oldCwd)
|
||||
})
|
||||
if err := os.Chdir(root); err != nil {
|
||||
t.Fatalf("chdir: %v", err)
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(root, "out", "ambientmusic.json")
|
||||
if err := RunConvertCommand([]string{
|
||||
"2da-to-json",
|
||||
"--namespace", "ambientmusic",
|
||||
"--key-field", "Resource",
|
||||
inputPath,
|
||||
outputPath,
|
||||
}, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("RunConvertCommand failed: %v", err)
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
var payload struct {
|
||||
Rows []map[string]any `json:"rows"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
t.Fatalf("unmarshal output: %v", err)
|
||||
}
|
||||
if got := payload.Rows[0]["key"]; got != "ambientmusic:cmp_reserved" {
|
||||
t.Fatalf("expected first duplicate key to remain unsuffixed, got %#v", got)
|
||||
}
|
||||
if got := payload.Rows[1]["key"]; got != "ambientmusic:cmp_reserved_2" {
|
||||
t.Fatalf("expected second duplicate key to gain row-id suffix, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConvertArgsSupportsEqualsSyntax(t *testing.T) {
|
||||
opts, input, output, err := parseConvertArgs([]string{
|
||||
"--namespace=environment",
|
||||
"--name=projectq",
|
||||
"--type=entries",
|
||||
"--collision=suffix",
|
||||
"--key-field=Label",
|
||||
"input.2da",
|
||||
"output.json",
|
||||
}, true, "error")
|
||||
if err != nil {
|
||||
t.Fatalf("parseConvertArgs failed: %v", err)
|
||||
}
|
||||
if opts.Namespace != "environment" {
|
||||
t.Fatalf("expected namespace from equals syntax, got %q", opts.Namespace)
|
||||
}
|
||||
if opts.Name != "projectq" {
|
||||
t.Fatalf("expected name from equals syntax, got %q", opts.Name)
|
||||
}
|
||||
if opts.Type != "entries" {
|
||||
t.Fatalf("expected type from equals syntax, got %q", opts.Type)
|
||||
}
|
||||
if opts.CollisionMode != "suffix" {
|
||||
t.Fatalf("expected collision from equals syntax, got %q", opts.CollisionMode)
|
||||
}
|
||||
if len(opts.KeyFields) != 1 || opts.KeyFields[0] != "Label" {
|
||||
t.Fatalf("expected key field from equals syntax, got %#v", opts.KeyFields)
|
||||
}
|
||||
if input != "input.2da" || output != "output.json" {
|
||||
t.Fatalf("unexpected positional parse result: input=%q output=%q", input, output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvert2DAToJSONSkipsConfiguredKeysWhenAnyKeyFieldIsNull(t *testing.T) {
|
||||
result, err := convert2DAToJSON(parsed2DA{
|
||||
Columns: []string{"LABEL", "RESREF", "STRREF"},
|
||||
Rows: []map[string]any{
|
||||
{"id": 309, "LABEL": "Unused", "RESREF": "unused", "STRREF": 1},
|
||||
{"id": 310, "LABEL": nullValue, "RESREF": "unused", "STRREF": 2},
|
||||
},
|
||||
}, "topdata/data/soundset/base.json", convertOptions{
|
||||
Namespace: "soundset",
|
||||
KeyFields: []string{"RESREF", "LABEL"},
|
||||
CollisionMode: "error",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("convert2DAToJSON failed: %v", err)
|
||||
}
|
||||
|
||||
rows := result["rows"].([]map[string]any)
|
||||
if got := rows[0]["key"]; got != "soundset:unused" {
|
||||
t.Fatalf("expected first row key to use the primary configured key field, got %#v", got)
|
||||
}
|
||||
if _, ok := rows[1]["key"]; ok {
|
||||
t.Fatalf("expected second row to remain unkeyed when one configured key field is null, got %#v", rows[1]["key"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConvertCommandFailsOnUnkeyedRows(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.json"), `{
|
||||
"module": {"name": "Test", "resref": "test"},
|
||||
"paths": {"source": "src", "assets": "assets", "build": "build"},
|
||||
"topdata": {"source": "topdata", "build": "build/topdata"}
|
||||
}`+"\n")
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "templates"))
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), `
|
||||
tables:
|
||||
template_appearance:
|
||||
namespace: appearance
|
||||
key_fields:
|
||||
- STRING_REF
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "template_appearance.2da"), "2DA V2.0\n\nLABEL STRING_REF NAME\n960 \"Zombie, Ash, Hot\" **** Zombie_Ash_Hot\n")
|
||||
|
||||
oldCwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(oldCwd)
|
||||
})
|
||||
if err := os.Chdir(root); err != nil {
|
||||
t.Fatalf("chdir: %v", err)
|
||||
}
|
||||
|
||||
err = RunConvertCommand([]string{
|
||||
"2da-to-module",
|
||||
"--name", "ashzombies",
|
||||
"topdata/templates/template_appearance.2da",
|
||||
}, &bytes.Buffer{})
|
||||
if err == nil || !strings.Contains(err.Error(), "unable to generate keys for row ids 960") {
|
||||
t.Fatalf("expected unkeyed row failure, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConvertCommandFallsBackToLegacyTemplateJSONConfig(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test
|
||||
resref: test
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
topdata:
|
||||
source: topdata
|
||||
build: build/topdata
|
||||
`)
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "templates"))
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "config.json"), `{
|
||||
"tables": {
|
||||
"template_appearance": {
|
||||
"namespace": "appearance",
|
||||
"key_fields": ["LABEL"]
|
||||
}
|
||||
}
|
||||
}`+"\n")
|
||||
inputPath := filepath.Join(root, "topdata", "templates", "template_appearance.2da")
|
||||
writeFile(t, inputPath, "2DA V2.0\n\nLABEL\n960 Zombie_Ash_Hot\n")
|
||||
|
||||
oldCwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(oldCwd)
|
||||
})
|
||||
if err := os.Chdir(root); err != nil {
|
||||
t.Fatalf("chdir: %v", err)
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(root, "out", "appearance.json")
|
||||
if err := RunConvertCommand([]string{
|
||||
"2da-to-json",
|
||||
"topdata/templates/template_appearance.2da",
|
||||
outputPath,
|
||||
}, &bytes.Buffer{}); err != nil {
|
||||
t.Fatalf("RunConvertCommand failed: %v", err)
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(outputPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(raw), `"appearance:zombie_ash_hot"`) {
|
||||
t.Fatalf("expected legacy JSON template config fallback to be honored, got %s", string(raw))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConvertCommandRejectsUnknownTemplateFilenamePrefixType(t *testing.T) {
|
||||
root := testProjectRoot(t)
|
||||
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||
module:
|
||||
name: Test
|
||||
resref: test
|
||||
paths:
|
||||
source: src
|
||||
assets: assets
|
||||
build: build
|
||||
topdata:
|
||||
source: topdata
|
||||
build: build/topdata
|
||||
`)
|
||||
mkdirAll(t, filepath.Join(root, "topdata", "templates"))
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "config.yaml"), `
|
||||
module_output:
|
||||
filename_prefixes:
|
||||
additions: add
|
||||
tables:
|
||||
template_appearance:
|
||||
namespace: appearance
|
||||
key_fields:
|
||||
- LABEL
|
||||
`)
|
||||
writeFile(t, filepath.Join(root, "topdata", "templates", "template_appearance.2da"), "2DA V2.0\n\nLABEL\n960 Zombie_Ash_Hot\n")
|
||||
|
||||
oldCwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(oldCwd)
|
||||
})
|
||||
if err := os.Chdir(root); err != nil {
|
||||
t.Fatalf("chdir: %v", err)
|
||||
}
|
||||
|
||||
err = RunConvertCommand([]string{
|
||||
"2da-to-module",
|
||||
"--name", "ashzombies",
|
||||
"topdata/templates/template_appearance.2da",
|
||||
}, &bytes.Buffer{})
|
||||
if err == nil || !strings.Contains(err.Error(), `unsupported module_output.filename_prefixes key "additions"`) {
|
||||
t.Fatalf("expected invalid filename prefix type error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvert2DAToModuleFailsOnDuplicateGeneratedKeysByDefault(t *testing.T) {
|
||||
_, err := convert2DAToModule(parsed2DA{
|
||||
Columns: []string{"LABEL"},
|
||||
Rows: []map[string]any{
|
||||
{"id": 1, "LABEL": "Zombie"},
|
||||
{"id": 2, "LABEL": "Zombie"},
|
||||
},
|
||||
}, "topdata/data/appearance/modules/add_appearance_ashzombies.json", convertOptions{
|
||||
Namespace: "appearance",
|
||||
CollisionMode: "error",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), `duplicate generated key "appearance:zombie"`) {
|
||||
t.Fatalf("expected duplicate generated key error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParse2DAFileRejectsRowsWithTooManyFields(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
inputPath := filepath.Join(root, "bad.2da")
|
||||
writeFile(t, inputPath, "2DA V2.0\n\nLABEL NAME\n0 one two three\n")
|
||||
|
||||
_, err := parse2DAFile(inputPath)
|
||||
if err == nil || !strings.Contains(err.Error(), "has 3 values but only 2 columns are declared") {
|
||||
t.Fatalf("expected row width error, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func importLegacyCreaturespeed(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
_ = legacyTLK
|
||||
|
||||
legacyDir := filepath.Join(referenceBuilderDir, "data", "creaturespeed")
|
||||
if _, err := os.Stat(legacyDir); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
targetDir := filepath.Join(dataDir, "creaturespeed")
|
||||
targetPath := filepath.Join(targetDir, "creaturespeed.json")
|
||||
if _, err := os.Stat(targetPath); err == nil {
|
||||
obj, err := loadJSONObject(targetPath)
|
||||
if err == nil && countLegacyTLKRefsInValue(obj) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := mergeCreaturespeedRows(baseObj, filepath.Join(legacyDir, "modules"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := removePathIfExists(filepath.Join(targetDir, "base.json")); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := removePathIfExists(filepath.Join(targetDir, "lock.json")); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := removePathIfExists(filepath.Join(targetDir, "modules")); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
out := map[string]any{
|
||||
"output": "creaturespeed.2da",
|
||||
"columns": baseObj["columns"],
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
raw, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(targetPath, raw, 0o644); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func mergeCreaturespeedRows(baseObj map[string]any, modulesDir string) ([]any, error) {
|
||||
rawRows, ok := baseObj["rows"].([]any)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
rows := make([]map[string]any, 0, len(rawRows))
|
||||
byID := map[int]map[string]any{}
|
||||
for _, raw := range rawRows {
|
||||
row, ok := deepCopyValue(raw).(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
delete(row, "key")
|
||||
rows = append(rows, row)
|
||||
if rawID, ok := row["id"]; ok {
|
||||
id, err := asInt(rawID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byID[id] = row
|
||||
}
|
||||
}
|
||||
|
||||
modulePaths, err := collectModulePaths(modulesDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, path := range modulePaths {
|
||||
obj, err := loadJSONObject(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
overrideList, ok := obj["overrides"].([]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, raw := range overrideList {
|
||||
override, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rawID, ok := override["id"]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
id, err := asInt(rawID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row := byID[id]
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
for key, value := range override {
|
||||
if key == "id" || key == "key" || key == "_tlk" {
|
||||
continue
|
||||
}
|
||||
row[key] = deepCopyValue(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]any, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func removePathIfExists(path string) error {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
@@ -1,288 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func importLegacyDamagetypes(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
_ = legacyTLK
|
||||
|
||||
legacyRoot := filepath.Join(referenceBuilderDir, "data", "damagetypes")
|
||||
if _, err := os.Stat(legacyRoot); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
targetDir := filepath.Join(dataDir, "damagetypes", "registry")
|
||||
targetPath := filepath.Join(targetDir, "types.json")
|
||||
if _, err := os.Stat(targetPath); err == nil {
|
||||
obj, err := loadJSONObject(targetPath)
|
||||
if err == nil && countLegacyTLKRefsInValue(obj) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
coreBase, err := loadJSONObject(filepath.Join(legacyRoot, "core", "base.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
groupBase, err := loadJSONObject(filepath.Join(legacyRoot, "groups", "base.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
hitvisualBase, err := loadJSONObject(filepath.Join(legacyRoot, "hitvisual", "base.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
coreModule, err := loadJSONObject(filepath.Join(legacyRoot, "core", "modules", "add_eos.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
coreLock, err := loadLockfile(filepath.Join(legacyRoot, "core", "lock.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
groupModule, err := loadJSONObject(filepath.Join(legacyRoot, "groups", "modules", "add_eos.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
groupLock, err := loadLockfile(filepath.Join(legacyRoot, "groups", "lock.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
hitvisualModule, err := loadJSONObject(filepath.Join(legacyRoot, "hitvisual", "modules", "add_eos.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
hitvisualLock, err := loadLockfile(filepath.Join(legacyRoot, "hitvisual", "lock.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
rows, lockData, err := mergeLegacyDamagetypes(coreBase, groupBase, hitvisualBase, coreModule, coreLock, groupModule, groupLock, hitvisualModule, hitvisualLock)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := removePathIfExists(filepath.Join(dataDir, "damagetypes", "core")); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := removePathIfExists(filepath.Join(dataDir, "damagetypes", "groups")); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := removePathIfExists(filepath.Join(dataDir, "damagetypes", "hitvisual")); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
typesOut := map[string]any{"rows": rows}
|
||||
raw, err := json.MarshalIndent(typesOut, "", " ")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(targetPath, raw, 0o644); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := saveLockfile(filepath.Join(targetDir, damagetypesRegistryLock), lockData); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 2, nil
|
||||
}
|
||||
|
||||
func mergeLegacyDamagetypes(coreBase, groupBase, hitvisualBase, coreModule map[string]any, coreLock map[string]int, groupModule map[string]any, groupLock map[string]int, hitvisualModule map[string]any, hitvisualLock map[string]int) ([]map[string]any, map[string]int, error) {
|
||||
coreRows, err := coerceRowMapSlice(coreBase["rows"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
groupRows, err := coerceRowMapSlice(groupBase["rows"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
hitRows, err := coerceRowMapSlice(hitvisualBase["rows"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
groupByID := map[int]map[string]any{}
|
||||
for _, row := range groupRows {
|
||||
id, err := asInt(row["id"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
groupByID[id] = row
|
||||
}
|
||||
hitByID := map[int]map[string]any{}
|
||||
for _, row := range hitRows {
|
||||
id, err := asInt(row["id"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
hitByID[id] = row
|
||||
}
|
||||
|
||||
outRows := make([]map[string]any, 0, len(coreRows))
|
||||
lockData := map[string]int{}
|
||||
for _, core := range coreRows {
|
||||
id, err := asInt(core["id"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
groupID, err := asInt(core["DamageTypeGroup"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
group := groupByID[groupID]
|
||||
if group == nil {
|
||||
return nil, nil, fmt.Errorf("core id %d references missing group %d", id, groupID)
|
||||
}
|
||||
hit := hitByID[id]
|
||||
if hit == nil {
|
||||
return nil, nil, fmt.Errorf("core id %d is missing hitvisual row", id)
|
||||
}
|
||||
key := "damagetype:" + normalizeDamagetypeKey(core["Label"])
|
||||
lockData[key] = id
|
||||
outRows = append(outRows, map[string]any{
|
||||
"key": key,
|
||||
"id": id,
|
||||
"label": deepCopyValue(core["Label"]),
|
||||
"charsheet_strref": deepCopyValue(core["CharsheetStrref"]),
|
||||
"damage_type_group": strconvString(groupID),
|
||||
"damage_ranged_projectile": deepCopyValue(core["DamageRangedProjectile"]),
|
||||
"group_label": deepCopyValue(group["Label"]),
|
||||
"feedback_strref": deepCopyValue(group["FeedbackStrref"]),
|
||||
"color_r": deepCopyValue(group["ColorR"]),
|
||||
"color_g": deepCopyValue(group["ColorG"]),
|
||||
"color_b": deepCopyValue(group["ColorB"]),
|
||||
"visual_effect_id": deepCopyValue(hit["VisualEffectID"]),
|
||||
"ranged_effect_id": deepCopyValue(hit["RangedEffectID"]),
|
||||
})
|
||||
}
|
||||
|
||||
coreEntries, err := coerceEntriesMap(coreModule["entries"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
groupEntries, err := coerceEntriesMap(groupModule["entries"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
hitEntries, err := coerceEntriesMap(hitvisualModule["entries"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(coreEntries))
|
||||
for key := range coreEntries {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
a := strings.TrimPrefix(keys[i], "damagetypes:")
|
||||
b := strings.TrimPrefix(keys[j], "damagetypes:")
|
||||
return a < b
|
||||
})
|
||||
for _, coreKey := range keys {
|
||||
core := coreEntries[coreKey]
|
||||
suffix := strings.TrimPrefix(coreKey, "damagetypes:")
|
||||
group := groupEntries["damagetypegroups:"+suffix]
|
||||
if group == nil {
|
||||
return nil, nil, fmt.Errorf("%s: missing matching group entry", coreKey)
|
||||
}
|
||||
hit := hitEntries["damagehitvisual:"+suffix]
|
||||
if hit == nil {
|
||||
return nil, nil, fmt.Errorf("%s: missing matching hitvisual entry", coreKey)
|
||||
}
|
||||
id, ok := coreLock[coreKey]
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("%s: missing core lock id", coreKey)
|
||||
}
|
||||
groupKey := "damagetypegroups:" + suffix
|
||||
groupID, ok := groupLock[groupKey]
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("%s: missing group lock id", groupKey)
|
||||
}
|
||||
hitKey := "damagehitvisual:" + suffix
|
||||
hitID, ok := hitvisualLock[hitKey]
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("%s: missing hitvisual lock id", hitKey)
|
||||
}
|
||||
if hitID != id {
|
||||
return nil, nil, fmt.Errorf("%s: core id %d does not match hitvisual id %d", suffix, id, hitID)
|
||||
}
|
||||
key := "damagetype:" + suffix
|
||||
lockData[key] = id
|
||||
outRows = append(outRows, map[string]any{
|
||||
"key": key,
|
||||
"id": id,
|
||||
"label": deepCopyValue(core["Label"]),
|
||||
"charsheet_strref": deepCopyValue(core["CharsheetStrref"]),
|
||||
"damage_type_group": strconvString(groupID),
|
||||
"damage_ranged_projectile": deepCopyValue(core["DamageRangedProjectile"]),
|
||||
"group_label": deepCopyValue(group["Label"]),
|
||||
"feedback_strref": deepCopyValue(group["FeedbackStrref"]),
|
||||
"color_r": deepCopyValue(group["ColorR"]),
|
||||
"color_g": deepCopyValue(group["ColorG"]),
|
||||
"color_b": deepCopyValue(group["ColorB"]),
|
||||
"visual_effect_id": deepCopyValue(hit["VisualEffectID"]),
|
||||
"ranged_effect_id": deepCopyValue(hit["RangedEffectID"]),
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(outRows, func(i, j int) bool {
|
||||
return outRows[i]["id"].(int) < outRows[j]["id"].(int)
|
||||
})
|
||||
return outRows, lockData, nil
|
||||
}
|
||||
|
||||
func coerceRowMapSlice(value any) ([]map[string]any, error) {
|
||||
rawRows, ok := value.([]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("rows must be an array")
|
||||
}
|
||||
rows := make([]map[string]any, 0, len(rawRows))
|
||||
for _, raw := range rawRows {
|
||||
row, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("row must be an object")
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func coerceEntriesMap(value any) (map[string]map[string]any, error) {
|
||||
rawEntries, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("entries must be an object")
|
||||
}
|
||||
entries := make(map[string]map[string]any, len(rawEntries))
|
||||
for key, raw := range rawEntries {
|
||||
row, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("entry %s must be an object", key)
|
||||
}
|
||||
entries[key] = row
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func normalizeDamagetypeKey(value any) string {
|
||||
text := strings.TrimSpace(strings.ToLower(format2DAValue(value)))
|
||||
text = strings.TrimSuffix(text, "damage")
|
||||
text = strings.ReplaceAll(text, " ", "")
|
||||
text = strings.ReplaceAll(text, "_", "")
|
||||
text = strings.ReplaceAll(text, "-", "")
|
||||
return text
|
||||
}
|
||||
|
||||
func strconvString(v int) string {
|
||||
return fmt.Sprintf("%d", v)
|
||||
}
|
||||
@@ -1,243 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
damagetypesRegistryDirName = "damagetypes/registry"
|
||||
damagetypesRegistryLock = "lock.json"
|
||||
)
|
||||
|
||||
type damagetypesRegistry struct {
|
||||
RootPath string
|
||||
LockPath string
|
||||
LockData map[string]int
|
||||
Types []map[string]any
|
||||
}
|
||||
|
||||
func collectGeneratedRegistryDatasets(dataDir string) ([]nativeCollectedDataset, error) {
|
||||
itempropsDatasets, err := collectItempropsRegistryDatasets(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
damagetypeDatasets, err := collectDamagetypesRegistryDatasets(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
racialtypesDatasets, err := collectRacialtypesRegistryDatasets(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]nativeCollectedDataset, 0, len(itempropsDatasets)+len(damagetypeDatasets)+len(racialtypesDatasets))
|
||||
out = append(out, itempropsDatasets...)
|
||||
out = append(out, damagetypeDatasets...)
|
||||
out = append(out, racialtypesDatasets...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func loadDamagetypesRegistry(dataDir string) (*damagetypesRegistry, error) {
|
||||
root := filepath.Join(dataDir, filepath.FromSlash(damagetypesRegistryDirName))
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("damagetypes registry root must be a directory: %s", root)
|
||||
}
|
||||
|
||||
lockData, err := loadLockfile(filepath.Join(root, damagetypesRegistryLock))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
types, err := loadRegistryRows(filepath.Join(root, "types.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &damagetypesRegistry{
|
||||
RootPath: root,
|
||||
LockPath: filepath.Join(root, damagetypesRegistryLock),
|
||||
LockData: lockData,
|
||||
Types: types,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func collectDamagetypesRegistryDatasets(dataDir string) ([]nativeCollectedDataset, error) {
|
||||
registry, err := loadDamagetypesRegistry(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if registry == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
typeIDs, lockModified, err := assignDamagetypeRegistryIDs(registry.Types, registry.LockData)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lockModified {
|
||||
if err := saveLockfile(registry.LockPath, registry.LockData); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
typeRowsSorted := slices.Clone(registry.Types)
|
||||
slices.SortFunc(typeRowsSorted, func(a, b map[string]any) int {
|
||||
return typeIDs[stringField(a, "key")] - typeIDs[stringField(b, "key")]
|
||||
})
|
||||
|
||||
coreRows := make([]map[string]any, 0, len(typeRowsSorted))
|
||||
hitvisualRows := make([]map[string]any, 0, len(typeRowsSorted))
|
||||
groupRowsByID := map[int]map[string]any{}
|
||||
groupOrder := make([]int, 0)
|
||||
for _, row := range typeRowsSorted {
|
||||
key := stringField(row, "key")
|
||||
rowID := typeIDs[key]
|
||||
|
||||
groupID, err := registryIntField(row, "damage_type_group")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
coreRows = append(coreRows, map[string]any{
|
||||
"id": rowID,
|
||||
"key": key,
|
||||
"Label": registryStringField(row, "label"),
|
||||
"CharsheetStrref": deepCopyValue(row["charsheet_strref"]),
|
||||
"DamageTypeGroup": strconv.Itoa(groupID),
|
||||
"DamageRangedProjectile": registryNullableField(row, "damage_ranged_projectile"),
|
||||
})
|
||||
hitvisualRows = append(hitvisualRows, map[string]any{
|
||||
"id": rowID,
|
||||
"key": strings.Replace(key, "damagetype:", "damagehitvisual:", 1),
|
||||
"Label": registryStringField(row, "label"),
|
||||
"VisualEffectID": registryNullableField(row, "visual_effect_id"),
|
||||
"RangedEffectID": registryNullableField(row, "ranged_effect_id"),
|
||||
})
|
||||
|
||||
if _, ok := groupRowsByID[groupID]; !ok {
|
||||
groupRowsByID[groupID] = map[string]any{
|
||||
"id": groupID,
|
||||
"key": strings.Replace(key, "damagetype:", "damagetypegroup:", 1),
|
||||
"Label": registryStringField(row, "group_label"),
|
||||
"FeedbackStrref": deepCopyValue(row["feedback_strref"]),
|
||||
"ColorR": registryNullableField(row, "color_r"),
|
||||
"ColorG": registryNullableField(row, "color_g"),
|
||||
"ColorB": registryNullableField(row, "color_b"),
|
||||
}
|
||||
groupOrder = append(groupOrder, groupID)
|
||||
continue
|
||||
}
|
||||
existing := groupRowsByID[groupID]
|
||||
for field, candidate := range map[string]any{
|
||||
"Label": registryStringField(row, "group_label"),
|
||||
"FeedbackStrref": deepCopyValue(row["feedback_strref"]),
|
||||
"ColorR": registryNullableField(row, "color_r"),
|
||||
"ColorG": registryNullableField(row, "color_g"),
|
||||
"ColorB": registryNullableField(row, "color_b"),
|
||||
} {
|
||||
if format2DAValue(existing[field]) != format2DAValue(candidate) {
|
||||
return nil, fmt.Errorf("%s: conflicting group projection for DamageTypeGroup %d field %s", key, groupID, field)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.Sort(groupOrder)
|
||||
groupRows := make([]map[string]any, 0, len(groupOrder))
|
||||
for _, id := range groupOrder {
|
||||
groupRows = append(groupRows, groupRowsByID[id])
|
||||
}
|
||||
|
||||
return []nativeCollectedDataset{
|
||||
newGeneratedDataset("damagetypes/registry/core", "damagetypes.2da", []string{"Label", "CharsheetStrref", "DamageTypeGroup", "DamageRangedProjectile"}, coreRows),
|
||||
newGeneratedDataset("damagetypes/registry/groups", "damagetypegroups.2da", []string{"Label", "FeedbackStrref", "ColorR", "ColorG", "ColorB"}, groupRows),
|
||||
newGeneratedDataset("damagetypes/registry/hitvisual", "damagehitvisual.2da", []string{"Label", "VisualEffectID", "RangedEffectID"}, hitvisualRows),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func assignDamagetypeRegistryIDs(rows []map[string]any, lockData map[string]int) (map[string]int, bool, error) {
|
||||
ids := map[string]int{}
|
||||
used := map[int]struct{}{}
|
||||
for key, id := range lockData {
|
||||
if strings.HasPrefix(key, "damagetype:") {
|
||||
used[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
modified := false
|
||||
for _, row := range rows {
|
||||
key := stringField(row, "key")
|
||||
if key == "" {
|
||||
return nil, false, fmt.Errorf("registry row is missing key")
|
||||
}
|
||||
if !strings.HasPrefix(key, "damagetype:") {
|
||||
return nil, false, fmt.Errorf("registry key %q must use prefix %q", key, "damagetype:")
|
||||
}
|
||||
if rawID, ok := row["id"]; ok {
|
||||
id, err := asInt(rawID)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s: invalid id: %w", key, err)
|
||||
}
|
||||
if existing, ok := lockData[key]; ok && existing != id {
|
||||
return nil, false, fmt.Errorf("%s: explicit id %d does not match lock id %d", key, id, existing)
|
||||
}
|
||||
lockData[key] = id
|
||||
ids[key] = id
|
||||
used[id] = struct{}{}
|
||||
modified = true
|
||||
continue
|
||||
}
|
||||
if id, ok := lockData[key]; ok {
|
||||
ids[key] = id
|
||||
used[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
nextID := nextAvailableID(used)
|
||||
for _, row := range rows {
|
||||
key := stringField(row, "key")
|
||||
if _, ok := ids[key]; ok {
|
||||
continue
|
||||
}
|
||||
lockData[key] = nextID
|
||||
ids[key] = nextID
|
||||
used[nextID] = struct{}{}
|
||||
nextID = nextAvailableID(used)
|
||||
modified = true
|
||||
}
|
||||
return ids, modified, nil
|
||||
}
|
||||
|
||||
func registryStringField(row map[string]any, field string) string {
|
||||
text := stringField(row, field)
|
||||
if text == "" {
|
||||
return nullValue
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func registryNullableField(row map[string]any, field string) any {
|
||||
value, ok := row[field]
|
||||
if !ok {
|
||||
return nullValue
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
if strings.TrimSpace(typed) == "" {
|
||||
return nullValue
|
||||
}
|
||||
}
|
||||
return deepCopyValue(value)
|
||||
}
|
||||
|
||||
func registryIntField(row map[string]any, field string) (int, error) {
|
||||
value, ok := row[field]
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("missing %s", field)
|
||||
}
|
||||
return asInt(value)
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func importLegacyDoortypes(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
_ = legacyTLK
|
||||
|
||||
legacyDir := filepath.Join(referenceBuilderDir, "data", "doortypes")
|
||||
if _, err := os.Stat(legacyDir); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
targetDir := filepath.Join(dataDir, "doortypes")
|
||||
targetBasePath := filepath.Join(targetDir, "base.json")
|
||||
targetLockPath := filepath.Join(targetDir, "lock.json")
|
||||
targetModulesDir := filepath.Join(targetDir, "modules")
|
||||
moduleNames := []string{
|
||||
"add_pld01.json",
|
||||
"add_sic11.json",
|
||||
"add_tapr.json",
|
||||
"add_tdm01.json",
|
||||
"add_tdx01.json",
|
||||
"add_tei01.json",
|
||||
"add_tfm01.json",
|
||||
}
|
||||
targetModulePaths := make([]string, 0, len(moduleNames))
|
||||
for _, name := range moduleNames {
|
||||
targetModulePaths = append(targetModulePaths, filepath.Join(targetModulesDir, name))
|
||||
}
|
||||
if fileExists(targetBasePath) && fileExists(targetLockPath) {
|
||||
allModulesPresent := true
|
||||
for _, path := range targetModulePaths {
|
||||
if !fileExists(path) {
|
||||
allModulesPresent = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allModulesPresent {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
baseObj["output"] = "doortypes.2da"
|
||||
|
||||
lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
moduleObjs := make([]map[string]any, 0, len(moduleNames))
|
||||
for _, name := range moduleNames {
|
||||
obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", name))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
moduleObjs = append(moduleObjs, obj)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(targetModulesDir, 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
writes := []struct {
|
||||
path string
|
||||
obj map[string]any
|
||||
}{
|
||||
{path: targetBasePath, obj: baseObj},
|
||||
{path: targetLockPath, obj: lockObj},
|
||||
}
|
||||
for i, path := range targetModulePaths {
|
||||
writes = append(writes, struct {
|
||||
path string
|
||||
obj map[string]any
|
||||
}{path: path, obj: moduleObjs[i]})
|
||||
}
|
||||
|
||||
for _, write := range writes {
|
||||
raw, err := json.MarshalIndent(write.obj, "", " ")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(write.path, raw, 0o644); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(writes), nil
|
||||
}
|
||||
@@ -1,246 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type jsonFileFormatting struct {
|
||||
Indent string
|
||||
LineEnding string
|
||||
FinalNewline bool
|
||||
}
|
||||
|
||||
type editorConfigDocument struct {
|
||||
Dir string
|
||||
Root bool
|
||||
Sections []editorConfigSection
|
||||
}
|
||||
|
||||
type editorConfigSection struct {
|
||||
Pattern string
|
||||
Properties map[string]string
|
||||
}
|
||||
|
||||
func defaultJSONFileFormatting() jsonFileFormatting {
|
||||
return jsonFileFormatting{
|
||||
Indent: " ",
|
||||
LineEnding: "\n",
|
||||
FinalNewline: true,
|
||||
}
|
||||
}
|
||||
|
||||
func resolveJSONFileFormatting(path string) (jsonFileFormatting, error) {
|
||||
formatting := defaultJSONFileFormatting()
|
||||
documents, err := editorConfigDocumentsForPath(path)
|
||||
if err != nil {
|
||||
return jsonFileFormatting{}, err
|
||||
}
|
||||
for _, document := range documents {
|
||||
relative, err := filepath.Rel(document.Dir, path)
|
||||
if err != nil {
|
||||
return jsonFileFormatting{}, fmt.Errorf("resolve editorconfig path for %s relative to %s: %w", path, document.Dir, err)
|
||||
}
|
||||
relative = filepath.ToSlash(relative)
|
||||
if strings.HasPrefix(relative, "../") || relative == ".." {
|
||||
continue
|
||||
}
|
||||
for _, section := range document.Sections {
|
||||
matches, err := editorConfigPatternMatches(section.Pattern, relative)
|
||||
if err != nil {
|
||||
return jsonFileFormatting{}, fmt.Errorf("%s/.editorconfig section [%s]: %w", document.Dir, section.Pattern, err)
|
||||
}
|
||||
if !matches {
|
||||
continue
|
||||
}
|
||||
next, err := applyEditorConfigProperties(formatting, section.Properties)
|
||||
if err != nil {
|
||||
return jsonFileFormatting{}, fmt.Errorf("%s/.editorconfig section [%s]: %w", document.Dir, section.Pattern, err)
|
||||
}
|
||||
formatting = next
|
||||
}
|
||||
}
|
||||
return formatting, nil
|
||||
}
|
||||
|
||||
func editorConfigDocumentsForPath(path string) ([]editorConfigDocument, error) {
|
||||
dir := filepath.Dir(path)
|
||||
documents := []editorConfigDocument{}
|
||||
for {
|
||||
configPath := filepath.Join(dir, ".editorconfig")
|
||||
raw, err := os.ReadFile(configPath)
|
||||
if err == nil {
|
||||
document, err := parseEditorConfig(configPath, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
document.Dir = dir
|
||||
documents = append(documents, document)
|
||||
if document.Root {
|
||||
break
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("read %s: %w", configPath, err)
|
||||
}
|
||||
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
break
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
for left, right := 0, len(documents)-1; left < right; left, right = left+1, right-1 {
|
||||
documents[left], documents[right] = documents[right], documents[left]
|
||||
}
|
||||
return documents, nil
|
||||
}
|
||||
|
||||
func parseEditorConfig(path string, raw []byte) (editorConfigDocument, error) {
|
||||
document := editorConfigDocument{}
|
||||
scanner := bufio.NewScanner(bytes.NewReader(raw))
|
||||
var current *editorConfigSection
|
||||
for lineNumber := 1; scanner.Scan(); lineNumber++ {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "[") {
|
||||
end := strings.LastIndex(line, "]")
|
||||
if end < 0 {
|
||||
return editorConfigDocument{}, fmt.Errorf("%s:%d: malformed section header", path, lineNumber)
|
||||
}
|
||||
section := editorConfigSection{
|
||||
Pattern: strings.TrimSpace(line[1:end]),
|
||||
Properties: map[string]string{},
|
||||
}
|
||||
document.Sections = append(document.Sections, section)
|
||||
current = &document.Sections[len(document.Sections)-1]
|
||||
continue
|
||||
}
|
||||
|
||||
index := strings.IndexAny(line, "=:")
|
||||
if index < 0 {
|
||||
return editorConfigDocument{}, fmt.Errorf("%s:%d: expected key/value property", path, lineNumber)
|
||||
}
|
||||
key := strings.ToLower(strings.TrimSpace(line[:index]))
|
||||
value := strings.TrimSpace(line[index+1:])
|
||||
if current == nil {
|
||||
if key == "root" {
|
||||
document.Root = strings.EqualFold(value, "true")
|
||||
}
|
||||
continue
|
||||
}
|
||||
current.Properties[key] = value
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return editorConfigDocument{}, fmt.Errorf("scan %s: %w", path, err)
|
||||
}
|
||||
return document, nil
|
||||
}
|
||||
|
||||
func applyEditorConfigProperties(formatting jsonFileFormatting, properties map[string]string) (jsonFileFormatting, error) {
|
||||
style := strings.ToLower(strings.TrimSpace(properties["indent_style"]))
|
||||
sizeText := strings.ToLower(strings.TrimSpace(properties["indent_size"]))
|
||||
if style == "tab" {
|
||||
formatting.Indent = "\t"
|
||||
} else if style != "" && style != "space" && style != "unset" {
|
||||
return jsonFileFormatting{}, fmt.Errorf("unsupported indent_style %q", properties["indent_style"])
|
||||
}
|
||||
if sizeText != "" && sizeText != "unset" && sizeText != "tab" {
|
||||
size, err := strconv.Atoi(sizeText)
|
||||
if err != nil || size < 1 {
|
||||
return jsonFileFormatting{}, fmt.Errorf("indent_size must be a positive integer, got %q", properties["indent_size"])
|
||||
}
|
||||
if style != "tab" {
|
||||
formatting.Indent = strings.Repeat(" ", size)
|
||||
}
|
||||
}
|
||||
|
||||
switch value := strings.ToLower(strings.TrimSpace(properties["end_of_line"])); value {
|
||||
case "", "unset":
|
||||
case "lf":
|
||||
formatting.LineEnding = "\n"
|
||||
case "crlf":
|
||||
formatting.LineEnding = "\r\n"
|
||||
case "cr":
|
||||
formatting.LineEnding = "\r"
|
||||
default:
|
||||
return jsonFileFormatting{}, fmt.Errorf("unsupported end_of_line %q", properties["end_of_line"])
|
||||
}
|
||||
|
||||
switch value := strings.ToLower(strings.TrimSpace(properties["insert_final_newline"])); value {
|
||||
case "", "unset":
|
||||
case "true":
|
||||
formatting.FinalNewline = true
|
||||
case "false":
|
||||
formatting.FinalNewline = false
|
||||
default:
|
||||
return jsonFileFormatting{}, fmt.Errorf("insert_final_newline must be true or false, got %q", properties["insert_final_newline"])
|
||||
}
|
||||
return formatting, nil
|
||||
}
|
||||
|
||||
func editorConfigPatternMatches(pattern, relativePath string) (bool, error) {
|
||||
pattern = filepath.ToSlash(strings.TrimSpace(pattern))
|
||||
pattern = strings.TrimPrefix(pattern, "/")
|
||||
if pattern == "" {
|
||||
return false, nil
|
||||
}
|
||||
if !strings.Contains(pattern, "/") {
|
||||
relativePath = pathBase(relativePath)
|
||||
}
|
||||
expression, err := editorConfigGlobRegexp(pattern)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return regexp.MatchString(expression, relativePath)
|
||||
}
|
||||
|
||||
func editorConfigGlobRegexp(pattern string) (string, error) {
|
||||
var builder strings.Builder
|
||||
builder.WriteByte('^')
|
||||
for index := 0; index < len(pattern); {
|
||||
char := pattern[index]
|
||||
switch char {
|
||||
case '*':
|
||||
if index+1 < len(pattern) && pattern[index+1] == '*' {
|
||||
if index+2 < len(pattern) && pattern[index+2] == '/' {
|
||||
builder.WriteString("(?:.*/)?")
|
||||
index += 3
|
||||
} else {
|
||||
builder.WriteString(".*")
|
||||
index += 2
|
||||
}
|
||||
continue
|
||||
}
|
||||
builder.WriteString("[^/]*")
|
||||
case '?':
|
||||
builder.WriteString("[^/]")
|
||||
case '.', '+', '(', ')', '|', '^', '$', '{', '}', '[', ']', '\\':
|
||||
builder.WriteByte('\\')
|
||||
builder.WriteByte(char)
|
||||
default:
|
||||
builder.WriteByte(char)
|
||||
}
|
||||
index++
|
||||
}
|
||||
builder.WriteByte('$')
|
||||
expression := builder.String()
|
||||
if _, err := regexp.Compile(expression); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return expression, nil
|
||||
}
|
||||
|
||||
func pathBase(path string) string {
|
||||
if index := strings.LastIndex(path, "/"); index >= 0 {
|
||||
return path[index+1:]
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -1,284 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func normalizeMetadataKey(key string) string {
|
||||
replacer := strings.NewReplacer("-", "_", " ", "_")
|
||||
return strings.ToLower(replacer.Replace(strings.TrimSpace(key)))
|
||||
}
|
||||
|
||||
func parseRowMetadata(raw any) (map[string]any, error) {
|
||||
if raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
obj, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("meta must be an object")
|
||||
}
|
||||
if len(obj) == 0 {
|
||||
return map[string]any{}, nil
|
||||
}
|
||||
|
||||
meta := make(map[string]any, len(obj))
|
||||
for key, value := range obj {
|
||||
switch normalizeMetadataKey(key) {
|
||||
case "family":
|
||||
family, err := parseFamilyMetadata(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("meta.family: %w", err)
|
||||
}
|
||||
meta["family"] = family
|
||||
case "wiki":
|
||||
wiki, err := parseWikiMetadata(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("meta.wiki: %w", err)
|
||||
}
|
||||
if len(wiki) > 0 {
|
||||
meta["wiki"] = wiki
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown metadata key %q", key)
|
||||
}
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) {
|
||||
datasetIndexMap := make(map[string]int)
|
||||
for i, ds := range collected {
|
||||
datasetIndexMap[ds.Dataset.Name] = i
|
||||
datasetIndexMap[ds.Dataset.OutputName] = i
|
||||
}
|
||||
|
||||
rowsWithExpansion := []struct {
|
||||
datasetName string
|
||||
row map[string]any
|
||||
}{}
|
||||
|
||||
for _, ds := range collected {
|
||||
for _, row := range ds.Rows {
|
||||
if hasExpansionData(row) {
|
||||
rowsWithExpansion = append(rowsWithExpansion, struct {
|
||||
datasetName string
|
||||
row map[string]any
|
||||
}{ds.Dataset.Name, row})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(rowsWithExpansion) == 0 {
|
||||
return collected, nil
|
||||
}
|
||||
|
||||
modifiedTargets := map[int]struct{}{}
|
||||
for _, src := range rowsWithExpansion {
|
||||
expansion, _ := extractExpansionData(src.row)
|
||||
for targetDatasetName, targetRows := range expansion.Data {
|
||||
targetIndex, ok := datasetIndexMap[targetDatasetName]
|
||||
if !ok {
|
||||
outputName := targetDatasetName
|
||||
if !strings.HasSuffix(outputName, ".2da") {
|
||||
outputName = outputName + ".2da"
|
||||
}
|
||||
targetIndex, ok = datasetIndexMap[outputName]
|
||||
}
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("expansion targets unknown dataset %q", targetDatasetName)
|
||||
}
|
||||
targetDS := collected[targetIndex]
|
||||
originalLockData, _ := loadLockfile(targetDS.Dataset.LockPath)
|
||||
if originalLockData == nil {
|
||||
originalLockData = map[string]int{}
|
||||
}
|
||||
lockModified := false
|
||||
usedIDs := map[int]struct{}{}
|
||||
usedKeys := map[string]struct{}{}
|
||||
for _, rowID := range targetDS.LockData {
|
||||
usedIDs[rowID] = struct{}{}
|
||||
}
|
||||
for _, row := range targetDS.Rows {
|
||||
if id, ok := row["id"].(int); ok {
|
||||
usedIDs[id] = struct{}{}
|
||||
}
|
||||
if key, ok := row["key"].(string); ok {
|
||||
usedKeys[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
nextID := nextAvailableIDAtLeast(usedIDs, targetDS.Dataset.RowGenerationMinRow)
|
||||
|
||||
for _, targetRow := range targetRows {
|
||||
var rowID int
|
||||
var hasID bool
|
||||
if rawID, ok := targetRow["id"]; ok {
|
||||
switch typed := rawID.(type) {
|
||||
case int:
|
||||
rowID = typed
|
||||
hasID = true
|
||||
case float64:
|
||||
rowID = int(typed)
|
||||
hasID = true
|
||||
case string:
|
||||
parsed, err := asInt(typed)
|
||||
if err == nil {
|
||||
rowID = parsed
|
||||
hasID = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
key, _ := targetRow["key"].(string)
|
||||
if strings.TrimSpace(key) == "" {
|
||||
return nil, fmt.Errorf("expansion into %s: injected rows must specify key", targetDatasetName)
|
||||
}
|
||||
|
||||
if key != "" {
|
||||
if existingID, exists := targetDS.LockData[key]; exists {
|
||||
rowID = existingID
|
||||
hasID = true
|
||||
} else if existingID, exists := originalLockData[key]; exists && targetDS.Dataset.RowGeneration != "first_null_row" {
|
||||
rowID = existingID
|
||||
hasID = true
|
||||
targetDS.LockData[key] = existingID
|
||||
lockModified = true
|
||||
targetDS.LockModified = true
|
||||
}
|
||||
if _, seen := usedKeys[key]; seen {
|
||||
continue
|
||||
}
|
||||
usedKeys[key] = struct{}{}
|
||||
}
|
||||
|
||||
if !hasID {
|
||||
rowID = nextID
|
||||
usedIDs[rowID] = struct{}{}
|
||||
nextID = nextAvailableIDAtLeast(usedIDs, targetDS.Dataset.RowGenerationMinRow)
|
||||
} else {
|
||||
if _, exists := usedIDs[rowID]; exists && key == "" {
|
||||
return nil, fmt.Errorf("expansion into %s: row id %d already exists", targetDatasetName, rowID)
|
||||
}
|
||||
usedIDs[rowID] = struct{}{}
|
||||
}
|
||||
|
||||
if key != "" {
|
||||
if _, exists := targetDS.LockData[key]; !exists {
|
||||
targetDS.LockData[key] = rowID
|
||||
lockModified = true
|
||||
targetDS.LockAdded++
|
||||
targetDS.LockModified = true
|
||||
}
|
||||
}
|
||||
|
||||
newRow := map[string]any{
|
||||
"id": rowID,
|
||||
}
|
||||
for k, v := range targetRow {
|
||||
if k != "id" {
|
||||
newRow[k] = v
|
||||
}
|
||||
}
|
||||
if key != "" {
|
||||
newRow["key"] = key
|
||||
}
|
||||
targetDS.Rows = append(targetDS.Rows, newRow)
|
||||
}
|
||||
|
||||
if lockModified {
|
||||
modifiedTargets[targetIndex] = struct{}{}
|
||||
}
|
||||
|
||||
slices.SortFunc(targetDS.Rows, func(a, b map[string]any) int {
|
||||
return a["id"].(int) - b["id"].(int)
|
||||
})
|
||||
collected[targetIndex] = targetDS
|
||||
}
|
||||
|
||||
if err := applyExpansionValueToRow(src.row, expansion.Value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
for targetIndex := range modifiedTargets {
|
||||
targetDS := collected[targetIndex]
|
||||
referencedKeys, err := collectReferencedLockKeys(nativeDatasetSourceDir(targetDS.Dataset))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dataset %s: collect referenced keys: %w", targetDS.Dataset.Name, err)
|
||||
}
|
||||
if pruned, updated := pruneLockDataToActiveRows(targetDS.LockData, targetDS.Rows, referencedKeys, targetDS.RetiredKeys); pruned > 0 || updated > 0 {
|
||||
targetDS.LockPruned += pruned
|
||||
targetDS.LockModified = true
|
||||
collected[targetIndex] = targetDS
|
||||
}
|
||||
}
|
||||
|
||||
return collected, nil
|
||||
}
|
||||
|
||||
type expansionSpec struct {
|
||||
Value string
|
||||
Data map[string][]map[string]any
|
||||
}
|
||||
|
||||
func hasExpansionData(row map[string]any) bool {
|
||||
for _, value := range row {
|
||||
if isExpansionValue(value) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isExpansionValue(value any) bool {
|
||||
obj, ok := value.(map[string]any)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, hasValue := obj["value"]
|
||||
_, hasData := obj["data"]
|
||||
return hasValue && hasData
|
||||
}
|
||||
|
||||
func extractExpansionData(row map[string]any) (expansionSpec, bool) {
|
||||
for _, value := range row {
|
||||
if isExpansionValue(value) {
|
||||
obj := value.(map[string]any)
|
||||
valueStr, _ := obj["value"].(string)
|
||||
dataMap, _ := obj["data"].(map[string]any)
|
||||
|
||||
result := expansionSpec{
|
||||
Value: valueStr,
|
||||
Data: make(map[string][]map[string]any),
|
||||
}
|
||||
|
||||
for datasetName, rawRows := range dataMap {
|
||||
switch typed := rawRows.(type) {
|
||||
case []any:
|
||||
rows := make([]map[string]any, 0, len(typed))
|
||||
for _, r := range typed {
|
||||
if rowObj, ok := r.(map[string]any); ok {
|
||||
rows = append(rows, rowObj)
|
||||
}
|
||||
}
|
||||
result.Data[datasetName] = rows
|
||||
case map[string]any:
|
||||
result.Data[datasetName] = []map[string]any{typed}
|
||||
}
|
||||
}
|
||||
|
||||
return result, true
|
||||
}
|
||||
}
|
||||
return expansionSpec{}, false
|
||||
}
|
||||
|
||||
func applyExpansionValueToRow(row map[string]any, value string) error {
|
||||
for field, oldValue := range row {
|
||||
if isExpansionValue(oldValue) {
|
||||
row[field] = value
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type familyIdentity struct {
|
||||
Parent string
|
||||
Child string
|
||||
}
|
||||
|
||||
type familyExpansionSource struct {
|
||||
Dataset string
|
||||
Column string
|
||||
Predicate string
|
||||
}
|
||||
|
||||
type familyExpansionTitleStyle struct {
|
||||
ChildCase string
|
||||
ChildParenthetical string
|
||||
}
|
||||
|
||||
type familyExpansionSpec struct {
|
||||
Family string
|
||||
FamilyKey string
|
||||
Template string
|
||||
ChildSource familyExpansionSource
|
||||
NamePrefix string
|
||||
LabelPrefix string
|
||||
ConstantPrefix string
|
||||
TemplateFields []string
|
||||
DefaultFields map[string]any
|
||||
ApplyAfterModules bool
|
||||
ChildRefField string
|
||||
IdentitySource string
|
||||
AllowExistingOnly bool
|
||||
AutoPrereqFields map[string]string
|
||||
LegacyFamilyKeys []string
|
||||
TitleStyle familyExpansionTitleStyle
|
||||
}
|
||||
|
||||
func splitFamilyExpansionIdentity(text string) familyIdentity {
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return familyIdentity{}
|
||||
}
|
||||
if idx := strings.Index(text, "_"); idx > 0 && idx < len(text)-1 {
|
||||
return familyIdentity{
|
||||
Parent: text[:idx],
|
||||
Child: text[idx+1:],
|
||||
}
|
||||
}
|
||||
return familyIdentity{Parent: text}
|
||||
}
|
||||
|
||||
func parseFamilyExpansionSource(raw any) (familyExpansionSource, error) {
|
||||
if raw == nil {
|
||||
return familyExpansionSource{}, fmt.Errorf("child_source is required")
|
||||
}
|
||||
obj, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return familyExpansionSource{}, fmt.Errorf("child_source must be an object")
|
||||
}
|
||||
dataset, ok := obj["dataset"].(string)
|
||||
if !ok || strings.TrimSpace(dataset) == "" {
|
||||
return familyExpansionSource{}, fmt.Errorf("child_source.dataset must be a non-empty string")
|
||||
}
|
||||
source := familyExpansionSource{
|
||||
Dataset: strings.TrimSpace(dataset),
|
||||
}
|
||||
if column, ok := obj["column"].(string); ok && strings.TrimSpace(column) != "" {
|
||||
source.Column = strings.TrimSpace(column)
|
||||
}
|
||||
if predicate, ok := obj["predicate"].(string); ok && strings.TrimSpace(predicate) != "" {
|
||||
source.Predicate = strings.TrimSpace(predicate)
|
||||
}
|
||||
return source, nil
|
||||
}
|
||||
|
||||
func parseFamilyExpansionSpec(path string, obj map[string]any) (familyExpansionSpec, error) {
|
||||
family, ok := obj["family"].(string)
|
||||
if !ok || strings.TrimSpace(family) == "" {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: family must be a non-empty string", path)
|
||||
}
|
||||
familyKey, ok := obj["family_key"].(string)
|
||||
if !ok || strings.TrimSpace(familyKey) == "" {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: family_key must be a non-empty string", path)
|
||||
}
|
||||
template, ok := obj["template"].(string)
|
||||
if !ok || strings.TrimSpace(template) == "" {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: template must be a non-empty string", path)
|
||||
}
|
||||
source, err := parseFamilyExpansionSource(obj["child_source"])
|
||||
if err != nil {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err)
|
||||
}
|
||||
namePrefix, _ := optionalTrimmedString(obj, "name_prefix")
|
||||
labelPrefix, _ := optionalTrimmedString(obj, "label_prefix")
|
||||
constantPrefix, _ := optionalTrimmedString(obj, "constant_prefix")
|
||||
templateFields, err := parseOptionalStringArray(obj["template_fields"], "template_fields")
|
||||
if err != nil {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err)
|
||||
}
|
||||
defaultFields, err := parseOptionalObject(obj["default_fields"], "default_fields")
|
||||
if err != nil {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err)
|
||||
}
|
||||
applyAfterModules, err := parseOptionalBoolField(obj["apply_after_modules"], "apply_after_modules")
|
||||
if err != nil {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err)
|
||||
}
|
||||
childRefField, _ := optionalTrimmedString(obj, "child_ref_field")
|
||||
identitySource, _ := optionalTrimmedString(obj, "identity_source")
|
||||
switch identitySource {
|
||||
case "", "child_source_value":
|
||||
default:
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: identity_source must be empty or child_source_value", path)
|
||||
}
|
||||
autoPrereqFields, err := parseOptionalStringMap(obj["auto_prereq_fields"], "auto_prereq_fields")
|
||||
if err != nil {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err)
|
||||
}
|
||||
legacyFamilyKeys, err := parseOptionalStringArray(obj["legacy_family_keys"], "legacy_family_keys")
|
||||
if err != nil {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err)
|
||||
}
|
||||
if err := validateLegacyFamilyKeys(familyKey, legacyFamilyKeys); err != nil {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err)
|
||||
}
|
||||
titleStyle, err := parseFamilyExpansionTitleStyle(obj["title_style"])
|
||||
if err != nil {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err)
|
||||
}
|
||||
allowExistingOnly, err := parseOptionalBoolField(obj["allow_existing_only"], "allow_existing_only")
|
||||
if err != nil {
|
||||
return familyExpansionSpec{}, fmt.Errorf("generated file %s: %w", path, err)
|
||||
}
|
||||
return familyExpansionSpec{
|
||||
Family: strings.TrimSpace(family),
|
||||
FamilyKey: strings.TrimSpace(familyKey),
|
||||
Template: strings.TrimSpace(template),
|
||||
ChildSource: source,
|
||||
NamePrefix: namePrefix,
|
||||
LabelPrefix: labelPrefix,
|
||||
ConstantPrefix: constantPrefix,
|
||||
TemplateFields: templateFields,
|
||||
DefaultFields: defaultFields,
|
||||
ApplyAfterModules: applyAfterModules,
|
||||
ChildRefField: childRefField,
|
||||
IdentitySource: identitySource,
|
||||
AllowExistingOnly: allowExistingOnly,
|
||||
AutoPrereqFields: autoPrereqFields,
|
||||
LegacyFamilyKeys: legacyFamilyKeys,
|
||||
TitleStyle: titleStyle,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseFamilyExpansionTitleStyle(raw any) (familyExpansionTitleStyle, error) {
|
||||
style := familyExpansionTitleStyle{
|
||||
ChildCase: "preserve",
|
||||
ChildParenthetical: "preserve",
|
||||
}
|
||||
if raw == nil {
|
||||
return style, nil
|
||||
}
|
||||
obj, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return familyExpansionTitleStyle{}, fmt.Errorf("title_style must be an object")
|
||||
}
|
||||
if childCase, ok := obj["child_case"].(string); ok && strings.TrimSpace(childCase) != "" {
|
||||
style.ChildCase = strings.TrimSpace(childCase)
|
||||
}
|
||||
switch style.ChildCase {
|
||||
case "preserve", "lower":
|
||||
default:
|
||||
return familyExpansionTitleStyle{}, fmt.Errorf("title_style.child_case must be preserve or lower")
|
||||
}
|
||||
if childParenthetical, ok := obj["child_parenthetical"].(string); ok && strings.TrimSpace(childParenthetical) != "" {
|
||||
style.ChildParenthetical = strings.TrimSpace(childParenthetical)
|
||||
}
|
||||
switch style.ChildParenthetical {
|
||||
case "preserve", "comma":
|
||||
default:
|
||||
return familyExpansionTitleStyle{}, fmt.Errorf("title_style.child_parenthetical must be preserve or comma")
|
||||
}
|
||||
return style, nil
|
||||
}
|
||||
|
||||
func validateLegacyFamilyKeys(familyKey string, legacyFamilyKeys []string) error {
|
||||
seen := map[string]string{}
|
||||
normalizedFamilyKey := normalizeKeyIdentity(familyKey)
|
||||
for _, legacyKey := range legacyFamilyKeys {
|
||||
normalizedLegacyKey := normalizeKeyIdentity(legacyKey)
|
||||
if normalizedLegacyKey == normalizedFamilyKey {
|
||||
return fmt.Errorf("legacy_family_keys must not include family_key %q", familyKey)
|
||||
}
|
||||
if previous, ok := seen[normalizedLegacyKey]; ok {
|
||||
return fmt.Errorf("legacy_family_keys contains duplicate-equivalent keys %q and %q", previous, legacyKey)
|
||||
}
|
||||
seen[normalizedLegacyKey] = legacyKey
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isFamilyExpansionObject(obj map[string]any) bool {
|
||||
_, hasFamilyKey := obj["family_key"]
|
||||
_, hasTemplate := obj["template"]
|
||||
_, hasChildSource := obj["child_source"]
|
||||
_, hasNamePrefix := obj["name_prefix"]
|
||||
_, hasLabelPrefix := obj["label_prefix"]
|
||||
_, hasConstantPrefix := obj["constant_prefix"]
|
||||
_, hasTemplateFields := obj["template_fields"]
|
||||
_, hasDefaultFields := obj["default_fields"]
|
||||
_, hasApplyAfterModules := obj["apply_after_modules"]
|
||||
_, hasChildRefField := obj["child_ref_field"]
|
||||
_, hasIdentitySource := obj["identity_source"]
|
||||
_, hasAllowExistingOnly := obj["allow_existing_only"]
|
||||
_, hasAutoPrereqFields := obj["auto_prereq_fields"]
|
||||
_, hasLegacyFamilyKeys := obj["legacy_family_keys"]
|
||||
_, hasTitleStyle := obj["title_style"]
|
||||
return hasFamilyKey || hasTemplate || hasChildSource || hasNamePrefix || hasLabelPrefix ||
|
||||
hasConstantPrefix || hasTemplateFields || hasDefaultFields || hasApplyAfterModules || hasChildRefField ||
|
||||
hasIdentitySource || hasAllowExistingOnly || hasAutoPrereqFields || hasLegacyFamilyKeys || hasTitleStyle
|
||||
}
|
||||
|
||||
func optionalTrimmedString(obj map[string]any, field string) (string, bool) {
|
||||
text, ok := obj[field].(string)
|
||||
if !ok || strings.TrimSpace(text) == "" {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(text), true
|
||||
}
|
||||
|
||||
func parseOptionalStringArray(raw any, field string) ([]string, error) {
|
||||
if raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
items, ok := raw.([]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s must be an array of strings", field)
|
||||
}
|
||||
out := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
text, ok := item.(string)
|
||||
if !ok || strings.TrimSpace(text) == "" {
|
||||
return nil, fmt.Errorf("%s must contain only non-empty strings", field)
|
||||
}
|
||||
out = append(out, strings.TrimSpace(text))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseOptionalObject(raw any, field string) (map[string]any, error) {
|
||||
if raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
obj, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s must be an object", field)
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
func parseOptionalStringMap(raw any, field string) (map[string]string, error) {
|
||||
if raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
obj, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s must be an object of strings", field)
|
||||
}
|
||||
out := make(map[string]string, len(obj))
|
||||
for key, value := range obj {
|
||||
text, ok := value.(string)
|
||||
if !ok || strings.TrimSpace(text) == "" {
|
||||
return nil, fmt.Errorf("%s must contain only non-empty string values", field)
|
||||
}
|
||||
out[key] = strings.TrimSpace(text)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseOptionalBoolField(raw any, field string) (bool, error) {
|
||||
if raw == nil {
|
||||
return false, nil
|
||||
}
|
||||
value, ok := raw.(bool)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("%s must be a boolean", field)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func familyMetadata(parent, child, source, template string) map[string]any {
|
||||
meta := map[string]any{
|
||||
"parent": strings.TrimSpace(parent),
|
||||
}
|
||||
if strings.TrimSpace(child) != "" {
|
||||
meta["child"] = strings.TrimSpace(child)
|
||||
}
|
||||
if strings.TrimSpace(source) != "" {
|
||||
meta["source"] = strings.TrimSpace(source)
|
||||
}
|
||||
if strings.TrimSpace(template) != "" {
|
||||
meta["template"] = strings.TrimSpace(template)
|
||||
}
|
||||
return map[string]any{"family": meta}
|
||||
}
|
||||
|
||||
func parseFamilyMetadata(raw any) (map[string]any, error) {
|
||||
obj, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("must be an object")
|
||||
}
|
||||
parent, ok := obj["parent"].(string)
|
||||
if !ok || strings.TrimSpace(parent) == "" {
|
||||
return nil, fmt.Errorf("parent must be a non-empty string")
|
||||
}
|
||||
meta := map[string]any{
|
||||
"parent": strings.TrimSpace(parent),
|
||||
}
|
||||
if child, ok := obj["child"].(string); ok && strings.TrimSpace(child) != "" {
|
||||
meta["child"] = strings.TrimSpace(child)
|
||||
}
|
||||
if source, ok := obj["source"].(string); ok && strings.TrimSpace(source) != "" {
|
||||
meta["source"] = strings.TrimSpace(source)
|
||||
}
|
||||
if template, ok := obj["template"].(string); ok && strings.TrimSpace(template) != "" {
|
||||
meta["template"] = strings.TrimSpace(template)
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
func childTokenFromExpandedKey(key, parent string) string {
|
||||
key = strings.TrimSpace(key)
|
||||
key = strings.TrimPrefix(key, "feat:")
|
||||
if strings.HasPrefix(key, parent+"_") {
|
||||
return strings.TrimPrefix(key, parent+"_")
|
||||
}
|
||||
if strings.HasPrefix(key, parent) {
|
||||
return strings.TrimPrefix(key, parent)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func importLegacyFeat(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
_ = legacyTLK
|
||||
|
||||
legacyDir := filepath.Join(referenceBuilderDir, "data", "feat")
|
||||
if _, err := os.Stat(legacyDir); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
legacyBasePath := filepath.Join(legacyDir, "base.json")
|
||||
if _, err := os.Stat(legacyBasePath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
targetDir := filepath.Join(dataDir, "feat")
|
||||
targetBasePath := filepath.Join(targetDir, "base.json")
|
||||
var targetObj map[string]any
|
||||
if _, err := os.Stat(targetBasePath); err == nil {
|
||||
targetObj, err = loadJSONObject(targetBasePath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return 0, err
|
||||
}
|
||||
if targetObj != nil {
|
||||
if rows, ok := targetObj["rows"].([]any); ok && len(rows) > 0 {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
baseObj, err := loadJSONObject(legacyBasePath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
baseObj["output"] = "feat.2da"
|
||||
baseObj["compare_reference"] = false
|
||||
canonicalizeWikiMetadataDocument(baseObj)
|
||||
|
||||
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw, err := json.MarshalIndent(baseObj, "", " ")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
if err := os.WriteFile(targetBasePath, raw, 0o644); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
type Generated2DAAsset struct {
|
||||
Rel string
|
||||
SourcePath string
|
||||
}
|
||||
|
||||
func BuildGenerated2DAAssets(p *project.Project, progress func(string)) ([]Generated2DAAsset, error) {
|
||||
if progress == nil {
|
||||
progress = func(string) {}
|
||||
}
|
||||
configs := p.EffectiveConfig().Generated.TopData2DA
|
||||
if len(configs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
results := make([]Generated2DAAsset, 0)
|
||||
for _, cfg := range configs {
|
||||
progress(fmt.Sprintf("Building generated 2DA assets for %s...", cfg.ID))
|
||||
generated, err := buildGenerated2DAAssetGroup(p, cfg, progress)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, generated...)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func buildGenerated2DAAssetGroup(p *project.Project, cfg project.GeneratedTopData2DAConfig, progress func(string)) ([]Generated2DAAsset, error) {
|
||||
sourceDir := filepath.Join(p.Root, filepath.FromSlash(strings.TrimSpace(cfg.Source)))
|
||||
dataDir := filepath.Join(sourceDir, "data")
|
||||
outputDir := filepath.Join(p.Root, filepath.FromSlash(strings.TrimSpace(cfg.Output)))
|
||||
|
||||
datasets, err := discoverNativeDatasets(dataDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("discover generated topdata 2DA datasets %s: %w", cfg.ID, err)
|
||||
}
|
||||
selected := make([]nativeDataset, 0, len(datasets))
|
||||
for _, dataset := range datasets {
|
||||
if generatedDatasetIncluded(dataset.Name, cfg.IncludeDatasets) {
|
||||
selected = append(selected, dataset)
|
||||
}
|
||||
}
|
||||
if len(selected) == 0 {
|
||||
return nil, fmt.Errorf("generated topdata 2DA config %s matched no datasets under %s", cfg.ID, dataDir)
|
||||
}
|
||||
|
||||
collected := make([]nativeCollectedDataset, 0, len(selected))
|
||||
for _, dataset := range selected {
|
||||
collectedDataset, err := collectNativeDataset(dataset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
collected = append(collected, collectedDataset)
|
||||
}
|
||||
|
||||
consumer := cfg.Autogen
|
||||
consumer.LocalOverrideRoot = p.AssetsDir()
|
||||
collected, err = applyAutogenConsumersForGeneratedAssets(collected, consumer, progress)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
collected, err = normalizePartsRowsACBonus(collected, consumer.PartsRows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
collected, err = applyPartOverridesWithConfig(sourceDir, collected, consumer.PartsRows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rejectGenerated2DATLKValues(cfg.ID, collected); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, _, err := saveNativeLockfiles(collected); err != nil {
|
||||
return nil, fmt.Errorf("save generated topdata 2DA lockfiles %s: %w", cfg.ID, err)
|
||||
}
|
||||
|
||||
tableRegistry, err := newResolvedTableRegistry(collected)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keyToID, rowByKey := generated2DAGlobalRows(collected)
|
||||
|
||||
if err := os.RemoveAll(outputDir); err != nil {
|
||||
return nil, fmt.Errorf("clean generated topdata 2DA output %s: %w", outputDir, err)
|
||||
}
|
||||
if err := os.MkdirAll(outputDir, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create generated topdata 2DA output %s: %w", outputDir, err)
|
||||
}
|
||||
|
||||
results := make([]Generated2DAAsset, 0, len(collected))
|
||||
for _, dataset := range collected {
|
||||
compiled, err := resolveNativeDataset(dataset, keyToID, rowByKey, tableRegistry, nil, project.TopDataClassFeatInjectionConfig{}, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outputPath := filepath.Join(outputDir, dataset.Dataset.OutputName)
|
||||
denseRows := dataset.Dataset.Kind == nativeDatasetBase ||
|
||||
(consumer.Mode == "parts_rows" && isPartsDataset(dataset.Dataset.Name))
|
||||
if err := write2DA(compiled, outputPath, denseRows, -1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, Generated2DAAsset{
|
||||
Rel: filepath.ToSlash(filepath.Join(cfg.PackageRoot, dataset.Dataset.OutputName)),
|
||||
SourcePath: outputPath,
|
||||
})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func applyAutogenConsumersForGeneratedAssets(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig, progress func(string)) ([]nativeCollectedDataset, error) {
|
||||
if strings.TrimSpace(consumer.Mode) == "" {
|
||||
return collected, nil
|
||||
}
|
||||
if progress == nil {
|
||||
progress = func(string) {}
|
||||
}
|
||||
if !autogenConsumerTargetsCollectedDataset(collected, consumer) {
|
||||
return collected, nil
|
||||
}
|
||||
if progress != nil {
|
||||
progress(fmt.Sprintf("Scanning local generated 2DA autogen input for %s from %s...", consumer.ID, consumer.LocalOverrideRoot))
|
||||
}
|
||||
entries, err := scanLocalAutogenEntries(consumer.LocalOverrideRoot, consumer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch consumer.Mode {
|
||||
case "parts_rows":
|
||||
return augmentWithAutogeneratedPartsWithConfig(collected, autogenPartsInventory(entries), consumer.PartsRows), nil
|
||||
case "cachedmodels_rows":
|
||||
return augmentWithAutogeneratedCachedModels(collected, entries)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported generated topdata 2DA autogen mode %q", consumer.Mode)
|
||||
}
|
||||
}
|
||||
|
||||
func generatedDatasetIncluded(name string, patterns []string) bool {
|
||||
name = filepath.ToSlash(strings.TrimSpace(name))
|
||||
for _, pattern := range patterns {
|
||||
pattern = filepath.ToSlash(strings.TrimSpace(pattern))
|
||||
if pattern == "" {
|
||||
continue
|
||||
}
|
||||
if pattern == name {
|
||||
return true
|
||||
}
|
||||
if strings.HasSuffix(pattern, "/**") {
|
||||
prefix := strings.TrimSuffix(pattern, "/**")
|
||||
if name == prefix || strings.HasPrefix(name, prefix+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func generated2DAGlobalRows(collected []nativeCollectedDataset) (map[string]int, map[string]map[string]any) {
|
||||
keyToID := map[string]int{}
|
||||
rowByKey := map[string]map[string]any{}
|
||||
for _, dataset := range collected {
|
||||
for key, rowID := range dataset.LockData {
|
||||
keyToID[key] = rowID
|
||||
}
|
||||
for _, row := range dataset.Rows {
|
||||
key, _ := row["key"].(string)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
if rowID, ok := row["id"].(int); ok {
|
||||
keyToID[key] = rowID
|
||||
rowByKey[key] = row
|
||||
}
|
||||
}
|
||||
}
|
||||
return keyToID, rowByKey
|
||||
}
|
||||
|
||||
func rejectGenerated2DATLKValues(configID string, collected []nativeCollectedDataset) error {
|
||||
for _, dataset := range collected {
|
||||
for _, row := range dataset.Rows {
|
||||
for field, value := range row {
|
||||
if field == "id" || field == "key" {
|
||||
continue
|
||||
}
|
||||
allowBare := columnMatchesSpec(dataset.Dataset.Spec, field)
|
||||
if _, ok, err := parseTLKPayload(value, allowBare); err != nil {
|
||||
return fmt.Errorf("generated topdata 2DA config %s dataset %s field %s has invalid TLK payload: %w", configID, dataset.Dataset.Name, field, err)
|
||||
} else if ok {
|
||||
return fmt.Errorf("generated topdata 2DA config %s dataset %s field %s uses TLK-backed text; 2DA-only generated asset builds do not generate TLK output", configID, dataset.Dataset.Name, field)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
package topdata
|
||||
|
||||
func importLegacyGenericdoors(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
|
||||
_ = legacyTLK
|
||||
return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "genericdoors", "genericdoors.2da", nil)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user