diff --git a/CONFIGURATION_HARDENING_CONTRACT.md b/CONFIGURATION_HARDENING_CONTRACT.md new file mode 100644 index 0000000..2a61aa5 --- /dev/null +++ b/CONFIGURATION_HARDENING_CONTRACT.md @@ -0,0 +1,782 @@ +Create a long-term multi-repository hardening plan focused on making YAML configuration the authoritative source of truth. + +# Goal + +Reduce hardcoded behavior across all repositories and toolkit code. + +All behavior that touches repository-specific configuration must be driven by YAML. If a behavior currently exists only as an explicit rule in application code, move it into configuration or create a new configuration option for it. + +# Core principle + +Configuration is fact. + +Code should interpret and enforce configuration, not silently override it. + +# Requirements + +- YAML is the only human-authored repository configuration format. +- JSON remains acceptable only for generated artifacts, datasets, manifests, caches, and machine-readable outputs. +- Any code path that touches configurable behavior must lock to YAML-defined values. +- If YAML defines a value, code must not substitute another value unless explicitly configured as a fallback. +- If behavior varies by repository, dataset, build target, asset type, or command, it belongs in YAML. +- If no configuration option exists for a behavior, create one. +- Remove implicit repository assumptions from code. +- Remove hidden defaults unless they are documented toolkit-level defaults. +- Make all defaults visible, documented, and overrideable. + +# Audit scope + +Audit all repositories and toolkit code for hardcoded or implicit behavior related to: + +- paths +- module metadata +- resrefs +- HAK names +- HAK order +- HAK grouping +- HAK priority +- HAK splitting +- HAK size limits +- optional HAKs +- include/exclude globs +- generated manifest names +- cache names +- release tags +- dataset names +- autogen producers +- autogen consumers +- derive rules +- music prefixes +- credits behavior +- extract behavior +- topdata behavior +- package names +- TLK names +- build directories +- source directories +- validation rules +- cleanup rules +- normalization rules +- naming schemes +- file extension rules +- ignore rules +- command-specific behavior +- logging verbosity +- deterministic ordering rules + +# Hardening rules + +1. YAML wins. If YAML declares a value, the application must use it exactly. + +2. No silent fallbacks. Fallbacks must be explicit, documented, and visible in logs. + +3. No hidden repository assumptions. Repository-specific behavior must not live in toolkit code. + +4. No magic names. Names like `sow_top`, `sow_core`, `content`, `build`, `.cache`, or specific manifest names must come from YAML unless they are generated from a configured rule. + +5. No implicit behavior drift. Existing behavior should remain stable unless the YAML explicitly changes it. + +6. Deterministic by default. Ordering, output paths, generated names, cache keys, manifests, and logs must be stable. + +7. Validation before execution. Invalid or incomplete configuration should fail early with clear errors. + +8. Configuration gaps become schema work. If code needs a behavior that YAML cannot express, extend the schema. + +# Implementation direction + +- Build a central configuration model. +- Route all repository behavior through that model. +- Add typed config objects or schema validation. +- Normalize config once at startup. +- Pass normalized config into commands instead of reading globals or hardcoded constants. +- Make command behavior depend on normalized config only. +- Add explicit toolkit defaults in one place. +- Generate an effective configuration view for debugging. +- Add a command such as: + +```text +toolkit config inspect +toolkit config validate +toolkit config explain +toolkit config effective +``` + +# Effective config + +Add support for printing the final resolved configuration after defaults, overrides, and dataset-specific settings have been applied. + +This should help verify that: + +- YAML was loaded. +- defaults were applied intentionally. +- no hidden behavior was used. +- paths resolved as expected. +- command behavior is explainable. + +# Repository hardening process + +For each repository: + +1. Inventory existing YAML config. +2. Inventory generated JSON artifacts separately. +3. Search code for hardcoded repository behavior. +4. Map each hardcoded behavior to an existing YAML entry. +5. If no YAML entry exists, add one. +6. Update schema and validation. +7. Update code to consume normalized config. +8. Add tests proving YAML controls the behavior. +9. Add regression tests for existing outputs. +10. Document the config option. + +# Testing requirements + +Add tests for: + +- YAML config loading +- config validation +- effective config generation +- deterministic ordering +- missing required config +- duplicate config entries +- invalid paths +- invalid HAK definitions +- invalid autogen definitions +- invalid dataset definitions +- command behavior matching YAML +- no hardcoded repository names +- JSON artifacts remaining generated data only + +Add regression tests where possible: + +- same YAML input produces same build plan +- same YAML input produces same HAK order +- same YAML input produces same manifest names +- same YAML input produces same cache paths +- same YAML input produces same autogen behavior + +# Code audit targets + +Look specifically for: + +- string literals that look like paths, names, prefixes, manifests, or cache files +- constants that duplicate config values +- default values inside command implementations +- repository-specific branch logic +- command-specific assumptions +- filename conventions not declared in config +- implicit glob patterns +- implicit output names +- implicit ordering rules +- duplicated config parsing +- separate code paths that bypass central config + +# Deliverables + +1. Long-term hardening plan +2. Config authority rules +3. Audit checklist +4. Proposed YAML schema expansions +5. Effective config design +6. Migration strategy per repository +7. Testing strategy +8. Documentation strategy +9. List of hardcoded assumptions found +10. Refactor tickets grouped by risk and priority + +# Acceptance criteria + +- YAML is the only human-authored repository config. +- Code does not override YAML silently. +- Repository-specific behavior is not hardcoded. +- Any behavior touching config is lock-set to the normalized YAML value. +- Missing config options are added instead of encoded in application logic. +- Generated JSON artifacts remain unchanged unless intentionally modified. +- Builds remain deterministic. +- Existing repositories continue to build. +- Developers can inspect the effective config and understand why each value was used. + +# Current project-needs analysis + +## Current state + +- Root repository configuration is already discovered as: + 1. `nwn-tool.yaml` + 2. `nwn-tool.yml` + 3. legacy `nwn-tool.json` +- YAML is decoded with `KnownFields(true)`. +- Legacy JSON is decoded with `DisallowUnknownFields()`. +- `internal/app.loadProject` prints the loaded config filename and warns when legacy JSON is used. +- The root config model currently lives in `internal/project.Config`. +- `internal/project.defaultConfig` currently only defaults `paths.build` to `build`. +- Generated and machine-readable JSON remains widespread and should stay JSON: + - HAK manifest: `build/haks.json` + - canonical GFF JSON documents + - topdata authored/generated datasets + - topdata wiki state + - topdata deploy manifest + - autogen manifests and release caches + - credits inventory +- Some behavior is already YAML-driven: + - module name/resref/description + - module HAK order + - source/assets/build paths + - HAK names, priorities, split behavior, max bytes, optional flag, include globs + - music prefix mapping + - topdata source/build/reference/assets/package names + - extract ignore extensions and post-extract module deletion + - autogen producer/consumer ids, roots, includes, modes, derive rules, release tags, asset names, cache names +- Some behavior is still implicit in code and should be made explicit or documented as toolkit defaults. + +## Important naming clarification + +The current repository root config file is `nwn-tool.yaml`, not `config.yaml`. + +Hardening assumption: + +- Keep `nwn-tool.yaml` as the canonical root config filename. +- Keep `nwn-tool.yml` as an accepted alternate YAML filename. +- Treat `nwn-tool.json` as temporary legacy compatibility only. +- Do not rename generated or nested JSON files. +- Do not treat topdata dataset JSON as root repository configuration. + +If a future decision changes the canonical name to `config.yaml`, update `project.ConfigFile`, docs, consumer wrappers, tests, and migration scripts together. + +# Current hardcoded assumptions found + +## Project config and path defaults + +- `paths.build` defaults to `build` in `internal/project.defaultConfig`. +- `HAKManifestPath` always returns `/haks.json`. +- `ModuleArchivePath` always uses `/.mod`. +- `HAKArchivePath` always uses `/.hak`. +- `SourceDir`, `AssetsDir`, and `BuildDir` directly join configured paths with the repo root and do not yet expose a richer path policy. +- `TopDataBuildDir` falls back to `/topdata` when `topdata.build` is empty. +- `TopDataUsesRepoCache` special-cases `topdata.build == .cache`. +- `TopDataCompiled2DADir` always appends `2da`. +- `TopDataCompiledTLKPath` uses `sow_tlk.tlk` either under `build` or topdata build output. +- `TopDataWikiRootDir` uses `.cache/wiki` for repo-cache mode and `/wiki` otherwise. +- `TopDataWikiStatePath` always uses `state.json`. +- `TopDataPackageHAKName` defaults to `sow_top.hak`. +- `TopDataPackageTLKName` defaults to `sow_tlk.tlk`. + +Hardening direction: + +- Split these into documented toolkit defaults and repository-overridable fields. +- Keep existing values as defaults only for compatibility. +- Show all derived values in effective config output. + +## Inventory and extension rules + +- Source extensions are hardcoded in `project.SourceExtensions`. +- Asset extensions are hardcoded in `project.AssetExtensions`. +- `Project.Scan` treats `.json` and `.nss` as source inputs. +- Source JSON extension detection assumes resource filenames like `..json`. +- Validator required fields are hardcoded, currently including `ifo -> Mod_Name`. +- Built-in script prefixes are hardcoded in `internal/validator`. +- Validator reference checks hardcode specific field labels and expected resource extensions: + - `Conversation -> .dlg` + - `Mod_Entry_Area -> .are` + - `Model -> .mdl` + - `Sound -> .wav` + +Hardening direction: + +- Add a `resources` or `inventory` config section for extension policy and source/asset classification. +- Keep NWN defaults built in, but make them inspectable and overrideable. +- Consider allowing validators to be named profiles, for example `nwn_module`, instead of requiring every repository to spell out base NWN rules. + +## Build and HAK behavior + +- `build-haks` reads/writes the hardcoded manifest name from `Project.HAKManifestPath`. +- HAK files are always emitted under `paths.build`. +- HAK archive filenames are always derived from `haks[].name`. +- `extract` scans all `*.hak` in the build directory rather than only configured/generated HAKs. +- `build-haks` has command flags that mutate behavior independently of config: + - `--hak` + - `--archive` + - `--source-manifest` + - `--plan-only` + - `--quiet` + - `--verbose` + - `--debug` +- `SOW_BUILD_HAKS_KEEP_EXISTING` controls HAK cleanup behavior outside YAML. + +Hardening direction: + +- Add configurable manifest names and cleanup policy. +- Define command-line flags as explicit per-run overrides in the effective config model. +- Log all active environment overrides. +- Consider config-controlled archive discovery for `extract`. + +## Script compiler behavior + +- Compiled scripts are cached under `.cache/ncs`. +- Compiler discovery is hardcoded to: + - `SOW_NWN_SCRIPT_COMPILER` + - `tools/script-compiler/nwn_script_comp` + - `tools/script-compiler/nwn_script_comp.exe` + - `tools/nwn_script_comp` + - `tools/nwn_script_comp.exe` + - `PATH` +- Script source directory is always `/scripts`. +- Compiler environment probing uses `SOW_NWN_USER_DIRECTORY`, `NWN_HOME`, `NWN_USER_DIRECTORY`, `SOW_NWN_ROOT`, and `NWN_ROOT`. + +Hardening direction: + +- Add `tools.script_compiler` and `script.cache` config. +- Keep environment variables as explicit override sources. +- Include resolved compiler path and cache path in effective config and debug logs. + +## Music behavior + +- Music conversion/scanning is hardcoded around `envi/music`. +- Music staging is hardcoded to `.cache/music`. +- Credits output is hardcoded to `.cache/credits`. +- Credits overlay filename is hardcoded to `CREDITS.md`. +- FFmpeg/ffprobe discovery is environment/PATH based. +- BMU output naming rules and 16-character NWN stem limits are hardcoded. + +Hardening direction: + +- Apply `MUSIC_REFACTOR_CONTRACT.md`. +- Convert music behavior to dataset configuration. +- Keep old `music.prefixes` only as compatibility shorthand until repositories migrate. + +## Topdata behavior + +- Topdata source layout is partly convention-based: + - `data` + - `tlk` + - `base_dialog.json` + - `lock.json` + - `.tlk_state.json` + - `templates` + - `assets` +- Topdata compiled output subdirectories are hardcoded: + - `2da` + - `wiki` + - `wiki/pages` + - `wiki/state.json` +- Wiki generation has hardcoded managed namespaces and status labels. +- Wiki deploy defaults are hardcoded: + - `.wiki_deploy_manifest.json` + - `Auto-generated from native builder` + - managed namespaces including `classes`, `feat`, `itemtypes`, `races`, `skills`, `spells`, `meta` +- Topdata package fallback names are `sow_top.hak` and `sow_tlk.tlk`. +- Autogen released manifest cache path is `.cache/`. +- Autogen released manifest max age is one hour. +- `SOW_AUTOGEN_MANIFEST_REFRESH` forces refresh outside YAML. +- Released autogen manifest discovery derives a Shadows Over Westgate assets repo spec from git remote and environment overrides. +- Supported autogen modes are hardcoded: + - `parts_rows` + - `head_visualeffects` +- Supported derive kinds are hardcoded: + - `trailing_numeric_suffix` + - `model_stem` +- Supported derive group source is hardcoded: + - `first_path_segment` + +Hardening direction: + +- Separate topdata toolkit defaults from repository-specific defaults. +- Add explicit config for output subpaths, wiki generation/deploy defaults, autogen cache policy, and release source. +- Keep dataset JSON shape unchanged. +- Treat topdata dataset internals as authored data, not root YAML config, unless the behavior genuinely varies by repository. + +## Extract behavior + +- Extract output layout is hardcoded: + - `.nss` -> `/scripts/.nss` + - GFF resources -> `//..json` + - other resources -> `//.` +- Stale cleanup uses the current project inventory. +- Ignore behavior is configurable only by extension. + +Hardening direction: + +- Add extract output layout settings or named extraction profiles. +- Keep current layout as `nwn_canonical_json`. +- Make cleanup scope explicit and visible. + +## App and logging behavior + +- Command descriptions mention hardcoded paths like `src/`, `assets/`, `build/`, `.cache/2da`, `.cache/wiki`, `sow_top.hak`, and `sow_tlk.tlk`. +- Verbosity flags are command-specific, especially for `build-haks`. +- Spinner and phase labels are hardcoded in `internal/app/app.go`. +- Config load diagnostics are not yet a full effective-config explanation. + +Hardening direction: + +- Update descriptions to avoid implying fixed repository layouts. +- Introduce shared global verbosity flags or a documented command-level policy. +- Expose effective config and active overrides through `config` commands. + +# Proposed schema expansions + +These are target concepts, not a requirement to implement every field in one pass. + +```yaml +toolkit: + defaults_profile: nwn + generated_artifacts: + format: json + logging: + default_verbosity: normal + paths: relative + +paths: + source: src + assets: assets + build: build + cache: .cache + tools: tools + +outputs: + module_archive: "{module.resref}.mod" + hak_manifest: haks.json + hak_archive: "{hak.name}.hak" + +inventory: + source_extensions: + - .are + - .dlg + - .fac + - .git + - .ifo + - .jrl + - .utc + - .utd + - .ute + - .uti + - .utm + - .utp + - .uts + - .utt + - .utw + asset_extensions: + - .2da + - .bmu + - .dds + - .mdl + - .wav + source_json_pattern: "{resref}.{extension}.json" + +validation: + profile: nwn_module + builtin_script_prefixes: + - nw_ + - x0_ + - x1_ + - x2_ + - x3_ + - ga_ + - gc_ + - gen_ + - gui_ + - nwg_ + - ta_ + +scripts: + source_dir: scripts + cache: "{paths.cache}/ncs" + compiler: + path: "" + env: + path: SOW_NWN_SCRIPT_COMPILER + nwn_root: SOW_NWN_ROOT + nwn_user_directory: SOW_NWN_USER_DIRECTORY + search: + - "{paths.tools}/script-compiler/nwn_script_comp" + - "{paths.tools}/script-compiler/nwn_script_comp.exe" + - "{paths.tools}/nwn_script_comp" + - "{paths.tools}/nwn_script_comp.exe" + - PATH:nwn_script_comp + +extract: + layout: nwn_canonical_json + ignore_extensions: [] + cleanup_stale: true + delete_module_archive_after_success: false + +topdata: + source: topdata + build: "{paths.build}/topdata" + compiled_2da_dir: 2da + compiled_tlk: sow_tlk.tlk + package_hak: sow_top.hak + package_tlk: sow_tlk.tlk + wiki: + enabled_by_default: false + output_root: wiki + pages_dir: pages + state_file: state.json + managed_namespaces: + - classes + - feat + - itemtypes + - races + - skills + - spells + deploy_manifest: .wiki_deploy_manifest.json + deploy_edit_summary: Auto-generated from native builder + +autogen: + cache: + root: "{paths.cache}" + max_age: 1h + refresh_env: SOW_AUTOGEN_MANIFEST_REFRESH + release_source: + provider: gitea + server_url_env: SOW_ASSETS_SERVER_URL + repo_env: SOW_ASSETS_REPO +``` + +Implementation does not need to preserve exactly this naming, but each hardcoded behavior above should either become: + +- a field in schema, +- a documented toolkit default visible in effective config, or +- a deliberate non-configurable engine invariant with a clear rationale. + +# Effective config design + +## Commands + +Add config inspection commands: + +```text +sow-toolkit config validate +sow-toolkit config effective [--json|--yaml] +sow-toolkit config inspect [] [--json] +sow-toolkit config explain +sow-toolkit config sources +``` + +`config validate` should load, normalize, and validate without scanning/building. + +`config effective` should print the complete normalized config after defaults and compatibility migrations. + +`config explain ` should show: + +- final value +- source of the value: + - toolkit default + - root YAML + - compatibility migration + - environment override + - command-line override +- whether the value is deprecated +- validation rules that apply + +`config sources` should print: + +- loaded config path and format +- legacy status +- accepted root config candidates +- active environment overrides +- command-line overrides used for the current invocation + +## Internal model + +Introduce a normalized/effective config layer separate from raw YAML structs. + +Suggested distinction: + +- `RawConfig`: direct YAML representation with pointer fields where inheritance or "unset" matters. +- `EffectiveConfig`: fully resolved values consumed by commands. +- `ConfigProvenance`: source metadata for inspect/explain commands. +- `ConfigDiagnostics`: warnings/errors/deprecations produced during normalization and validation. + +Commands should consume `EffectiveConfig` or methods derived from it, not raw YAML fields. + +## Override rules + +Order of authority: + +1. toolkit built-in defaults +2. YAML root config +3. compatibility migrations from deprecated YAML fields +4. environment overrides +5. command-line overrides + +For each override: + +- document whether it is allowed +- record provenance +- show it in debug/effective output +- avoid silent behavior changes + +Environment variables should not be removed immediately, but they should be treated as visible overrides, not hidden behavior. + +# Hardening phases + +## Phase 1: Baseline audit and effective config shell + +- Add a config audit document generated from code search. +- Implement `config validate`. +- Implement `config effective --json` using current config and current defaults. +- Add provenance for: + - config filename + - legacy JSON use + - `paths.build` default + - topdata default package names + - HAK manifest default name +- Do not change build behavior in this phase. + +## Phase 2: Centralize derived paths and names + +- Move path/name derivation out of scattered methods and into effective config. +- Cover: + - module archive path + - HAK archive path template + - HAK manifest path + - cache root + - script cache + - topdata compiled dirs + - topdata package names + - wiki output paths + - autogen cache root +- Keep old methods as wrappers around effective config during migration. +- Add tests proving old and new derivations match. + +## Phase 3: Schema expansion for current hardcoded behavior + +- Add schema fields for outputs, cache roots, inventory extensions, script compiler settings, extract layout, topdata output subpaths, wiki deploy defaults, and autogen cache policy. +- Preserve current defaults. +- Add strict validation for new fields. +- Update README examples. + +## Phase 4: Command integration + +- Update build, extract, validate, compare, topdata, autogen, music, and wiki commands to consume effective config. +- Convert environment variables into explicit override records. +- Update logs to report configured names/paths instead of hardcoded labels. +- Add `--debug` output for effective values used by each command. + +## Phase 5: Repository migrations + +- Update consumer repository YAML to explicitly declare any behavior that is repository-specific. +- Remove compatibility shims only after all active repositories have migrated. +- Keep generated JSON artifacts stable unless a separate contract says otherwise. +- Add regression checks comparing pre/post build plans and generated manifests. + +## Phase 6: Enforcement + +- Add tests or static checks that reject new repository-specific literals in core command paths. +- Remove legacy JSON loading after the agreed migration window. +- Turn deprecated YAML shorthand warnings into validation errors where appropriate. + +# Repository-by-repository checklist + +For each consumer repository: + +1. Identify root config file and confirm it is `nwn-tool.yaml` or `nwn-tool.yml`. +2. Confirm no root `nwn-tool.json` remains except as deliberate migration fixture. +3. List all wrapper scripts and environment variables they set. +4. Record local tool paths and whether they need YAML fields. +5. List generated artifact paths and confirm they are not treated as config. +6. Run current build and save: + - HAK manifest + - module HAK order + - topdata package paths + - autogen manifest paths + - credits inventory paths +7. Convert implicit behavior to explicit YAML. +8. Run `config effective` and verify values. +9. Run build/validate workflows and compare generated outputs. +10. Document intentional diffs. + +# Test strategy expansion + +Add tests for: + +- `config validate` succeeds without scanning missing optional trees. +- `config effective --json` is deterministic. +- `config explain paths.build` identifies toolkit default when omitted. +- `config explain topdata.package_hak` identifies YAML source when configured. +- legacy `nwn-tool.json` emits warning and provenance. +- YAML wins over legacy JSON when both exist. +- unknown YAML fields fail before any scan/build side effects. +- environment overrides appear in effective config. +- command-line overrides appear in effective config for the command invocation. +- HAK manifest name can be configured without changing generated JSON schema. +- topdata package names can be configured and all command output follows them. +- script compiler cache path can be configured. +- extract layout profile preserves current output paths. +- autogen cache root and max age are configurable. +- wiki deploy manifest name is configurable. +- source/asset extension defaults match current scanning behavior. +- invalid path traversal in output/cache config fails early. +- absolute paths are rejected unless the field explicitly allows them. +- duplicate or case-colliding generated output paths fail validation. +- config-generated effective values are sorted deterministically. + +Regression tests: + +- current minimal YAML still builds to the same default paths. +- current Shadows Over Westgate-style topdata package output remains `sow_top.hak` / `sow_tlk.tlk` unless YAML changes it. +- current HAK manifest remains `haks.json` unless YAML changes it. +- current `.cache` outputs remain stable unless YAML changes them. +- old `music.prefixes` compatibility still maps to current behavior until removed. + +# Priority refactor tickets + +## P0: Authority and safety + +- Add effective config command output. +- Record config provenance and legacy warnings. +- Centralize output/cache path derivation. +- Validate path traversal and unsafe output roots. +- Make environment overrides visible. + +## P1: Repository-specific names and paths + +- Configure HAK manifest filename. +- Configure cache root. +- Configure script compiler path/search/cache. +- Configure topdata compiled output subpaths. +- Configure wiki deploy manifest/edit summary/namespaces. +- Configure autogen cache root/max age/release source. + +## P2: Inventory and validation profiles + +- Configure source and asset extension lists. +- Configure or profile validator reference rules. +- Configure built-in script prefixes. +- Configure extract output layout. + +## P3: Migration cleanup + +- Deprecate and remove legacy JSON root config loading. +- Deprecate music prefix shorthand after dataset migration. +- Replace command descriptions that mention hardcoded defaults. +- Add static/lint checks for new repository-specific literals. + +# Documentation strategy + +- Document `nwn-tool.yaml` as the canonical root config file. +- Maintain a generated effective-config reference in docs. +- Document every toolkit default separately from repository examples. +- For every config field, document: + - type + - default + - whether relative paths are repo-relative or output-root-relative + - whether absolute paths are allowed + - validation rules + - generated artifacts affected + - migration notes +- Keep a clear line between: + - human-authored root YAML config + - human-authored topdata dataset JSON + - generated JSON artifacts + - compatibility/legacy files + +# Expanded acceptance criteria + +- `sow-toolkit config effective --json` shows all values that commands use. +- Every effective value can be traced to YAML, toolkit default, environment override, compatibility migration, or command override. +- Existing hardcoded defaults are either configurable or documented as toolkit defaults. +- Command implementations do not invent repository-specific paths after config normalization. +- Topdata, HAK, music, extract, autogen, wiki, and script compiler paths are inspectable. +- Legacy JSON root config use is visible and eventually removable. +- Generated JSON schemas remain stable unless intentionally changed. +- Existing consumer repositories build before and after migration. diff --git a/MUSIC_REFACTOR_CONTRACT.md b/MUSIC_REFACTOR_CONTRACT.md new file mode 100644 index 0000000..1d8228f --- /dev/null +++ b/MUSIC_REFACTOR_CONTRACT.md @@ -0,0 +1,738 @@ +Refactor the music generation / editing / credits pipeline so it exists entirely inside the shared toolkit and is no longer hardcoded or repository-specific. + +Goal: Turn the current Shadows Over Westgate-specific music pipeline into a reusable, repository-agnostic toolkit subsystem that can operate across multiple projects and datasets with configuration-driven behavior. + +# Requirements + +## Architecture + +- Move all music pipeline logic into the toolkit itself. +- Eliminate repository-specific assumptions, paths, naming conventions, or hardcoded behavior. +- Repositories should only provide configuration and assets. +- The toolkit should expose reusable commands/services for: + - music scanning + - metadata extraction + - credits generation + - track normalization + - BMU generation/mapping + - music editing/transcoding + - manifest generation + - cache generation + - validation + +## Configuration + +Support configuration at multiple levels: + +1. Global toolkit defaults +2. Repository-level configuration +3. Dataset-level overrides + +Example hierarchy: + +- toolkit defaults +- repo config +- dataset config +- per-task overrides + +The system should merge configuration predictably. + +## Datasets + +A "dataset" should represent an isolated music collection/pipeline target. + +Examples: + +- envi/music/westgate +- modules/oc/music +- premium/music/bardpack + +Each dataset should support: + +- independent input/output paths +- naming rules +- normalization settings +- credits settings +- encoding/transcoding options +- cache behavior +- manifest generation +- validation rules + +# Desired Features + +## Reusable pipeline stages + +Design the pipeline as composable stages. + +Example: + +- discover +- scan metadata +- normalize +- transcode +- generate identifiers +- build mappings +- generate credits +- write manifests +- validate outputs + +Stages should: + +- be independently callable +- support dry-run +- support incremental rebuilds +- support verbose/debug logging +- expose machine-readable results + +## Incremental behavior + +Avoid unnecessary rebuilds. + +The pipeline should: + +- detect unchanged inputs +- reuse generated artifacts +- avoid duplicate rescans +- avoid rewriting unchanged outputs +- report concise summaries + +## Repository independence + +Do not assume: + +- NWN-specific paths +- westgate naming +- BMU naming formats +- specific cache directories +- specific manifest names + +All of these should be configurable. + +# CLI design + +Design clean toolkit commands. + +Example direction: + +- toolkit music scan +- toolkit music build +- toolkit music credits +- toolkit music normalize +- toolkit music validate + +Or: + +- toolkit dataset build + +The exact CLI structure may change if a cleaner abstraction exists. + +# Configuration examples + +Support something like: + +```yaml +music: + defaults: + normalization: + enabled: true + + datasets: + westgate: + source: envi/music/westgate + output: build/music/westgate + + naming: + scheme: nwn_bmu + + credits: + enabled: true + output: .cache/credits/westgate + + premium_bardpack: + source: premium/music/bardpack + + normalization: + loudness: -14 + + transcoding: + format: ogg +``` + +# Implementation expectations + +- Separate orchestration from implementation details. +- Separate scanning, transformation, and output stages cleanly. +- Introduce typed dataset definitions/config models. +- Centralize logging and cache management. +- Prefer declarative configuration over imperative repository scripts. +- Keep generated outputs deterministic. +- Preserve compatibility where possible through adapters/shims. +- Minimize repository glue code. + +# Migration + +- Migrate existing Shadows Over Westgate behavior onto the new system. +- Preserve existing outputs unless intentionally improved. +- Add compatibility wrappers if needed. +- Ensure old workflows continue functioning during transition. + +# Deliverables + +1. Proposed architecture +2. Configuration schema +3. CLI design +4. Migration strategy +5. Refactor plan +6. Example configs +7. Before/after repository responsibilities +8. Suggested module/package layout +9. Identification of reusable abstractions +10. Example logs/output + +Focus on maintainability, reuse, composability, and repository independence. + +# Initial project-needs analysis + +## Current state + +- Music processing is currently implemented in `internal/pipeline/music.go`. +- The music pipeline is not independently callable. It is invoked as part of: + - `BuildHAKs` / `planOrBuildHAKs` + - `plannedModuleHAKOrder` +- The current behavior assumes the repository has an asset tree and that music lives under `envi/music`. +- Convertible source formats are currently hardcoded to: + - `.mp3` + - `.ogg` +- Passthrough audio/resource formats are currently hardcoded to: + - `.bmu` + - `.wav` +- Generated BMU-like resources are staged under `.cache/music`, loaded into HAK resources, and then the staging root is deleted. +- Generated credits are written under `.cache/credits`. +- Credits inventory is written to `.cache/credits/credits.json`. +- Authored credit overlays are discovered by walking the whole configured assets directory for `CREDITS.md`. +- A per-directory `CREDITS.md` can override probed metadata and manually pin the generated output filename. +- FFmpeg and ffprobe are resolved from: + - `SOW_FFMPEG` + - `SOW_FFPROBE` + - `PATH` +- The ffmpeg command currently strips metadata and writes MP3-compatible audio bytes to a file named `.bmu`. +- Output stems are constrained to 16 characters, lower-case ASCII letters, digits, and underscores. +- Generated output names are reserved against every existing asset stem, not just music files. +- Current generated credits writes are already incremental: desired files are rendered in memory, unchanged files are not rewritten, and stale generated artifacts are removed. +- App-layer output for music is currently coupled to `build-haks` summary output in `internal/app/app.go`. + +## Existing behavior to preserve during migration + +- `build-haks` must continue to convert configured music sources into HAK resources without requiring repository scripts. +- Existing Shadows Over Westgate outputs should remain stable by default: + - source root: `envi/music` + - westgate prefix mapping: `envi/music/westgate -> mus_wg_` + - generated output extension: `.bmu` + - generated cache inventory: `.cache/credits/credits.json` + - generated credits markdown shape and columns + - manual `CREDITS.md` overlay semantics + - duplicate output-name collision behavior + - 16-character NWN resource stem limit +- Existing concise/verbose `build-haks` log behavior should keep working. +- Existing environment variable overrides for ffmpeg/ffprobe should continue to work as compatibility aliases. +- Machine-readable generated artifacts should remain JSON unless there is a separate explicit migration contract. + +## Current code areas affected + +- `internal/project/project.go` + - `Config.Music` currently only supports `prefixes`. + - `ValidateLayout` calls `validateMusicConfig`. + - `AssetExtensions` currently defines which music files appear in project inventory. + - `Scan` only scans the configured assets directory, so datasets outside `paths.assets` need a deliberate design. +- `internal/pipeline/music.go` + - all scanning, naming, metadata, conversion, credits, cache, and cleanup logic is currently one private pipeline helper. + - hardcoded assumptions include `envi/music`, `.cache/music`, `.cache/credits`, `CREDITS.md`, BMU output, ffmpeg args, metadata tags, and NWN stem rules. +- `internal/pipeline/build.go` + - HAK planning/building depends on `prepareMusicAssets`. + - plan-only HAK workflows can still invoke music probing/conversion today through HAK planning paths. +- `internal/app/app.go` + - `build-haks` owns the only user-facing music summary. + - New music commands should reuse the same logging/verbosity model instead of inventing unrelated output. +- `internal/pipeline/pipeline_test.go` + - currently has regression coverage for conversion, generated credits, manual overlay, inventory inclusion, staging cleanup, and unchanged second pass. +- `internal/app/app_test.go` + - currently checks normal versus verbose mapping output. +- `README.md` + - currently documents only `music.prefixes`. + +# Design constraints and decisions + +## Dataset roots + +Datasets should explicitly state where they read from and where generated artifacts go. + +Important distinction: + +- `source` is the authored input root for a dataset. +- `output` is the generated publish/staging root for transformed audio resources. +- `credits.output` is where generated human-readable credits go. +- `manifest.output` is where generated machine-readable dataset state goes. +- `cache.root` is where reusable implementation caches live. + +Do not assume dataset sources are under `paths.assets`. Support both: + +- asset-relative dataset roots for existing HAK packaging behavior +- project-relative dataset roots for future non-HAK or external datasets + +For HAK builds, the pipeline must know whether generated outputs are meant to be injected into the project asset inventory and under which resource-relative path. + +## NWN-specific behavior + +The toolkit can support NWN conventions, but they must be selected by configuration instead of implicit global behavior. + +Suggested built-in profiles: + +- `nwn_bmu` + - output extension: `.bmu` + - max stem length: 16 + - allowed stem characters: `[a-z0-9_]` + - lower-case output names + - default ffmpeg encoder compatible with current behavior +- `passthrough` + - no renaming unless configured + - no transcoding unless configured +- `audio_export` + - configurable format, stem length, and codec rules without NWN resource constraints + +Do not make `nwn_bmu` the only naming implementation. + +## Configuration merge semantics + +Merge configuration in this order: + +1. toolkit defaults +2. repository `music.defaults` +3. dataset definition +4. command-line/per-task overrides + +Merge rules should be explicit: + +- scalar fields replace inherited values +- maps merge by key +- lists replace by default unless a field is documented as appendable +- empty strings mean "not set", not "clear this inherited field" +- booleans that need tri-state inheritance should use pointer/nullable semantics internally +- command-line overrides should be represented in result metadata so logs can show effective behavior + +Reject unknown fields via strict YAML decode, matching the rest of the project config philosophy. + +## Compatibility config shape + +The current config: + +```yaml +music: + prefixes: + envi/music/westgate: mus_wg_ +``` + +should be treated as a compatibility shorthand equivalent to: + +```yaml +music: + datasets: + envi_music: + source: envi/music + package: + mode: hak_asset + output_root: envi/music + naming: + scheme: nwn_bmu + prefixes: + westgate: mus_wg_ +``` + +Exact normalized dataset ids may differ, but the migration must define how old `music.prefixes` maps into the new dataset model. + +## Proposed expanded schema + +Example target: + +```yaml +music: + tools: + ffmpeg: "" # optional; fallback to SOW_FFMPEG then PATH + ffprobe: "" # optional; fallback to SOW_FFPROBE then PATH + + defaults: + discover: + include: + - "**/*.mp3" + - "**/*.ogg" + passthrough: + - "**/*.bmu" + - "**/*.wav" + exclude: [] + credits_file: CREDITS.md + + naming: + scheme: nwn_bmu + max_stem_length: 16 + prefix: "" + prefixes: {} + drop_words: [] + reserved_stems_from_assets: true + + metadata: + enabled: true + prefer_overlay: true + probe_tags: + title: + - title + artist: + - artist + - album_artist + - composer + rights: + - license + - license_url + - copyright + + transcoding: + enabled: true + output_extension: .bmu + codec: libmp3lame + format: mp3 + sample_rate: 44100 + channels: 2 + bitrate: 192k + strip_metadata: true + ffmpeg_args: [] + + credits: + enabled: true + output: .cache/credits + inventory: .cache/credits/credits.json + format: nwn_markdown_table + + cache: + root: .cache/music + mode: content_hash + keep_staging: false + + manifest: + enabled: true + output: .cache/music/manifest.json + format: json + + validation: + require_credits_for_generated: false + reject_unknown_overlay_rows: true + reject_output_collisions: true + + datasets: + westgate: + source: envi/music/westgate + output: envi/music/westgate + package: + mode: hak_asset + naming: + prefix: mus_wg_ +``` + +Schema names can change during implementation, but the final shape should preserve these concepts. + +# Pipeline architecture + +## Suggested package layout + +Prefer a dedicated internal subsystem instead of expanding `internal/pipeline/music.go`. + +Suggested layout: + +```text +internal/music/ + config.go typed config, defaults, merge, validation + dataset.go resolved dataset model and effective settings + discover.go source discovery and file classification + metadata.go ffprobe integration and metadata normalization + credits.go markdown parsing/rendering and inventory generation + naming.go naming schemes, collision detection, compatibility prefix logic + transcode.go ffmpeg integration and output writing + cache.go content hashing and incremental state + manifest.go machine-readable dataset/stage manifests + validate.go dataset validation diagnostics + pipeline.go stage orchestration + result.go machine-readable stage/build result types +``` + +Then keep `internal/pipeline` responsible for HAK/module orchestration and call `internal/music` through a narrow adapter. + +## Stage model + +Each stage should have: + +- typed input +- typed output +- deterministic ordering +- dry-run support where it can avoid writes +- changed/unchanged reporting +- structured diagnostics +- stable machine-readable result data + +Suggested stages: + +1. Resolve datasets and effective config. +2. Discover candidate files. +3. Load authored overlays. +4. Probe source metadata. +5. Plan output names and detect collisions. +6. Plan transformations. +7. Reuse cached outputs or transcode changed files. +8. Render generated credits. +9. Write dataset manifest/inventory. +10. Return generated package resources to the HAK pipeline. +11. Validate outputs and report diagnostics. + +## Result model + +Music command results should be structured enough for both CLI output and tests. + +Include: + +- dataset id +- source root +- output root +- files discovered +- files converted +- files reused from cache +- passthrough files +- skipped files +- generated resources +- credit artifacts +- manifest paths +- changed files +- warnings/errors +- source-to-output mappings +- effective naming/transcoding profile + +Do not require the app layer to parse free-form progress strings. + +# Incremental and cache behavior + +## Content identity + +Incremental rebuilds should be based on content and effective configuration, not only mtimes. + +Cache keys should include: + +- source content hash +- source relative path +- effective dataset id +- effective naming settings +- effective transcoding settings +- ffmpeg command/profile version +- toolkit music pipeline version +- manual overlay data that affects output metadata/name + +Changing an output prefix, codec, bitrate, overlay output filename, or metadata rendering rules must invalidate the relevant planned output. + +## Output writes + +- Avoid rewriting unchanged credits, manifests, and transformed outputs. +- Write through temporary files and atomic rename where practical. +- Preserve deterministic JSON indentation and key ordering. +- Remove stale generated artifacts inside owned generated roots. +- Never remove authored source files. +- Never remove files outside configured owned cache/output roots. + +## Plan-only behavior + +Current HAK planning can trigger conversion work. The refactor should define stricter behavior: + +- `music scan` and HAK `--plan-only` should not transcode by default. +- They may probe metadata only if needed and allowed. +- They should be able to produce planned mappings without writing transformed audio. +- If actual resource size is needed for HAK chunk planning, use cached transformed size when available and report when exact planning requires a build. + +# CLI design expansion + +Add first-class commands while preserving `build-haks` compatibility. + +Suggested commands: + +```text +sow-toolkit music list-datasets +sow-toolkit music scan [--dataset ] [--json] +sow-toolkit music build [--dataset ] [--dry-run] [--force] +sow-toolkit music credits [--dataset ] [--check] +sow-toolkit music validate [--dataset ] [--json] +sow-toolkit music manifest [--dataset ] [--check] +``` + +Integration commands: + +```text +sow-toolkit build-haks +sow-toolkit build-haks --skip-music +sow-toolkit build-haks --music-dataset +``` + +Command output should follow the existing log refactor direction: + +- normal mode: phase summaries and counts +- verbose mode: mappings and per-file actions +- debug mode: resolved config, tool paths, ffmpeg args, cache keys +- quiet mode: final summary only +- `--json`: machine-readable result on stdout and human diagnostics on stderr + +# Validation requirements + +Validate repository config: + +- duplicate dataset ids +- invalid dataset source/output paths +- absolute paths unless explicitly allowed +- path traversal outside project root for owned outputs/caches +- duplicate or ambiguous prefix mappings +- prefix too long for selected naming scheme +- unsupported output extension for selected package mode +- invalid codec/profile fields +- impossible inherited config combinations +- `package.mode: hak_asset` without an asset-relative output root +- unknown naming/transcoding/credits/manifest schemes + +Validate discovered music: + +- source files with unsupported extensions +- duplicate case-insensitive output stems +- output name collision with existing assets and generated outputs +- manual overlay rows referencing unknown originals or outputs +- manual overlay output filenames violating selected naming scheme +- metadata probe failures +- source paths differing only by case on case-insensitive targets +- generated resource names that exceed NWN constraints +- generated output extension/resource type compatibility + +Validation severity should distinguish: + +- errors that block deterministic build output +- warnings for incomplete credits or missing optional metadata +- info for skipped files and unchanged caches + +# Fringe considerations + +## Credits and licensing + +- Preserve authored `CREDITS.md` files as source overlays. +- Generated credits should clearly identify generated versus authored sources in inventory. +- Missing artist/title/license can be warning-level or error-level per dataset. +- Overlay parsing should keep accepting backtick-wrapped filenames. +- Markdown parsing should continue to support escaped pipes and `
` line breaks. +- Generated credits should be deterministic across platforms. +- Consider whether aggregate credits inventory should include datasets disabled for build but enabled for credits. + +## Filename and path edge cases + +- Normalize separators to forward slashes in config, manifests, and logs. +- Treat paths case-insensitively for resource collision checks. +- Reject NUL bytes and unsafe relative paths. +- Define behavior for spaces, Unicode, apostrophes, brackets, years, and punctuation in stems. +- Keep existing slug/drop-word behavior for `nwn_bmu` compatibility, but make drop words configurable. +- Avoid Windows-reserved names and problematic trailing dots/spaces if outputs may be checked out or built on Windows. +- Detect collisions after truncation and suffixing. +- Validate suffixing still respects max stem length. + +## Audio tooling + +- Keep `SOW_FFMPEG` and `SOW_FFPROBE` as compatibility fallbacks. +- Add config-level tool paths for repositories that pin local toolchain binaries. +- Expose ffmpeg/ffprobe version in debug output and manifest metadata. +- Capture ffmpeg failures with enough context to diagnose source file, dataset, and command profile. +- Do not print enormous ffmpeg logs in normal mode unless the command fails. +- Decide whether `.ogg -> .bmu` should continue transcoding or become configurable passthrough for datasets that want native OGG output. + +## HAK/resource integration + +- Generated music resources must enter collision detection alongside authored resources. +- Source `.mp3`/`.ogg` files that are converted must be omitted from HAK packaging unless config says otherwise. +- Passthrough `.bmu` and `.wav` handling should remain distinct from generated outputs. +- Resource sorting should stay deterministic and reuse existing HAK resource ordering rules. +- Generated music should be represented in build manifests with enough source provenance to debug. +- HAK include globs should match generated output paths, not source paths, unless explicitly configured otherwise. + +## Cross-repository behavior + +- A repository with no `music` config should not pay a heavy ffmpeg/ffprobe cost. +- A repository with authored `.bmu` files but no generated music config should preserve current passthrough packaging. +- Multiple datasets should be able to target different output roots without collisions. +- Multiple repositories should be able to use different naming schemes without code changes. +- Dataset config should be portable between Linux and Windows path conventions. +- The toolkit should not assume Shadows Over Westgate names, HAK names, prefixes, or source layout. + +# Migration strategy + +## Phase 1: Extract without behavior changes + +- Move existing private music functions from `internal/pipeline/music.go` into `internal/music`. +- Keep the current `prepareMusicAssets` adapter in `internal/pipeline` to avoid broad HAK changes. +- Preserve every existing test expectation. +- Add focused unit tests around moved naming, overlay, metadata, and credits functions. + +## Phase 2: Introduce dataset config + +- Extend `project.MusicConfig` with typed defaults and datasets. +- Keep `music.prefixes` as deprecated compatibility shorthand. +- Implement config normalization from legacy prefixes into a default dataset. +- Add strict validation for dataset ids, paths, naming, and prefix constraints. +- Update README examples. + +## Phase 3: Add first-class music commands + +- Wire `music scan`, `music build`, `music credits`, and `music validate` into `internal/app`. +- Reuse existing verbosity levels and summary formatting. +- Add JSON output support if the command result model is ready. +- Ensure `build-haks` calls the same service API as the music commands. + +## Phase 4: Incremental generated output cache + +- Replace temporary-only `.cache/music` staging with a content-addressed cache when configured. +- Keep cleanup behavior compatible for existing builds unless `cache.keep_staging` is enabled. +- Avoid conversion during dry-run/plan-only unless exact output artifacts are explicitly requested. +- Add stale generated artifact cleanup scoped only to owned roots. + +## Phase 5: Repository migration + +- Convert Shadows Over Westgate config from `music.prefixes` to explicit dataset config. +- Keep compatibility warning for one release window. +- Update consumer wrapper scripts only if command names change. +- Verify existing generated HAK manifests and credits inventories remain stable or document intentional diffs. + +# Test plan + +Add or preserve tests for: + +- legacy `music.prefixes` compatibility +- explicit dataset config equivalent to current Westgate behavior +- dataset source outside `envi/music` +- multiple datasets with independent prefixes and outputs +- no music config with authored `.bmu` passthrough assets +- manual overlay output filename validation +- manual overlay references unknown source/output +- duplicate generated output collisions +- collision with existing authored asset stem +- max stem length and suffix collision behavior +- unchanged second build does not rewrite credits or manifests +- dry-run/plan-only does not transcode +- missing ffmpeg/ffprobe failure messages +- configured ffmpeg/ffprobe paths +- normal, verbose, quiet, debug, and JSON CLI output +- Windows-style path separators in config normalize deterministically +- generated credits and inventory are deterministic + +# Acceptance criteria + +- Music functionality is reusable through a dedicated toolkit subsystem. +- No repository-specific path or prefix is hardcoded in music implementation. +- Existing `build-haks` behavior remains compatible for Shadows Over Westgate. +- First-class music commands can scan, build, generate credits, and validate a configured dataset. +- Dataset configuration can express current Westgate behavior and at least one non-Westgate layout. +- Generated outputs are deterministic and unchanged files are not rewritten. +- Machine-readable artifacts remain JSON. +- Logs are concise by default and detailed under verbose/debug. +- Tests cover the compatibility path and the new dataset-driven path. diff --git a/internal/app/app.go b/internal/app/app.go index 07b3f38..d946b60 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1,6 +1,7 @@ package app import ( + "encoding/json" "errors" "fmt" "io" @@ -16,6 +17,7 @@ import ( "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/topdata" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/validator" + "gopkg.in/yaml.v3" ) type command struct { @@ -132,17 +134,17 @@ func isInteractiveTTY(w io.Writer) bool { var commands = []command{ { name: "build", - description: "Build module and HAK outputs into build/, plus top package output when topdata is configured.", + description: "Build module, HAK, and configured top package outputs.", run: runBuild, }, { name: "build-module", - description: "Build only the module archive from src/ into build/.", + description: "Build only the configured module archive.", run: runBuildModule, }, { name: "build-haks", - description: "Build only HAK archives and manifest from assets/ into build/.", + description: "Build only configured HAK archives and manifest.", run: runBuildHAKs, }, { @@ -155,6 +157,11 @@ var commands = []command{ description: "Validate project layout, config, and source inventory.", run: runValidate, }, + { + name: "config", + description: "Inspect, explain, and validate the resolved project configuration.", + run: runConfig, + }, { name: "compare", description: "Compare current source content against the built archives.", @@ -172,12 +179,12 @@ var commands = []command{ }, { name: "build-topdata", - description: "Build canonical topdata outputs into .cache/2da and .cache/wiki, then refresh build/sow_top.hak and build/sow_tlk.tlk.", + description: "Build configured topdata outputs, then refresh the configured top package.", run: runBuildTopData, }, { name: "build-top-package", - description: "Build build/sow_top.hak and build/sow_tlk.tlk from cached native topdata output and authored topdata/assets.", + description: "Build the configured top package from cached native topdata output and authored topdata assets.", run: runBuildTopPackage, }, { @@ -670,6 +677,140 @@ func runValidate(ctx context) error { return nil } +func runConfig(ctx context) error { + if len(ctx.args) < 2 { + return errors.New("usage: config ...") + } + root, err := project.FindRoot(ctx.cwd) + if err != nil { + return err + } + p, err := project.Load(root) + if err != nil { + return err + } + + switch ctx.args[1] { + case "validate": + if len(ctx.args) > 2 { + return fmt.Errorf("usage: config validate") + } + if err := p.ValidateLayout(); err != nil { + return err + } + fmt.Fprintf(ctx.stdout, "config: ok\n") + fmt.Fprintf(ctx.stdout, "source: %s\n", p.ConfigSource.Path) + return nil + case "effective": + format, err := parseConfigFormatArgs("config effective", ctx.args[2:]) + if err != nil { + return err + } + return emitConfigValue(ctx.stdout, p.EffectiveConfig(), format) + case "inspect": + key := "" + format := "yaml" + for _, arg := range ctx.args[2:] { + switch arg { + case "--json": + format = "json" + case "--yaml": + format = "yaml" + default: + if key != "" { + return fmt.Errorf("usage: config inspect [] [--json|--yaml]") + } + key = arg + } + } + if key == "" { + return emitConfigValue(ctx.stdout, p.EffectiveConfig(), format) + } + value, _, ok := p.ExplainConfig(key) + if !ok { + return fmt.Errorf("unknown config key %q", key) + } + return emitConfigValue(ctx.stdout, value, format) + case "explain": + if len(ctx.args) != 3 { + return fmt.Errorf("usage: config explain ") + } + key := ctx.args[2] + value, provenance, ok := p.ExplainConfig(key) + if !ok { + return fmt.Errorf("unknown config key %q", key) + } + fmt.Fprintf(ctx.stdout, "key: %s\n", key) + fmt.Fprintf(ctx.stdout, "value: %s\n", formatConfigScalar(value)) + fmt.Fprintf(ctx.stdout, "source: %s\n", provenance.Source) + if provenance.Detail != "" { + fmt.Fprintf(ctx.stdout, "detail: %s\n", provenance.Detail) + } + if provenance.Deprecated { + fmt.Fprintf(ctx.stdout, "deprecated: true\n") + } + return nil + case "sources": + if len(ctx.args) > 2 { + return fmt.Errorf("usage: config sources") + } + fmt.Fprintf(ctx.stdout, "loaded: %s\n", p.ConfigSource.Path) + fmt.Fprintf(ctx.stdout, "name: %s\n", p.ConfigSource.Name) + fmt.Fprintf(ctx.stdout, "format: %s\n", p.ConfigSource.Format) + fmt.Fprintf(ctx.stdout, "legacy: %t\n", p.ConfigSource.Legacy) + fmt.Fprintf(ctx.stdout, "candidates: %s\n", strings.Join(project.ConfigFileCandidates, ", ")) + return nil + default: + return fmt.Errorf("unknown config subcommand %q", ctx.args[1]) + } +} + +func parseConfigFormatArgs(commandName string, args []string) (string, error) { + format := "yaml" + for _, arg := range args { + switch arg { + case "--json": + format = "json" + case "--yaml": + format = "yaml" + default: + return "", fmt.Errorf("usage: %s [--json|--yaml]", commandName) + } + } + return format, nil +} + +func emitConfigValue(w io.Writer, value any, format string) error { + var ( + raw []byte + err error + ) + switch format { + case "json": + raw, err = json.MarshalIndent(value, "", " ") + case "yaml": + raw, err = yaml.Marshal(value) + default: + return fmt.Errorf("unsupported config output format %q", format) + } + if err != nil { + return err + } + if len(raw) == 0 || raw[len(raw)-1] != '\n' { + raw = append(raw, '\n') + } + _, err = w.Write(raw) + return err +} + +func formatConfigScalar(value any) string { + raw, err := json.Marshal(value) + if err != nil { + return fmt.Sprint(value) + } + return string(raw) +} + func emitValidatorReport(w io.Writer, report validator.Report) { lines := report.SummaryLines(5) emitSummaryLines(w, lines, 20, "validation diagnostics") diff --git a/internal/app/app_test.go b/internal/app/app_test.go index adca376..4c09a03 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -198,6 +198,94 @@ func TestRunBuildHAKsVerboseListsMappings(t *testing.T) { } } +func TestRunConfigEffectiveJSONShowsDefaultsAndProvenance(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src +`) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"config", "effective", "--json"}, + } + + if err := runConfig(ctx); err != nil { + t.Fatalf("runConfig failed: %v", err) + } + output := stdout.String() + if !strings.Contains(output, `"hak_manifest": "haks.json"`) { + t.Fatalf("expected HAK manifest default in effective config, got %q", output) + } + if !strings.Contains(output, `"paths.build":`) || !strings.Contains(output, `"toolkit default"`) { + t.Fatalf("expected default provenance in effective config, got %q", output) + } +} + +func TestRunConfigExplainReportsYAMLSource(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + build: output +`) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"config", "explain", "paths.build"}, + } + + if err := runConfig(ctx); err != nil { + t.Fatalf("runConfig failed: %v", err) + } + output := stdout.String() + if !strings.Contains(output, "value: \"output\"") { + t.Fatalf("expected configured build value, got %q", output) + } + if !strings.Contains(output, "source: yaml") { + t.Fatalf("expected YAML source, got %q", output) + } +} + +func TestRunConfigValidateDoesNotScanSourceInventory(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "build")) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: missing-src + build: build +`) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"config", "validate"}, + } + + if err := runConfig(ctx); err != nil { + t.Fatalf("runConfig failed: %v", err) + } + if !strings.Contains(stdout.String(), "config: ok") { + t.Fatalf("expected config validation output, got %q", stdout.String()) + } +} + func mkdirAll(t *testing.T, path string) { t.Helper() if err := os.MkdirAll(path, 0o755); err != nil { diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index 508fda1..b3fa307 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -167,6 +167,9 @@ func buildModule(p *project.Project, progress ProgressFunc) (BuildResult, error) progressf(progress, "Writing module archive...") outputPath := p.ModuleArchivePath() + if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create module output dir: %w", err) + } output, err := os.Create(outputPath) if err != nil { return BuildResult{}, fmt.Errorf("create module archive: %w", err) @@ -321,6 +324,9 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo return BuildResult{}, err } progressf(progress, "Writing HAK manifest...") + if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create hak manifest dir: %w", err) + } if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil { return BuildResult{}, fmt.Errorf("write hak manifest: %w", err) } @@ -345,6 +351,9 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo } else { result.HAKSummary.Written++ progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets))) + if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err) + } hakOutput, err := os.Create(hakPath) if err != nil { return BuildResult{}, fmt.Errorf("create hak archive: %w", err) @@ -374,6 +383,9 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo } progressf(progress, "Writing HAK manifest...") + if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create hak manifest dir: %w", err) + } if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil { return BuildResult{}, fmt.Errorf("write hak manifest: %w", err) } @@ -558,7 +570,7 @@ func compileReferencedScripts(p *project.Project) ([]erf.Resource, error) { } resources := make([]erf.Resource, 0, len(referenced)) - scriptsDir := filepath.Join(p.SourceDir(), "scripts") + scriptsDir := p.ScriptSourceDir() for _, resref := range referenced { sourcePath, ok := scriptSources[resref] if !ok { @@ -586,7 +598,7 @@ func compileReferencedScripts(p *project.Project) ([]erf.Resource, error) { } func compiledScriptCacheDir(p *project.Project) string { - return filepath.Join(p.Root, ".cache", "ncs") + return p.ScriptCacheDir() } func referencedScriptResrefs(p *project.Project) ([]string, error) { diff --git a/internal/pipeline/music.go b/internal/pipeline/music.go index b8e940f..9a80f49 100644 --- a/internal/pipeline/music.go +++ b/internal/pipeline/music.go @@ -128,7 +128,7 @@ func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) { sort.Strings(dirs) generatedByDir := make(map[string][]creditsEntry) - stageRoot := filepath.Join(p.Root, ".cache", "music") + stageRoot := p.MusicStageDir() result.TempRoots = append(result.TempRoots, stageRoot) ffmpegPath, err := resolveFFmpegBinary() @@ -144,7 +144,7 @@ func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) { sources := append([]string(nil), dirSources[dir]...) sort.Strings(sources) prefix := musicPrefixForDir(p, dir) - overlayPath := filepath.Join(p.AssetsDir(), filepath.FromSlash(dir), "CREDITS.md") + overlayPath := filepath.Join(p.AssetsDir(), filepath.FromSlash(dir), p.EffectiveConfig().Music.CreditsOverlay) overlay, err := parseCreditsOverlay(overlayPath) if err != nil { return nil, fmt.Errorf("parse credits overlay %s: %w", overlayPath, err) @@ -239,7 +239,7 @@ type creditsArtifactWriteResult struct { } func writeCreditsArtifacts(p *project.Project, generatedByDir map[string][]creditsEntry) (creditsArtifactWriteResult, error) { - creditsRoot := filepath.Join(p.Root, ".cache", "credits") + creditsRoot := p.CreditsDir() sources := make([]creditsSource, 0) desiredFiles := make(map[string][]byte) summary := CreditsRefreshSummary{} @@ -260,7 +260,7 @@ func writeCreditsArtifacts(p *project.Project, generatedByDir map[string][]credi } return strings.ToLower(entries[i].OutputFile) < strings.ToLower(entries[j].OutputFile) }) - outputPath := filepath.Join(creditsRoot, filepath.FromSlash(dir), "CREDITS.md") + outputPath := filepath.Join(creditsRoot, filepath.FromSlash(dir), p.EffectiveConfig().Music.CreditsOverlay) desiredFiles[outputPath] = []byte(renderCreditsMarkdown(entries)) summary.GeneratedPaths = append(summary.GeneratedPaths, outputPath) for _, entry := range entries { diff --git a/internal/project/effective.go b/internal/project/effective.go new file mode 100644 index 0000000..3ee23b1 --- /dev/null +++ b/internal/project/effective.go @@ -0,0 +1,318 @@ +package project + +import ( + "encoding/json" + "fmt" + "path/filepath" + "slices" + "strings" + "time" +) + +const ( + DefaultBuildPath = "build" + DefaultCachePath = ".cache" + DefaultToolsPath = "tools" + DefaultModuleArchiveTemplate = "{module.resref}.mod" + DefaultHAKManifest = "haks.json" + DefaultHAKArchiveTemplate = "{hak.name}.hak" + DefaultSourceJSONPattern = "{resref}.{extension}.json" + DefaultScriptsSourceDir = "scripts" + DefaultScriptsCache = "{paths.cache}/ncs" + DefaultMusicStageRoot = "{paths.cache}/music" + DefaultCreditsRoot = "{paths.cache}/credits" + DefaultCreditsOverlayFile = "CREDITS.md" + DefaultTopDataBuild = "{paths.build}/topdata" + DefaultTopDataCompiled2DADir = "2da" + DefaultTopDataCompiledTLK = "sow_tlk.tlk" + DefaultTopDataWikiOutputRoot = "wiki" + DefaultTopDataWikiPagesDir = "pages" + DefaultTopDataWikiStateFile = "state.json" + DefaultTopDataPackageHAK = "sow_top.hak" + DefaultTopDataPackageTLK = "sow_tlk.tlk" + DefaultAutogenCacheRoot = "{paths.cache}" + DefaultAutogenCacheMaxAge = time.Hour + DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH" +) + +type ConfigProvenance map[string]ConfigValueProvenance + +type ConfigValueProvenance struct { + Source string `json:"source" yaml:"source"` + Detail string `json:"detail,omitempty" yaml:"detail,omitempty"` + Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"` +} + +type EffectiveConfig struct { + ConfigSource ConfigSource `json:"config_source" yaml:"config_source"` + Module ModuleConfig `json:"module" yaml:"module"` + Paths EffectivePathConfig `json:"paths" yaml:"paths"` + Outputs EffectiveOutputConfig `json:"outputs" yaml:"outputs"` + Inventory InventoryConfig `json:"inventory" yaml:"inventory"` + Scripts EffectiveScriptsConfig `json:"scripts" yaml:"scripts"` + Music EffectiveMusicConfig `json:"music" yaml:"music"` + TopData EffectiveTopDataConfig `json:"topdata" yaml:"topdata"` + Extract ExtractConfig `json:"extract" yaml:"extract"` + Autogen EffectiveAutogenConfig `json:"autogen" yaml:"autogen"` + HAKs []HAKConfig `json:"haks" yaml:"haks"` + Provenance ConfigProvenance `json:"provenance" yaml:"provenance"` +} + +type EffectivePathConfig struct { + Source string `json:"source" yaml:"source"` + Assets string `json:"assets" yaml:"assets"` + Build string `json:"build" yaml:"build"` + Cache string `json:"cache" yaml:"cache"` + Tools string `json:"tools" yaml:"tools"` +} + +type EffectiveOutputConfig struct { + ModuleArchive string `json:"module_archive" yaml:"module_archive"` + HAKManifest string `json:"hak_manifest" yaml:"hak_manifest"` + HAKArchive string `json:"hak_archive" yaml:"hak_archive"` +} + +type EffectiveScriptsConfig struct { + SourceDir string `json:"source_dir" yaml:"source_dir"` + Cache string `json:"cache" yaml:"cache"` +} + +type EffectiveMusicConfig struct { + Prefixes map[string]string `json:"prefixes" yaml:"prefixes"` + StageRoot string `json:"stage_root" yaml:"stage_root"` + CreditsRoot string `json:"credits_root" yaml:"credits_root"` + CreditsOverlay string `json:"credits_overlay" yaml:"credits_overlay"` + MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"` +} + +type EffectiveTopDataConfig struct { + Source string `json:"source" yaml:"source"` + Build string `json:"build" yaml:"build"` + ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"` + Assets string `json:"assets" yaml:"assets"` + Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"` + CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"` + PackageHAK string `json:"package_hak" yaml:"package_hak"` + PackageTLK string `json:"package_tlk" yaml:"package_tlk"` + Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"` +} + +type EffectiveAutogenConfig struct { + Producers []AutogenProducerConfig `json:"producers" yaml:"producers"` + Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"` + Cache EffectiveAutogenCacheConfig `json:"cache" yaml:"cache"` +} + +type EffectiveAutogenCacheConfig struct { + Root string `json:"root" yaml:"root"` + MaxAge string `json:"max_age" yaml:"max_age"` + RefreshEnv string `json:"refresh_env" yaml:"refresh_env"` +} + +func (p *Project) EffectiveConfig() EffectiveConfig { + provenance := cloneProvenance(p.Provenance) + markMissingDefaults(provenance) + + paths := EffectivePathConfig{ + Source: strings.TrimSpace(p.Config.Paths.Source), + Assets: strings.TrimSpace(p.Config.Paths.Assets), + Build: defaultString(p.Config.Paths.Build, DefaultBuildPath), + Cache: defaultString(p.Config.Paths.Cache, DefaultCachePath), + Tools: defaultString(p.Config.Paths.Tools, DefaultToolsPath), + } + outputs := EffectiveOutputConfig{ + ModuleArchive: defaultString(p.Config.Outputs.ModuleArchive, DefaultModuleArchiveTemplate), + HAKManifest: defaultString(p.Config.Outputs.HAKManifest, DefaultHAKManifest), + HAKArchive: defaultString(p.Config.Outputs.HAKArchive, DefaultHAKArchiveTemplate), + } + inventory := InventoryConfig{ + SourceExtensions: defaultStringSlice(p.Config.Inventory.SourceExtensions, SourceExtensions), + AssetExtensions: defaultStringSlice(p.Config.Inventory.AssetExtensions, AssetExtensions), + SourceJSONPattern: defaultString(p.Config.Inventory.SourceJSONPattern, DefaultSourceJSONPattern), + } + scripts := EffectiveScriptsConfig{ + SourceDir: defaultString(p.Config.Scripts.SourceDir, DefaultScriptsSourceDir), + Cache: expandPathTemplate(defaultString(p.Config.Scripts.Cache, DefaultScriptsCache), paths), + } + topBuild := expandPathTemplate(defaultString(p.Config.TopData.Build, DefaultTopDataBuild), paths) + top := EffectiveTopDataConfig{ + Source: strings.TrimSpace(p.Config.TopData.Source), + Build: topBuild, + ReferenceBuilder: strings.TrimSpace(p.Config.TopData.ReferenceBuilder), + Assets: strings.TrimSpace(p.Config.TopData.Assets), + Compiled2DADir: defaultString(p.Config.TopData.Compiled2DADir, DefaultTopDataCompiled2DADir), + CompiledTLK: defaultString(p.Config.TopData.CompiledTLK, DefaultTopDataCompiledTLK), + PackageHAK: defaultString(p.Config.TopData.PackageHAK, DefaultTopDataPackageHAK), + PackageTLK: defaultString(p.Config.TopData.PackageTLK, DefaultTopDataPackageTLK), + Wiki: TopDataWikiConfig{ + OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot), + PagesDir: defaultString(p.Config.TopData.Wiki.PagesDir, DefaultTopDataWikiPagesDir), + StateFile: defaultString(p.Config.TopData.Wiki.StateFile, DefaultTopDataWikiStateFile), + }, + } + + return EffectiveConfig{ + ConfigSource: p.ConfigSource, + Module: p.Config.Module, + Paths: paths, + Outputs: outputs, + Inventory: inventory, + Scripts: scripts, + Music: EffectiveMusicConfig{ + Prefixes: cloneStringMap(p.Config.Music.Prefixes), + StageRoot: expandPathTemplate(DefaultMusicStageRoot, paths), + CreditsRoot: expandPathTemplate(DefaultCreditsRoot, paths), + CreditsOverlay: DefaultCreditsOverlayFile, + MaxStemLength: 16, + }, + TopData: top, + Extract: p.Config.Extract, + Autogen: EffectiveAutogenConfig{ + Producers: slices.Clone(p.Config.Autogen.Producers), + Consumers: slices.Clone(p.Config.Autogen.Consumers), + Cache: EffectiveAutogenCacheConfig{ + Root: expandPathTemplate(defaultString(p.Config.Autogen.Cache.Root, DefaultAutogenCacheRoot), paths), + MaxAge: DefaultAutogenCacheMaxAge.String(), + RefreshEnv: DefaultAutogenRefreshEnv, + }, + }, + HAKs: slices.Clone(p.Config.HAKs), + Provenance: provenance, + } +} + +func (p *Project) EffectiveConfigJSON() ([]byte, error) { + return json.MarshalIndent(p.EffectiveConfig(), "", " ") +} + +func (p *Project) ExplainConfig(key string) (any, ConfigValueProvenance, bool) { + effective := p.EffectiveConfig() + value, ok := lookupEffectiveValue(effective, key) + if !ok { + return nil, ConfigValueProvenance{}, false + } + prov := effective.Provenance[key] + if prov.Source == "" { + prov = ConfigValueProvenance{Source: "yaml", Detail: p.ConfigSource.Name} + } + return value, prov, true +} + +func lookupEffectiveValue(effective EffectiveConfig, key string) (any, bool) { + raw, err := json.Marshal(effective) + if err != nil { + return nil, false + } + var data any + if err := json.Unmarshal(raw, &data); err != nil { + return nil, false + } + current := data + for _, part := range strings.Split(key, ".") { + object, ok := current.(map[string]any) + if !ok { + return nil, false + } + current, ok = object[part] + if !ok { + return nil, false + } + } + return current, true +} + +func markMissingDefaults(provenance ConfigProvenance) { + defaults := map[string]string{ + "paths.build": "build", + "paths.cache": ".cache", + "paths.tools": "tools", + "outputs.module_archive": DefaultModuleArchiveTemplate, + "outputs.hak_manifest": DefaultHAKManifest, + "outputs.hak_archive": DefaultHAKArchiveTemplate, + "inventory.source_extensions": "NWN source resource extensions", + "inventory.asset_extensions": "NWN asset resource extensions", + "inventory.source_json_pattern": DefaultSourceJSONPattern, + "scripts.source_dir": DefaultScriptsSourceDir, + "scripts.cache": DefaultScriptsCache, + "music.stage_root": DefaultMusicStageRoot, + "music.credits_root": DefaultCreditsRoot, + "music.credits_overlay": DefaultCreditsOverlayFile, + "music.max_stem_length": "16", + "topdata.build": DefaultTopDataBuild, + "topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir, + "topdata.compiled_tlk": DefaultTopDataCompiledTLK, + "topdata.package_hak": DefaultTopDataPackageHAK, + "topdata.package_tlk": DefaultTopDataPackageTLK, + "topdata.wiki.output_root": DefaultTopDataWikiOutputRoot, + "topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir, + "topdata.wiki.state_file": DefaultTopDataWikiStateFile, + "autogen.cache.root": DefaultAutogenCacheRoot, + "autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(), + "autogen.cache.refresh_env": DefaultAutogenRefreshEnv, + } + for key, detail := range defaults { + if _, ok := provenance[key]; !ok { + provenance[key] = ConfigValueProvenance{Source: "toolkit default", Detail: detail} + } + } +} + +func cloneProvenance(input ConfigProvenance) ConfigProvenance { + out := ConfigProvenance{} + for key, value := range input { + out[key] = value + } + return out +} + +func defaultString(value, fallback string) string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return fallback + } + return trimmed +} + +func defaultStringSlice(value, fallback []string) []string { + if len(value) == 0 { + return slices.Clone(fallback) + } + return slices.Clone(value) +} + +func cloneStringMap(input map[string]string) map[string]string { + if len(input) == 0 { + return nil + } + out := make(map[string]string, len(input)) + for key, value := range input { + out[key] = value + } + return out +} + +func expandPathTemplate(template string, paths EffectivePathConfig) string { + replacer := strings.NewReplacer( + "{paths.source}", paths.Source, + "{paths.assets}", paths.Assets, + "{paths.build}", paths.Build, + "{paths.cache}", paths.Cache, + "{paths.tools}", paths.Tools, + ) + return filepath.ToSlash(replacer.Replace(template)) +} + +func (o EffectiveOutputConfig) ModuleArchiveName(module ModuleConfig) string { + return strings.NewReplacer("{module.resref}", strings.TrimSpace(module.ResRef)).Replace(o.ModuleArchive) +} + +func (o EffectiveOutputConfig) HAKArchiveName(name string) string { + return strings.NewReplacer("{hak.name}", strings.TrimSpace(name)).Replace(o.HAKArchive) +} + +func (p ConfigValueProvenance) String() string { + if p.Detail == "" { + return p.Source + } + return fmt.Sprintf("%s (%s)", p.Source, p.Detail) +} diff --git a/internal/project/project.go b/internal/project/project.go index 48044ba..ef54e21 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -36,6 +36,7 @@ type Project struct { Root string Config Config ConfigSource ConfigSource + Provenance ConfigProvenance Inventory Inventory } @@ -47,13 +48,16 @@ type ConfigSource struct { } type Config struct { - Module ModuleConfig `json:"module" yaml:"module"` - Paths PathConfig `json:"paths" yaml:"paths"` - HAKs []HAKConfig `json:"haks" yaml:"haks"` - Music MusicConfig `json:"music" yaml:"music"` - TopData TopDataConfig `json:"topdata" yaml:"topdata"` - Extract ExtractConfig `json:"extract" yaml:"extract"` - Autogen AutogenConfig `json:"autogen" yaml:"autogen"` + Module ModuleConfig `json:"module" yaml:"module"` + Paths PathConfig `json:"paths" yaml:"paths"` + Outputs OutputConfig `json:"outputs" yaml:"outputs"` + HAKs []HAKConfig `json:"haks" yaml:"haks"` + Inventory InventoryConfig `json:"inventory" yaml:"inventory"` + Scripts ScriptsConfig `json:"scripts" yaml:"scripts"` + Music MusicConfig `json:"music" yaml:"music"` + TopData TopDataConfig `json:"topdata" yaml:"topdata"` + Extract ExtractConfig `json:"extract" yaml:"extract"` + Autogen AutogenConfig `json:"autogen" yaml:"autogen"` } type ModuleConfig struct { @@ -67,6 +71,25 @@ type PathConfig struct { Source string `json:"source" yaml:"source"` Assets string `json:"assets" yaml:"assets"` Build string `json:"build" yaml:"build"` + Cache string `json:"cache" yaml:"cache"` + Tools string `json:"tools" yaml:"tools"` +} + +type OutputConfig struct { + ModuleArchive string `json:"module_archive" yaml:"module_archive"` + HAKManifest string `json:"hak_manifest" yaml:"hak_manifest"` + HAKArchive string `json:"hak_archive" yaml:"hak_archive"` +} + +type InventoryConfig struct { + SourceExtensions []string `json:"source_extensions" yaml:"source_extensions"` + AssetExtensions []string `json:"asset_extensions" yaml:"asset_extensions"` + SourceJSONPattern string `json:"source_json_pattern" yaml:"source_json_pattern"` +} + +type ScriptsConfig struct { + SourceDir string `json:"source_dir" yaml:"source_dir"` + Cache string `json:"cache" yaml:"cache"` } type HAKConfig struct { @@ -83,12 +106,21 @@ type MusicConfig struct { } type TopDataConfig struct { - Source string `json:"source" yaml:"source"` - Build string `json:"build" yaml:"build"` - ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"` - Assets string `json:"assets" yaml:"assets"` - PackageHAK string `json:"package_hak" yaml:"package_hak"` - PackageTLK string `json:"package_tlk" yaml:"package_tlk"` + Source string `json:"source" yaml:"source"` + Build string `json:"build" yaml:"build"` + ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"` + Assets string `json:"assets" yaml:"assets"` + Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"` + CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"` + PackageHAK string `json:"package_hak" yaml:"package_hak"` + PackageTLK string `json:"package_tlk" yaml:"package_tlk"` + Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"` +} + +type TopDataWikiConfig struct { + OutputRoot string `json:"output_root" yaml:"output_root"` + PagesDir string `json:"pages_dir" yaml:"pages_dir"` + StateFile string `json:"state_file" yaml:"state_file"` } type ExtractConfig struct { @@ -99,6 +131,11 @@ type ExtractConfig struct { type AutogenConfig struct { Producers []AutogenProducerConfig `json:"producers" yaml:"producers"` Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"` + Cache AutogenCacheConfig `json:"cache" yaml:"cache"` +} + +type AutogenCacheConfig struct { + Root string `json:"root" yaml:"root"` } type AutogenProducerConfig struct { @@ -175,6 +212,10 @@ func Load(root string) (*Project, error) { } cfg := defaultConfig() + provenance, err := configProvenance(raw, source) + if err != nil { + return nil, err + } if source.Format == "yaml" { decoder := yaml.NewDecoder(bytes.NewReader(raw)) decoder.KnownFields(true) @@ -202,6 +243,7 @@ func Load(root string) (*Project, error) { Root: root, Config: cfg, ConfigSource: source, + Provenance: provenance, }, nil } @@ -226,6 +268,51 @@ func findConfigSource(root string) (ConfigSource, error) { return ConfigSource{}, fmt.Errorf("could not find %s, %s, or legacy %s in %s", ConfigFile, ConfigFileYML, LegacyConfigFile, root) } +func configProvenance(raw []byte, source ConfigSource) (ConfigProvenance, error) { + var data any + if source.Format == "yaml" { + if err := yaml.Unmarshal(raw, &data); err != nil { + return nil, fmt.Errorf("parse %s for provenance: %w", source.Name, err) + } + } else { + if err := json.Unmarshal(raw, &data); err != nil { + return nil, fmt.Errorf("parse legacy %s for provenance: %w", source.Name, err) + } + } + provenance := ConfigProvenance{} + sourceName := "yaml" + if source.Legacy { + sourceName = "legacy json" + } + recordProvenance(provenance, "", data, ConfigValueProvenance{ + Source: sourceName, + Detail: source.Name, + Deprecated: source.Legacy, + }) + return provenance, nil +} + +func recordProvenance(provenance ConfigProvenance, prefix string, value any, source ConfigValueProvenance) { + switch typed := value.(type) { + case map[string]any: + for key, nested := range typed { + next := key + if prefix != "" { + next = prefix + "." + key + } + recordProvenance(provenance, next, nested, source) + } + case []any: + if prefix != "" { + provenance[prefix] = source + } + default: + if prefix != "" { + provenance[prefix] = source + } + } +} + func (p *Project) ValidateLayout() error { var failures []error @@ -245,15 +332,49 @@ func (p *Project) ValidateLayout() error { "paths.source": p.Config.Paths.Source, "paths.assets": p.Config.Paths.Assets, "paths.build": p.Config.Paths.Build, + "paths.cache": p.Config.Paths.Cache, + "paths.tools": p.Config.Paths.Tools, + "outputs.module_archive": p.Config.Outputs.ModuleArchive, + "outputs.hak_manifest": p.Config.Outputs.HAKManifest, + "outputs.hak_archive": p.Config.Outputs.HAKArchive, + "scripts.source_dir": p.Config.Scripts.SourceDir, + "scripts.cache": p.Config.Scripts.Cache, "topdata.source": p.Config.TopData.Source, "topdata.build": p.Config.TopData.Build, "topdata.reference_builder": p.Config.TopData.ReferenceBuilder, "topdata.assets": p.Config.TopData.Assets, + "topdata.compiled_2da_dir": p.Config.TopData.Compiled2DADir, + "topdata.compiled_tlk": p.Config.TopData.CompiledTLK, + "topdata.wiki.output_root": p.Config.TopData.Wiki.OutputRoot, + "topdata.wiki.pages_dir": p.Config.TopData.Wiki.PagesDir, + "topdata.wiki.state_file": p.Config.TopData.Wiki.StateFile, + "autogen.cache.root": p.Config.Autogen.Cache.Root, } { if strings.Contains(value, "\x00") { failures = append(failures, fmt.Errorf("%s must not contain NUL bytes", field)) } } + effective := p.EffectiveConfig() + failures = append(failures, validateRelativePath("paths.cache", effective.Paths.Cache)...) + failures = append(failures, validateRelativePath("outputs.module_archive", effective.Outputs.ModuleArchive)...) + failures = append(failures, validateRelativePath("outputs.hak_manifest", effective.Outputs.HAKManifest)...) + failures = append(failures, validateRelativePath("outputs.hak_archive", effective.Outputs.HAKArchive)...) + failures = append(failures, validateRelativePath("scripts.cache", effective.Scripts.Cache)...) + failures = append(failures, validateRelativePath("topdata.compiled_2da_dir", effective.TopData.Compiled2DADir)...) + failures = append(failures, validateRelativePath("topdata.compiled_tlk", effective.TopData.CompiledTLK)...) + failures = append(failures, validateRelativePath("topdata.wiki.output_root", effective.TopData.Wiki.OutputRoot)...) + failures = append(failures, validateRelativePath("topdata.wiki.pages_dir", effective.TopData.Wiki.PagesDir)...) + failures = append(failures, validateRelativePath("topdata.wiki.state_file", effective.TopData.Wiki.StateFile)...) + failures = append(failures, validateRelativePath("autogen.cache.root", effective.Autogen.Cache.Root)...) + if err := validateOutputFileName("outputs.module_archive", filepath.Base(effective.Outputs.ModuleArchive), ".mod"); err != nil { + failures = append(failures, err) + } + if err := validateOutputFileName("outputs.hak_manifest", filepath.Base(effective.Outputs.HAKManifest), ".json"); err != nil { + failures = append(failures, err) + } + if err := validateOutputFileName("outputs.hak_archive", filepath.Base(effective.Outputs.HAKArchive), ".hak"); err != nil { + failures = append(failures, err) + } if p.HasTopData() { if err := validateOutputFileName("topdata.package_hak", p.TopDataPackageHAKName(), ".hak"); err != nil { failures = append(failures, err) @@ -322,6 +443,7 @@ func (p *Project) ValidateLayout() error { func (p *Project) Scan() error { var sourceFiles []string var sourceExts []string + effective := p.EffectiveConfig() sourceDir := p.SourceDir() if sourceDir != "" && filepath.Clean(sourceDir) != p.Root { @@ -339,7 +461,7 @@ func (p *Project) Scan() error { assetFiles, _, err := scanDir(p.AssetsDir(), func(path string) bool { ext := strings.ToLower(filepath.Ext(path)) - return slices.Contains(AssetExtensions, ext) + return slices.Contains(effective.Inventory.AssetExtensions, ext) }) if err != nil { if errors.Is(err, os.ErrNotExist) { @@ -381,31 +503,36 @@ func (p *Project) Scan() error { } func (p *Project) SourceDir() string { - return filepath.Join(p.Root, p.Config.Paths.Source) + return p.rootPath(p.EffectiveConfig().Paths.Source) } func (p *Project) AssetsDir() string { - assets := strings.TrimSpace(p.Config.Paths.Assets) + assets := p.EffectiveConfig().Paths.Assets if assets == "" { return "" } - return filepath.Join(p.Root, assets) + return p.rootPath(assets) } func (p *Project) BuildDir() string { - return filepath.Join(p.Root, p.Config.Paths.Build) + return p.rootPath(p.EffectiveConfig().Paths.Build) +} + +func (p *Project) CacheDir() string { + return p.rootPath(p.EffectiveConfig().Paths.Cache) } func (p *Project) ModuleArchivePath() string { - return filepath.Join(p.BuildDir(), p.Config.Module.ResRef+".mod") + effective := p.EffectiveConfig() + return filepath.Join(p.BuildDir(), effective.Outputs.ModuleArchiveName(effective.Module)) } func (p *Project) HAKManifestPath() string { - return filepath.Join(p.BuildDir(), "haks.json") + return filepath.Join(p.BuildDir(), filepath.FromSlash(p.EffectiveConfig().Outputs.HAKManifest)) } func (p *Project) HAKArchivePath(name string) string { - return filepath.Join(p.BuildDir(), strings.TrimSpace(name)+".hak") + return filepath.Join(p.BuildDir(), filepath.FromSlash(p.EffectiveConfig().Outputs.HAKArchiveName(name))) } func (p *Project) CloneWithHAKNames(names []string) (*Project, error) { @@ -469,14 +596,7 @@ func (p *Project) TopDataSourceDir() string { } func (p *Project) TopDataBuildDir() string { - buildPath := strings.TrimSpace(p.Config.TopData.Build) - if buildPath == "" { - buildPath = filepath.Join(p.Config.Paths.Build, "topdata") - } - if filepath.IsAbs(buildPath) { - return buildPath - } - return filepath.Join(p.Root, buildPath) + return p.rootPath(p.EffectiveConfig().TopData.Build) } func (p *Project) TopDataReferenceBuilderDir() string { @@ -506,18 +626,19 @@ func (p *Project) TopDataAssetsDir() string { } func (p *Project) TopDataUsesRepoCache() bool { - return sameCleanPath(p.TopDataBuildDir(), filepath.Join(p.Root, ".cache")) + return sameCleanPath(p.TopDataBuildDir(), p.CacheDir()) } func (p *Project) TopDataCompiled2DADir() string { - return filepath.Join(p.TopDataBuildDir(), "2da") + return filepath.Join(p.TopDataBuildDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Compiled2DADir)) } func (p *Project) TopDataCompiledTLKPath() string { + tlkName := p.EffectiveConfig().TopData.CompiledTLK if p.TopDataUsesRepoCache() { - return filepath.Join(p.BuildDir(), "sow_tlk.tlk") + return filepath.Join(p.BuildDir(), filepath.FromSlash(tlkName)) } - return filepath.Join(p.TopDataBuildDir(), "sow_tlk.tlk") + return filepath.Join(p.TopDataBuildDir(), filepath.FromSlash(tlkName)) } func (p *Project) TopDataCompiledTLKDir() string { @@ -525,34 +646,27 @@ func (p *Project) TopDataCompiledTLKDir() string { } func (p *Project) TopDataWikiRootDir() string { + wikiRoot := filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.OutputRoot) if p.TopDataUsesRepoCache() { - return filepath.Join(p.Root, ".cache", "wiki") + return filepath.Join(p.CacheDir(), wikiRoot) } - return filepath.Join(p.TopDataBuildDir(), "wiki") + return filepath.Join(p.TopDataBuildDir(), wikiRoot) } func (p *Project) TopDataWikiPagesDir() string { - return filepath.Join(p.TopDataWikiRootDir(), "pages") + return filepath.Join(p.TopDataWikiRootDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.PagesDir)) } func (p *Project) TopDataWikiStatePath() string { - return filepath.Join(p.TopDataWikiRootDir(), "state.json") + return filepath.Join(p.TopDataWikiRootDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.StateFile)) } func (p *Project) TopDataPackageHAKName() string { - name := strings.TrimSpace(p.Config.TopData.PackageHAK) - if name == "" { - name = "sow_top.hak" - } - return name + return p.EffectiveConfig().TopData.PackageHAK } func (p *Project) TopDataPackageTLKName() string { - name := strings.TrimSpace(p.Config.TopData.PackageTLK) - if name == "" { - name = "sow_tlk.tlk" - } - return name + return p.EffectiveConfig().TopData.PackageTLK } func (p *Project) TopDataPackageHAKPath() string { @@ -563,6 +677,40 @@ func (p *Project) TopDataPackageTLKPath() string { return filepath.Join(p.BuildDir(), p.TopDataPackageTLKName()) } +func (p *Project) ScriptSourceDir() string { + return filepath.Join(p.SourceDir(), filepath.FromSlash(p.EffectiveConfig().Scripts.SourceDir)) +} + +func (p *Project) ScriptCacheDir() string { + return p.rootPath(p.EffectiveConfig().Scripts.Cache) +} + +func (p *Project) MusicStageDir() string { + return p.rootPath(p.EffectiveConfig().Music.StageRoot) +} + +func (p *Project) CreditsDir() string { + return p.rootPath(p.EffectiveConfig().Music.CreditsRoot) +} + +func (p *Project) AutogenCacheDir() string { + return p.rootPath(p.EffectiveConfig().Autogen.Cache.Root) +} + +func (p *Project) AutogenCachePath(name string) string { + return filepath.Join(p.AutogenCacheDir(), filepath.FromSlash(strings.TrimSpace(name))) +} + +func (p *Project) rootPath(path string) string { + if strings.TrimSpace(path) == "" { + return p.Root + } + if filepath.IsAbs(path) { + return path + } + return filepath.Join(p.Root, filepath.FromSlash(path)) +} + func (i Inventory) Report() InventoryReport { return InventoryReport{ SourceFiles: len(i.SourceFiles), @@ -575,12 +723,14 @@ func (i Inventory) Report() InventoryReport { func defaultConfig() Config { return Config{ Paths: PathConfig{ - Build: "build", + Build: DefaultBuildPath, }, } } func normalizeConfig(cfg *Config) { + cfg.Inventory.SourceExtensions = normalizeExtensionSlice(cfg.Inventory.SourceExtensions) + cfg.Inventory.AssetExtensions = normalizeExtensionSlice(cfg.Inventory.AssetExtensions) for i := range cfg.HAKs { cfg.HAKs[i].Include = normalizeStringSlice(cfg.HAKs[i].Include) } @@ -604,6 +754,22 @@ func normalizeStringSlice(input []string) []string { return out } +func normalizeExtensionSlice(input []string) []string { + if len(input) == 0 { + return input + } + out := make([]string, 0, len(input)) + for _, value := range input { + trimmed := strings.ToLower(strings.TrimSpace(value)) + if trimmed != "" && !strings.HasPrefix(trimmed, ".") { + trimmed = "." + trimmed + } + out = append(out, trimmed) + } + slices.Sort(out) + return out +} + func validateAutogenConfig(cfg AutogenConfig) []error { var failures []error producerIDs := map[string]struct{}{} @@ -816,6 +982,25 @@ func validateOutputFileName(field, name, expectedExt string) error { return nil } +func validateRelativePath(field, path string) []error { + var failures []error + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return failures + } + if filepath.IsAbs(trimmed) { + failures = append(failures, fmt.Errorf("%s must be relative to the repository root: %q", field, path)) + } + clean := filepath.Clean(filepath.FromSlash(trimmed)) + if clean == "." { + return failures + } + if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + failures = append(failures, fmt.Errorf("%s must not escape the repository root: %q", field, path)) + } + return failures +} + func sameCleanPath(a, b string) bool { return filepath.Clean(a) == filepath.Clean(b) } diff --git a/internal/project/project_test.go b/internal/project/project_test.go index d7c89a5..fe10d89 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -1,6 +1,7 @@ package project import ( + "bytes" "errors" "os" "path/filepath" @@ -43,6 +44,128 @@ haks: } } +func TestEffectiveConfigAppliesVisibleToolkitDefaults(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + effective := proj.EffectiveConfig() + if got, want := effective.Paths.Build, "build"; got != want { + t.Fatalf("expected default build path %q, got %q", want, got) + } + if got, want := effective.Outputs.HAKManifest, "haks.json"; got != want { + t.Fatalf("expected default HAK manifest %q, got %q", want, got) + } + if got, want := effective.TopData.PackageHAK, "sow_top.hak"; got != want { + t.Fatalf("expected default topdata HAK %q, got %q", want, got) + } + if prov := effective.Provenance["paths.build"]; prov.Source != "toolkit default" { + t.Fatalf("expected paths.build toolkit default provenance, got %#v", prov) + } + if prov := effective.Provenance["module.name"]; prov.Source != "yaml" { + t.Fatalf("expected module.name YAML provenance, got %#v", prov) + } +} + +func TestEffectiveConfigHonorsConfiguredOutputAndCacheFields(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + build: output + cache: cache +outputs: + module_archive: modules/{module.resref}.mod + hak_manifest: manifests/haks.json + hak_archive: haks/{hak.name}.hak +scripts: + cache: "{paths.cache}/compiled" +topdata: + source: topdata + build: "{paths.cache}" + compiled_2da_dir: tables + compiled_tlk: dialog.tlk + package_hak: custom_top.hak + package_tlk: custom_dialog.tlk + wiki: + output_root: docs + pages_dir: pages + state_file: wiki-state.json +autogen: + cache: + root: "{paths.cache}/autogen" +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if got, want := proj.ModuleArchivePath(), filepath.Join(root, "output", "modules", "testmod.mod"); got != want { + t.Fatalf("expected module archive path %s, got %s", want, got) + } + if got, want := proj.HAKManifestPath(), filepath.Join(root, "output", "manifests", "haks.json"); got != want { + t.Fatalf("expected HAK manifest path %s, got %s", want, got) + } + if got, want := proj.HAKArchivePath("core_01"), filepath.Join(root, "output", "haks", "core_01.hak"); got != want { + t.Fatalf("expected HAK archive path %s, got %s", want, got) + } + if got, want := proj.ScriptCacheDir(), filepath.Join(root, "cache", "compiled"); got != want { + t.Fatalf("expected script cache path %s, got %s", want, got) + } + if got, want := proj.TopDataCompiled2DADir(), filepath.Join(root, "cache", "tables"); got != want { + t.Fatalf("expected topdata 2da path %s, got %s", want, got) + } + if got, want := proj.TopDataWikiStatePath(), filepath.Join(root, "cache", "docs", "wiki-state.json"); got != want { + t.Fatalf("expected wiki state path %s, got %s", want, got) + } + if got, want := proj.AutogenCachePath("manifest.json"), filepath.Join(root, "cache", "autogen", "manifest.json"); got != want { + t.Fatalf("expected autogen cache path %s, got %s", want, got) + } +} + +func TestEffectiveConfigJSONIsDeterministic(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + first, err := proj.EffectiveConfigJSON() + if err != nil { + t.Fatalf("EffectiveConfigJSON returned error: %v", err) + } + second, err := proj.EffectiveConfigJSON() + if err != nil { + t.Fatalf("EffectiveConfigJSON returned error: %v", err) + } + if !bytes.Equal(first, second) { + t.Fatalf("effective config JSON is not deterministic") + } + if !bytes.Contains(first, []byte(`"hak_manifest": "haks.json"`)) { + t.Fatalf("effective config JSON missing HAK manifest default: %s", first) + } +} + func TestLoadYMLConfig(t *testing.T) { root := t.TempDir() writeProjectFile(t, filepath.Join(root, ConfigFileYML), ` @@ -346,6 +469,38 @@ func TestValidateLayoutRejectsInvalidTopDataPackageOutputNames(t *testing.T) { } } +func TestValidateLayoutRejectsUnsafeGeneratedOutputPaths(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + Outputs: OutputConfig{ + HAKManifest: "../haks.json", + }, + TopData: TopDataConfig{ + Source: "topdata", + Compiled2DADir: "../2da", + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected unsafe output path validation error") + } + if !strings.Contains(err.Error(), "outputs.hak_manifest") { + t.Fatalf("expected HAK manifest path validation error, got %v", err) + } + if !strings.Contains(err.Error(), "topdata.compiled_2da_dir") { + t.Fatalf("expected topdata path validation error, got %v", err) + } +} + func TestValidateLayoutRejectsInvalidAutogenConsumerConfig(t *testing.T) { root := t.TempDir() mkdirAll(t, filepath.Join(root, "src")) diff --git a/internal/topdata/autogen.go b/internal/topdata/autogen.go index a7fdfe8..9fed478 100644 --- a/internal/topdata/autogen.go +++ b/internal/topdata/autogen.go @@ -328,7 +328,7 @@ func deriveAutogenManifestEntry(rel string, derive project.AutogenDeriveConfig) } func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) { - cachePath := filepath.Join(p.Root, ".cache", consumer.Manifest.CacheName) + cachePath := p.AutogenCachePath(consumer.Manifest.CacheName) if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) == "" { manifest, fresh, err := readFreshAutogenManifestCache(cachePath, autogenManifestCacheMaxAge) if err != nil { diff --git a/internal/topdata/parts_manifest.go b/internal/topdata/parts_manifest.go index e03272a..f8a1a17 100644 --- a/internal/topdata/parts_manifest.go +++ b/internal/topdata/parts_manifest.go @@ -45,7 +45,7 @@ func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (ma if progress == nil { progress = func(string) {} } - cachePath := filepath.Join(p.Root, ".cache", partsManifestCacheFileName) + cachePath := p.AutogenCachePath(partsManifestCacheFileName) if strings.TrimSpace(os.Getenv("SOW_PARTS_MANIFEST_REFRESH")) == "" { manifest, fresh, err := readFreshPartsManifestCache(cachePath, partsManifestCacheMaxAge) if err != nil { diff --git a/internal/topdata/top_package.go b/internal/topdata/top_package.go index d171bf9..50e7dda 100644 --- a/internal/topdata/top_package.go +++ b/internal/topdata/top_package.go @@ -322,7 +322,7 @@ func newestAutogenInput(p *project.Project, now time.Time) (time.Time, string, e continue } - cachePath := filepath.Join(p.Root, ".cache", consumer.Manifest.CacheName) + cachePath := p.AutogenCachePath(consumer.Manifest.CacheName) candidateTime, candidatePath, err := newestReleasedAutogenManifestInput(p, consumer, now, cachePath) if err != nil { return time.Time{}, "", err