diff --git a/CONFIGURATION_REFACTOR_CONTRACT.md b/CONFIGURATION_REFACTOR_CONTRACT.md new file mode 100644 index 0000000..8dcebbf --- /dev/null +++ b/CONFIGURATION_REFACTOR_CONTRACT.md @@ -0,0 +1,325 @@ +Refactor toolkit configuration from JSON to YAML only. + +# Context + +Repositories currently use `config.json` for human-authored toolkit configuration. + +However, JSON is also used heavily for generated datasets, manifests, caches, and machine-readable artifacts. This makes repository configuration harder to distinguish from generated data. + +# Goal + +Make YAML the source of truth for all human-authored repository configuration. + +JSON should remain valid for generated data artifacts, but not as the preferred format for repository configuration. + +# Core requirements + +- Replace `config.json` with `config.yaml`. +- Load repository configuration from YAML. +- Treat YAML as the canonical source-of-truth. +- Keep generated datasets, manifests, caches, and machine-readable outputs as JSON. +- Do not move dataset artifacts to YAML. +- Do not hardcode configuration behavior in code if it can reasonably be expressed in config. +- The toolkit should be deterministic. + +# Configuration philosophy + +Configuration should control repository behavior. + +Avoid hardcoded assumptions for: + +- module names +- resrefs +- paths +- HAK names +- HAK priority +- HAK split behavior +- HAK size limits +- include/exclude globs +- optional HAKs +- music prefixes +- extract rules +- topdata paths/packages +- autogen producers +- autogen consumers +- manifest names +- cache names +- release tags +- dataset names +- derivation rules + +If behavior varies by repository, it belongs in YAML. + +# Example target config + +Convert this kind of repository config: + +```json +{ + "module": { + "name": "Shadows Over Westgate", + "resref": "sow_module", + "description": "Shadows Over Westgate asset pipeline.", + "hak_order": ["sow_top", "group:sow_over", "group:sow_core"] + }, + "paths": { + "assets": "content", + "build": "build" + }, + "music": { + "prefixes": { + "envi/music/westgate": "mus_wg_" + } + } +} +``` + +Into: + +```yaml +module: + name: Shadows Over Westgate + resref: sow_module + description: Shadows Over Westgate asset pipeline. + hak_order: + - sow_top + - group:sow_over + - group:sow_core + +paths: + assets: content + build: build + +music: + prefixes: + envi/music/westgate: mus_wg_ +``` + +# Implementation requirements + +- Add YAML config loading. +- Prefer `config.yaml`. +- Optionally accept `config.yml`. +- Remove reliance on `config.json` for repository configuration. +- If `config.json` support is kept temporarily, mark it as legacy/deprecated. +- If both YAML and JSON configs exist, YAML must win. +- Emit a clear warning when legacy JSON config is used. +- Preserve existing schema semantics. +- Preserve existing defaults only where they are intentional toolkit defaults. +- Move repository-specific defaults into YAML. +- Keep validation strict. +- Fail clearly on malformed YAML or unknown required fields. +- Preserve deterministic ordering where configs are serialized, logged, or normalized. + +# Validation requirements + +- Validate YAML against the same schema expectations currently used for JSON. +- Ensure required sections are present when needed. +- Ensure paths are strings. +- Ensure HAK entries have required fields. +- Ensure autogen producers/consumers have valid ids, roots, includes, derive rules, and manifest config. +- Ensure music prefixes are deterministic and unambiguous. +- Ensure duplicate HAK names, producer ids, consumer ids, or dataset ids are rejected. +- Ensure priority ordering is deterministic even when priorities match. +- Ensure include glob ordering is stable. + +# Migration requirements + +- Convert existing `config.json` files to `config.yaml`. +- Preserve all existing values exactly unless a change is intentional. +- Update code paths, scripts, tests, and documentation to reference YAML. +- Add or update tests for: + - loading `config.yaml` + - loading `config.yml` + - legacy `config.json` fallback, if retained + - YAML precedence over JSON + - validation failures + - deterministic ordering + - representative configs matching the current repository examples + +- Do not change generated artifact formats unless explicitly required. + +# Generated artifacts + +Keep these as JSON unless there is a separate reason to change them: + +- datasets +- manifests +- caches +- credits output +- build summaries +- machine-readable reports + +Rationale: + +- YAML is for human-authored repository configuration. +- JSON is for generated or machine-readable data. + +# Deliverables + +1. YAML config loader design +2. Updated config schema +3. JSON-to-YAML migration plan +4. Converted example configs +5. Legacy JSON handling policy +6. Validation updates +7. Test plan +8. Documentation updates +9. List of code-level assumptions that should be moved into config +10. Before/after examples + +# Initial project-needs analysis + +## Current state + +- Main repository configuration is currently modeled by `internal/project.Config`. +- The toolkit currently discovers and loads `nwn-tool.json`, not `config.json`. +- Config loading is centralized in `internal/project.FindRoot` and `internal/project.Load`. +- Config structs currently have JSON tags only. +- There is no YAML dependency in `go.mod`. +- Generated and machine-readable JSON is widespread and should remain JSON: + - build HAK manifest: `build/haks.json` + - topdata authored/generated datasets + - topdata template conversion metadata: `topdata/templates/config.json` + - topdata wiki state and deploy manifests + - autogen manifest caches under `.cache` + - credits inventory output + - GFF canonical JSON documents + +## Naming decision needed + +The contract says to replace `config.json` with `config.yaml`, but the current toolkit root config file is `nwn-tool.json`. + +Plan assumption: + +- Treat `nwn-tool.yaml` as the direct YAML replacement for the existing root config. +- Optionally accept `nwn-tool.yml`. +- Keep temporary legacy fallback to `nwn-tool.json` with a warning. +- Do not rename generated or nested metadata files unless they are confirmed to be human-authored repository root configuration. + +If the intended final filename is instead literally `config.yaml`, update the loader plan before implementation. + +## Code areas affected + +- `internal/project/project.go` + - config file discovery + - YAML/legacy JSON loading + - schema tags and strict decode behavior + - validation for duplicate names, path fields, include ordering, and required config sections + - currently hardcoded defaults such as `build`, `.cache`, `sow_top.hak`, `sow_tlk.tlk`, and `haks.json` +- `internal/app/app.go` + - project-load diagnostics and logging so commands clearly state which config file was loaded + - user-facing descriptions that mention hardcoded paths or outputs +- `internal/pipeline/*` + - HAK ordering, manifest naming, music cache/credits paths, and music prefix behavior + - generated JSON outputs must remain JSON +- `internal/topdata/*` + - package names, cache names, released manifest metadata, wiki state paths, and autogen consumer cache paths + - topdata datasets and manifests must remain JSON +- `internal/validator/*` + - validation tests and duplicate/ordering rules + - potential config control for built-in script prefixes if repository-specific +- Tests under `internal/**/*_test.go` + - many tests currently write `nwn-tool.json`; representative tests should move to YAML with targeted legacy JSON coverage retained +- `README.md` and workflow docs + - update root configuration references to YAML + - explicitly document JSON artifacts that remain JSON + +## Hardcoded assumptions to audit for config ownership + +Move to YAML if repository-specific: + +- root config filename policy and migration behavior +- build output directory and HAK manifest filename +- topdata build/cache paths +- topdata package HAK/TLK filenames and derived TLK behavior +- autogen release tags, asset names, cache names, producers, consumers, roots, includes, modes, and derive rules +- released parts manifest repository/server metadata when not a universal toolkit default +- HAK names, priorities, split behavior, optional flags, include globs, and module HAK order +- music source roots, prefixes, credit overlay names, generated credits paths, and music stem constraints where repository-specific +- source/asset include extension sets where repository-specific +- validator script-prefix exclusions if they vary by repository + +Keep in code if toolkit-intrinsic: + +- generated JSON encoding for datasets, manifests, caches, GFF documents, and reports +- NWN resource format constraints such as resref length limits +- deterministic sort rules after config values are loaded +- supported autogen derive kinds and consumer modes unless extensibility is implemented + +# Implementation plan + +## Phase 1: Loader and schema foundation + +- Add YAML parsing dependency, likely `gopkg.in/yaml.v3`. +- Add YAML tags to all config structs while preserving JSON tags for legacy decode. +- Introduce config discovery order: + 1. `nwn-tool.yaml` + 2. `nwn-tool.yml` + 3. legacy `nwn-tool.json` +- Return loaded config metadata from `project.Load`, including path, format, and legacy flag. +- Make YAML win when both YAML and JSON are present. +- Emit a clear legacy warning when `nwn-tool.json` is used. +- Emit/log the selected config path for normal command execution. +- Fail clearly for malformed YAML and unknown fields. + +## Phase 2: Validation and determinism + +- Preserve existing schema semantics while tightening validation. +- Validate required module fields and path field types. +- Reject duplicate HAK names, autogen producer IDs, autogen consumer IDs, and dataset IDs where represented in root config. +- Validate HAK entries for name, priority, max bytes, include globs, and output-safe names. +- Validate autogen manifest filenames and deterministic producer/consumer references. +- Validate music prefixes for ambiguous duplicate normalized roots. +- Ensure include globs and map-derived outputs are sorted before use where order matters. +- Add deterministic tie-breakers for HAK priority sorting. + +## Phase 3: Move repository-specific defaults into YAML + +- Keep only intentional toolkit defaults in `defaultConfig`. +- Move Shadows Over Westgate-specific defaults into repo YAML: + - top package names + - topdata cache/build paths + - autogen manifest metadata + - music prefixes + - HAK ordering and include globs +- Add config fields where needed before removing hardcoded assumptions. +- Leave generated artifact paths as JSON outputs unless the path itself must be configurable. + +## Phase 4: Migration + +- Convert consumer root configs from `nwn-tool.json` to `nwn-tool.yaml`. +- Preserve existing values exactly unless a separate behavior change is documented. +- Keep `topdata/templates/config.json` unchanged for now because it is topdata conversion metadata, not root repository configuration. +- Keep generated datasets, manifests, caches, credits inventory, and GFF JSON unchanged. +- Add before/after examples for root config migration. + +## Phase 5: Tests + +- Add focused loader tests for: + - `nwn-tool.yaml` + - `nwn-tool.yml` + - legacy `nwn-tool.json` + - YAML precedence over JSON + - malformed YAML + - unknown YAML fields +- Update representative build, validator, pipeline, topdata, and app tests to use YAML root config. +- Keep targeted JSON tests only for legacy fallback and generated artifact behavior. +- Add validation tests for duplicates, path type failures, deterministic HAK ordering, and ambiguous music prefixes. +- Run `go test ./...`. + +## Phase 6: Documentation + +- Update `README.md` repository layout and consumer instructions to reference `nwn-tool.yaml`. +- Document legacy JSON fallback and planned removal policy. +- Document which JSON files remain intentional generated or machine-readable artifacts. +- Document config precedence and command log behavior. + +# Acceptance criteria + +- A repository can build from the chosen YAML root config file (`nwn-tool.yaml` under the current plan, or `config.yaml` if the naming decision changes). +- Existing generated JSON datasets and manifests still work. +- No repository-specific behavior remains hardcoded if it can be expressed in YAML. +- Builds remain deterministic. +- Existing behavior is preserved after migration. +- Logs clearly state which config file was loaded. diff --git a/README.md b/README.md index 1da83f2..e17bb89 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,45 @@ internal/validator/validation and diagnostics tools/ built development binary output ``` +## Repository Configuration + +Human-authored consumer repository configuration is YAML. The toolkit discovers +configuration in this order: + +1. `nwn-tool.yaml` +2. `nwn-tool.yml` +3. legacy `nwn-tool.json` + +YAML is canonical. If both YAML and legacy JSON files are present, the YAML file +wins. Legacy `nwn-tool.json` still loads for migration, but commands print a +warning and should be updated to `nwn-tool.yaml`. + +Example: + +```yaml +module: + name: Shadows Over Westgate + resref: sow_module + description: Shadows Over Westgate asset pipeline. + hak_order: + - sow_top + - group:sow_over + - group:sow_core + +paths: + assets: content + build: build + +music: + prefixes: + envi/music/westgate: mus_wg_ +``` + +Generated and machine-readable artifacts remain JSON, including HAK manifests, +topdata datasets, caches, credits inventories, build reports, and canonical GFF +documents. Nested metadata such as `topdata/templates/config.json` is not root +repository configuration and is intentionally unchanged. + ## Intended Consumers - `sow-module` uses `sow-toolkit` for `build-module`, `extract`, `validate`, `compare`, and `apply-hak-manifest` diff --git a/go.mod b/go.mod index 66cc59b..3751969 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module gitea.westgate.pw/ShadowsOverWestgate/sow-tools go 1.26.0 -require golang.org/x/text v0.35.0 // indirect +require ( + golang.org/x/text v0.35.0 + gopkg.in/yaml.v3 v3.0.1 +) diff --git a/go.sum b/go.sum index b95faad..fece86f 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,6 @@ golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/app.go b/internal/app/app.go index 4d71560..07b3f38 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -255,7 +255,7 @@ func newContext() (context, error) { } func runBuild(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -281,7 +281,7 @@ func runBuild(ctx context) error { } func runBuildModule(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -514,7 +514,7 @@ func formatCount(value int) string { } func runBuildHAKs(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -618,7 +618,7 @@ func parseInlineFlagValue(arg, name string) (string, bool) { } func runExtract(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -645,7 +645,7 @@ func runExtract(ctx context) error { } func runValidate(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -697,7 +697,7 @@ func emitSummaryLines(w io.Writer, lines []string, maxLines int, label string) { } func runCompare(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -718,7 +718,7 @@ func runCompare(ctx context) error { } func runApplyHAKManifest(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -744,7 +744,7 @@ func runApplyHAKManifest(ctx context) error { } func runValidateTopData(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -768,7 +768,7 @@ func runValidateTopData(ctx context) error { } func runBuildTopData(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -804,7 +804,7 @@ func runBuildTopData(ctx context) error { } func runBuildTopPackage(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -861,7 +861,7 @@ func parseBuildTopDataArgs(commandName string, args []string) (topdata.BuildAndP } func runCompareTopData(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -892,7 +892,7 @@ func runConvertTopData(ctx context) error { } func runBuildWiki(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -932,7 +932,7 @@ func parseBuildWikiArgs(commandName string, args []string) (topdata.BuildWikiOpt } func runDeployWiki(ctx context) error { - p, err := loadProject(ctx.cwd) + p, err := loadProject(ctx) if err != nil { return err } @@ -1200,8 +1200,8 @@ func parseWikiCategoryArg(raw string, target map[string]int) error { return nil } -func loadProject(cwd string) (*project.Project, error) { - root, err := project.FindRoot(cwd) +func loadProject(ctx context) (*project.Project, error) { + root, err := project.FindRoot(ctx.cwd) if err != nil { return nil, err } @@ -1210,6 +1210,12 @@ func loadProject(cwd string) (*project.Project, error) { if err != nil { return nil, err } + if ctx.stderr != nil { + fmt.Fprintf(ctx.stderr, "Loaded config: %s\n", p.ConfigSource.Name) + if p.ConfigSource.Legacy { + fmt.Fprintf(ctx.stderr, "Warning: %s is legacy repository configuration; migrate to %s.\n", project.LegacyConfigFile, project.ConfigFile) + } + } var failures []error if err := p.ValidateLayout(); err != nil { diff --git a/internal/pipeline/music.go b/internal/pipeline/music.go index 5c14a91..b8e940f 100644 --- a/internal/pipeline/music.go +++ b/internal/pipeline/music.go @@ -739,7 +739,13 @@ func musicPrefixForDir(p *project.Project, dir string) string { dir = trimSlashes(filepath.ToSlash(dir)) bestPrefix := "" bestLen := -1 - for key, value := range p.Config.Music.Prefixes { + keys := make([]string, 0, len(p.Config.Music.Prefixes)) + for key := range p.Config.Music.Prefixes { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + value := p.Config.Music.Prefixes[key] normalizedKey := trimSlashes(filepath.ToSlash(key)) if normalizedKey == "" { continue diff --git a/internal/project/project.go b/internal/project/project.go index ac3e6a7..48044ba 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -1,17 +1,27 @@ package project import ( + "bytes" "encoding/json" "errors" "fmt" + "io" "io/fs" "os" "path/filepath" "slices" "strings" + + "gopkg.in/yaml.v3" ) -const ConfigFile = "nwn-tool.json" +const ( + ConfigFile = "nwn-tool.yaml" + ConfigFileYML = "nwn-tool.yml" + LegacyConfigFile = "nwn-tool.json" +) + +var ConfigFileCandidates = []string{ConfigFile, ConfigFileYML, LegacyConfigFile} var SourceExtensions = []string{ ".are", ".dlg", ".fac", ".git", ".ifo", ".jrl", @@ -23,96 +33,104 @@ var AssetExtensions = []string{ } type Project struct { - Root string - Config Config - Inventory Inventory + Root string + Config Config + ConfigSource ConfigSource + Inventory Inventory +} + +type ConfigSource struct { + Path string + Name string + Format string + Legacy bool } type Config struct { - Module ModuleConfig `json:"module"` - Paths PathConfig `json:"paths"` - HAKs []HAKConfig `json:"haks"` - Music MusicConfig `json:"music"` - TopData TopDataConfig `json:"topdata"` - Extract ExtractConfig `json:"extract"` - Autogen AutogenConfig `json:"autogen"` + Module ModuleConfig `json:"module" yaml:"module"` + Paths PathConfig `json:"paths" yaml:"paths"` + HAKs []HAKConfig `json:"haks" yaml:"haks"` + Music MusicConfig `json:"music" yaml:"music"` + TopData TopDataConfig `json:"topdata" yaml:"topdata"` + Extract ExtractConfig `json:"extract" yaml:"extract"` + Autogen AutogenConfig `json:"autogen" yaml:"autogen"` } type ModuleConfig struct { - Name string `json:"name"` - ResRef string `json:"resref"` - Description string `json:"description"` - HAKOrder []string `json:"hak_order"` + Name string `json:"name" yaml:"name"` + ResRef string `json:"resref" yaml:"resref"` + Description string `json:"description" yaml:"description"` + HAKOrder []string `json:"hak_order" yaml:"hak_order"` } type PathConfig struct { - Source string `json:"source"` - Assets string `json:"assets"` - Build string `json:"build"` + Source string `json:"source" yaml:"source"` + Assets string `json:"assets" yaml:"assets"` + Build string `json:"build" yaml:"build"` } type HAKConfig struct { - Name string `json:"name"` - Priority int `json:"priority"` - MaxBytes int64 `json:"max_bytes"` - Split bool `json:"split"` - Optional bool `json:"optional,omitempty"` - Include []string `json:"include"` + Name string `json:"name" yaml:"name"` + Priority int `json:"priority" yaml:"priority"` + MaxBytes int64 `json:"max_bytes" yaml:"max_bytes"` + Split bool `json:"split" yaml:"split"` + Optional bool `json:"optional,omitempty" yaml:"optional,omitempty"` + Include []string `json:"include" yaml:"include"` } type MusicConfig struct { - Prefixes map[string]string `json:"prefixes"` + Prefixes map[string]string `json:"prefixes" yaml:"prefixes"` } type TopDataConfig struct { - Source string `json:"source"` - Build string `json:"build"` - ReferenceBuilder string `json:"reference_builder"` - Assets string `json:"assets"` - PackageHAK string `json:"package_hak"` - PackageTLK string `json:"package_tlk"` + Source string `json:"source" yaml:"source"` + Build string `json:"build" yaml:"build"` + ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"` + Assets string `json:"assets" yaml:"assets"` + PackageHAK string `json:"package_hak" yaml:"package_hak"` + PackageTLK string `json:"package_tlk" yaml:"package_tlk"` } type ExtractConfig struct { - IgnoreExtensions []string `json:"ignore_extensions"` - DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty"` + IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"` + DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"` } type AutogenConfig struct { - Producers []AutogenProducerConfig `json:"producers"` - Consumers []AutogenConsumerConfig `json:"consumers"` + Producers []AutogenProducerConfig `json:"producers" yaml:"producers"` + Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"` } type AutogenProducerConfig struct { - ID string `json:"id"` - Root string `json:"root"` - Include []string `json:"include"` - Derive AutogenDeriveConfig `json:"derive"` - Manifest AutogenManifestConfig `json:"manifest"` + ID string `json:"id" yaml:"id"` + Root string `json:"root" yaml:"root"` + Include []string `json:"include" yaml:"include"` + Derive AutogenDeriveConfig `json:"derive" yaml:"derive"` + Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"` } type AutogenConsumerConfig struct { - ID string `json:"id"` - Producer string `json:"producer"` - Dataset string `json:"dataset"` - Mode string `json:"mode"` - Optional bool `json:"optional,omitempty"` - Root string `json:"root"` - Include []string `json:"include"` - Derive AutogenDeriveConfig `json:"derive"` - Manifest AutogenManifestConfig `json:"manifest"` - LocalOverrideRoot string `json:"local_override_root,omitempty"` + ID string `json:"id" yaml:"id"` + Producer string `json:"producer" yaml:"producer"` + Dataset string `json:"dataset" yaml:"dataset"` + Mode string `json:"mode" yaml:"mode"` + Optional bool `json:"optional,omitempty" yaml:"optional,omitempty"` + Root string `json:"root" yaml:"root"` + Include []string `json:"include" yaml:"include"` + Derive AutogenDeriveConfig `json:"derive" yaml:"derive"` + Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"` + LocalOverrideRoot string `json:"local_override_root,omitempty" yaml:"local_override_root,omitempty"` } type AutogenDeriveConfig struct { - Kind string `json:"kind"` - GroupFrom string `json:"group_from,omitempty"` + Kind string `json:"kind" yaml:"kind"` + GroupFrom string `json:"group_from,omitempty" yaml:"group_from,omitempty"` } type AutogenManifestConfig struct { - ReleaseTag string `json:"release_tag"` - AssetName string `json:"asset_name"` - CacheName string `json:"cache_name"` + ReleaseTag string `json:"release_tag" yaml:"release_tag"` + AssetName string `json:"asset_name" yaml:"asset_name"` + CacheName string `json:"cache_name" yaml:"cache_name"` } type Inventory struct { @@ -132,35 +150,82 @@ type InventoryReport struct { func FindRoot(start string) (string, error) { current := start for { - candidate := filepath.Join(current, ConfigFile) - if _, err := os.Stat(candidate); err == nil { - return current, nil + for _, name := range ConfigFileCandidates { + candidate := filepath.Join(current, name) + if _, err := os.Stat(candidate); err == nil { + return current, nil + } } parent := filepath.Dir(current) if parent == current { - return "", fmt.Errorf("could not find %s from %s", ConfigFile, start) + return "", fmt.Errorf("could not find %s, %s, or legacy %s from %s", ConfigFile, ConfigFileYML, LegacyConfigFile, start) } current = parent } } func Load(root string) (*Project, error) { - raw, err := os.ReadFile(filepath.Join(root, ConfigFile)) + source, err := findConfigSource(root) if err != nil { - return nil, fmt.Errorf("read %s: %w", ConfigFile, err) + return nil, err + } + raw, err := os.ReadFile(source.Path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", source.Name, err) } cfg := defaultConfig() - if err := json.Unmarshal(raw, &cfg); err != nil { - return nil, fmt.Errorf("parse %s: %w", ConfigFile, err) + if source.Format == "yaml" { + decoder := yaml.NewDecoder(bytes.NewReader(raw)) + decoder.KnownFields(true) + if err := decoder.Decode(&cfg); err != nil { + return nil, fmt.Errorf("parse %s: %w", source.Name, err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return nil, fmt.Errorf("parse %s: multiple YAML documents are not supported", source.Name) + } + } else { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&cfg); err != nil { + return nil, fmt.Errorf("parse legacy %s: %w", source.Name, err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return nil, fmt.Errorf("parse legacy %s: multiple JSON documents are not supported", source.Name) + } } + normalizeConfig(&cfg) return &Project{ - Root: root, - Config: cfg, + Root: root, + Config: cfg, + ConfigSource: source, }, nil } +func findConfigSource(root string) (ConfigSource, error) { + for _, name := range ConfigFileCandidates { + path := filepath.Join(root, name) + if _, err := os.Stat(path); err == nil { + format := "yaml" + legacy := false + if name == LegacyConfigFile { + format = "json" + legacy = true + } + return ConfigSource{ + Path: path, + Name: name, + Format: format, + Legacy: legacy, + }, nil + } + } + return ConfigSource{}, fmt.Errorf("could not find %s, %s, or legacy %s in %s", ConfigFile, ConfigFileYML, LegacyConfigFile, root) +} + func (p *Project) ValidateLayout() error { var failures []error @@ -173,6 +238,22 @@ func (p *Project) ValidateLayout() error { if len(p.Config.Module.ResRef) > 16 { failures = append(failures, fmt.Errorf("module.resref %q exceeds 16 characters", p.Config.Module.ResRef)) } + if strings.TrimSpace(p.Config.Paths.Source) == "" && strings.TrimSpace(p.Config.Paths.Assets) == "" && !p.HasTopData() { + failures = append(failures, errors.New("at least one of paths.source, paths.assets, or topdata.source is required")) + } + for field, value := range map[string]string{ + "paths.source": p.Config.Paths.Source, + "paths.assets": p.Config.Paths.Assets, + "paths.build": p.Config.Paths.Build, + "topdata.source": p.Config.TopData.Source, + "topdata.build": p.Config.TopData.Build, + "topdata.reference_builder": p.Config.TopData.ReferenceBuilder, + "topdata.assets": p.Config.TopData.Assets, + } { + if strings.Contains(value, "\x00") { + failures = append(failures, fmt.Errorf("%s must not contain NUL bytes", field)) + } + } if p.HasTopData() { if err := validateOutputFileName("topdata.package_hak", p.TopDataPackageHAKName(), ".hak"); err != nil { failures = append(failures, err) @@ -185,10 +266,22 @@ func (p *Project) ValidateLayout() error { } } failures = append(failures, validateAutogenConfig(p.Config.Autogen)...) + failures = append(failures, validateMusicConfig(p.Config.Music)...) + hakNames := map[string]struct{}{} for index, hak := range p.Config.HAKs { + name := strings.TrimSpace(hak.Name) if strings.TrimSpace(hak.Name) == "" { failures = append(failures, fmt.Errorf("haks[%d].name is required", index)) + } else { + normalizedName := strings.ToLower(name) + if _, exists := hakNames[normalizedName]; exists { + failures = append(failures, fmt.Errorf("haks[%d].name %q is duplicated", index, name)) + } + hakNames[normalizedName] = struct{}{} + if err := validateOutputFileName(fmt.Sprintf("haks[%d].name", index), name+".hak", ".hak"); err != nil { + failures = append(failures, err) + } } if hak.Priority <= 0 { failures = append(failures, fmt.Errorf("haks[%d].priority must be greater than zero", index)) @@ -199,6 +292,7 @@ func (p *Project) ValidateLayout() error { if len(hak.Include) == 0 { failures = append(failures, fmt.Errorf("haks[%d].include must have at least one pattern", index)) } + failures = append(failures, validateGlobList(fmt.Sprintf("haks[%d].include", index), hak.Include)...) } for label, pathFn := range map[string]func() string{ @@ -483,18 +577,38 @@ func defaultConfig() Config { Paths: PathConfig{ Build: "build", }, - TopData: TopDataConfig{ - Build: ".cache", - PackageHAK: "sow_top.hak", - PackageTLK: "sow_tlk.tlk", - }, } } +func normalizeConfig(cfg *Config) { + for i := range cfg.HAKs { + cfg.HAKs[i].Include = normalizeStringSlice(cfg.HAKs[i].Include) + } + cfg.Extract.IgnoreExtensions = normalizeStringSlice(cfg.Extract.IgnoreExtensions) + for i := range cfg.Autogen.Producers { + cfg.Autogen.Producers[i].Include = normalizeStringSlice(cfg.Autogen.Producers[i].Include) + } + for i := range cfg.Autogen.Consumers { + cfg.Autogen.Consumers[i].Include = normalizeStringSlice(cfg.Autogen.Consumers[i].Include) + } +} + +func normalizeStringSlice(input []string) []string { + if len(input) == 0 { + return input + } + out := slices.Clone(input) + slices.SortFunc(out, func(a, b string) int { + return strings.Compare(strings.TrimSpace(a), strings.TrimSpace(b)) + }) + return out +} + func validateAutogenConfig(cfg AutogenConfig) []error { var failures []error producerIDs := map[string]struct{}{} consumerIDs := map[string]struct{}{} + datasetIDs := map[string]struct{}{} for index, producer := range cfg.Producers { fieldPrefix := fmt.Sprintf("autogen.producers[%d]", index) @@ -513,6 +627,7 @@ func validateAutogenConfig(cfg AutogenConfig) []error { if len(producer.Include) == 0 { failures = append(failures, fmt.Errorf("%s.include must contain at least one glob", fieldPrefix)) } + failures = append(failures, validateGlobList(fieldPrefix+".include", producer.Include)...) failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", producer.Derive)...) failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", producer.Manifest)...) } @@ -533,6 +648,17 @@ func validateAutogenConfig(cfg AutogenConfig) []error { } if strings.TrimSpace(consumer.Dataset) == "" { failures = append(failures, fmt.Errorf("%s.dataset is required", fieldPrefix)) + } else { + dataset := strings.TrimSpace(consumer.Dataset) + if _, exists := datasetIDs[dataset]; exists { + failures = append(failures, fmt.Errorf("%s.dataset %q is duplicated", fieldPrefix, dataset)) + } + datasetIDs[dataset] = struct{}{} + } + if strings.TrimSpace(consumer.Producer) != "" { + if _, exists := producerIDs[strings.TrimSpace(consumer.Producer)]; len(producerIDs) > 0 && !exists { + failures = append(failures, fmt.Errorf("%s.producer %q does not match a configured producer", fieldPrefix, consumer.Producer)) + } } switch strings.TrimSpace(consumer.Mode) { case "parts_rows", "head_visualeffects": @@ -547,6 +673,7 @@ func validateAutogenConfig(cfg AutogenConfig) []error { if len(consumer.Include) == 0 { failures = append(failures, fmt.Errorf("%s.include must contain at least one glob", fieldPrefix)) } + failures = append(failures, validateGlobList(fieldPrefix+".include", consumer.Include)...) failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", consumer.Derive)...) failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", consumer.Manifest)...) } @@ -554,6 +681,50 @@ func validateAutogenConfig(cfg AutogenConfig) []error { return failures } +func validateMusicConfig(cfg MusicConfig) []error { + var failures []error + normalizedRoots := map[string]string{} + for root, prefix := range cfg.Prefixes { + normalizedRoot := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(root))) + if normalizedRoot == "" { + failures = append(failures, errors.New("music.prefixes contains an empty root")) + continue + } + if prior, exists := normalizedRoots[normalizedRoot]; exists { + failures = append(failures, fmt.Errorf("music.prefixes roots %q and %q normalize to the same path", prior, root)) + } + normalizedRoots[normalizedRoot] = root + if strings.TrimSpace(prefix) == "" { + failures = append(failures, fmt.Errorf("music.prefixes[%q] must not be empty", root)) + } + } + return failures +} + +func validateGlobList(field string, patterns []string) []error { + var failures []error + seen := map[string]struct{}{} + for index, pattern := range patterns { + trimmed := strings.TrimSpace(pattern) + if trimmed == "" { + failures = append(failures, fmt.Errorf("%s[%d] must not be empty", field, index)) + continue + } + if strings.Contains(trimmed, "\x00") { + failures = append(failures, fmt.Errorf("%s[%d] must not contain NUL bytes", field, index)) + } + if _, exists := seen[trimmed]; exists { + failures = append(failures, fmt.Errorf("%s[%d] duplicates %q", field, index, trimmed)) + } + seen[trimmed] = struct{}{} + } + return failures +} + +func trimConfigSlashes(path string) string { + return strings.Trim(path, "/") +} + func validateAutogenDeriveConfig(fieldPrefix string, cfg AutogenDeriveConfig) []error { var failures []error switch strings.TrimSpace(cfg.Kind) { diff --git a/internal/project/project_test.go b/internal/project/project_test.go index 3eba387..d7c89a5 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -9,6 +9,156 @@ import ( "testing" ) +func TestLoadYAMLConfig(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: output +haks: + - name: core + priority: 1 + max_bytes: 1024 + split: false + include: + - core/** +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if proj.ConfigSource.Name != ConfigFile || proj.ConfigSource.Format != "yaml" || proj.ConfigSource.Legacy { + t.Fatalf("unexpected config source: %#v", proj.ConfigSource) + } + if got, want := proj.Config.Module.Name, "Test Module"; got != want { + t.Fatalf("expected module name %q, got %q", want, got) + } + if got, want := proj.BuildDir(), filepath.Join(root, "output"); got != want { + t.Fatalf("expected build dir %s, got %s", want, got) + } +} + +func TestLoadYMLConfig(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFileYML), ` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if proj.ConfigSource.Name != ConfigFileYML || proj.ConfigSource.Format != "yaml" || proj.ConfigSource.Legacy { + t.Fatalf("unexpected config source: %#v", proj.ConfigSource) + } +} + +func TestLoadLegacyJSONConfig(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, LegacyConfigFile), `{ + "module": { + "name": "Legacy Module", + "resref": "legacy" + }, + "paths": { + "source": "src", + "build": "build" + } +}`+"\n") + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if proj.ConfigSource.Name != LegacyConfigFile || proj.ConfigSource.Format != "json" || !proj.ConfigSource.Legacy { + t.Fatalf("unexpected config source: %#v", proj.ConfigSource) + } +} + +func TestLoadPrefersYAMLOverLegacyJSON(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, LegacyConfigFile), `{ + "module": { + "name": "Legacy Module", + "resref": "legacy" + } +}`+"\n") + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: YAML Module + resref: yamlmod +paths: + source: src + build: build +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if proj.ConfigSource.Name != ConfigFile { + t.Fatalf("expected YAML config to win, got %#v", proj.ConfigSource) + } + if got, want := proj.Config.Module.ResRef, "yamlmod"; got != want { + t.Fatalf("expected resref %q, got %q", want, got) + } +} + +func TestLoadRejectsMalformedYAML(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), "module:\n name: Test\n resref: bad\n") + + if _, err := Load(root); err == nil { + t.Fatal("expected malformed YAML error") + } +} + +func TestLoadRejectsUnknownYAMLFields(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +unexpected: true +`) + + err := loadAndValidate(root) + if err == nil { + t.Fatal("expected unknown field error") + } + if !strings.Contains(err.Error(), "field unexpected not found") { + t.Fatalf("expected unknown field error, got %v", err) + } +} + +func TestFindRootUsesYAMLConfig(t *testing.T) { + root := t.TempDir() + child := filepath.Join(root, "nested", "dir") + mkdirAll(t, child) + writeProjectFile(t, filepath.Join(root, ConfigFile), "module:\n name: Test\n resref: test\n") + + got, err := FindRoot(child) + if err != nil { + t.Fatalf("FindRoot returned error: %v", err) + } + if got != root { + t.Fatalf("expected root %s, got %s", root, got) + } +} + func TestTopDataAssetsDirPrefersExplicitConfig(t *testing.T) { root := t.TempDir() mkdirAll(t, filepath.Join(root, "src")) @@ -242,6 +392,60 @@ func TestValidateLayoutRejectsInvalidAutogenConsumerConfig(t *testing.T) { } } +func TestValidateLayoutRejectsDuplicateHAKNames(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + HAKs: []HAKConfig{ + {Name: "core", Priority: 1, Include: []string{"core/**"}}, + {Name: "CORE", Priority: 2, Include: []string{"other/**"}}, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected duplicate hak validation error") + } + if !strings.Contains(err.Error(), "duplicated") { + t.Fatalf("expected duplicate validation error, got %v", err) + } +} + +func TestValidateLayoutRejectsAmbiguousMusicPrefixes(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Assets: "assets", Build: "build"}, + Music: MusicConfig{ + Prefixes: map[string]string{ + "envi/music": "mus_", + "/envi/music": "mus2_", + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected music prefix validation error") + } + if !strings.Contains(err.Error(), "normalize to the same path") { + t.Fatalf("expected ambiguous prefix validation error, got %v", err) + } +} + func TestScanAllowsMissingAssetsDir(t *testing.T) { root := t.TempDir() mkdirAll(t, filepath.Join(root, "src")) @@ -320,3 +524,18 @@ func TestCloneWithHAKNamesRejectsUnknownHAKs(t *testing.T) { t.Fatal("expected error for unknown hak name") } } + +func loadAndValidate(root string) error { + proj, err := Load(root) + if err != nil { + return err + } + return proj.ValidateLayout() +} + +func writeProjectFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(strings.TrimPrefix(content, "\n")), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +}