From d1b20684f566639fac4194fc45f8677f991a7470 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Thu, 7 May 2026 21:33:53 +0200 Subject: [PATCH] Configuration hardening --- CONFIGURATION_HARDENING_AUDIT.md | 7 +- README.md | 44 ++- internal/app/app.go | 177 +++++++++++- internal/app/app_test.go | 74 +++++ internal/music/music.go | 23 ++ internal/music/naming.go | 43 ++- internal/pipeline/build.go | 26 +- internal/pipeline/music.go | 393 ++++++++++++++++++++++----- internal/pipeline/pipeline_test.go | 149 ++++++++++ internal/project/effective.go | 116 ++++++-- internal/project/project.go | 261 ++++++++++++++++-- internal/project/project_test.go | 37 +++ internal/validator/validator.go | 65 +++-- internal/validator/validator_test.go | 96 +++++++ 14 files changed, 1335 insertions(+), 176 deletions(-) diff --git a/CONFIGURATION_HARDENING_AUDIT.md b/CONFIGURATION_HARDENING_AUDIT.md index b5c1e99..1bac2ab 100644 --- a/CONFIGURATION_HARDENING_AUDIT.md +++ b/CONFIGURATION_HARDENING_AUDIT.md @@ -25,6 +25,8 @@ This audit records the current repository-configuration authority model for - `build.keep_existing_haks` - `inventory.source_extensions`, `inventory.asset_extensions`, `inventory.source_json_pattern` +- `validation.profile`, `validation.builtin_script_prefixes`, and + `validation.required_fields` - `scripts.source_dir`, `scripts.cache`, `scripts.compiler` - `extract.layout`, `extract.hak_discovery`, `extract.cleanup_stale` - `topdata.compiled_2da_dir`, `topdata.compiled_tlk`, package names, and wiki @@ -42,9 +44,8 @@ repository-specific YAML: - Supported autogen algorithms: `parts_rows`, `head_visualeffects`, `trailing_numeric_suffix`, `model_stem`, and `first_path_segment`. - Native topdata dataset internals and generated JSON dataset formats. -- NWN validator semantics for required module fields and resource-reference - checks. These are candidates for future validation profiles, but are not - repository-specific defaults. +- NWN validator resource-reference checks. These are candidates for future + validation profiles, but are not repository-specific defaults. ## Migration Checklist diff --git a/README.md b/README.md index 16f9ae1..aae9474 100644 --- a/README.md +++ b/README.md @@ -66,12 +66,23 @@ music: envi/music/westgate: mus_wg_ # Explicit dataset config (replaces prefixes above). + tools: + ffmpeg: tools/ffmpeg + ffprobe: tools/ffprobe defaults: + stage_root: "{paths.cache}/music" + credits_root: "{paths.cache}/credits" + credits_overlay: CREDITS.md max_stem_length: 16 + naming_scheme: nwn_bmu + output_extension: .bmu + convert_extensions: [.mp3, .ogg] datasets: westgate: source: envi/music/westgate + output: envi/music/westgate prefix: mus_wg_ + package_mode: hak_asset ``` Resolved configuration can be inspected without scanning or building: @@ -112,6 +123,12 @@ inventory: asset_extensions: [] # empty means built-in NWN asset extensions source_json_pattern: "{resref}.{extension}.json" +validation: + profile: nwn_module + builtin_script_prefixes: [nw_, x0_, x1_, x2_, x3_, ga_, gc_, gen_, gui_, nwg_, ta_] + required_fields: + ifo: [Mod_Name] + scripts: source_dir: scripts cache: "{paths.cache}/ncs" @@ -169,11 +186,34 @@ music: credits_root: "{paths.cache}/credits" credits_overlay: CREDITS.md max_stem_length: 16 + naming_scheme: nwn_bmu + output_extension: .bmu + convert_extensions: [.mp3, .ogg] ``` Music datasets define individual music collections with independent sources and -naming prefixes. The legacy `music.prefixes` shorthand is automatically normalized -into a dataset. +naming prefixes. `source` is the authored input root under `paths.assets`; +`output` is the generated HAK asset root matched by HAK include globs. If +`output` is omitted, it defaults to `source`. `package_mode: hak_asset` converts +configured source extensions into generated HAK resources; `package_mode: none` +scans and writes credits without adding generated HAK resources. The legacy +`music.prefixes` shorthand is automatically normalized into datasets. + +Music can also be run directly: + +```bash +sow-toolkit music list-datasets +sow-toolkit music scan --dataset westgate +sow-toolkit music build --dataset westgate --dry-run +sow-toolkit music build --dataset westgate +sow-toolkit music credits --dataset westgate --check +sow-toolkit music validate --dataset westgate --json +``` + +`build-haks` uses the same music pipeline. Use `--plan-only` or music +`--dry-run` to compute mappings without transcoding. Use `build-haks +--skip-music` to package authored assets only, or `build-haks --music-dataset +westgate` to limit conversion to a configured dataset. Generated and machine-readable artifacts remain JSON, including HAK manifests, topdata datasets, caches, credits inventories, build reports, and canonical GFF diff --git a/internal/app/app.go b/internal/app/app.go index a448a57..f87734a 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "sort" "strconv" "strings" "sync" @@ -163,6 +164,11 @@ var commands = []command{ description: "Inspect, explain, and validate the resolved project configuration.", run: runConfig, }, + { + name: "music", + description: "List, scan, build, and validate configured music datasets.", + run: runMusic, + }, { name: "compare", description: "Compare current source content against the built archives.", @@ -336,6 +342,8 @@ type buildHAKOptions struct { filteredHAKs []string filteredArchives []string sourceManifest string + musicDatasets []string + skipMusic bool planOnly bool logLevel logLevel } @@ -567,14 +575,17 @@ func runBuildHAKs(ctx context) error { spin.configure(ctx.stderr, console.spinnerEnabled) spin.update("Build HAKs: starting") var result pipeline.BuildResult + pipelineOpts := pipeline.BuildHAKOptions{ + Progress: console.progress, + ArchiveNames: opts.filteredArchives, + SourceManifestPath: opts.sourceManifest, + SkipMusic: opts.skipMusic, + MusicDatasetIDs: opts.musicDatasets, + } if opts.planOnly { - result, err = pipeline.PlanHAKsWithProgress(p, console.progress) + result, err = pipeline.PlanHAKsWithOptions(p, pipelineOpts) } else { - result, err = pipeline.BuildHAKsWithOptions(p, pipeline.BuildHAKOptions{ - Progress: console.progress, - ArchiveNames: opts.filteredArchives, - SourceManifestPath: opts.sourceManifest, - }) + result, err = pipeline.BuildHAKsWithOptions(p, pipelineOpts) } if err != nil { return err @@ -594,7 +605,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) { arg := args[index] switch arg { case "-h", "--help": - return opts, errors.New("usage: build-haks [--hak ...] [--archive ...] [--source-manifest ] [--plan-only] [--quiet|--verbose|--debug]") + return opts, errors.New("usage: build-haks [--hak ...] [--archive ...] [--source-manifest ] [--plan-only] [--skip-music] [--music-dataset ...] [--quiet|--verbose|--debug]") case "--hak": index++ if index >= len(args) { @@ -609,6 +620,14 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) { opts.filteredArchives = append(opts.filteredArchives, args[index]) case "--plan-only": opts.planOnly = true + case "--skip-music": + opts.skipMusic = true + case "--music-dataset": + index++ + if index >= len(args) { + return opts, errors.New("--music-dataset requires a value") + } + opts.musicDatasets = append(opts.musicDatasets, args[index]) case "--quiet": opts.logLevel = logLevelQuiet case "--verbose": @@ -634,6 +653,10 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) { opts.sourceManifest = value continue } + if value, ok := parseInlineFlagValue(arg, "--music-dataset"); ok { + opts.musicDatasets = append(opts.musicDatasets, value) + continue + } if arg == "--quiet=true" { opts.logLevel = logLevelQuiet continue @@ -652,6 +675,146 @@ func parseInlineFlagValue(arg, name string) (string, bool) { return strings.TrimSpace(strings.TrimPrefix(arg, prefix)), true } +type musicCommandOptions struct { + datasets []string + json bool + dryRun bool + check bool + force bool +} + +func runMusic(ctx context) error { + if len(ctx.args) < 2 { + return errors.New("usage: music [--dataset ] [--json] [--dry-run] [--check] [--force]") + } + p, err := loadProject(ctx) + if err != nil { + return err + } + if err := p.Scan(); err != nil { + return err + } + subcommand := ctx.args[1] + opts, err := parseMusicCommandArgs(ctx.args[2:]) + if err != nil { + return err + } + switch subcommand { + case "list-datasets": + return emitMusicDatasets(ctx, p, opts) + case "scan": + result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false}) + if err != nil { + return err + } + return emitMusicResult(ctx, "scan", result, opts) + case "build", "credits": + write := true + if subcommand == "build" && opts.dryRun { + write = false + } + if subcommand == "credits" && opts.check { + write = false + } + result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: write}) + if err != nil { + return err + } + return emitMusicResult(ctx, subcommand, result, opts) + case "manifest", "normalize": + result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false}) + if err != nil { + return err + } + return emitMusicResult(ctx, subcommand, result, opts) + case "validate": + if err := p.ValidateLayout(); err != nil { + return err + } + result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false}) + if err != nil { + return err + } + return emitMusicResult(ctx, "validate", result, opts) + default: + return fmt.Errorf("unknown music subcommand %q", subcommand) + } +} + +func parseMusicCommandArgs(args []string) (musicCommandOptions, error) { + opts := musicCommandOptions{} + for index := 0; index < len(args); index++ { + arg := args[index] + switch arg { + case "--dataset": + index++ + if index >= len(args) || strings.TrimSpace(args[index]) == "" { + return opts, errors.New("--dataset requires a value") + } + opts.datasets = append(opts.datasets, args[index]) + case "--json": + opts.json = true + case "--dry-run": + opts.dryRun = true + case "--check": + opts.check = true + case "--force": + opts.force = true + default: + if value, ok := parseInlineFlagValue(arg, "--dataset"); ok { + opts.datasets = append(opts.datasets, value) + continue + } + return opts, fmt.Errorf("unknown music argument %q", arg) + } + } + return opts, nil +} + +func emitMusicDatasets(ctx context, p *project.Project, opts musicCommandOptions) error { + effective := p.EffectiveConfig() + if opts.json { + return emitConfigValue(ctx.stdout, effective.Music.Datasets, "json") + } + if len(effective.Music.Datasets) == 0 { + fmt.Fprintln(ctx.stdout, "music datasets: none") + return nil + } + fmt.Fprintln(ctx.stdout, "music datasets:") + ids := make([]string, 0, len(effective.Music.Datasets)) + for id := range effective.Music.Datasets { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + ds := effective.Music.Datasets[id] + fmt.Fprintf(ctx.stdout, " %s: source=%s output=%s prefix=%s\n", id, ds.Source, ds.Output, ds.Prefix) + } + return nil +} + +func emitMusicResult(ctx context, action string, result pipeline.BuildResult, opts musicCommandOptions) error { + if opts.json { + return emitConfigValue(ctx.stdout, result.CreditsSummary, "json") + } + fmt.Fprintf(ctx.stdout, "music %s\n", action) + if len(result.CreditsSummary.ScannedDirs) > 0 { + fmt.Fprintf(ctx.stdout, "scanned: %s\n", strings.Join(result.CreditsSummary.ScannedDirs, ", ")) + } + fmt.Fprintf(ctx.stdout, "tracks: %d\n", result.CreditsSummary.TrackCount) + if action != "scan" && len(result.CreditsArtifactPaths) > 0 { + fmt.Fprintf(ctx.stdout, "credits artifacts: %d\n", len(result.CreditsArtifactPaths)) + } + if len(result.CreditsSummary.Mappings) > 0 && ctx.logLevel >= logLevelVerbose { + fmt.Fprintln(ctx.stdout, "mappings:") + for _, mapping := range result.CreditsSummary.Mappings { + fmt.Fprintf(ctx.stdout, " %s -> %s\n", mapping.Source, mapping.Output) + } + } + fmt.Fprintln(ctx.stdout, "status: ok") + return nil +} + func runExtract(ctx context) error { p, err := loadProject(ctx) if err != nil { diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 4817b7f..c6d6ad9 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -198,6 +198,80 @@ func TestRunBuildHAKsVerboseListsMappings(t *testing.T) { } } +func TestRunMusicScanUsesConfiguredDatasetsWithoutWriting(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets", "audio", "westgate")) + mkdirAll(t, filepath.Join(root, "build")) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + assets: assets + build: build +music: + datasets: + westgate_audio: + source: audio/westgate + output: generated/music + prefix: wg_ +`) + writeFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3") + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"music", "scan", "--dataset", "westgate_audio"}, + } + + if err := runMusic(ctx); err != nil { + t.Fatalf("runMusic failed: %v", err) + } + output := stdout.String() + if !strings.Contains(output, "music scan") || !strings.Contains(output, "tracks: 1") { + t.Fatalf("unexpected music scan output: %q", output) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "credits")); !os.IsNotExist(err) { + t.Fatalf("music scan should not write credits artifacts, stat err=%v", err) + } +} + +func TestRunMusicListDatasets(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + assets: assets +music: + datasets: + westgate_audio: + source: audio/westgate + output: generated/music + prefix: wg_ +`) + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"music", "list-datasets"}, + } + + if err := runMusic(ctx); err != nil { + t.Fatalf("runMusic failed: %v", err) + } + if !strings.Contains(stdout.String(), "westgate_audio: source=audio/westgate output=generated/music prefix=wg_") { + t.Fatalf("unexpected dataset list: %q", stdout.String()) + } +} + func TestRunConfigEffectiveJSONShowsDefaultsAndProvenance(t *testing.T) { root := t.TempDir() writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` diff --git a/internal/music/music.go b/internal/music/music.go index 000715b..e9ce566 100644 --- a/internal/music/music.go +++ b/internal/music/music.go @@ -78,6 +78,13 @@ type CreditsOverlay struct { } func ResolveFFmpeg() (string, error) { + return ResolveFFmpegWithConfig("") +} + +func ResolveFFmpegWithConfig(configuredPath string) (string, error) { + if configured := strings.TrimSpace(configuredPath); configured != "" { + return configured, nil + } if configured := strings.TrimSpace(os.Getenv("SOW_FFMPEG")); configured != "" { return configured, nil } @@ -89,6 +96,13 @@ func ResolveFFmpeg() (string, error) { } func ResolveFFprobe() (string, error) { + return ResolveFFprobeWithConfig("") +} + +func ResolveFFprobeWithConfig(configuredPath string) (string, error) { + if configured := strings.TrimSpace(configuredPath); configured != "" { + return configured, nil + } if configured := strings.TrimSpace(os.Getenv("SOW_FFPROBE")); configured != "" { return configured, nil } @@ -104,6 +118,15 @@ func IsMusicAssetPath(rel string) bool { return rel == "envi/music" || strings.HasPrefix(rel, "envi/music/") } +func PathIsUnder(root, rel string) bool { + root = TrimSlashes(filepath.ToSlash(root)) + rel = TrimSlashes(filepath.ToSlash(rel)) + if root == "" || rel == "" { + return false + } + return rel == root || strings.HasPrefix(rel, root+"/") +} + func PrefixForDir(prefixes map[string]string, dir string) string { dir = TrimSlashes(filepath.ToSlash(dir)) bestPrefix := "" diff --git a/internal/music/naming.go b/internal/music/naming.go index 2874bc7..798f190 100644 --- a/internal/music/naming.go +++ b/internal/music/naming.go @@ -8,7 +8,14 @@ import ( ) func GenerateStem(fileName, prefix string, used map[string]struct{}) (string, error) { - maxCore := MaxStemLen - len(prefix) + return GenerateStemWithMax(fileName, prefix, MaxStemLen, used) +} + +func GenerateStemWithMax(fileName, prefix string, maxStemLen int, used map[string]struct{}) (string, error) { + if maxStemLen <= 0 { + maxStemLen = MaxStemLen + } + maxCore := maxStemLen - len(prefix) if maxCore < 3 { return "", fmt.Errorf("prefix too long: %q", prefix) } @@ -19,11 +26,11 @@ func GenerateStem(fileName, prefix string, used map[string]struct{}) (string, er sum := sha1.Sum([]byte(fileName)) core = fmt.Sprintf("%x", sum[:])[:maxCore] } - stem := strings.TrimRight((prefix + core)[:Min(MaxStemLen, len(prefix)+len(core))], "_") + stem := strings.TrimRight((prefix + core)[:Min(maxStemLen, len(prefix)+len(core))], "_") if stem == "" { return "", fmt.Errorf("could not derive music stem for %s", fileName) } - return UniqueName(stem, used), nil + return UniqueNameWithMax(stem, maxStemLen, used), nil } func SlugWords(value string) []string { @@ -119,13 +126,20 @@ func AbbreviateWord(word string, maxLen int) string { } func UniqueName(stem string, used map[string]struct{}) string { + return UniqueNameWithMax(stem, MaxStemLen, used) +} + +func UniqueNameWithMax(stem string, maxStemLen int, used map[string]struct{}) string { + if maxStemLen <= 0 { + maxStemLen = MaxStemLen + } if _, exists := used[stem]; !exists { used[stem] = struct{}{} return stem } for index := 1; ; index++ { suffix := fmt.Sprintf("_%d", index) - prefixLen := Min(len(stem), Max(0, MaxStemLen-len(suffix))) + prefixLen := Min(len(stem), Max(0, maxStemLen-len(suffix))) candidate := strings.TrimRight(stem[:prefixLen], "_") + suffix if _, exists := used[candidate]; exists { continue @@ -148,16 +162,27 @@ func ReserveName(stem string, used map[string]struct{}) error { } func ValidateManualOutputFile(name string) error { + return ValidateManualOutputFileWithRules(name, ".bmu", MaxStemLen) +} + +func ValidateManualOutputFileWithRules(name, extension string, maxStemLen int) error { name = strings.ToLower(strings.TrimSpace(name)) - if !strings.HasSuffix(name, ".bmu") { - return fmt.Errorf("output file must end with .bmu") + extension = strings.ToLower(strings.TrimSpace(extension)) + if extension == "" { + extension = ".bmu" } - stem := strings.TrimSuffix(name, ".bmu") + if maxStemLen <= 0 { + maxStemLen = MaxStemLen + } + if !strings.HasSuffix(name, extension) { + return fmt.Errorf("output file must end with %s", extension) + } + stem := strings.TrimSuffix(name, extension) if stem == "" { return fmt.Errorf("output file stem is empty") } - if len(stem) > MaxStemLen { - return fmt.Errorf("output file stem %q exceeds %d characters", stem, MaxStemLen) + if len(stem) > maxStemLen { + return fmt.Errorf("output file stem %q exceeds %d characters", stem, maxStemLen) } for _, r := range stem { switch { diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index dc25205..b80b461 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -110,6 +110,8 @@ type BuildHAKOptions struct { Progress ProgressFunc ArchiveNames []string SourceManifestPath string + SkipMusic bool + MusicDatasetIDs []string } func Build(p *project.Project) (BuildResult, error) { @@ -195,7 +197,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu } func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) { - return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath) + return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs) } func PlanHAKs(p *project.Project) (BuildResult, error) { @@ -206,15 +208,19 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul return planHAKs(p, progress) } +func PlanHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) { + return planOrBuildHAKs(p, opts.Progress, false, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs) +} + func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { - return planOrBuildHAKs(p, progress, true, nil, "") + return planOrBuildHAKs(p, progress, true, nil, "", false, nil) } func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { - return planOrBuildHAKs(p, progress, false, nil, "") + return planOrBuildHAKs(p, progress, false, nil, "", false, nil) } -func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string) (result BuildResult, err error) { +func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string, skipMusic bool, musicDatasetIDs []string) (result BuildResult, err error) { preserveExistingHAKs := p.EffectiveConfig().Build.KeepExistingHAKs || envBool("SOW_BUILD_HAKS_KEEP_EXISTING") progressf(progress, "Validating project...") @@ -227,9 +233,15 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo if err != nil { return BuildResult{}, err } - musicAssets, err := prepareMusicAssets(p) - if err != nil { - return BuildResult{}, err + musicAssets := &preparedMusicAssets{SkipSourceRel: map[string]struct{}{}} + if !skipMusic { + musicAssets, err = prepareMusicAssetsWithOptions(p, musicPrepareOptions{ + datasetIDs: musicDatasetIDs, + write: writeArchives, + }) + if err != nil { + return BuildResult{}, err + } } defer func() { if cleanupErr := cleanupPreparedMusicAssets(musicAssets); cleanupErr != nil && err == nil { diff --git a/internal/pipeline/music.go b/internal/pipeline/music.go index b6e4152..f9d2d07 100644 --- a/internal/pipeline/music.go +++ b/internal/pipeline/music.go @@ -21,16 +21,193 @@ type preparedMusicAssets struct { Summary CreditsRefreshSummary } +type MusicBuildOptions struct { + DatasetIDs []string + Write bool +} + +type musicPrepareOptions struct { + datasetIDs []string + write bool +} + +type musicSourceGroup struct { + DatasetID string + Dataset project.EffectiveMusicDataset + SourceDir string + OutputDir string + Sources []string +} + +type musicCreditGroup struct { + CreditsRoot string + SourceDir string + Entries []music.CreditsEntry +} + +func musicRootPath(p *project.Project, path string) string { + if strings.TrimSpace(path) == "" { + return p.Root + } + if filepath.IsAbs(path) { + return path + } + return filepath.Join(p.Root, filepath.FromSlash(path)) +} + +func effectiveMusicDatasets(p *project.Project, onlyIDs []string) (map[string]project.EffectiveMusicDataset, error) { + effective := p.EffectiveConfig() + wanted := map[string]struct{}{} + for _, id := range onlyIDs { + id = strings.TrimSpace(id) + if id != "" { + wanted[id] = struct{}{} + } + } + datasets := map[string]project.EffectiveMusicDataset{} + for id, dataset := range effective.Music.Datasets { + if len(wanted) > 0 { + if _, ok := wanted[id]; !ok { + continue + } + delete(wanted, id) + } + datasets[id] = dataset + } + if len(wanted) > 0 { + unknown := make([]string, 0, len(wanted)) + for id := range wanted { + unknown = append(unknown, id) + } + sort.Strings(unknown) + return nil, fmt.Errorf("unknown music dataset(s): %s", strings.Join(unknown, ", ")) + } + if len(datasets) == 0 && len(wanted) == 0 { + for _, rel := range p.Inventory.AssetFiles { + ext := strings.ToLower(filepath.Ext(rel)) + if !music.PathIsUnder("envi/music", rel) { + continue + } + if _, ok := extensionSet(effective.Music.ConvertExtensions)[ext]; ok { + datasets["legacy_envi_music"] = project.EffectiveMusicDataset{ + Source: "envi/music", + Output: "envi/music", + StageRoot: effective.Music.StageRoot, + CreditsRoot: effective.Music.CreditsRoot, + NamingScheme: effective.Music.NamingScheme, + OutputExtension: effective.Music.OutputExtension, + MaxStemLength: effective.Music.MaxStemLength, + PackageMode: "hak_asset", + ConvertExtensions: effective.Music.ConvertExtensions, + } + break + } + } + } + return datasets, nil +} + +func extensionSet(values []string) map[string]struct{} { + out := make(map[string]struct{}, len(values)) + for _, value := range values { + value = strings.ToLower(strings.TrimSpace(value)) + if value != "" { + out[value] = struct{}{} + } + } + return out +} + +func datasetForMusicSource(datasets map[string]project.EffectiveMusicDataset, rel, ext string) (string, project.EffectiveMusicDataset, bool) { + bestID := "" + var best project.EffectiveMusicDataset + bestLen := -1 + for id, dataset := range datasets { + if _, ok := extensionSet(dataset.ConvertExtensions)[ext]; !ok { + continue + } + if !music.PathIsUnder(dataset.Source, rel) { + continue + } + if len(dataset.Source) > bestLen { + bestID = id + best = dataset + bestLen = len(dataset.Source) + } + } + return bestID, best, bestID != "" +} + +func plannedMusicAsset(outputRel, sourceRel string) (assetResource, error) { + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(outputRel)), ".") + resourceType, ok := erf.HAKResourceTypeForExtension(extension) + if !ok { + return assetResource{}, fmt.Errorf("unsupported generated music resource extension %q", filepath.Ext(outputRel)) + } + name := strings.ToLower(strings.TrimSuffix(filepath.Base(outputRel), filepath.Ext(outputRel))) + return assetResource{ + Rel: outputRel, + Resource: erf.Resource{ + Name: name, + Type: resourceType, + }, + ContentID: "music-plan:" + filepath.ToSlash(sourceRel), + }, nil +} + +func BuildMusic(p *project.Project, opts MusicBuildOptions) (BuildResult, error) { + prepared, err := prepareMusicAssetsWithOptions(p, musicPrepareOptions{datasetIDs: opts.DatasetIDs, write: opts.Write}) + if err != nil { + return BuildResult{}, err + } + if err := cleanupPreparedMusicAssets(prepared); err != nil { + return BuildResult{}, err + } + return BuildResult{ + CreditsArtifactPaths: prepared.Artifacts, + CreditsSummary: prepared.Summary, + HAKAssets: len(prepared.Generated), + }, nil +} + func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) { + return prepareMusicAssetsWithOptions(p, musicPrepareOptions{write: true}) +} + +func prepareMusicAssetsWithOptions(p *project.Project, opts musicPrepareOptions) (*preparedMusicAssets, error) { usedNames := make(map[string]struct{}) - dirSources := make(map[string][]string) + groupsByKey := make(map[string]*musicSourceGroup) + datasets, err := effectiveMusicDatasets(p, opts.datasetIDs) + if err != nil { + return nil, err + } for _, rel := range p.Inventory.AssetFiles { rel = filepath.ToSlash(rel) ext := strings.ToLower(filepath.Ext(rel)) name := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))) - if _, convertible := music.ConvertExtensions[ext]; convertible && music.IsMusicAssetPath(rel) { - dir := filepath.ToSlash(filepath.Dir(rel)) - dirSources[dir] = append(dirSources[dir], rel) + if datasetID, dataset, ok := datasetForMusicSource(datasets, rel, ext); ok { + sourceDir := filepath.ToSlash(filepath.Dir(rel)) + relDir, err := filepath.Rel(filepath.FromSlash(dataset.Source), filepath.FromSlash(sourceDir)) + if err != nil { + return nil, fmt.Errorf("resolve music source directory %s: %w", sourceDir, err) + } + relDir = filepath.ToSlash(relDir) + outputDir := dataset.Output + if relDir != "." && relDir != "" { + outputDir = filepath.ToSlash(filepath.Join(outputDir, filepath.FromSlash(relDir))) + } + key := datasetID + "\x00" + sourceDir + group := groupsByKey[key] + if group == nil { + group = &musicSourceGroup{ + DatasetID: datasetID, + Dataset: dataset, + SourceDir: sourceDir, + OutputDir: outputDir, + } + groupsByKey[key] = group + } + group.Sources = append(group.Sources, rel) continue } if name != "" { @@ -41,7 +218,10 @@ func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) { result := &preparedMusicAssets{ SkipSourceRel: make(map[string]struct{}), } - if len(dirSources) == 0 { + if len(groupsByKey) == 0 { + if !opts.write { + return result, nil + } writeResult, err := writeCreditsArtifacts(p, nil) if err != nil { return nil, err @@ -51,57 +231,86 @@ func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) { return result, nil } - dirs := make([]string, 0, len(dirSources)) - for dir := range dirSources { - dirs = append(dirs, dir) + groups := make([]*musicSourceGroup, 0, len(groupsByKey)) + for _, group := range groupsByKey { + sort.Strings(group.Sources) + groups = append(groups, group) } - sort.Strings(dirs) + sort.Slice(groups, func(i, j int) bool { + if groups[i].SourceDir != groups[j].SourceDir { + return groups[i].SourceDir < groups[j].SourceDir + } + return groups[i].DatasetID < groups[j].DatasetID + }) - generatedByDir := make(map[string][]music.CreditsEntry) - stageRoot := p.MusicStageDir() - result.TempRoots = append(result.TempRoots, stageRoot) - - ffmpegPath, err := music.ResolveFFmpeg() - if err != nil { - return nil, err + creditGroups := make([]musicCreditGroup, 0, len(groups)) + for _, group := range groups { + result.Summary.ScannedDirs = append(result.Summary.ScannedDirs, group.SourceDir) + result.Summary.TrackCount += len(group.Sources) } - ffprobePath, err := music.ResolveFFprobe() - if err != nil { - return nil, err + ffmpegPath := "" + ffprobePath := "" + if opts.write { + var err error + ffmpegPath, err = music.ResolveFFmpegWithConfig(p.MusicFFmpegPath()) + if err != nil { + return nil, err + } + ffprobePath, err = music.ResolveFFprobeWithConfig(p.MusicFFprobePath()) + if err != nil { + return nil, err + } } - for _, dir := range dirs { - sources := append([]string(nil), dirSources[dir]...) - sort.Strings(sources) - prefix := music.PrefixForDir(p.EffectiveConfig().Music.Prefixes, dir) - overlayPath := filepath.Join(p.AssetsDir(), filepath.FromSlash(dir), p.EffectiveConfig().Music.CreditsOverlay) + for _, group := range groups { + stageRoot := musicRootPath(p, group.Dataset.StageRoot) + result.TempRoots = append(result.TempRoots, stageRoot) + prefix := music.SanitizePrefix(group.Dataset.Prefix) + overlayPath := filepath.Join(p.AssetsDir(), filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay) overlay, err := music.ParseCreditsOverlay(overlayPath) if err != nil { return nil, fmt.Errorf("parse credits overlay %s: %w", overlayPath, err) } - entries := make([]music.CreditsEntry, 0, len(sources)) - for _, rel := range sources { + entries := make([]music.CreditsEntry, 0, len(group.Sources)) + for _, rel := range group.Sources { base := filepath.Base(rel) manual := overlay.Match(base, "") outputFile := "" switch { case manual != nil && strings.TrimSpace(manual.OutputFile) != "": outputFile = strings.ToLower(strings.TrimSpace(manual.OutputFile)) - if err := music.ValidateManualOutputFile(outputFile); err != nil { + if err := music.ValidateManualOutputFileWithRules(outputFile, group.Dataset.OutputExtension, group.Dataset.MaxStemLength); err != nil { return nil, fmt.Errorf("manual credits output for %s: %w", rel, err) } if err := music.ReserveName(strings.TrimSuffix(outputFile, filepath.Ext(outputFile)), usedNames); err != nil { return nil, fmt.Errorf("manual credits output for %s: %w", rel, err) } default: - stem, err := music.GenerateStem(base, prefix, usedNames) + stem, err := music.GenerateStemWithMax(base, prefix, group.Dataset.MaxStemLength, usedNames) if err != nil { return nil, fmt.Errorf("generate music name for %s: %w", rel, err) } - outputFile = stem + ".bmu" + outputFile = stem + group.Dataset.OutputExtension } - outputRel := filepath.ToSlash(filepath.Join(dir, outputFile)) + outputRel := filepath.ToSlash(filepath.Join(group.OutputDir, outputFile)) + result.Summary.Mappings = append(result.Summary.Mappings, MusicFileMapping{ + Source: rel, + Output: outputRel, + }) + result.SkipSourceRel[rel] = struct{}{} + + if !opts.write { + if group.Dataset.PackageMode == "hak_asset" { + asset, err := plannedMusicAsset(outputRel, rel) + if err != nil { + return nil, err + } + result.Generated = append(result.Generated, asset) + } + continue + } + metadata, err := music.ProbeMetadata(ffprobePath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) if err != nil { return nil, fmt.Errorf("probe music metadata for %s: %w", rel, err) @@ -125,36 +334,50 @@ func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) { if err := music.ConvertSource(ffmpegPath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)), stagePath); err != nil { return nil, fmt.Errorf("convert music %s: %w", rel, err) } - resourceInfo, err := assetResourceFromPath(stagePath, outputRel, lfsAssetInfo{}, false) - if err != nil { - return nil, fmt.Errorf("load converted music %s: %w", outputRel, err) + if group.Dataset.PackageMode == "hak_asset" { + resourceInfo, err := assetResourceFromPath(stagePath, outputRel, lfsAssetInfo{}, false) + if err != nil { + return nil, fmt.Errorf("load converted music %s: %w", outputRel, err) + } + info, err := os.Stat(stagePath) + if err != nil { + return nil, fmt.Errorf("stat converted music %s: %w", stagePath, err) + } + result.Generated = append(result.Generated, assetResource{ + Rel: outputRel, + Resource: resourceInfo.Resource, + Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}), + CreatedAt: info.ModTime(), + ContentID: resourceInfo.ContentID, + }) } - info, err := os.Stat(stagePath) - if err != nil { - return nil, fmt.Errorf("stat converted music %s: %w", stagePath, err) - } - result.Generated = append(result.Generated, assetResource{ - Rel: outputRel, - Resource: resourceInfo.Resource, - Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}), - CreatedAt: info.ModTime(), - ContentID: resourceInfo.ContentID, - }) - result.SkipSourceRel[rel] = struct{}{} entries = append(entries, entry) } - if err := music.ValidateOverlayEntries(dir, overlay, entries); err != nil { - return nil, err + if opts.write { + if err := music.ValidateOverlayEntries(group.SourceDir, overlay, entries); err != nil { + return nil, err + } + creditGroups = append(creditGroups, musicCreditGroup{ + CreditsRoot: group.Dataset.CreditsRoot, + SourceDir: group.SourceDir, + Entries: entries, + }) } - generatedByDir[dir] = entries } - writeResult, err := writeCreditsArtifacts(p, generatedByDir) + if !opts.write { + sort.Slice(result.Generated, func(i, j int) bool { + return compareResourceKeys(result.Generated[i].Resource, result.Generated[j].Resource) < 0 + }) + return result, nil + } + + writeResult, err := writeCreditsArtifacts(p, creditGroups) if err != nil { return nil, err } - writeResult.Summary.ScannedDirs = append(writeResult.Summary.ScannedDirs, dirs...) - writeResult.Summary.TrackCount = len(result.Generated) + writeResult.Summary.ScannedDirs = append(writeResult.Summary.ScannedDirs, result.Summary.ScannedDirs...) + writeResult.Summary.TrackCount = result.Summary.TrackCount result.Artifacts = writeResult.Artifacts result.Summary = writeResult.Summary sort.Slice(result.Generated, func(i, j int) bool { @@ -168,19 +391,24 @@ type creditsArtifactWriteResult struct { Summary CreditsRefreshSummary } -func writeCreditsArtifacts(p *project.Project, generatedByDir map[string][]music.CreditsEntry) (creditsArtifactWriteResult, error) { - creditsRoot := p.CreditsDir() +func writeCreditsArtifacts(p *project.Project, generated []musicCreditGroup) (creditsArtifactWriteResult, error) { + defaultCreditsRoot := p.CreditsDir() sources := make([]music.CreditsSource, 0) - desiredFiles := make(map[string][]byte) + desiredByRoot := map[string]map[string][]byte{} summary := CreditsRefreshSummary{} - if generatedByDir != nil { - dirs := make([]string, 0, len(generatedByDir)) - for dir := range generatedByDir { - dirs = append(dirs, dir) - } - sort.Strings(dirs) - for _, dir := range dirs { - entries := append([]music.CreditsEntry(nil), generatedByDir[dir]...) + if generated != nil { + sort.Slice(generated, func(i, j int) bool { + if generated[i].SourceDir != generated[j].SourceDir { + return generated[i].SourceDir < generated[j].SourceDir + } + return generated[i].CreditsRoot < generated[j].CreditsRoot + }) + for _, group := range generated { + creditsRoot := musicRootPath(p, group.CreditsRoot) + if desiredByRoot[creditsRoot] == nil { + desiredByRoot[creditsRoot] = map[string][]byte{} + } + entries := append([]music.CreditsEntry(nil), group.Entries...) sort.Slice(entries, func(i, j int) bool { if strings.ToLower(entries[i].Artist) != strings.ToLower(entries[j].Artist) { return strings.ToLower(entries[i].Artist) < strings.ToLower(entries[j].Artist) @@ -190,8 +418,8 @@ func writeCreditsArtifacts(p *project.Project, generatedByDir map[string][]music } return strings.ToLower(entries[i].OutputFile) < strings.ToLower(entries[j].OutputFile) }) - outputPath := filepath.Join(creditsRoot, filepath.FromSlash(dir), p.EffectiveConfig().Music.CreditsOverlay) - desiredFiles[outputPath] = []byte(music.RenderCreditsMarkdown(entries)) + outputPath := filepath.Join(creditsRoot, filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay) + desiredByRoot[creditsRoot][outputPath] = []byte(music.RenderCreditsMarkdown(entries)) summary.GeneratedPaths = append(summary.GeneratedPaths, outputPath) for _, entry := range entries { summary.Mappings = append(summary.Mappings, MusicFileMapping{ @@ -237,27 +465,42 @@ func writeCreditsArtifacts(p *project.Project, generatedByDir map[string][]music } sort.Slice(sources, func(i, j int) bool { return sources[i].Path < sources[j].Path }) - inventoryPath := filepath.Join(creditsRoot, "credits.json") payload, err := json.MarshalIndent(music.CreditsInventory{Sources: sources}, "", " ") if err != nil { return creditsArtifactWriteResult{}, fmt.Errorf("marshal credits inventory: %w", err) } payload = append(payload, '\n') - desiredFiles[inventoryPath] = payload - if err := os.MkdirAll(creditsRoot, 0o755); err != nil { - return creditsArtifactWriteResult{}, fmt.Errorf("create credits dir: %w", err) + if len(desiredByRoot) == 0 { + desiredByRoot[defaultCreditsRoot] = map[string][]byte{} } - changedFiles, err := music.SyncGeneratedCreditsArtifacts(creditsRoot, desiredFiles) - if err != nil { - return creditsArtifactWriteResult{}, err + artifacts := make([]string, 0) + changedFiles := 0 + roots := make([]string, 0, len(desiredByRoot)) + for root := range desiredByRoot { + roots = append(roots, root) } - artifacts := make([]string, 0, len(desiredFiles)) - for path := range desiredFiles { - artifacts = append(artifacts, path) + sort.Strings(roots) + for _, creditsRoot := range roots { + desiredFiles := desiredByRoot[creditsRoot] + inventoryPath := filepath.Join(creditsRoot, "credits.json") + desiredFiles[inventoryPath] = payload + if err := os.MkdirAll(creditsRoot, 0o755); err != nil { + return creditsArtifactWriteResult{}, fmt.Errorf("create credits dir: %w", err) + } + changed, err := music.SyncGeneratedCreditsArtifacts(creditsRoot, desiredFiles) + if err != nil { + return creditsArtifactWriteResult{}, err + } + changedFiles += changed + for path := range desiredFiles { + artifacts = append(artifacts, path) + } + if summary.InventoryPath == "" { + summary.InventoryPath = inventoryPath + } } sort.Strings(artifacts) summary.ArtifactPaths = append(summary.ArtifactPaths, artifacts...) - summary.InventoryPath = inventoryPath summary.ChangedFiles = changedFiles return creditsArtifactWriteResult{ Artifacts: artifacts, diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 203873a..b4a55f1 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -2769,4 +2769,153 @@ func TestBuildHAKsConvertsMusicSourcesAndWritesCreditsArtifacts(t *testing.T) { } } +func TestBuildHAKsUsesExplicitMusicDatasetOutsideLegacyRoot(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate")) + mustMkdir(t, filepath.Join(root, "build")) + ffprobePath := filepath.Join(root, "tools", "ffprobe") + ffmpegPath := filepath.Join(root, "tools", "ffmpeg") + mustMkdir(t, filepath.Dir(ffprobePath)) + mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Dataset Artist\",\"title\":\"Dataset Title\"}}}'\n") + mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'dataset-bmu' > \"$last\"\n") + if err := os.Chmod(ffprobePath, 0o755); err != nil { + t.Fatalf("chmod ffprobe: %v", err) + } + if err := os.Chmod(ffmpegPath, 0o755); err != nil { + t.Fatalf("chmod ffmpeg: %v", err) + } + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Assets + resref: testassets +paths: + assets: assets + build: build +music: + tools: + ffmpeg: tools/ffmpeg + ffprobe: tools/ffprobe + defaults: + credits_root: custom-credits + datasets: + westgate_audio: + source: audio/westgate + output: generated/music + prefix: wg_ +haks: + - name: music + priority: 1 + max_bytes: 1048576 + split: false + include: + - generated/music/** +`) + mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKs(p) + if err != nil { + t.Fatalf("build haks: %v", err) + } + if result.CreditsSummary.TrackCount != 1 { + t.Fatalf("expected one music track, got %#v", result.CreditsSummary) + } + manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + manifestText := string(manifestRaw) + if !strings.Contains(manifestText, "generated/music/wg_theme.bmu") { + t.Fatalf("expected dataset output resource in manifest, got:\n%s", manifestText) + } + if strings.Contains(manifestText, "audio/westgate/Theme Song.mp3") { + t.Fatalf("did not expect source mp3 in manifest, got:\n%s", manifestText) + } + creditsRaw, err := os.ReadFile(filepath.Join(root, "custom-credits", "audio", "westgate", "CREDITS.md")) + if err != nil { + t.Fatalf("read generated dataset credits: %v", err) + } + if !strings.Contains(string(creditsRaw), "Dataset Artist") || !strings.Contains(string(creditsRaw), "wg_theme.bmu") { + t.Fatalf("unexpected generated credits:\n%s", string(creditsRaw)) + } +} + +func TestPlanHAKsDoesNotTranscodeMusicSources(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate")) + mustMkdir(t, filepath.Join(root, "build")) + + ffmpegPath := filepath.Join(root, "tools", "ffmpeg") + mustMkdir(t, filepath.Dir(ffmpegPath)) + mustWriteFile(t, ffmpegPath, "#!/bin/sh\necho should-not-run >&2\nexit 42\n") + if err := os.Chmod(ffmpegPath, 0o755); err != nil { + t.Fatalf("chmod ffmpeg: %v", err) + } + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Assets + resref: testassets +paths: + assets: assets + build: build +music: + tools: + ffmpeg: tools/ffmpeg + ffprobe: tools/ffprobe + datasets: + westgate_audio: + source: audio/westgate + output: generated/music + prefix: wg_ +haks: + - name: music + priority: 1 + max_bytes: 1048576 + split: false + include: + - generated/music/** +`) + mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := PlanHAKs(p) + if err != nil { + t.Fatalf("plan haks: %v", err) + } + if result.CreditsSummary.TrackCount != 1 { + t.Fatalf("expected one planned music track, got %#v", result.CreditsSummary) + } + manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + if !strings.Contains(string(manifestRaw), "generated/music/wg_theme.bmu") { + t.Fatalf("expected planned generated music in manifest, got:\n%s", string(manifestRaw)) + } + if _, err := os.Stat(filepath.Join(root, ".cache", "music")); !os.IsNotExist(err) { + t.Fatalf("plan-only should not stage music, stat err=%v", err) + } +} diff --git a/internal/project/effective.go b/internal/project/effective.go index 9c2bfb3..671a014 100644 --- a/internal/project/effective.go +++ b/internal/project/effective.go @@ -18,12 +18,15 @@ const ( DefaultHAKManifest = "haks.json" DefaultHAKArchiveTemplate = "{hak.name}.hak" DefaultSourceJSONPattern = "{resref}.{extension}.json" + DefaultValidationProfile = "nwn_module" DefaultScriptsSourceDir = "scripts" DefaultScriptsCache = "{paths.cache}/ncs" DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER" DefaultMusicStageRoot = "{paths.cache}/music" DefaultCreditsRoot = "{paths.cache}/credits" DefaultCreditsOverlayFile = "CREDITS.md" + DefaultMusicNamingScheme = "nwn_bmu" + DefaultMusicOutputExtension = ".bmu" DefaultTopDataBuild = "{paths.build}/topdata" DefaultTopDataCompiled2DADir = "2da" DefaultTopDataCompiledTLK = "sow_tlk.tlk" @@ -60,6 +63,7 @@ type EffectiveConfig struct { Outputs EffectiveOutputConfig `json:"outputs" yaml:"outputs"` Build BuildConfig `json:"build" yaml:"build"` Inventory InventoryConfig `json:"inventory" yaml:"inventory"` + Validation ValidationConfig `json:"validation" yaml:"validation"` Scripts EffectiveScriptsConfig `json:"scripts" yaml:"scripts"` Music EffectiveMusicConfig `json:"music" yaml:"music"` TopData EffectiveTopDataConfig `json:"topdata" yaml:"topdata"` @@ -103,19 +107,29 @@ type EffectiveScriptCompilerEnvConfig struct { } type EffectiveMusicConfig struct { - Prefixes map[string]string `json:"prefixes" yaml:"prefixes"` - StageRoot string `json:"stage_root" yaml:"stage_root"` - CreditsRoot string `json:"credits_root" yaml:"credits_root"` - CreditsOverlay string `json:"credits_overlay" yaml:"credits_overlay"` - MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"` - Datasets map[string]EffectiveMusicDataset `json:"datasets,omitempty" yaml:"datasets,omitempty"` + Prefixes map[string]string `json:"prefixes" yaml:"prefixes"` + Tools MusicToolsConfig `json:"tools" yaml:"tools"` + StageRoot string `json:"stage_root" yaml:"stage_root"` + CreditsRoot string `json:"credits_root" yaml:"credits_root"` + CreditsOverlay string `json:"credits_overlay" yaml:"credits_overlay"` + MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"` + NamingScheme string `json:"naming_scheme" yaml:"naming_scheme"` + OutputExtension string `json:"output_extension" yaml:"output_extension"` + ConvertExtensions []string `json:"convert_extensions" yaml:"convert_extensions"` + Datasets map[string]EffectiveMusicDataset `json:"datasets,omitempty" yaml:"datasets,omitempty"` } type EffectiveMusicDataset struct { - Source string `json:"source" yaml:"source"` - Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"` - StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` - CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` + Source string `json:"source" yaml:"source"` + Output string `json:"output" yaml:"output"` + Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"` + StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` + CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` + NamingScheme string `json:"naming_scheme" yaml:"naming_scheme"` + OutputExtension string `json:"output_extension" yaml:"output_extension"` + MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"` + PackageMode string `json:"package_mode" yaml:"package_mode"` + ConvertExtensions []string `json:"convert_extensions" yaml:"convert_extensions"` } type EffectiveTopDataConfig struct { @@ -177,6 +191,11 @@ func (p *Project) EffectiveConfig() EffectiveConfig { AssetExtensions: defaultStringSlice(p.Config.Inventory.AssetExtensions, AssetExtensions), SourceJSONPattern: defaultString(p.Config.Inventory.SourceJSONPattern, DefaultSourceJSONPattern), } + validation := ValidationConfig{ + Profile: defaultString(p.Config.Validation.Profile, DefaultValidationProfile), + BuiltinScriptPrefixes: normalizeLowerStringSlice(defaultStringSlice(p.Config.Validation.BuiltinScriptPrefixes, DefaultBuiltinScriptPrefixes())), + RequiredFields: defaultRequiredFields(p.Config.Validation.RequiredFields), + } scripts := EffectiveScriptsConfig{ SourceDir: defaultString(p.Config.Scripts.SourceDir, DefaultScriptsSourceDir), Cache: expandPathTemplate(defaultString(p.Config.Scripts.Cache, DefaultScriptsCache), paths), @@ -221,6 +240,9 @@ func (p *Project) EffectiveConfig() EffectiveConfig { musicCreditsRoot := expandPathTemplate(DefaultCreditsRoot, paths) musicCreditsOverlay := DefaultCreditsOverlayFile musicMaxStemLength := 16 + musicNamingScheme := DefaultMusicNamingScheme + musicOutputExtension := DefaultMusicOutputExtension + musicConvertExtensions := []string{".mp3", ".ogg"} if d := p.Config.Music.Defaults; d != nil { if d.StageRoot != "" { musicStageRoot = expandPathTemplate(d.StageRoot, paths) @@ -234,6 +256,15 @@ func (p *Project) EffectiveConfig() EffectiveConfig { if d.MaxStemLength != nil { musicMaxStemLength = *d.MaxStemLength } + if d.NamingScheme != "" { + musicNamingScheme = d.NamingScheme + } + if d.OutputExtension != "" { + musicOutputExtension = normalizeExtension(d.OutputExtension) + } + if len(d.ConvertExtensions) > 0 { + musicConvertExtensions = normalizeExtensionSlice(d.ConvertExtensions) + } } musicPrefixes := cloneStringMap(p.Config.Music.Prefixes) if len(musicPrefixes) == 0 { @@ -254,11 +285,30 @@ func (p *Project) EffectiveConfig() EffectiveConfig { if dsCreditsRoot == "" { dsCreditsRoot = musicCreditsRoot } + dsOutput := strings.TrimSpace(ds.Output) + if dsOutput == "" { + dsOutput = ds.Source + } + dsNamingScheme := defaultString(ds.NamingScheme, musicNamingScheme) + dsOutputExtension := musicOutputExtension + if ds.OutputExtension != "" { + dsOutputExtension = normalizeExtension(ds.OutputExtension) + } + dsConvertExtensions := musicConvertExtensions + if len(ds.ConvertExtensions) > 0 { + dsConvertExtensions = normalizeExtensionSlice(ds.ConvertExtensions) + } effectiveDatasets[id] = EffectiveMusicDataset{ - Source: ds.Source, - Prefix: ds.Prefix, - StageRoot: dsStageRoot, - CreditsRoot: dsCreditsRoot, + Source: ds.Source, + Output: dsOutput, + Prefix: ds.Prefix, + StageRoot: dsStageRoot, + CreditsRoot: dsCreditsRoot, + NamingScheme: dsNamingScheme, + OutputExtension: dsOutputExtension, + MaxStemLength: musicMaxStemLength, + PackageMode: defaultString(ds.PackageMode, "hak_asset"), + ConvertExtensions: dsConvertExtensions, } } @@ -269,14 +319,19 @@ func (p *Project) EffectiveConfig() EffectiveConfig { Outputs: outputs, Build: p.Config.Build, Inventory: inventory, + Validation: validation, Scripts: scripts, Music: EffectiveMusicConfig{ - Prefixes: musicPrefixes, - StageRoot: musicStageRoot, - CreditsRoot: musicCreditsRoot, - CreditsOverlay: musicCreditsOverlay, - MaxStemLength: musicMaxStemLength, - Datasets: effectiveDatasets, + Prefixes: musicPrefixes, + Tools: p.Config.Music.Tools, + StageRoot: musicStageRoot, + CreditsRoot: musicCreditsRoot, + CreditsOverlay: musicCreditsOverlay, + MaxStemLength: musicMaxStemLength, + NamingScheme: musicNamingScheme, + OutputExtension: musicOutputExtension, + ConvertExtensions: musicConvertExtensions, + Datasets: effectiveDatasets, }, TopData: top, Extract: extract, @@ -353,6 +408,9 @@ func markMissingDefaults(provenance ConfigProvenance) { "inventory.source_extensions": "NWN source resource extensions", "inventory.asset_extensions": "NWN asset resource extensions", "inventory.source_json_pattern": DefaultSourceJSONPattern, + "validation.profile": DefaultValidationProfile, + "validation.builtin_script_prefixes": "NWN built-in script prefixes", + "validation.required_fields": "NWN required GFF fields", "scripts.source_dir": DefaultScriptsSourceDir, "scripts.cache": DefaultScriptsCache, "scripts.compiler.env.path": DefaultScriptCompilerEnvPath, @@ -361,6 +419,9 @@ func markMissingDefaults(provenance ConfigProvenance) { "music.credits_root": DefaultCreditsRoot, "music.credits_overlay": DefaultCreditsOverlayFile, "music.max_stem_length": "16", + "music.naming_scheme": DefaultMusicNamingScheme, + "music.output_extension": DefaultMusicOutputExtension, + "music.convert_extensions": ".mp3, .ogg", "topdata.build": DefaultTopDataBuild, "topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir, "topdata.compiled_tlk": DefaultTopDataCompiledTLK, @@ -413,6 +474,21 @@ func defaultStringSlice(value, fallback []string) []string { return slices.Clone(value) } +func defaultRequiredFields(configured map[string][]string) map[string][]string { + if len(configured) > 0 { + return cloneStringSliceMap(normalizeRequiredFields(configured)) + } + return DefaultRequiredFields() +} + +func cloneStringSliceMap(input map[string][]string) map[string][]string { + out := make(map[string][]string, len(input)) + for key, values := range input { + out[key] = slices.Clone(values) + } + return out +} + func cloneStringMap(input map[string]string) map[string]string { if len(input) == 0 { return nil diff --git a/internal/project/project.go b/internal/project/project.go index 0a24741..1f690e1 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -33,6 +33,36 @@ var AssetExtensions = []string{ ".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".gr2", ".itp", ".jpg", ".lod", ".lyt", ".mdb", ".mdl", ".mdx", ".mp3", ".mtr", ".ogg", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wlk", ".wok", ".xml", } +var BuiltinScriptPrefixes = []string{ + "nw_", + "x0_", + "x1_", + "x2_", + "x3_", + "ga_", + "gc_", + "gen_", + "gui_", + "nwg_", + "ta_", +} + +var RequiredFields = map[string][]string{ + "ifo": {"Mod_Name"}, +} + +func DefaultBuiltinScriptPrefixes() []string { + return slices.Clone(BuiltinScriptPrefixes) +} + +func DefaultRequiredFields() map[string][]string { + out := make(map[string][]string, len(RequiredFields)) + for extension, fields := range RequiredFields { + out[extension] = slices.Clone(fields) + } + return out +} + type Project struct { Root string Config Config @@ -49,17 +79,18 @@ type ConfigSource struct { } type Config struct { - Module ModuleConfig `json:"module" yaml:"module"` - Paths PathConfig `json:"paths" yaml:"paths"` - Outputs OutputConfig `json:"outputs" yaml:"outputs"` - Build BuildConfig `json:"build" yaml:"build"` - HAKs []HAKConfig `json:"haks" yaml:"haks"` - Inventory InventoryConfig `json:"inventory" yaml:"inventory"` - Scripts ScriptsConfig `json:"scripts" yaml:"scripts"` - Music MusicConfig `json:"music" yaml:"music"` - TopData TopDataConfig `json:"topdata" yaml:"topdata"` - Extract ExtractConfig `json:"extract" yaml:"extract"` - Autogen AutogenConfig `json:"autogen" yaml:"autogen"` + Module ModuleConfig `json:"module" yaml:"module"` + Paths PathConfig `json:"paths" yaml:"paths"` + Outputs OutputConfig `json:"outputs" yaml:"outputs"` + Build BuildConfig `json:"build" yaml:"build"` + HAKs []HAKConfig `json:"haks" yaml:"haks"` + Inventory InventoryConfig `json:"inventory" yaml:"inventory"` + Validation ValidationConfig `json:"validation" yaml:"validation"` + Scripts ScriptsConfig `json:"scripts" yaml:"scripts"` + Music MusicConfig `json:"music" yaml:"music"` + TopData TopDataConfig `json:"topdata" yaml:"topdata"` + Extract ExtractConfig `json:"extract" yaml:"extract"` + Autogen AutogenConfig `json:"autogen" yaml:"autogen"` } type ModuleConfig struct { @@ -93,6 +124,12 @@ type InventoryConfig struct { SourceJSONPattern string `json:"source_json_pattern" yaml:"source_json_pattern"` } +type ValidationConfig struct { + Profile string `json:"profile" yaml:"profile"` + BuiltinScriptPrefixes []string `json:"builtin_script_prefixes" yaml:"builtin_script_prefixes"` + RequiredFields map[string][]string `json:"required_fields" yaml:"required_fields"` +} + type ScriptsConfig struct { SourceDir string `json:"source_dir" yaml:"source_dir"` Cache string `json:"cache" yaml:"cache"` @@ -121,23 +158,37 @@ type HAKConfig struct { } type MusicConfig struct { - Prefixes map[string]string `json:"prefixes,omitempty" yaml:"prefixes,omitempty"` + Prefixes map[string]string `json:"prefixes,omitempty" yaml:"prefixes,omitempty"` + Tools MusicToolsConfig `json:"tools,omitempty" yaml:"tools,omitempty"` Defaults *MusicDefaultsConfig `json:"defaults,omitempty" yaml:"defaults,omitempty"` Datasets map[string]MusicDatasetConfig `json:"datasets,omitempty" yaml:"datasets,omitempty"` } +type MusicToolsConfig struct { + FFmpeg string `json:"ffmpeg,omitempty" yaml:"ffmpeg,omitempty"` + FFprobe string `json:"ffprobe,omitempty" yaml:"ffprobe,omitempty"` +} + type MusicDefaultsConfig struct { - StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` - CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` - CreditsOverlay string `json:"credits_overlay,omitempty" yaml:"credits_overlay,omitempty"` - MaxStemLength *int `json:"max_stem_length,omitempty" yaml:"max_stem_length,omitempty"` + StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` + CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` + CreditsOverlay string `json:"credits_overlay,omitempty" yaml:"credits_overlay,omitempty"` + MaxStemLength *int `json:"max_stem_length,omitempty" yaml:"max_stem_length,omitempty"` + NamingScheme string `json:"naming_scheme,omitempty" yaml:"naming_scheme,omitempty"` + OutputExtension string `json:"output_extension,omitempty" yaml:"output_extension,omitempty"` + ConvertExtensions []string `json:"convert_extensions,omitempty" yaml:"convert_extensions,omitempty"` } type MusicDatasetConfig struct { - Source string `json:"source" yaml:"source"` - Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"` - StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` - CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` + Source string `json:"source" yaml:"source"` + Output string `json:"output,omitempty" yaml:"output,omitempty"` + Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"` + StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"` + CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"` + NamingScheme string `json:"naming_scheme,omitempty" yaml:"naming_scheme,omitempty"` + OutputExtension string `json:"output_extension,omitempty" yaml:"output_extension,omitempty"` + PackageMode string `json:"package_mode,omitempty" yaml:"package_mode,omitempty"` + ConvertExtensions []string `json:"convert_extensions,omitempty" yaml:"convert_extensions,omitempty"` } type TopDataConfig struct { @@ -387,6 +438,7 @@ func (p *Project) ValidateLayout() error { "outputs.module_archive": p.Config.Outputs.ModuleArchive, "outputs.hak_manifest": p.Config.Outputs.HAKManifest, "outputs.hak_archive": p.Config.Outputs.HAKArchive, + "validation.profile": p.Config.Validation.Profile, "scripts.source_dir": p.Config.Scripts.SourceDir, "scripts.cache": p.Config.Scripts.Cache, "scripts.compiler.path": p.Config.Scripts.Compiler.Path, @@ -415,6 +467,7 @@ func (p *Project) ValidateLayout() error { failures = append(failures, fmt.Errorf("%s must not contain NUL bytes", field)) } } + failures = append(failures, validateValidationConfig(p.Config.Validation)...) effective := p.EffectiveConfig() failures = append(failures, validateRelativePath("paths.cache", effective.Paths.Cache)...) failures = append(failures, validateRelativePath("outputs.module_archive", effective.Outputs.ModuleArchive)...) @@ -815,6 +868,25 @@ func (p *Project) CreditsDir() string { return p.rootPath(p.EffectiveConfig().Music.CreditsRoot) } +func (p *Project) MusicToolPath(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return "" + } + if filepath.IsAbs(path) { + return path + } + return filepath.Join(p.Root, filepath.FromSlash(path)) +} + +func (p *Project) MusicFFmpegPath() string { + return p.MusicToolPath(p.EffectiveConfig().Music.Tools.FFmpeg) +} + +func (p *Project) MusicFFprobePath() string { + return p.MusicToolPath(p.EffectiveConfig().Music.Tools.FFprobe) +} + func (p *Project) AutogenCacheDir() string { return p.rootPath(p.EffectiveConfig().Autogen.Cache.Root) } @@ -877,6 +949,9 @@ func defaultConfig() Config { func normalizeConfig(cfg *Config) { cfg.Inventory.SourceExtensions = normalizeExtensionSlice(cfg.Inventory.SourceExtensions) cfg.Inventory.AssetExtensions = normalizeExtensionSlice(cfg.Inventory.AssetExtensions) + cfg.Validation.BuiltinScriptPrefixes = normalizeLowerStringSlice(cfg.Validation.BuiltinScriptPrefixes) + cfg.Validation.Profile = strings.ToLower(strings.TrimSpace(cfg.Validation.Profile)) + cfg.Validation.RequiredFields = normalizeRequiredFields(cfg.Validation.RequiredFields) for i := range cfg.HAKs { cfg.HAKs[i].Include = normalizeStringSlice(cfg.HAKs[i].Include) } @@ -891,6 +966,10 @@ func normalizeConfig(cfg *Config) { } func normalizeMusicConfig(music *MusicConfig) { + if music.Defaults != nil { + music.Defaults.ConvertExtensions = normalizeExtensionSlice(music.Defaults.ConvertExtensions) + music.Defaults.OutputExtension = normalizeExtension(music.Defaults.OutputExtension) + } if len(music.Prefixes) > 0 && len(music.Datasets) == 0 { datasets := make(map[string]MusicDatasetConfig, len(music.Prefixes)) for dir, prefix := range music.Prefixes { @@ -906,6 +985,13 @@ func normalizeMusicConfig(music *MusicConfig) { } music.Datasets = datasets } + for id, ds := range music.Datasets { + ds.Source = trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Source))) + ds.Output = trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Output))) + ds.OutputExtension = normalizeExtension(ds.OutputExtension) + ds.ConvertExtensions = normalizeExtensionSlice(ds.ConvertExtensions) + music.Datasets[id] = ds + } } func normalizeStringSlice(input []string) []string { @@ -935,6 +1021,108 @@ func normalizeExtensionSlice(input []string) []string { return out } +func normalizeExtension(input string) string { + trimmed := strings.ToLower(strings.TrimSpace(input)) + if trimmed != "" && !strings.HasPrefix(trimmed, ".") { + trimmed = "." + trimmed + } + return trimmed +} + +func normalizeLowerStringSlice(input []string) []string { + if len(input) == 0 { + return input + } + out := make([]string, 0, len(input)) + for _, value := range input { + trimmed := strings.ToLower(strings.TrimSpace(value)) + if trimmed != "" { + out = append(out, trimmed) + } + } + slices.Sort(out) + return out +} + +func normalizeRequiredFields(input map[string][]string) map[string][]string { + if len(input) == 0 { + return input + } + out := make(map[string][]string, len(input)) + for extension, labels := range input { + key := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(extension)), ".") + if key == "" { + continue + } + normalizedLabels := make([]string, 0, len(labels)) + for _, label := range labels { + label = strings.TrimSpace(label) + if label != "" { + normalizedLabels = append(normalizedLabels, label) + } + } + slices.Sort(normalizedLabels) + out[key] = normalizedLabels + } + return out +} + +func validateValidationConfig(cfg ValidationConfig) []error { + var failures []error + switch strings.TrimSpace(cfg.Profile) { + case "", DefaultValidationProfile: + default: + failures = append(failures, fmt.Errorf("validation.profile %q is not supported", cfg.Profile)) + } + seen := map[string]struct{}{} + for index, prefix := range cfg.BuiltinScriptPrefixes { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + failures = append(failures, fmt.Errorf("validation.builtin_script_prefixes[%d] must not be empty", index)) + continue + } + if strings.Contains(prefix, "\x00") { + failures = append(failures, fmt.Errorf("validation.builtin_script_prefixes[%d] must not contain NUL bytes", index)) + } + normalized := strings.ToLower(prefix) + if _, exists := seen[normalized]; exists { + failures = append(failures, fmt.Errorf("validation.builtin_script_prefixes[%d] %q is duplicated", index, prefix)) + } + seen[normalized] = struct{}{} + } + seenExtensions := map[string]struct{}{} + for extension, labels := range cfg.RequiredFields { + normalizedExtension := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(extension)), ".") + if normalizedExtension == "" { + failures = append(failures, errors.New("validation.required_fields contains an empty extension")) + continue + } + if strings.Contains(normalizedExtension, "\x00") { + failures = append(failures, fmt.Errorf("validation.required_fields[%q] must not contain NUL bytes", extension)) + } + if _, exists := seenExtensions[normalizedExtension]; exists { + failures = append(failures, fmt.Errorf("validation.required_fields[%q] is duplicated after normalization", extension)) + } + seenExtensions[normalizedExtension] = struct{}{} + seenLabels := map[string]struct{}{} + for index, label := range labels { + label = strings.TrimSpace(label) + if label == "" { + failures = append(failures, fmt.Errorf("validation.required_fields[%q][%d] must not be empty", extension, index)) + continue + } + if strings.Contains(label, "\x00") { + failures = append(failures, fmt.Errorf("validation.required_fields[%q][%d] must not contain NUL bytes", extension, index)) + } + if _, exists := seenLabels[label]; exists { + failures = append(failures, fmt.Errorf("validation.required_fields[%q][%d] %q is duplicated", extension, index, label)) + } + seenLabels[label] = struct{}{} + } + } + return failures +} + func validateAutogenConfig(cfg AutogenConfig) []error { var failures []error producerIDs := map[string]struct{}{} @@ -1014,6 +1202,17 @@ func validateAutogenConfig(cfg AutogenConfig) []error { func validateMusicConfig(cfg MusicConfig) []error { var failures []error + if cfg.Defaults != nil { + if cfg.Defaults.MaxStemLength != nil && *cfg.Defaults.MaxStemLength < 3 { + failures = append(failures, errors.New("music.defaults.max_stem_length must be at least 3")) + } + if cfg.Defaults.NamingScheme != "" && cfg.Defaults.NamingScheme != "nwn_bmu" && cfg.Defaults.NamingScheme != "passthrough" { + failures = append(failures, fmt.Errorf("music.defaults.naming_scheme %q is not supported", cfg.Defaults.NamingScheme)) + } + if cfg.Defaults.OutputExtension != "" && !strings.HasPrefix(cfg.Defaults.OutputExtension, ".") { + failures = append(failures, errors.New("music.defaults.output_extension must start with .")) + } + } normalizedRoots := map[string]string{} for root, prefix := range cfg.Prefixes { normalizedRoot := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(root))) @@ -1033,19 +1232,41 @@ func validateMusicConfig(cfg MusicConfig) []error { for id, ds := range cfg.Datasets { if strings.TrimSpace(id) == "" { failures = append(failures, errors.New("music.datasets contains an empty key")) + } else if strings.Contains(id, "/") || strings.Contains(id, "\\") { + failures = append(failures, fmt.Errorf("music.datasets id %q must not contain path separators", id)) } source := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Source))) if source == "" { failures = append(failures, fmt.Errorf("music.datasets[%q].source is required", id)) + } else if pathEscapesRoot(source) { + failures = append(failures, fmt.Errorf("music.datasets[%q].source must not escape project root", id)) } else if prior, exists := seenSources[source]; exists { failures = append(failures, fmt.Errorf("music.datasets[%q].source %q conflicts with dataset %q", id, ds.Source, prior)) } else { seenSources[source] = id } + output := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Output))) + if output != "" && pathEscapesRoot(output) { + failures = append(failures, fmt.Errorf("music.datasets[%q].output must not escape project root", id)) + } + if ds.NamingScheme != "" && ds.NamingScheme != "nwn_bmu" && ds.NamingScheme != "passthrough" { + failures = append(failures, fmt.Errorf("music.datasets[%q].naming_scheme %q is not supported", id, ds.NamingScheme)) + } + if ds.PackageMode != "" && ds.PackageMode != "hak_asset" && ds.PackageMode != "none" { + failures = append(failures, fmt.Errorf("music.datasets[%q].package_mode %q is not supported", id, ds.PackageMode)) + } + if ds.OutputExtension != "" && !strings.HasPrefix(ds.OutputExtension, ".") { + failures = append(failures, fmt.Errorf("music.datasets[%q].output_extension must start with .", id)) + } } return failures } +func pathEscapesRoot(path string) bool { + clean := filepath.Clean(filepath.FromSlash(path)) + return clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) +} + func validateGlobList(field string, patterns []string) []error { var failures []error seen := map[string]struct{}{} diff --git a/internal/project/project_test.go b/internal/project/project_test.go index 3fb6a5f..e3e75f4 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -69,6 +69,12 @@ paths: if got, want := effective.TopData.PackageHAK, "sow_top.hak"; got != want { t.Fatalf("expected default topdata HAK %q, got %q", want, got) } + if got, want := strings.Join(effective.Validation.BuiltinScriptPrefixes, ","), "ga_,gc_,gen_,gui_,nw_,nwg_,ta_,x0_,x1_,x2_,x3_"; got != want { + t.Fatalf("expected default built-in script prefixes %q, got %q", want, got) + } + if got := strings.Join(effective.Validation.RequiredFields["ifo"], ","); got != "Mod_Name" { + t.Fatalf("expected default IFO required fields, got %#v", effective.Validation.RequiredFields) + } if prov := effective.Provenance["paths.build"]; prov.Source != "toolkit default" { t.Fatalf("expected paths.build toolkit default provenance, got %#v", prov) } @@ -77,6 +83,37 @@ paths: } } +func TestEffectiveConfigHonorsValidationConfig(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src +validation: + builtin_script_prefixes: + - custom_ + - NW_ + required_fields: + ifo: + - Mod_Name + - Mod_Hak +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + got := strings.Join(proj.EffectiveConfig().Validation.BuiltinScriptPrefixes, ",") + if want := "custom_,nw_"; got != want { + t.Fatalf("expected configured validation prefixes %q, got %q", want, got) + } + if got := strings.Join(proj.EffectiveConfig().Validation.RequiredFields["ifo"], ","); got != "Mod_Hak,Mod_Name" { + t.Fatalf("expected configured required fields, got %#v", proj.EffectiveConfig().Validation.RequiredFields) + } +} + func TestEffectiveConfigHonorsConfiguredOutputAndCacheFields(t *testing.T) { root := t.TempDir() writeProjectFile(t, filepath.Join(root, ConfigFile), ` diff --git a/internal/validator/validator.go b/internal/validator/validator.go index d27f80c..3c85604 100644 --- a/internal/validator/validator.go +++ b/internal/validator/validator.go @@ -10,6 +10,7 @@ import ( "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) @@ -49,24 +50,6 @@ type assetGroupResolver struct { defaultGroup string } -var requiredFields = map[string][]string{ - "ifo": {"Mod_Name"}, -} - -var builtinScriptPrefixes = []string{ - "nw_", - "x0_", - "x1_", - "x2_", - "x3_", - "ga_", - "gc_", - "gen_", - "gui_", - "nwg_", - "ta_", -} - func ValidateProject(p *project.Project) Report { report := Report{} resourceIndex := map[string]string{} @@ -75,6 +58,7 @@ func ValidateProject(p *project.Project) Report { assetOccurrences := map[string][]assetOccurrence{} documents := make([]loadedDocument, 0, len(p.Inventory.SourceFiles)) resolver := newAssetGroupResolver(p) + effective := p.EffectiveConfig() for _, rel := range p.Inventory.SourceFiles { if hasUppercaseResourceName(rel) { @@ -101,7 +85,7 @@ func ValidateProject(p *project.Project) Report { resourceIndex[key] = rel } - validateDocumentStructure(&report, rel, extension, document) + validateDocumentStructure(&report, rel, extension, document, effective.Validation.RequiredFields) } for _, rel := range p.Inventory.ScriptFiles { @@ -117,7 +101,8 @@ func ValidateProject(p *project.Project) Report { } for _, rel := range p.Inventory.AssetFiles { - if hasUppercaseResourceName(rel) && !isMusicSourceAsset(rel) { + musicSource := isMusicSourceAsset(p, rel) + if hasUppercaseResourceName(rel) && !musicSource { report.add(rel, "resource filenames should be lowercase", SeverityWarning) } base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))) @@ -131,6 +116,9 @@ func ValidateProject(p *project.Project) Report { if _, exists := assetIndex[key]; !exists { assetIndex[key] = rel } + if musicSource { + continue + } group, err := resolver.groupFor(rel) if err != nil { @@ -147,7 +135,7 @@ func ValidateProject(p *project.Project) Report { classifyAssetDuplicates(&report, assetOccurrences) for _, document := range documents { - validateReferences(&report, document, resourceIndex, scriptIndex, assetIndex) + validateReferences(&report, document, resourceIndex, scriptIndex, assetIndex, effective.Validation.BuiltinScriptPrefixes) } slices.SortFunc(report.Diagnostics, func(a, b Diagnostic) int { @@ -296,7 +284,7 @@ func loadDocument(path string) (gff.Document, string, string, error) { return document, resref, extension, nil } -func validateDocumentStructure(report *Report, path, extension string, document gff.Document) { +func validateDocumentStructure(report *Report, path, extension string, document gff.Document, requiredFields map[string][]string) { expectedType := strings.ToUpper(extension) if strings.TrimSpace(document.FileType) != "" && strings.TrimSpace(document.FileType) != expectedType { report.add(path, fmt.Sprintf("file_type %q does not match expected %q", document.FileType, expectedType), SeverityError) @@ -338,7 +326,7 @@ func validateUniqueLabels(report *Report, path, scope string, s gff.Struct) { } } -func validateReferences(report *Report, document loadedDocument, resources, scripts, assets map[string]string) { +func validateReferences(report *Report, document loadedDocument, resources, scripts, assets map[string]string, builtinScriptPrefixes []string) { walkFields(document.Document.Root, func(field gff.Field) { value, ok := fieldStringValue(field.Value) if !ok || value == "" { @@ -347,7 +335,7 @@ func validateReferences(report *Report, document loadedDocument, resources, scri switch { case isScriptField(field.Label): - if isBuiltinScript(value) { + if isBuiltinScript(value, builtinScriptPrefixes) { return } if _, exists := scripts[strings.ToLower(value)]; !exists { @@ -519,20 +507,31 @@ func hasAsset(name string, extensions []string, assets map[string]string) bool { return false } -func isMusicSourceAsset(rel string) bool { +func isMusicSourceAsset(p *project.Project, rel string) bool { rel = filepath.ToSlash(strings.TrimSpace(rel)) - if rel != "envi/music" && !strings.HasPrefix(rel, "envi/music/") { - return false + ext := strings.ToLower(filepath.Ext(rel)) + effective := p.EffectiveConfig() + for _, dataset := range effective.Music.Datasets { + if !music.PathIsUnder(dataset.Source, rel) { + continue + } + for _, candidate := range dataset.ConvertExtensions { + if ext == strings.ToLower(strings.TrimSpace(candidate)) { + return true + } + } } - switch strings.ToLower(filepath.Ext(rel)) { - case ".mp3", ".ogg": - return true - default: - return false + if music.PathIsUnder("envi/music", rel) { + for _, candidate := range effective.Music.ConvertExtensions { + if ext == strings.ToLower(strings.TrimSpace(candidate)) { + return true + } + } } + return false } -func isBuiltinScript(name string) bool { +func isBuiltinScript(name string, builtinScriptPrefixes []string) bool { lower := strings.ToLower(name) for _, prefix := range builtinScriptPrefixes { if strings.HasPrefix(lower, prefix) { diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index a7030c2..c5045d9 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -199,6 +199,102 @@ func TestValidateProjectWarnsForMissingScriptReferences(t *testing.T) { } } +func TestValidateProjectUsesConfiguredBuiltinScriptPrefixes(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "dialogs")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +validation: + builtin_script_prefixes: + - custom_ +`) + mustWriteFile(t, filepath.Join(root, "src", "dialogs", "merchant.dlg.json"), `{ + "file_type": "DLG ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Script", + "type": "ResRef", + "value": "custom_open_store" + } + ] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if report.HasErrors() || report.WarningCount() != 0 { + t.Fatalf("expected configured script prefix to suppress warning, got %#v", report.Diagnostics) + } +} + +func TestValidateProjectUsesConfiguredRequiredFields(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +validation: + required_fields: + ifo: [] +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if report.HasErrors() { + t.Fatalf("expected configured required fields to suppress IFO error, got %#v", report.Diagnostics) + } +} + func TestValidateProjectReportsDecompiledModelsWithoutFailingValidation(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src"))