ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Configuration hardening

Browse Source
This commit is contained in:
archvillainette
2026-05-07 21:33:53 +02:00
parent bba0d2fb45
commit d1b20684f5
14 changed files with 1335 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CONFIGURATION_HARDENING_AUDIT.md
+4 -3
View File
@@ -25,6 +25,8 @@ This audit records the current repository-configuration authority model for
- `build.keep_existing_haks`
- `inventory.source_extensions`, `inventory.asset_extensions`,
`inventory.source_json_pattern`
- `validation.profile`, `validation.builtin_script_prefixes`, and
`validation.required_fields`
- `scripts.source_dir`, `scripts.cache`, `scripts.compiler`
- `extract.layout`, `extract.hak_discovery`, `extract.cleanup_stale`
- `topdata.compiled_2da_dir`, `topdata.compiled_tlk`, package names, and wiki
@@ -42,9 +44,8 @@ repository-specific YAML:
- Supported autogen algorithms: `parts_rows`, `head_visualeffects`,
`trailing_numeric_suffix`, `model_stem`, and `first_path_segment`.
- Native topdata dataset internals and generated JSON dataset formats.
- NWN validator semantics for required module fields and resource-reference
checks. These are candidates for future validation profiles, but are not
repository-specific defaults.
- NWN validator resource-reference checks. These are candidates for future
validation profiles, but are not repository-specific defaults.
## Migration Checklist
README.md
+42 -2
View File
@@ -66,12 +66,23 @@ music:
envi/music/westgate: mus_wg_
# Explicit dataset config (replaces prefixes above).
tools:
ffmpeg: tools/ffmpeg
ffprobe: tools/ffprobe
defaults:
stage_root: "{paths.cache}/music"
credits_root: "{paths.cache}/credits"
credits_overlay: CREDITS.md
max_stem_length: 16
naming_scheme: nwn_bmu
output_extension: .bmu
convert_extensions: [.mp3, .ogg]
datasets:
westgate:
source: envi/music/westgate
output: envi/music/westgate
prefix: mus_wg_
package_mode: hak_asset
```
Resolved configuration can be inspected without scanning or building:
@@ -112,6 +123,12 @@ inventory:
asset_extensions: [] # empty means built-in NWN asset extensions
source_json_pattern: "{resref}.{extension}.json"
validation:
profile: nwn_module
builtin_script_prefixes: [nw_, x0_, x1_, x2_, x3_, ga_, gc_, gen_, gui_, nwg_, ta_]
required_fields:
ifo: [Mod_Name]
scripts:
source_dir: scripts
cache: "{paths.cache}/ncs"
@@ -169,11 +186,34 @@ music:
credits_root: "{paths.cache}/credits"
credits_overlay: CREDITS.md
max_stem_length: 16
naming_scheme: nwn_bmu
output_extension: .bmu
convert_extensions: [.mp3, .ogg]
```
Music datasets define individual music collections with independent sources and
naming prefixes. The legacy `music.prefixes` shorthand is automatically normalized
into a dataset.
naming prefixes. `source` is the authored input root under `paths.assets`;
`output` is the generated HAK asset root matched by HAK include globs. If
`output` is omitted, it defaults to `source`. `package_mode: hak_asset` converts
configured source extensions into generated HAK resources; `package_mode: none`
scans and writes credits without adding generated HAK resources. The legacy
`music.prefixes` shorthand is automatically normalized into datasets.
Music can also be run directly:
```bash
sow-toolkit music list-datasets
sow-toolkit music scan --dataset westgate
sow-toolkit music build --dataset westgate --dry-run
sow-toolkit music build --dataset westgate
sow-toolkit music credits --dataset westgate --check
sow-toolkit music validate --dataset westgate --json
```
`build-haks` uses the same music pipeline. Use `--plan-only` or music
`--dry-run` to compute mappings without transcoding. Use `build-haks
--skip-music` to package authored assets only, or `build-haks --music-dataset
westgate` to limit conversion to a configured dataset.
Generated and machine-readable artifacts remain JSON, including HAK manifests,
topdata datasets, caches, credits inventories, build reports, and canonical GFF
internal/app/app.go
+170 -7
View File
@@ -7,6 +7,7 @@ import (
"io"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
@@ -163,6 +164,11 @@ var commands = []command{
description: "Inspect, explain, and validate the resolved project configuration.",
run: runConfig,
},
{
name: "music",
description: "List, scan, build, and validate configured music datasets.",
run: runMusic,
},
{
name: "compare",
description: "Compare current source content against the built archives.",
@@ -336,6 +342,8 @@ type buildHAKOptions struct {
filteredHAKs []string
filteredArchives []string
sourceManifest string
musicDatasets []string
skipMusic bool
planOnly bool
logLevel logLevel
}
@@ -567,14 +575,17 @@ func runBuildHAKs(ctx context) error {
spin.configure(ctx.stderr, console.spinnerEnabled)
spin.update("Build HAKs: starting")
var result pipeline.BuildResult
pipelineOpts := pipeline.BuildHAKOptions{
Progress: console.progress,
ArchiveNames: opts.filteredArchives,
SourceManifestPath: opts.sourceManifest,
SkipMusic: opts.skipMusic,
MusicDatasetIDs: opts.musicDatasets,
}
if opts.planOnly {
result, err = pipeline.PlanHAKsWithProgress(p, console.progress)
result, err = pipeline.PlanHAKsWithOptions(p, pipelineOpts)
} else {
result, err = pipeline.BuildHAKsWithOptions(p, pipeline.BuildHAKOptions{
Progress: console.progress,
ArchiveNames: opts.filteredArchives,
SourceManifestPath: opts.sourceManifest,
})
result, err = pipeline.BuildHAKsWithOptions(p, pipelineOpts)
}
if err != nil {
return err
@@ -594,7 +605,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
arg := args[index]
switch arg {
case "-h", "--help":
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--plan-only] [--quiet|--verbose|--debug]")
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]")
case "--hak":
index++
if index >= len(args) {
@@ -609,6 +620,14 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
opts.filteredArchives = append(opts.filteredArchives, args[index])
case "--plan-only":
opts.planOnly = true
case "--skip-music":
opts.skipMusic = true
case "--music-dataset":
index++
if index >= len(args) {
return opts, errors.New("--music-dataset requires a value")
}
opts.musicDatasets = append(opts.musicDatasets, args[index])
case "--quiet":
opts.logLevel = logLevelQuiet
case "--verbose":
@@ -634,6 +653,10 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
opts.sourceManifest = value
continue
}
if value, ok := parseInlineFlagValue(arg, "--music-dataset"); ok {
opts.musicDatasets = append(opts.musicDatasets, value)
continue
}
if arg == "--quiet=true" {
opts.logLevel = logLevelQuiet
continue
@@ -652,6 +675,146 @@ func parseInlineFlagValue(arg, name string) (string, bool) {
return strings.TrimSpace(strings.TrimPrefix(arg, prefix)), true
}
type musicCommandOptions struct {
datasets []string
json bool
dryRun bool
check bool
force bool
}
func runMusic(ctx context) error {
if len(ctx.args) < 2 {
return errors.New("usage: music <list-datasets|scan|build|credits|validate|manifest|normalize> [--dataset <id>] [--json] [--dry-run] [--check] [--force]")
}
p, err := loadProject(ctx)
if err != nil {
return err
}
if err := p.Scan(); err != nil {
return err
}
subcommand := ctx.args[1]
opts, err := parseMusicCommandArgs(ctx.args[2:])
if err != nil {
return err
}
switch subcommand {
case "list-datasets":
return emitMusicDatasets(ctx, p, opts)
case "scan":
result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false})
if err != nil {
return err
}
return emitMusicResult(ctx, "scan", result, opts)
case "build", "credits":
write := true
if subcommand == "build" && opts.dryRun {
write = false
}
if subcommand == "credits" && opts.check {
write = false
}
result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: write})
if err != nil {
return err
}
return emitMusicResult(ctx, subcommand, result, opts)
case "manifest", "normalize":
result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false})
if err != nil {
return err
}
return emitMusicResult(ctx, subcommand, result, opts)
case "validate":
if err := p.ValidateLayout(); err != nil {
return err
}
result, err := pipeline.BuildMusic(p, pipeline.MusicBuildOptions{DatasetIDs: opts.datasets, Write: false})
if err != nil {
return err
}
return emitMusicResult(ctx, "validate", result, opts)
default:
return fmt.Errorf("unknown music subcommand %q", subcommand)
}
}
func parseMusicCommandArgs(args []string) (musicCommandOptions, error) {
opts := musicCommandOptions{}
for index := 0; index < len(args); index++ {
arg := args[index]
switch arg {
case "--dataset":
index++
if index >= len(args) || strings.TrimSpace(args[index]) == "" {
return opts, errors.New("--dataset requires a value")
}
opts.datasets = append(opts.datasets, args[index])
case "--json":
opts.json = true
case "--dry-run":
opts.dryRun = true
case "--check":
opts.check = true
case "--force":
opts.force = true
default:
if value, ok := parseInlineFlagValue(arg, "--dataset"); ok {
opts.datasets = append(opts.datasets, value)
continue
}
return opts, fmt.Errorf("unknown music argument %q", arg)
}
}
return opts, nil
}
func emitMusicDatasets(ctx context, p *project.Project, opts musicCommandOptions) error {
effective := p.EffectiveConfig()
if opts.json {
return emitConfigValue(ctx.stdout, effective.Music.Datasets, "json")
}
if len(effective.Music.Datasets) == 0 {
fmt.Fprintln(ctx.stdout, "music datasets: none")
return nil
}
fmt.Fprintln(ctx.stdout, "music datasets:")
ids := make([]string, 0, len(effective.Music.Datasets))
for id := range effective.Music.Datasets {
ids = append(ids, id)
}
sort.Strings(ids)
for _, id := range ids {
ds := effective.Music.Datasets[id]
fmt.Fprintf(ctx.stdout, " %s: source=%s output=%s prefix=%s\n", id, ds.Source, ds.Output, ds.Prefix)
}
return nil
}
func emitMusicResult(ctx context, action string, result pipeline.BuildResult, opts musicCommandOptions) error {
if opts.json {
return emitConfigValue(ctx.stdout, result.CreditsSummary, "json")
}
fmt.Fprintf(ctx.stdout, "music %s\n", action)
if len(result.CreditsSummary.ScannedDirs) > 0 {
fmt.Fprintf(ctx.stdout, "scanned: %s\n", strings.Join(result.CreditsSummary.ScannedDirs, ", "))
}
fmt.Fprintf(ctx.stdout, "tracks: %d\n", result.CreditsSummary.TrackCount)
if action != "scan" && len(result.CreditsArtifactPaths) > 0 {
fmt.Fprintf(ctx.stdout, "credits artifacts: %d\n", len(result.CreditsArtifactPaths))
}
if len(result.CreditsSummary.Mappings) > 0 && ctx.logLevel >= logLevelVerbose {
fmt.Fprintln(ctx.stdout, "mappings:")
for _, mapping := range result.CreditsSummary.Mappings {
fmt.Fprintf(ctx.stdout, " %s -> %s\n", mapping.Source, mapping.Output)
}
}
fmt.Fprintln(ctx.stdout, "status: ok")
return nil
}
func runExtract(ctx context) error {
p, err := loadProject(ctx)
if err != nil {
internal/app/app_test.go
+74
View File
@@ -198,6 +198,80 @@ func TestRunBuildHAKsVerboseListsMappings(t *testing.T) {
}
}
func TestRunMusicScanUsesConfiguredDatasetsWithoutWriting(t *testing.T) {
root := t.TempDir()
mkdirAll(t, filepath.Join(root, "assets", "audio", "westgate"))
mkdirAll(t, filepath.Join(root, "build"))
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Module
resref: testmod
paths:
assets: assets
build: build
music:
datasets:
westgate_audio:
source: audio/westgate
output: generated/music
prefix: wg_
`)
writeFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3")
var stdout bytes.Buffer
ctx := context{
stdout: &stdout,
stderr: &bytes.Buffer{},
cwd: root,
args: []string{"music", "scan", "--dataset", "westgate_audio"},
}
if err := runMusic(ctx); err != nil {
t.Fatalf("runMusic failed: %v", err)
}
output := stdout.String()
if !strings.Contains(output, "music scan") || !strings.Contains(output, "tracks: 1") {
t.Fatalf("unexpected music scan output: %q", output)
}
if _, err := os.Stat(filepath.Join(root, ".cache", "credits")); !os.IsNotExist(err) {
t.Fatalf("music scan should not write credits artifacts, stat err=%v", err)
}
}
func TestRunMusicListDatasets(t *testing.T) {
root := t.TempDir()
mkdirAll(t, filepath.Join(root, "assets"))
mkdirAll(t, filepath.Join(root, "build"))
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Module
resref: testmod
paths:
assets: assets
music:
datasets:
westgate_audio:
source: audio/westgate
output: generated/music
prefix: wg_
`)
var stdout bytes.Buffer
ctx := context{
stdout: &stdout,
stderr: &bytes.Buffer{},
cwd: root,
args: []string{"music", "list-datasets"},
}
if err := runMusic(ctx); err != nil {
t.Fatalf("runMusic failed: %v", err)
}
if !strings.Contains(stdout.String(), "westgate_audio: source=audio/westgate output=generated/music prefix=wg_") {
t.Fatalf("unexpected dataset list: %q", stdout.String())
}
}
func TestRunConfigEffectiveJSONShowsDefaultsAndProvenance(t *testing.T) {
root := t.TempDir()
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
internal/music/music.go
+23
View File
@@ -78,6 +78,13 @@ type CreditsOverlay struct {
}
func ResolveFFmpeg() (string, error) {
return ResolveFFmpegWithConfig("")
}
func ResolveFFmpegWithConfig(configuredPath string) (string, error) {
if configured := strings.TrimSpace(configuredPath); configured != "" {
return configured, nil
}
if configured := strings.TrimSpace(os.Getenv("SOW_FFMPEG")); configured != "" {
return configured, nil
}
@@ -89,6 +96,13 @@ func ResolveFFmpeg() (string, error) {
}
func ResolveFFprobe() (string, error) {
return ResolveFFprobeWithConfig("")
}
func ResolveFFprobeWithConfig(configuredPath string) (string, error) {
if configured := strings.TrimSpace(configuredPath); configured != "" {
return configured, nil
}
if configured := strings.TrimSpace(os.Getenv("SOW_FFPROBE")); configured != "" {
return configured, nil
}
@@ -104,6 +118,15 @@ func IsMusicAssetPath(rel string) bool {
return rel == "envi/music" || strings.HasPrefix(rel, "envi/music/")
}
func PathIsUnder(root, rel string) bool {
root = TrimSlashes(filepath.ToSlash(root))
rel = TrimSlashes(filepath.ToSlash(rel))
if root == "" || rel == "" {
return false
}
return rel == root || strings.HasPrefix(rel, root+"/")
}
func PrefixForDir(prefixes map[string]string, dir string) string {
dir = TrimSlashes(filepath.ToSlash(dir))
bestPrefix := ""
internal/music/naming.go
+34 -9
View File
@@ -8,7 +8,14 @@ import (
)
func GenerateStem(fileName, prefix string, used map[string]struct{}) (string, error) {
maxCore := MaxStemLen - len(prefix)
return GenerateStemWithMax(fileName, prefix, MaxStemLen, used)
}
func GenerateStemWithMax(fileName, prefix string, maxStemLen int, used map[string]struct{}) (string, error) {
if maxStemLen <= 0 {
maxStemLen = MaxStemLen
}
maxCore := maxStemLen - len(prefix)
if maxCore < 3 {
return "", fmt.Errorf("prefix too long: %q", prefix)
}
@@ -19,11 +26,11 @@ func GenerateStem(fileName, prefix string, used map[string]struct{}) (string, er
sum := sha1.Sum([]byte(fileName))
core = fmt.Sprintf("%x", sum[:])[:maxCore]
}
stem := strings.TrimRight((prefix + core)[:Min(MaxStemLen, len(prefix)+len(core))], "_")
stem := strings.TrimRight((prefix + core)[:Min(maxStemLen, len(prefix)+len(core))], "_")
if stem == "" {
return "", fmt.Errorf("could not derive music stem for %s", fileName)
}
return UniqueName(stem, used), nil
return UniqueNameWithMax(stem, maxStemLen, used), nil
}
func SlugWords(value string) []string {
@@ -119,13 +126,20 @@ func AbbreviateWord(word string, maxLen int) string {
}
func UniqueName(stem string, used map[string]struct{}) string {
return UniqueNameWithMax(stem, MaxStemLen, used)
}
func UniqueNameWithMax(stem string, maxStemLen int, used map[string]struct{}) string {
if maxStemLen <= 0 {
maxStemLen = MaxStemLen
}
if _, exists := used[stem]; !exists {
used[stem] = struct{}{}
return stem
}
for index := 1; ; index++ {
suffix := fmt.Sprintf("_%d", index)
prefixLen := Min(len(stem), Max(0, MaxStemLen-len(suffix)))
prefixLen := Min(len(stem), Max(0, maxStemLen-len(suffix)))
candidate := strings.TrimRight(stem[:prefixLen], "_") + suffix
if _, exists := used[candidate]; exists {
continue
@@ -148,16 +162,27 @@ func ReserveName(stem string, used map[string]struct{}) error {
}
func ValidateManualOutputFile(name string) error {
return ValidateManualOutputFileWithRules(name, ".bmu", MaxStemLen)
}
func ValidateManualOutputFileWithRules(name, extension string, maxStemLen int) error {
name = strings.ToLower(strings.TrimSpace(name))
if !strings.HasSuffix(name, ".bmu") {
return fmt.Errorf("output file must end with .bmu")
extension = strings.ToLower(strings.TrimSpace(extension))
if extension == "" {
extension = ".bmu"
}
stem := strings.TrimSuffix(name, ".bmu")
if maxStemLen <= 0 {
maxStemLen = MaxStemLen
}
if !strings.HasSuffix(name, extension) {
return fmt.Errorf("output file must end with %s", extension)
}
stem := strings.TrimSuffix(name, extension)
if stem == "" {
return fmt.Errorf("output file stem is empty")
}
if len(stem) > MaxStemLen {
return fmt.Errorf("output file stem %q exceeds %d characters", stem, MaxStemLen)
if len(stem) > maxStemLen {
return fmt.Errorf("output file stem %q exceeds %d characters", stem, maxStemLen)
}
for _, r := range stem {
switch {
internal/pipeline/build.go
+19 -7
View File
@@ -110,6 +110,8 @@ type BuildHAKOptions struct {
Progress ProgressFunc
ArchiveNames []string
SourceManifestPath string
SkipMusic bool
MusicDatasetIDs []string
}
func Build(p *project.Project) (BuildResult, error) {
@@ -195,7 +197,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu
}
func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath)
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs)
}
func PlanHAKs(p *project.Project) (BuildResult, error) {
@@ -206,15 +208,19 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul
return planHAKs(p, progress)
}
func PlanHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
return planOrBuildHAKs(p, opts.Progress, false, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs)
}
func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
return planOrBuildHAKs(p, progress, true, nil, "")
return planOrBuildHAKs(p, progress, true, nil, "", false, nil)
}
func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
return planOrBuildHAKs(p, progress, false, nil, "")
return planOrBuildHAKs(p, progress, false, nil, "", false, nil)
}
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string) (result BuildResult, err error) {
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string, skipMusic bool, musicDatasetIDs []string) (result BuildResult, err error) {
preserveExistingHAKs := p.EffectiveConfig().Build.KeepExistingHAKs || envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
progressf(progress, "Validating project...")
@@ -227,9 +233,15 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
if err != nil {
return BuildResult{}, err
}
musicAssets, err := prepareMusicAssets(p)
if err != nil {
return BuildResult{}, err
musicAssets := &preparedMusicAssets{SkipSourceRel: map[string]struct{}{}}
if !skipMusic {
musicAssets, err = prepareMusicAssetsWithOptions(p, musicPrepareOptions{
datasetIDs: musicDatasetIDs,
write: writeArchives,
})
if err != nil {
return BuildResult{}, err
}
}
defer func() {
if cleanupErr := cleanupPreparedMusicAssets(musicAssets); cleanupErr != nil && err == nil {
internal/pipeline/music.go
+318 -75
View File
@@ -21,16 +21,193 @@ type preparedMusicAssets struct {
Summary CreditsRefreshSummary
}
type MusicBuildOptions struct {
DatasetIDs []string
Write bool
}
type musicPrepareOptions struct {
datasetIDs []string
write bool
}
type musicSourceGroup struct {
DatasetID string
Dataset project.EffectiveMusicDataset
SourceDir string
OutputDir string
Sources []string
}
type musicCreditGroup struct {
CreditsRoot string
SourceDir string
Entries []music.CreditsEntry
}
func musicRootPath(p *project.Project, path string) string {
if strings.TrimSpace(path) == "" {
return p.Root
}
if filepath.IsAbs(path) {
return path
}
return filepath.Join(p.Root, filepath.FromSlash(path))
}
func effectiveMusicDatasets(p *project.Project, onlyIDs []string) (map[string]project.EffectiveMusicDataset, error) {
effective := p.EffectiveConfig()
wanted := map[string]struct{}{}
for _, id := range onlyIDs {
id = strings.TrimSpace(id)
if id != "" {
wanted[id] = struct{}{}
}
}
datasets := map[string]project.EffectiveMusicDataset{}
for id, dataset := range effective.Music.Datasets {
if len(wanted) > 0 {
if _, ok := wanted[id]; !ok {
continue
}
delete(wanted, id)
}
datasets[id] = dataset
}
if len(wanted) > 0 {
unknown := make([]string, 0, len(wanted))
for id := range wanted {
unknown = append(unknown, id)
}
sort.Strings(unknown)
return nil, fmt.Errorf("unknown music dataset(s): %s", strings.Join(unknown, ", "))
}
if len(datasets) == 0 && len(wanted) == 0 {
for _, rel := range p.Inventory.AssetFiles {
ext := strings.ToLower(filepath.Ext(rel))
if !music.PathIsUnder("envi/music", rel) {
continue
}
if _, ok := extensionSet(effective.Music.ConvertExtensions)[ext]; ok {
datasets["legacy_envi_music"] = project.EffectiveMusicDataset{
Source: "envi/music",
Output: "envi/music",
StageRoot: effective.Music.StageRoot,
CreditsRoot: effective.Music.CreditsRoot,
NamingScheme: effective.Music.NamingScheme,
OutputExtension: effective.Music.OutputExtension,
MaxStemLength: effective.Music.MaxStemLength,
PackageMode: "hak_asset",
ConvertExtensions: effective.Music.ConvertExtensions,
}
break
}
}
}
return datasets, nil
}
func extensionSet(values []string) map[string]struct{} {
out := make(map[string]struct{}, len(values))
for _, value := range values {
value = strings.ToLower(strings.TrimSpace(value))
if value != "" {
out[value] = struct{}{}
}
}
return out
}
func datasetForMusicSource(datasets map[string]project.EffectiveMusicDataset, rel, ext string) (string, project.EffectiveMusicDataset, bool) {
bestID := ""
var best project.EffectiveMusicDataset
bestLen := -1
for id, dataset := range datasets {
if _, ok := extensionSet(dataset.ConvertExtensions)[ext]; !ok {
continue
}
if !music.PathIsUnder(dataset.Source, rel) {
continue
}
if len(dataset.Source) > bestLen {
bestID = id
best = dataset
bestLen = len(dataset.Source)
}
}
return bestID, best, bestID != ""
}
func plannedMusicAsset(outputRel, sourceRel string) (assetResource, error) {
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(outputRel)), ".")
resourceType, ok := erf.HAKResourceTypeForExtension(extension)
if !ok {
return assetResource{}, fmt.Errorf("unsupported generated music resource extension %q", filepath.Ext(outputRel))
}
name := strings.ToLower(strings.TrimSuffix(filepath.Base(outputRel), filepath.Ext(outputRel)))
return assetResource{
Rel: outputRel,
Resource: erf.Resource{
Name: name,
Type: resourceType,
},
ContentID: "music-plan:" + filepath.ToSlash(sourceRel),
}, nil
}
func BuildMusic(p *project.Project, opts MusicBuildOptions) (BuildResult, error) {
prepared, err := prepareMusicAssetsWithOptions(p, musicPrepareOptions{datasetIDs: opts.DatasetIDs, write: opts.Write})
if err != nil {
return BuildResult{}, err
}
if err := cleanupPreparedMusicAssets(prepared); err != nil {
return BuildResult{}, err
}
return BuildResult{
CreditsArtifactPaths: prepared.Artifacts,
CreditsSummary: prepared.Summary,
HAKAssets: len(prepared.Generated),
}, nil
}
func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) {
return prepareMusicAssetsWithOptions(p, musicPrepareOptions{write: true})
}
func prepareMusicAssetsWithOptions(p *project.Project, opts musicPrepareOptions) (*preparedMusicAssets, error) {
usedNames := make(map[string]struct{})
dirSources := make(map[string][]string)
groupsByKey := make(map[string]*musicSourceGroup)
datasets, err := effectiveMusicDatasets(p, opts.datasetIDs)
if err != nil {
return nil, err
}
for _, rel := range p.Inventory.AssetFiles {
rel = filepath.ToSlash(rel)
ext := strings.ToLower(filepath.Ext(rel))
name := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
if _, convertible := music.ConvertExtensions[ext]; convertible && music.IsMusicAssetPath(rel) {
dir := filepath.ToSlash(filepath.Dir(rel))
dirSources[dir] = append(dirSources[dir], rel)
if datasetID, dataset, ok := datasetForMusicSource(datasets, rel, ext); ok {
sourceDir := filepath.ToSlash(filepath.Dir(rel))
relDir, err := filepath.Rel(filepath.FromSlash(dataset.Source), filepath.FromSlash(sourceDir))
if err != nil {
return nil, fmt.Errorf("resolve music source directory %s: %w", sourceDir, err)
}
relDir = filepath.ToSlash(relDir)
outputDir := dataset.Output
if relDir != "." && relDir != "" {
outputDir = filepath.ToSlash(filepath.Join(outputDir, filepath.FromSlash(relDir)))
}
key := datasetID + "\x00" + sourceDir
group := groupsByKey[key]
if group == nil {
group = &musicSourceGroup{
DatasetID: datasetID,
Dataset: dataset,
SourceDir: sourceDir,
OutputDir: outputDir,
}
groupsByKey[key] = group
}
group.Sources = append(group.Sources, rel)
continue
}
if name != "" {
@@ -41,7 +218,10 @@ func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) {
result := &preparedMusicAssets{
SkipSourceRel: make(map[string]struct{}),
}
if len(dirSources) == 0 {
if len(groupsByKey) == 0 {
if !opts.write {
return result, nil
}
writeResult, err := writeCreditsArtifacts(p, nil)
if err != nil {
return nil, err
@@ -51,57 +231,86 @@ func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) {
return result, nil
}
dirs := make([]string, 0, len(dirSources))
for dir := range dirSources {
dirs = append(dirs, dir)
groups := make([]*musicSourceGroup, 0, len(groupsByKey))
for _, group := range groupsByKey {
sort.Strings(group.Sources)
groups = append(groups, group)
}
sort.Strings(dirs)
sort.Slice(groups, func(i, j int) bool {
if groups[i].SourceDir != groups[j].SourceDir {
return groups[i].SourceDir < groups[j].SourceDir
}
return groups[i].DatasetID < groups[j].DatasetID
})
generatedByDir := make(map[string][]music.CreditsEntry)
stageRoot := p.MusicStageDir()
result.TempRoots = append(result.TempRoots, stageRoot)
ffmpegPath, err := music.ResolveFFmpeg()
if err != nil {
return nil, err
creditGroups := make([]musicCreditGroup, 0, len(groups))
for _, group := range groups {
result.Summary.ScannedDirs = append(result.Summary.ScannedDirs, group.SourceDir)
result.Summary.TrackCount += len(group.Sources)
}
ffprobePath, err := music.ResolveFFprobe()
if err != nil {
return nil, err
ffmpegPath := ""
ffprobePath := ""
if opts.write {
var err error
ffmpegPath, err = music.ResolveFFmpegWithConfig(p.MusicFFmpegPath())
if err != nil {
return nil, err
}
ffprobePath, err = music.ResolveFFprobeWithConfig(p.MusicFFprobePath())
if err != nil {
return nil, err
}
}
for _, dir := range dirs {
sources := append([]string(nil), dirSources[dir]...)
sort.Strings(sources)
prefix := music.PrefixForDir(p.EffectiveConfig().Music.Prefixes, dir)
overlayPath := filepath.Join(p.AssetsDir(), filepath.FromSlash(dir), p.EffectiveConfig().Music.CreditsOverlay)
for _, group := range groups {
stageRoot := musicRootPath(p, group.Dataset.StageRoot)
result.TempRoots = append(result.TempRoots, stageRoot)
prefix := music.SanitizePrefix(group.Dataset.Prefix)
overlayPath := filepath.Join(p.AssetsDir(), filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay)
overlay, err := music.ParseCreditsOverlay(overlayPath)
if err != nil {
return nil, fmt.Errorf("parse credits overlay %s: %w", overlayPath, err)
}
entries := make([]music.CreditsEntry, 0, len(sources))
for _, rel := range sources {
entries := make([]music.CreditsEntry, 0, len(group.Sources))
for _, rel := range group.Sources {
base := filepath.Base(rel)
manual := overlay.Match(base, "")
outputFile := ""
switch {
case manual != nil && strings.TrimSpace(manual.OutputFile) != "":
outputFile = strings.ToLower(strings.TrimSpace(manual.OutputFile))
if err := music.ValidateManualOutputFile(outputFile); err != nil {
if err := music.ValidateManualOutputFileWithRules(outputFile, group.Dataset.OutputExtension, group.Dataset.MaxStemLength); err != nil {
return nil, fmt.Errorf("manual credits output for %s: %w", rel, err)
}
if err := music.ReserveName(strings.TrimSuffix(outputFile, filepath.Ext(outputFile)), usedNames); err != nil {
return nil, fmt.Errorf("manual credits output for %s: %w", rel, err)
}
default:
stem, err := music.GenerateStem(base, prefix, usedNames)
stem, err := music.GenerateStemWithMax(base, prefix, group.Dataset.MaxStemLength, usedNames)
if err != nil {
return nil, fmt.Errorf("generate music name for %s: %w", rel, err)
}
outputFile = stem + ".bmu"
outputFile = stem + group.Dataset.OutputExtension
}
outputRel := filepath.ToSlash(filepath.Join(dir, outputFile))
outputRel := filepath.ToSlash(filepath.Join(group.OutputDir, outputFile))
result.Summary.Mappings = append(result.Summary.Mappings, MusicFileMapping{
Source: rel,
Output: outputRel,
})
result.SkipSourceRel[rel] = struct{}{}
if !opts.write {
if group.Dataset.PackageMode == "hak_asset" {
asset, err := plannedMusicAsset(outputRel, rel)
if err != nil {
return nil, err
}
result.Generated = append(result.Generated, asset)
}
continue
}
metadata, err := music.ProbeMetadata(ffprobePath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
if err != nil {
return nil, fmt.Errorf("probe music metadata for %s: %w", rel, err)
@@ -125,36 +334,50 @@ func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) {
if err := music.ConvertSource(ffmpegPath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)), stagePath); err != nil {
return nil, fmt.Errorf("convert music %s: %w", rel, err)
}
resourceInfo, err := assetResourceFromPath(stagePath, outputRel, lfsAssetInfo{}, false)
if err != nil {
return nil, fmt.Errorf("load converted music %s: %w", outputRel, err)
if group.Dataset.PackageMode == "hak_asset" {
resourceInfo, err := assetResourceFromPath(stagePath, outputRel, lfsAssetInfo{}, false)
if err != nil {
return nil, fmt.Errorf("load converted music %s: %w", outputRel, err)
}
info, err := os.Stat(stagePath)
if err != nil {
return nil, fmt.Errorf("stat converted music %s: %w", stagePath, err)
}
result.Generated = append(result.Generated, assetResource{
Rel: outputRel,
Resource: resourceInfo.Resource,
Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}),
CreatedAt: info.ModTime(),
ContentID: resourceInfo.ContentID,
})
}
info, err := os.Stat(stagePath)
if err != nil {
return nil, fmt.Errorf("stat converted music %s: %w", stagePath, err)
}
result.Generated = append(result.Generated, assetResource{
Rel: outputRel,
Resource: resourceInfo.Resource,
Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}),
CreatedAt: info.ModTime(),
ContentID: resourceInfo.ContentID,
})
result.SkipSourceRel[rel] = struct{}{}
entries = append(entries, entry)
}
if err := music.ValidateOverlayEntries(dir, overlay, entries); err != nil {
return nil, err
if opts.write {
if err := music.ValidateOverlayEntries(group.SourceDir, overlay, entries); err != nil {
return nil, err
}
creditGroups = append(creditGroups, musicCreditGroup{
CreditsRoot: group.Dataset.CreditsRoot,
SourceDir: group.SourceDir,
Entries: entries,
})
}
generatedByDir[dir] = entries
}
writeResult, err := writeCreditsArtifacts(p, generatedByDir)
if !opts.write {
sort.Slice(result.Generated, func(i, j int) bool {
return compareResourceKeys(result.Generated[i].Resource, result.Generated[j].Resource) < 0
})
return result, nil
}
writeResult, err := writeCreditsArtifacts(p, creditGroups)
if err != nil {
return nil, err
}
writeResult.Summary.ScannedDirs = append(writeResult.Summary.ScannedDirs, dirs...)
writeResult.Summary.TrackCount = len(result.Generated)
writeResult.Summary.ScannedDirs = append(writeResult.Summary.ScannedDirs, result.Summary.ScannedDirs...)
writeResult.Summary.TrackCount = result.Summary.TrackCount
result.Artifacts = writeResult.Artifacts
result.Summary = writeResult.Summary
sort.Slice(result.Generated, func(i, j int) bool {
@@ -168,19 +391,24 @@ type creditsArtifactWriteResult struct {
Summary CreditsRefreshSummary
}
func writeCreditsArtifacts(p *project.Project, generatedByDir map[string][]music.CreditsEntry) (creditsArtifactWriteResult, error) {
creditsRoot := p.CreditsDir()
func writeCreditsArtifacts(p *project.Project, generated []musicCreditGroup) (creditsArtifactWriteResult, error) {
defaultCreditsRoot := p.CreditsDir()
sources := make([]music.CreditsSource, 0)
desiredFiles := make(map[string][]byte)
desiredByRoot := map[string]map[string][]byte{}
summary := CreditsRefreshSummary{}
if generatedByDir != nil {
dirs := make([]string, 0, len(generatedByDir))
for dir := range generatedByDir {
dirs = append(dirs, dir)
}
sort.Strings(dirs)
for _, dir := range dirs {
entries := append([]music.CreditsEntry(nil), generatedByDir[dir]...)
if generated != nil {
sort.Slice(generated, func(i, j int) bool {
if generated[i].SourceDir != generated[j].SourceDir {
return generated[i].SourceDir < generated[j].SourceDir
}
return generated[i].CreditsRoot < generated[j].CreditsRoot
})
for _, group := range generated {
creditsRoot := musicRootPath(p, group.CreditsRoot)
if desiredByRoot[creditsRoot] == nil {
desiredByRoot[creditsRoot] = map[string][]byte{}
}
entries := append([]music.CreditsEntry(nil), group.Entries...)
sort.Slice(entries, func(i, j int) bool {
if strings.ToLower(entries[i].Artist) != strings.ToLower(entries[j].Artist) {
return strings.ToLower(entries[i].Artist) < strings.ToLower(entries[j].Artist)
@@ -190,8 +418,8 @@ func writeCreditsArtifacts(p *project.Project, generatedByDir map[string][]music
}
return strings.ToLower(entries[i].OutputFile) < strings.ToLower(entries[j].OutputFile)
})
outputPath := filepath.Join(creditsRoot, filepath.FromSlash(dir), p.EffectiveConfig().Music.CreditsOverlay)
desiredFiles[outputPath] = []byte(music.RenderCreditsMarkdown(entries))
outputPath := filepath.Join(creditsRoot, filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay)
desiredByRoot[creditsRoot][outputPath] = []byte(music.RenderCreditsMarkdown(entries))
summary.GeneratedPaths = append(summary.GeneratedPaths, outputPath)
for _, entry := range entries {
summary.Mappings = append(summary.Mappings, MusicFileMapping{
@@ -237,27 +465,42 @@ func writeCreditsArtifacts(p *project.Project, generatedByDir map[string][]music
}
sort.Slice(sources, func(i, j int) bool { return sources[i].Path < sources[j].Path })
inventoryPath := filepath.Join(creditsRoot, "credits.json")
payload, err := json.MarshalIndent(music.CreditsInventory{Sources: sources}, "", " ")
if err != nil {
return creditsArtifactWriteResult{}, fmt.Errorf("marshal credits inventory: %w", err)
}
payload = append(payload, '\n')
desiredFiles[inventoryPath] = payload
if err := os.MkdirAll(creditsRoot, 0o755); err != nil {
return creditsArtifactWriteResult{}, fmt.Errorf("create credits dir: %w", err)
if len(desiredByRoot) == 0 {
desiredByRoot[defaultCreditsRoot] = map[string][]byte{}
}
changedFiles, err := music.SyncGeneratedCreditsArtifacts(creditsRoot, desiredFiles)
if err != nil {
return creditsArtifactWriteResult{}, err
artifacts := make([]string, 0)
changedFiles := 0
roots := make([]string, 0, len(desiredByRoot))
for root := range desiredByRoot {
roots = append(roots, root)
}
artifacts := make([]string, 0, len(desiredFiles))
for path := range desiredFiles {
artifacts = append(artifacts, path)
sort.Strings(roots)
for _, creditsRoot := range roots {
desiredFiles := desiredByRoot[creditsRoot]
inventoryPath := filepath.Join(creditsRoot, "credits.json")
desiredFiles[inventoryPath] = payload
if err := os.MkdirAll(creditsRoot, 0o755); err != nil {
return creditsArtifactWriteResult{}, fmt.Errorf("create credits dir: %w", err)
}
changed, err := music.SyncGeneratedCreditsArtifacts(creditsRoot, desiredFiles)
if err != nil {
return creditsArtifactWriteResult{}, err
}
changedFiles += changed
for path := range desiredFiles {
artifacts = append(artifacts, path)
}
if summary.InventoryPath == "" {
summary.InventoryPath = inventoryPath
}
}
sort.Strings(artifacts)
summary.ArtifactPaths = append(summary.ArtifactPaths, artifacts...)
summary.InventoryPath = inventoryPath
summary.ChangedFiles = changedFiles
return creditsArtifactWriteResult{
Artifacts: artifacts,
internal/pipeline/pipeline_test.go
+149
View File
@@ -2769,4 +2769,153 @@ func TestBuildHAKsConvertsMusicSourcesAndWritesCreditsArtifacts(t *testing.T) {
}
}
func TestBuildHAKsUsesExplicitMusicDatasetOutsideLegacyRoot(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate"))
mustMkdir(t, filepath.Join(root, "build"))
ffprobePath := filepath.Join(root, "tools", "ffprobe")
ffmpegPath := filepath.Join(root, "tools", "ffmpeg")
mustMkdir(t, filepath.Dir(ffprobePath))
mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"Dataset Artist\",\"title\":\"Dataset Title\"}}}'\n")
mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'dataset-bmu' > \"$last\"\n")
if err := os.Chmod(ffprobePath, 0o755); err != nil {
t.Fatalf("chmod ffprobe: %v", err)
}
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
t.Fatalf("chmod ffmpeg: %v", err)
}
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Assets
resref: testassets
paths:
assets: assets
build: build
music:
tools:
ffmpeg: tools/ffmpeg
ffprobe: tools/ffprobe
defaults:
credits_root: custom-credits
datasets:
westgate_audio:
source: audio/westgate
output: generated/music
prefix: wg_
haks:
- name: music
priority: 1
max_bytes: 1048576
split: false
include:
- generated/music/**
`)
mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildHAKs(p)
if err != nil {
t.Fatalf("build haks: %v", err)
}
if result.CreditsSummary.TrackCount != 1 {
t.Fatalf("expected one music track, got %#v", result.CreditsSummary)
}
manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
if err != nil {
t.Fatalf("read manifest: %v", err)
}
manifestText := string(manifestRaw)
if !strings.Contains(manifestText, "generated/music/wg_theme.bmu") {
t.Fatalf("expected dataset output resource in manifest, got:\n%s", manifestText)
}
if strings.Contains(manifestText, "audio/westgate/Theme Song.mp3") {
t.Fatalf("did not expect source mp3 in manifest, got:\n%s", manifestText)
}
creditsRaw, err := os.ReadFile(filepath.Join(root, "custom-credits", "audio", "westgate", "CREDITS.md"))
if err != nil {
t.Fatalf("read generated dataset credits: %v", err)
}
if !strings.Contains(string(creditsRaw), "Dataset Artist") || !strings.Contains(string(creditsRaw), "wg_theme.bmu") {
t.Fatalf("unexpected generated credits:\n%s", string(creditsRaw))
}
}
func TestPlanHAKsDoesNotTranscodeMusicSources(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate"))
mustMkdir(t, filepath.Join(root, "build"))
ffmpegPath := filepath.Join(root, "tools", "ffmpeg")
mustMkdir(t, filepath.Dir(ffmpegPath))
mustWriteFile(t, ffmpegPath, "#!/bin/sh\necho should-not-run >&2\nexit 42\n")
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
t.Fatalf("chmod ffmpeg: %v", err)
}
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Assets
resref: testassets
paths:
assets: assets
build: build
music:
tools:
ffmpeg: tools/ffmpeg
ffprobe: tools/ffprobe
datasets:
westgate_audio:
source: audio/westgate
output: generated/music
prefix: wg_
haks:
- name: music
priority: 1
max_bytes: 1048576
split: false
include:
- generated/music/**
`)
mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.mp3"), "source-mp3")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := PlanHAKs(p)
if err != nil {
t.Fatalf("plan haks: %v", err)
}
if result.CreditsSummary.TrackCount != 1 {
t.Fatalf("expected one planned music track, got %#v", result.CreditsSummary)
}
manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
if err != nil {
t.Fatalf("read manifest: %v", err)
}
if !strings.Contains(string(manifestRaw), "generated/music/wg_theme.bmu") {
t.Fatalf("expected planned generated music in manifest, got:\n%s", string(manifestRaw))
}
if _, err := os.Stat(filepath.Join(root, ".cache", "music")); !os.IsNotExist(err) {
t.Fatalf("plan-only should not stage music, stat err=%v", err)
}
}
internal/project/effective.go
+96 -20
View File
@@ -18,12 +18,15 @@ const (
DefaultHAKManifest = "haks.json"
DefaultHAKArchiveTemplate = "{hak.name}.hak"
DefaultSourceJSONPattern = "{resref}.{extension}.json"
DefaultValidationProfile = "nwn_module"
DefaultScriptsSourceDir = "scripts"
DefaultScriptsCache = "{paths.cache}/ncs"
DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER"
DefaultMusicStageRoot = "{paths.cache}/music"
DefaultCreditsRoot = "{paths.cache}/credits"
DefaultCreditsOverlayFile = "CREDITS.md"
DefaultMusicNamingScheme = "nwn_bmu"
DefaultMusicOutputExtension = ".bmu"
DefaultTopDataBuild = "{paths.build}/topdata"
DefaultTopDataCompiled2DADir = "2da"
DefaultTopDataCompiledTLK = "sow_tlk.tlk"
@@ -60,6 +63,7 @@ type EffectiveConfig struct {
Outputs EffectiveOutputConfig `json:"outputs" yaml:"outputs"`
Build BuildConfig `json:"build" yaml:"build"`
Inventory InventoryConfig `json:"inventory" yaml:"inventory"`
Validation ValidationConfig `json:"validation" yaml:"validation"`
Scripts EffectiveScriptsConfig `json:"scripts" yaml:"scripts"`
Music EffectiveMusicConfig `json:"music" yaml:"music"`
TopData EffectiveTopDataConfig `json:"topdata" yaml:"topdata"`
@@ -103,19 +107,29 @@ type EffectiveScriptCompilerEnvConfig struct {
}
type EffectiveMusicConfig struct {
Prefixes map[string]string `json:"prefixes" yaml:"prefixes"`
StageRoot string `json:"stage_root" yaml:"stage_root"`
CreditsRoot string `json:"credits_root" yaml:"credits_root"`
CreditsOverlay string `json:"credits_overlay" yaml:"credits_overlay"`
MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"`
Datasets map[string]EffectiveMusicDataset `json:"datasets,omitempty" yaml:"datasets,omitempty"`
Prefixes map[string]string `json:"prefixes" yaml:"prefixes"`
Tools MusicToolsConfig `json:"tools" yaml:"tools"`
StageRoot string `json:"stage_root" yaml:"stage_root"`
CreditsRoot string `json:"credits_root" yaml:"credits_root"`
CreditsOverlay string `json:"credits_overlay" yaml:"credits_overlay"`
MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"`
NamingScheme string `json:"naming_scheme" yaml:"naming_scheme"`
OutputExtension string `json:"output_extension" yaml:"output_extension"`
ConvertExtensions []string `json:"convert_extensions" yaml:"convert_extensions"`
Datasets map[string]EffectiveMusicDataset `json:"datasets,omitempty" yaml:"datasets,omitempty"`
}
type EffectiveMusicDataset struct {
Source string `json:"source" yaml:"source"`
Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
Source string `json:"source" yaml:"source"`
Output string `json:"output" yaml:"output"`
Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
NamingScheme string `json:"naming_scheme" yaml:"naming_scheme"`
OutputExtension string `json:"output_extension" yaml:"output_extension"`
MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"`
PackageMode string `json:"package_mode" yaml:"package_mode"`
ConvertExtensions []string `json:"convert_extensions" yaml:"convert_extensions"`
}
type EffectiveTopDataConfig struct {
@@ -177,6 +191,11 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
AssetExtensions: defaultStringSlice(p.Config.Inventory.AssetExtensions, AssetExtensions),
SourceJSONPattern: defaultString(p.Config.Inventory.SourceJSONPattern, DefaultSourceJSONPattern),
}
validation := ValidationConfig{
Profile: defaultString(p.Config.Validation.Profile, DefaultValidationProfile),
BuiltinScriptPrefixes: normalizeLowerStringSlice(defaultStringSlice(p.Config.Validation.BuiltinScriptPrefixes, DefaultBuiltinScriptPrefixes())),
RequiredFields: defaultRequiredFields(p.Config.Validation.RequiredFields),
}
scripts := EffectiveScriptsConfig{
SourceDir: defaultString(p.Config.Scripts.SourceDir, DefaultScriptsSourceDir),
Cache: expandPathTemplate(defaultString(p.Config.Scripts.Cache, DefaultScriptsCache), paths),
@@ -221,6 +240,9 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
musicCreditsRoot := expandPathTemplate(DefaultCreditsRoot, paths)
musicCreditsOverlay := DefaultCreditsOverlayFile
musicMaxStemLength := 16
musicNamingScheme := DefaultMusicNamingScheme
musicOutputExtension := DefaultMusicOutputExtension
musicConvertExtensions := []string{".mp3", ".ogg"}
if d := p.Config.Music.Defaults; d != nil {
if d.StageRoot != "" {
musicStageRoot = expandPathTemplate(d.StageRoot, paths)
@@ -234,6 +256,15 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
if d.MaxStemLength != nil {
musicMaxStemLength = *d.MaxStemLength
}
if d.NamingScheme != "" {
musicNamingScheme = d.NamingScheme
}
if d.OutputExtension != "" {
musicOutputExtension = normalizeExtension(d.OutputExtension)
}
if len(d.ConvertExtensions) > 0 {
musicConvertExtensions = normalizeExtensionSlice(d.ConvertExtensions)
}
}
musicPrefixes := cloneStringMap(p.Config.Music.Prefixes)
if len(musicPrefixes) == 0 {
@@ -254,11 +285,30 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
if dsCreditsRoot == "" {
dsCreditsRoot = musicCreditsRoot
}
dsOutput := strings.TrimSpace(ds.Output)
if dsOutput == "" {
dsOutput = ds.Source
}
dsNamingScheme := defaultString(ds.NamingScheme, musicNamingScheme)
dsOutputExtension := musicOutputExtension
if ds.OutputExtension != "" {
dsOutputExtension = normalizeExtension(ds.OutputExtension)
}
dsConvertExtensions := musicConvertExtensions
if len(ds.ConvertExtensions) > 0 {
dsConvertExtensions = normalizeExtensionSlice(ds.ConvertExtensions)
}
effectiveDatasets[id] = EffectiveMusicDataset{
Source: ds.Source,
Prefix: ds.Prefix,
StageRoot: dsStageRoot,
CreditsRoot: dsCreditsRoot,
Source: ds.Source,
Output: dsOutput,
Prefix: ds.Prefix,
StageRoot: dsStageRoot,
CreditsRoot: dsCreditsRoot,
NamingScheme: dsNamingScheme,
OutputExtension: dsOutputExtension,
MaxStemLength: musicMaxStemLength,
PackageMode: defaultString(ds.PackageMode, "hak_asset"),
ConvertExtensions: dsConvertExtensions,
}
}
@@ -269,14 +319,19 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
Outputs: outputs,
Build: p.Config.Build,
Inventory: inventory,
Validation: validation,
Scripts: scripts,
Music: EffectiveMusicConfig{
Prefixes: musicPrefixes,
StageRoot: musicStageRoot,
CreditsRoot: musicCreditsRoot,
CreditsOverlay: musicCreditsOverlay,
MaxStemLength: musicMaxStemLength,
Datasets: effectiveDatasets,
Prefixes: musicPrefixes,
Tools: p.Config.Music.Tools,
StageRoot: musicStageRoot,
CreditsRoot: musicCreditsRoot,
CreditsOverlay: musicCreditsOverlay,
MaxStemLength: musicMaxStemLength,
NamingScheme: musicNamingScheme,
OutputExtension: musicOutputExtension,
ConvertExtensions: musicConvertExtensions,
Datasets: effectiveDatasets,
},
TopData: top,
Extract: extract,
@@ -353,6 +408,9 @@ func markMissingDefaults(provenance ConfigProvenance) {
"inventory.source_extensions": "NWN source resource extensions",
"inventory.asset_extensions": "NWN asset resource extensions",
"inventory.source_json_pattern": DefaultSourceJSONPattern,
"validation.profile": DefaultValidationProfile,
"validation.builtin_script_prefixes": "NWN built-in script prefixes",
"validation.required_fields": "NWN required GFF fields",
"scripts.source_dir": DefaultScriptsSourceDir,
"scripts.cache": DefaultScriptsCache,
"scripts.compiler.env.path": DefaultScriptCompilerEnvPath,
@@ -361,6 +419,9 @@ func markMissingDefaults(provenance ConfigProvenance) {
"music.credits_root": DefaultCreditsRoot,
"music.credits_overlay": DefaultCreditsOverlayFile,
"music.max_stem_length": "16",
"music.naming_scheme": DefaultMusicNamingScheme,
"music.output_extension": DefaultMusicOutputExtension,
"music.convert_extensions": ".mp3, .ogg",
"topdata.build": DefaultTopDataBuild,
"topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir,
"topdata.compiled_tlk": DefaultTopDataCompiledTLK,
@@ -413,6 +474,21 @@ func defaultStringSlice(value, fallback []string) []string {
return slices.Clone(value)
}
func defaultRequiredFields(configured map[string][]string) map[string][]string {
if len(configured) > 0 {
return cloneStringSliceMap(normalizeRequiredFields(configured))
}
return DefaultRequiredFields()
}
func cloneStringSliceMap(input map[string][]string) map[string][]string {
out := make(map[string][]string, len(input))
for key, values := range input {
out[key] = slices.Clone(values)
}
return out
}
func cloneStringMap(input map[string]string) map[string]string {
if len(input) == 0 {
return nil
internal/project/project.go
+241 -20
View File
@@ -33,6 +33,36 @@ var AssetExtensions = []string{
".2da", ".bik", ".bmp", ".bmu", ".dds", ".dwk", ".gr2", ".itp", ".jpg", ".lod", ".lyt", ".mdb", ".mdl", ".mdx", ".mp3", ".mtr", ".ogg", ".plt", ".png", ".pwk", ".set", ".shd", ".tga", ".txi", ".uti", ".vis", ".wav", ".wlk", ".wok", ".xml",
}
var BuiltinScriptPrefixes = []string{
"nw_",
"x0_",
"x1_",
"x2_",
"x3_",
"ga_",
"gc_",
"gen_",
"gui_",
"nwg_",
"ta_",
}
var RequiredFields = map[string][]string{
"ifo": {"Mod_Name"},
}
func DefaultBuiltinScriptPrefixes() []string {
return slices.Clone(BuiltinScriptPrefixes)
}
func DefaultRequiredFields() map[string][]string {
out := make(map[string][]string, len(RequiredFields))
for extension, fields := range RequiredFields {
out[extension] = slices.Clone(fields)
}
return out
}
type Project struct {
Root string
Config Config
@@ -49,17 +79,18 @@ type ConfigSource struct {
}
type Config struct {
Module ModuleConfig `json:"module" yaml:"module"`
Paths PathConfig `json:"paths" yaml:"paths"`
Outputs OutputConfig `json:"outputs" yaml:"outputs"`
Build BuildConfig `json:"build" yaml:"build"`
HAKs []HAKConfig `json:"haks" yaml:"haks"`
Inventory InventoryConfig `json:"inventory" yaml:"inventory"`
Scripts ScriptsConfig `json:"scripts" yaml:"scripts"`
Music MusicConfig `json:"music" yaml:"music"`
TopData TopDataConfig `json:"topdata" yaml:"topdata"`
Extract ExtractConfig `json:"extract" yaml:"extract"`
Autogen AutogenConfig `json:"autogen" yaml:"autogen"`
Module ModuleConfig `json:"module" yaml:"module"`
Paths PathConfig `json:"paths" yaml:"paths"`
Outputs OutputConfig `json:"outputs" yaml:"outputs"`
Build BuildConfig `json:"build" yaml:"build"`
HAKs []HAKConfig `json:"haks" yaml:"haks"`
Inventory InventoryConfig `json:"inventory" yaml:"inventory"`
Validation ValidationConfig `json:"validation" yaml:"validation"`
Scripts ScriptsConfig `json:"scripts" yaml:"scripts"`
Music MusicConfig `json:"music" yaml:"music"`
TopData TopDataConfig `json:"topdata" yaml:"topdata"`
Extract ExtractConfig `json:"extract" yaml:"extract"`
Autogen AutogenConfig `json:"autogen" yaml:"autogen"`
}
type ModuleConfig struct {
@@ -93,6 +124,12 @@ type InventoryConfig struct {
SourceJSONPattern string `json:"source_json_pattern" yaml:"source_json_pattern"`
}
type ValidationConfig struct {
Profile string `json:"profile" yaml:"profile"`
BuiltinScriptPrefixes []string `json:"builtin_script_prefixes" yaml:"builtin_script_prefixes"`
RequiredFields map[string][]string `json:"required_fields" yaml:"required_fields"`
}
type ScriptsConfig struct {
SourceDir string `json:"source_dir" yaml:"source_dir"`
Cache string `json:"cache" yaml:"cache"`
@@ -121,23 +158,37 @@ type HAKConfig struct {
}
type MusicConfig struct {
Prefixes map[string]string `json:"prefixes,omitempty" yaml:"prefixes,omitempty"`
Prefixes map[string]string `json:"prefixes,omitempty" yaml:"prefixes,omitempty"`
Tools MusicToolsConfig `json:"tools,omitempty" yaml:"tools,omitempty"`
Defaults *MusicDefaultsConfig `json:"defaults,omitempty" yaml:"defaults,omitempty"`
Datasets map[string]MusicDatasetConfig `json:"datasets,omitempty" yaml:"datasets,omitempty"`
}
type MusicToolsConfig struct {
FFmpeg string `json:"ffmpeg,omitempty" yaml:"ffmpeg,omitempty"`
FFprobe string `json:"ffprobe,omitempty" yaml:"ffprobe,omitempty"`
}
type MusicDefaultsConfig struct {
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
CreditsOverlay string `json:"credits_overlay,omitempty" yaml:"credits_overlay,omitempty"`
MaxStemLength *int `json:"max_stem_length,omitempty" yaml:"max_stem_length,omitempty"`
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
CreditsOverlay string `json:"credits_overlay,omitempty" yaml:"credits_overlay,omitempty"`
MaxStemLength *int `json:"max_stem_length,omitempty" yaml:"max_stem_length,omitempty"`
NamingScheme string `json:"naming_scheme,omitempty" yaml:"naming_scheme,omitempty"`
OutputExtension string `json:"output_extension,omitempty" yaml:"output_extension,omitempty"`
ConvertExtensions []string `json:"convert_extensions,omitempty" yaml:"convert_extensions,omitempty"`
}
type MusicDatasetConfig struct {
Source string `json:"source" yaml:"source"`
Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
Source string `json:"source" yaml:"source"`
Output string `json:"output,omitempty" yaml:"output,omitempty"`
Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
StageRoot string `json:"stage_root,omitempty" yaml:"stage_root,omitempty"`
CreditsRoot string `json:"credits_root,omitempty" yaml:"credits_root,omitempty"`
NamingScheme string `json:"naming_scheme,omitempty" yaml:"naming_scheme,omitempty"`
OutputExtension string `json:"output_extension,omitempty" yaml:"output_extension,omitempty"`
PackageMode string `json:"package_mode,omitempty" yaml:"package_mode,omitempty"`
ConvertExtensions []string `json:"convert_extensions,omitempty" yaml:"convert_extensions,omitempty"`
}
type TopDataConfig struct {
@@ -387,6 +438,7 @@ func (p *Project) ValidateLayout() error {
"outputs.module_archive": p.Config.Outputs.ModuleArchive,
"outputs.hak_manifest": p.Config.Outputs.HAKManifest,
"outputs.hak_archive": p.Config.Outputs.HAKArchive,
"validation.profile": p.Config.Validation.Profile,
"scripts.source_dir": p.Config.Scripts.SourceDir,
"scripts.cache": p.Config.Scripts.Cache,
"scripts.compiler.path": p.Config.Scripts.Compiler.Path,
@@ -415,6 +467,7 @@ func (p *Project) ValidateLayout() error {
failures = append(failures, fmt.Errorf("%s must not contain NUL bytes", field))
}
}
failures = append(failures, validateValidationConfig(p.Config.Validation)...)
effective := p.EffectiveConfig()
failures = append(failures, validateRelativePath("paths.cache", effective.Paths.Cache)...)
failures = append(failures, validateRelativePath("outputs.module_archive", effective.Outputs.ModuleArchive)...)
@@ -815,6 +868,25 @@ func (p *Project) CreditsDir() string {
return p.rootPath(p.EffectiveConfig().Music.CreditsRoot)
}
func (p *Project) MusicToolPath(path string) string {
path = strings.TrimSpace(path)
if path == "" {
return ""
}
if filepath.IsAbs(path) {
return path
}
return filepath.Join(p.Root, filepath.FromSlash(path))
}
func (p *Project) MusicFFmpegPath() string {
return p.MusicToolPath(p.EffectiveConfig().Music.Tools.FFmpeg)
}
func (p *Project) MusicFFprobePath() string {
return p.MusicToolPath(p.EffectiveConfig().Music.Tools.FFprobe)
}
func (p *Project) AutogenCacheDir() string {
return p.rootPath(p.EffectiveConfig().Autogen.Cache.Root)
}
@@ -877,6 +949,9 @@ func defaultConfig() Config {
func normalizeConfig(cfg *Config) {
cfg.Inventory.SourceExtensions = normalizeExtensionSlice(cfg.Inventory.SourceExtensions)
cfg.Inventory.AssetExtensions = normalizeExtensionSlice(cfg.Inventory.AssetExtensions)
cfg.Validation.BuiltinScriptPrefixes = normalizeLowerStringSlice(cfg.Validation.BuiltinScriptPrefixes)
cfg.Validation.Profile = strings.ToLower(strings.TrimSpace(cfg.Validation.Profile))
cfg.Validation.RequiredFields = normalizeRequiredFields(cfg.Validation.RequiredFields)
for i := range cfg.HAKs {
cfg.HAKs[i].Include = normalizeStringSlice(cfg.HAKs[i].Include)
}
@@ -891,6 +966,10 @@ func normalizeConfig(cfg *Config) {
}
func normalizeMusicConfig(music *MusicConfig) {
if music.Defaults != nil {
music.Defaults.ConvertExtensions = normalizeExtensionSlice(music.Defaults.ConvertExtensions)
music.Defaults.OutputExtension = normalizeExtension(music.Defaults.OutputExtension)
}
if len(music.Prefixes) > 0 && len(music.Datasets) == 0 {
datasets := make(map[string]MusicDatasetConfig, len(music.Prefixes))
for dir, prefix := range music.Prefixes {
@@ -906,6 +985,13 @@ func normalizeMusicConfig(music *MusicConfig) {
}
music.Datasets = datasets
}
for id, ds := range music.Datasets {
ds.Source = trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Source)))
ds.Output = trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Output)))
ds.OutputExtension = normalizeExtension(ds.OutputExtension)
ds.ConvertExtensions = normalizeExtensionSlice(ds.ConvertExtensions)
music.Datasets[id] = ds
}
}
func normalizeStringSlice(input []string) []string {
@@ -935,6 +1021,108 @@ func normalizeExtensionSlice(input []string) []string {
return out
}
func normalizeExtension(input string) string {
trimmed := strings.ToLower(strings.TrimSpace(input))
if trimmed != "" && !strings.HasPrefix(trimmed, ".") {
trimmed = "." + trimmed
}
return trimmed
}
func normalizeLowerStringSlice(input []string) []string {
if len(input) == 0 {
return input
}
out := make([]string, 0, len(input))
for _, value := range input {
trimmed := strings.ToLower(strings.TrimSpace(value))
if trimmed != "" {
out = append(out, trimmed)
}
}
slices.Sort(out)
return out
}
func normalizeRequiredFields(input map[string][]string) map[string][]string {
if len(input) == 0 {
return input
}
out := make(map[string][]string, len(input))
for extension, labels := range input {
key := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(extension)), ".")
if key == "" {
continue
}
normalizedLabels := make([]string, 0, len(labels))
for _, label := range labels {
label = strings.TrimSpace(label)
if label != "" {
normalizedLabels = append(normalizedLabels, label)
}
}
slices.Sort(normalizedLabels)
out[key] = normalizedLabels
}
return out
}
func validateValidationConfig(cfg ValidationConfig) []error {
var failures []error
switch strings.TrimSpace(cfg.Profile) {
case "", DefaultValidationProfile:
default:
failures = append(failures, fmt.Errorf("validation.profile %q is not supported", cfg.Profile))
}
seen := map[string]struct{}{}
for index, prefix := range cfg.BuiltinScriptPrefixes {
prefix = strings.TrimSpace(prefix)
if prefix == "" {
failures = append(failures, fmt.Errorf("validation.builtin_script_prefixes[%d] must not be empty", index))
continue
}
if strings.Contains(prefix, "\x00") {
failures = append(failures, fmt.Errorf("validation.builtin_script_prefixes[%d] must not contain NUL bytes", index))
}
normalized := strings.ToLower(prefix)
if _, exists := seen[normalized]; exists {
failures = append(failures, fmt.Errorf("validation.builtin_script_prefixes[%d] %q is duplicated", index, prefix))
}
seen[normalized] = struct{}{}
}
seenExtensions := map[string]struct{}{}
for extension, labels := range cfg.RequiredFields {
normalizedExtension := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(extension)), ".")
if normalizedExtension == "" {
failures = append(failures, errors.New("validation.required_fields contains an empty extension"))
continue
}
if strings.Contains(normalizedExtension, "\x00") {
failures = append(failures, fmt.Errorf("validation.required_fields[%q] must not contain NUL bytes", extension))
}
if _, exists := seenExtensions[normalizedExtension]; exists {
failures = append(failures, fmt.Errorf("validation.required_fields[%q] is duplicated after normalization", extension))
}
seenExtensions[normalizedExtension] = struct{}{}
seenLabels := map[string]struct{}{}
for index, label := range labels {
label = strings.TrimSpace(label)
if label == "" {
failures = append(failures, fmt.Errorf("validation.required_fields[%q][%d] must not be empty", extension, index))
continue
}
if strings.Contains(label, "\x00") {
failures = append(failures, fmt.Errorf("validation.required_fields[%q][%d] must not contain NUL bytes", extension, index))
}
if _, exists := seenLabels[label]; exists {
failures = append(failures, fmt.Errorf("validation.required_fields[%q][%d] %q is duplicated", extension, index, label))
}
seenLabels[label] = struct{}{}
}
}
return failures
}
func validateAutogenConfig(cfg AutogenConfig) []error {
var failures []error
producerIDs := map[string]struct{}{}
@@ -1014,6 +1202,17 @@ func validateAutogenConfig(cfg AutogenConfig) []error {
func validateMusicConfig(cfg MusicConfig) []error {
var failures []error
if cfg.Defaults != nil {
if cfg.Defaults.MaxStemLength != nil && *cfg.Defaults.MaxStemLength < 3 {
failures = append(failures, errors.New("music.defaults.max_stem_length must be at least 3"))
}
if cfg.Defaults.NamingScheme != "" && cfg.Defaults.NamingScheme != "nwn_bmu" && cfg.Defaults.NamingScheme != "passthrough" {
failures = append(failures, fmt.Errorf("music.defaults.naming_scheme %q is not supported", cfg.Defaults.NamingScheme))
}
if cfg.Defaults.OutputExtension != "" && !strings.HasPrefix(cfg.Defaults.OutputExtension, ".") {
failures = append(failures, errors.New("music.defaults.output_extension must start with ."))
}
}
normalizedRoots := map[string]string{}
for root, prefix := range cfg.Prefixes {
normalizedRoot := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(root)))
@@ -1033,19 +1232,41 @@ func validateMusicConfig(cfg MusicConfig) []error {
for id, ds := range cfg.Datasets {
if strings.TrimSpace(id) == "" {
failures = append(failures, errors.New("music.datasets contains an empty key"))
} else if strings.Contains(id, "/") || strings.Contains(id, "\\") {
failures = append(failures, fmt.Errorf("music.datasets id %q must not contain path separators", id))
}
source := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Source)))
if source == "" {
failures = append(failures, fmt.Errorf("music.datasets[%q].source is required", id))
} else if pathEscapesRoot(source) {
failures = append(failures, fmt.Errorf("music.datasets[%q].source must not escape project root", id))
} else if prior, exists := seenSources[source]; exists {
failures = append(failures, fmt.Errorf("music.datasets[%q].source %q conflicts with dataset %q", id, ds.Source, prior))
} else {
seenSources[source] = id
}
output := trimConfigSlashes(filepath.ToSlash(strings.TrimSpace(ds.Output)))
if output != "" && pathEscapesRoot(output) {
failures = append(failures, fmt.Errorf("music.datasets[%q].output must not escape project root", id))
}
if ds.NamingScheme != "" && ds.NamingScheme != "nwn_bmu" && ds.NamingScheme != "passthrough" {
failures = append(failures, fmt.Errorf("music.datasets[%q].naming_scheme %q is not supported", id, ds.NamingScheme))
}
if ds.PackageMode != "" && ds.PackageMode != "hak_asset" && ds.PackageMode != "none" {
failures = append(failures, fmt.Errorf("music.datasets[%q].package_mode %q is not supported", id, ds.PackageMode))
}
if ds.OutputExtension != "" && !strings.HasPrefix(ds.OutputExtension, ".") {
failures = append(failures, fmt.Errorf("music.datasets[%q].output_extension must start with .", id))
}
}
return failures
}
func pathEscapesRoot(path string) bool {
clean := filepath.Clean(filepath.FromSlash(path))
return clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator))
}
func validateGlobList(field string, patterns []string) []error {
var failures []error
seen := map[string]struct{}{}
internal/project/project_test.go
+37
View File
@@ -69,6 +69,12 @@ paths:
if got, want := effective.TopData.PackageHAK, "sow_top.hak"; got != want {
t.Fatalf("expected default topdata HAK %q, got %q", want, got)
}
if got, want := strings.Join(effective.Validation.BuiltinScriptPrefixes, ","), "ga_,gc_,gen_,gui_,nw_,nwg_,ta_,x0_,x1_,x2_,x3_"; got != want {
t.Fatalf("expected default built-in script prefixes %q, got %q", want, got)
}
if got := strings.Join(effective.Validation.RequiredFields["ifo"], ","); got != "Mod_Name" {
t.Fatalf("expected default IFO required fields, got %#v", effective.Validation.RequiredFields)
}
if prov := effective.Provenance["paths.build"]; prov.Source != "toolkit default" {
t.Fatalf("expected paths.build toolkit default provenance, got %#v", prov)
}
@@ -77,6 +83,37 @@ paths:
}
}
func TestEffectiveConfigHonorsValidationConfig(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `
module:
name: Test Module
resref: testmod
paths:
source: src
validation:
builtin_script_prefixes:
- custom_
- NW_
required_fields:
ifo:
- Mod_Name
- Mod_Hak
`)
proj, err := Load(root)
if err != nil {
t.Fatalf("Load returned error: %v", err)
}
got := strings.Join(proj.EffectiveConfig().Validation.BuiltinScriptPrefixes, ",")
if want := "custom_,nw_"; got != want {
t.Fatalf("expected configured validation prefixes %q, got %q", want, got)
}
if got := strings.Join(proj.EffectiveConfig().Validation.RequiredFields["ifo"], ","); got != "Mod_Hak,Mod_Name" {
t.Fatalf("expected configured required fields, got %#v", proj.EffectiveConfig().Validation.RequiredFields)
}
}
func TestEffectiveConfigHonorsConfiguredOutputAndCacheFields(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `
internal/validator/validator.go
+32 -33
View File
@@ -10,6 +10,7 @@ import (
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
@@ -49,24 +50,6 @@ type assetGroupResolver struct {
defaultGroup string
}
var requiredFields = map[string][]string{
"ifo": {"Mod_Name"},
}
var builtinScriptPrefixes = []string{
"nw_",
"x0_",
"x1_",
"x2_",
"x3_",
"ga_",
"gc_",
"gen_",
"gui_",
"nwg_",
"ta_",
}
func ValidateProject(p *project.Project) Report {
report := Report{}
resourceIndex := map[string]string{}
@@ -75,6 +58,7 @@ func ValidateProject(p *project.Project) Report {
assetOccurrences := map[string][]assetOccurrence{}
documents := make([]loadedDocument, 0, len(p.Inventory.SourceFiles))
resolver := newAssetGroupResolver(p)
effective := p.EffectiveConfig()
for _, rel := range p.Inventory.SourceFiles {
if hasUppercaseResourceName(rel) {
@@ -101,7 +85,7 @@ func ValidateProject(p *project.Project) Report {
resourceIndex[key] = rel
}
validateDocumentStructure(&report, rel, extension, document)
validateDocumentStructure(&report, rel, extension, document, effective.Validation.RequiredFields)
}
for _, rel := range p.Inventory.ScriptFiles {
@@ -117,7 +101,8 @@ func ValidateProject(p *project.Project) Report {
}
for _, rel := range p.Inventory.AssetFiles {
if hasUppercaseResourceName(rel) && !isMusicSourceAsset(rel) {
musicSource := isMusicSourceAsset(p, rel)
if hasUppercaseResourceName(rel) && !musicSource {
report.add(rel, "resource filenames should be lowercase", SeverityWarning)
}
base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
@@ -131,6 +116,9 @@ func ValidateProject(p *project.Project) Report {
if _, exists := assetIndex[key]; !exists {
assetIndex[key] = rel
}
if musicSource {
continue
}
group, err := resolver.groupFor(rel)
if err != nil {
@@ -147,7 +135,7 @@ func ValidateProject(p *project.Project) Report {
classifyAssetDuplicates(&report, assetOccurrences)
for _, document := range documents {
validateReferences(&report, document, resourceIndex, scriptIndex, assetIndex)
validateReferences(&report, document, resourceIndex, scriptIndex, assetIndex, effective.Validation.BuiltinScriptPrefixes)
}
slices.SortFunc(report.Diagnostics, func(a, b Diagnostic) int {
@@ -296,7 +284,7 @@ func loadDocument(path string) (gff.Document, string, string, error) {
return document, resref, extension, nil
}
func validateDocumentStructure(report *Report, path, extension string, document gff.Document) {
func validateDocumentStructure(report *Report, path, extension string, document gff.Document, requiredFields map[string][]string) {
expectedType := strings.ToUpper(extension)
if strings.TrimSpace(document.FileType) != "" && strings.TrimSpace(document.FileType) != expectedType {
report.add(path, fmt.Sprintf("file_type %q does not match expected %q", document.FileType, expectedType), SeverityError)
@@ -338,7 +326,7 @@ func validateUniqueLabels(report *Report, path, scope string, s gff.Struct) {
}
}
func validateReferences(report *Report, document loadedDocument, resources, scripts, assets map[string]string) {
func validateReferences(report *Report, document loadedDocument, resources, scripts, assets map[string]string, builtinScriptPrefixes []string) {
walkFields(document.Document.Root, func(field gff.Field) {
value, ok := fieldStringValue(field.Value)
if !ok || value == "" {
@@ -347,7 +335,7 @@ func validateReferences(report *Report, document loadedDocument, resources, scri
switch {
case isScriptField(field.Label):
if isBuiltinScript(value) {
if isBuiltinScript(value, builtinScriptPrefixes) {
return
}
if _, exists := scripts[strings.ToLower(value)]; !exists {
@@ -519,20 +507,31 @@ func hasAsset(name string, extensions []string, assets map[string]string) bool {
return false
}
func isMusicSourceAsset(rel string) bool {
func isMusicSourceAsset(p *project.Project, rel string) bool {
rel = filepath.ToSlash(strings.TrimSpace(rel))
if rel != "envi/music" && !strings.HasPrefix(rel, "envi/music/") {
return false
ext := strings.ToLower(filepath.Ext(rel))
effective := p.EffectiveConfig()
for _, dataset := range effective.Music.Datasets {
if !music.PathIsUnder(dataset.Source, rel) {
continue
}
for _, candidate := range dataset.ConvertExtensions {
if ext == strings.ToLower(strings.TrimSpace(candidate)) {
return true
}
}
}
switch strings.ToLower(filepath.Ext(rel)) {
case ".mp3", ".ogg":
return true
default:
return false
if music.PathIsUnder("envi/music", rel) {
for _, candidate := range effective.Music.ConvertExtensions {
if ext == strings.ToLower(strings.TrimSpace(candidate)) {
return true
}
}
}
return false
}
func isBuiltinScript(name string) bool {
func isBuiltinScript(name string, builtinScriptPrefixes []string) bool {
lower := strings.ToLower(name)
for _, prefix := range builtinScriptPrefixes {
if strings.HasPrefix(lower, prefix) {
internal/validator/validator_test.go
+96
View File
@@ -199,6 +199,102 @@ func TestValidateProjectWarnsForMissingScriptReferences(t *testing.T) {
}
}
func TestValidateProjectUsesConfiguredBuiltinScriptPrefixes(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "dialogs"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Module
resref: testmod
paths:
source: src
assets: assets
build: build
validation:
builtin_script_prefixes:
- custom_
`)
mustWriteFile(t, filepath.Join(root, "src", "dialogs", "merchant.dlg.json"), `{
"file_type": "DLG ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Script",
"type": "ResRef",
"value": "custom_open_store"
}
]
}
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
report := ValidateProject(p)
if report.HasErrors() || report.WarningCount() != 0 {
t.Fatalf("expected configured script prefix to suppress warning, got %#v", report.Diagnostics)
}
}
func TestValidateProjectUsesConfiguredRequiredFields(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Module
resref: testmod
paths:
source: src
assets: assets
build: build
validation:
required_fields:
ifo: []
`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": []
}
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
report := ValidateProject(p)
if report.HasErrors() {
t.Fatalf("expected configured required fields to suppress IFO error, got %#v", report.Diagnostics)
}
}
func TestValidateProjectReportsDecompiledModelsWithoutFailingValidation(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))