ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Music Pipeline Configuration Hardening

Browse Source
This commit is contained in:
archvillainette
2026-05-08 11:22:57 +02:00
parent 75942b0940
commit c7fd63a80c
6 changed files with 265 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/music/music.go
+16 -2
View File
@@ -15,9 +15,19 @@ const (
)
var (
defaultConvertExtensions = []string{
".flac",
".m4a",
".mp3",
".ogg",
".wav",
}
ConvertExtensions = map[string]struct{}{
".mp3": {},
".ogg": {},
".flac": {},
".m4a": {},
".mp3": {},
".ogg": {},
".wav": {},
}
PassthroughExtensions = map[string]struct{}{
".bmu": {},
@@ -38,6 +48,10 @@ var (
VowelRE = regexp.MustCompile(`[aeiou]`)
)
func DefaultConvertExtensions() []string {
return append([]string(nil), defaultConvertExtensions...)
}
type Metadata struct {
Title string
Artist string
internal/pipeline/pipeline_test.go
+140
View File
@@ -3447,6 +3447,146 @@ haks:
}
}
func TestBuildHAKsConvertsFLACMusicSourcesByDefault(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate"))
mustMkdir(t, filepath.Join(root, "build"))
ffprobePath := filepath.Join(root, "tools", "ffprobe")
ffmpegPath := filepath.Join(root, "tools", "ffmpeg")
mustMkdir(t, filepath.Dir(ffprobePath))
mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"FLAC Artist\",\"title\":\"FLAC Title\"}}}'\n")
mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'dataset-bmu' > \"$last\"\n")
if err := os.Chmod(ffprobePath, 0o755); err != nil {
t.Fatalf("chmod ffprobe: %v", err)
}
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
t.Fatalf("chmod ffmpeg: %v", err)
}
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Assets
resref: testassets
paths:
assets: assets
build: build
music:
tools:
ffmpeg: tools/ffmpeg
ffprobe: tools/ffprobe
datasets:
westgate_audio:
source: audio/westgate
output: generated/music
prefix: wg_
haks:
- name: music
priority: 1
max_bytes: 1048576
split: false
include:
- generated/music/**
`)
mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.flac"), "source-flac")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildHAKs(p)
if err != nil {
t.Fatalf("build haks: %v", err)
}
if result.CreditsSummary.TrackCount != 1 {
t.Fatalf("expected one music track, got %#v", result.CreditsSummary)
}
manifestRaw, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
if err != nil {
t.Fatalf("read manifest: %v", err)
}
manifestText := string(manifestRaw)
if !strings.Contains(manifestText, "generated/music/wg_theme.bmu") {
t.Fatalf("expected generated FLAC music in manifest, got:\n%s", manifestText)
}
if strings.Contains(manifestText, "audio/westgate/Theme Song.flac") {
t.Fatalf("did not expect source flac in manifest, got:\n%s", manifestText)
}
}
func TestBuildHAKsUsesYAMLMusicConvertExtensionsForDiscovery(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate"))
mustMkdir(t, filepath.Join(root, "build"))
ffprobePath := filepath.Join(root, "tools", "ffprobe")
ffmpegPath := filepath.Join(root, "tools", "ffmpeg")
mustMkdir(t, filepath.Dir(ffprobePath))
mustWriteFile(t, ffprobePath, "#!/bin/sh\nprintf '%s\\n' '{\"format\":{\"tags\":{\"artist\":\"AAC Artist\",\"title\":\"AAC Title\"}}}'\n")
mustWriteFile(t, ffmpegPath, "#!/bin/sh\nfor last; do :; done\nprintf 'dataset-bmu' > \"$last\"\n")
if err := os.Chmod(ffprobePath, 0o755); err != nil {
t.Fatalf("chmod ffprobe: %v", err)
}
if err := os.Chmod(ffmpegPath, 0o755); err != nil {
t.Fatalf("chmod ffmpeg: %v", err)
}
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Assets
resref: testassets
paths:
assets: assets
build: build
music:
defaults:
convert_extensions:
- .aac
tools:
ffmpeg: tools/ffmpeg
ffprobe: tools/ffprobe
datasets:
westgate_audio:
source: audio/westgate
output: generated/music
prefix: wg_
haks:
- name: music
priority: 1
max_bytes: 1048576
split: false
include:
- generated/music/**
`)
mustWriteFile(t, filepath.Join(root, "assets", "audio", "westgate", "Theme Song.aac"), "source-aac")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildHAKs(p)
if err != nil {
t.Fatalf("build haks: %v", err)
}
if result.CreditsSummary.TrackCount != 1 {
t.Fatalf("expected one music track discovered through YAML convert_extensions, got %#v", result.CreditsSummary)
}
}
func TestPlanHAKsDoesNotTranscodeMusicSources(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "assets", "audio", "westgate"))
internal/project/effective.go
+4 -3
View File
@@ -8,6 +8,8 @@ import (
"slices"
"strings"
"time"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music"
)
const (
@@ -249,7 +251,7 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
musicMaxStemLength := 16
musicNamingScheme := DefaultMusicNamingScheme
musicOutputExtension := DefaultMusicOutputExtension
musicConvertExtensions := []string{".mp3", ".ogg"}
musicConvertExtensions := music.DefaultConvertExtensions()
if d := p.Config.Music.Defaults; d != nil {
if d.StageRoot != "" {
musicStageRoot = expandPathTemplate(d.StageRoot, paths)
@@ -318,7 +320,6 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
ConvertExtensions: dsConvertExtensions,
}
}
effective := EffectiveConfig{
ConfigSource: p.ConfigSource,
Module: p.Config.Module,
@@ -428,7 +429,7 @@ func markMissingDefaults(provenance ConfigProvenance) {
"music.max_stem_length": "16",
"music.naming_scheme": DefaultMusicNamingScheme,
"music.output_extension": DefaultMusicOutputExtension,
"music.convert_extensions": ".mp3, .ogg",
"music.convert_extensions": strings.Join(music.DefaultConvertExtensions(), ", "),
"topdata.build": DefaultTopDataBuild,
"topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir,
"topdata.compiled_tlk": DefaultTopDataCompiledTLK,
internal/project/project.go
+47 -2
View File
@@ -602,10 +602,46 @@ func (p *Project) Scan() error {
assetDir := p.AssetsDir()
var assetFiles []string
if assetDir != "" && filepath.Clean(assetDir) != filepath.Clean(p.Root) {
musicSourceExts := make(map[string]struct{})
for _, ext := range effective.Music.ConvertExtensions {
musicSourceExts[ext] = struct{}{}
}
musicSourceRoots := make(map[string]map[string]struct{})
for _, ds := range effective.Music.Datasets {
if strings.TrimSpace(ds.Source) == "" {
continue
}
exts := musicSourceRoots[ds.Source]
if exts == nil {
exts = make(map[string]struct{})
musicSourceRoots[ds.Source] = exts
}
for _, ext := range ds.ConvertExtensions {
exts[ext] = struct{}{}
}
for ext := range musicSourceExts {
exts[ext] = struct{}{}
}
}
var err error
assetFiles, _, err = scanDir(assetDir, func(path string) bool {
ext := strings.ToLower(filepath.Ext(path))
return slices.Contains(effective.Inventory.AssetExtensions, ext)
rel, err := filepath.Rel(assetDir, path)
if err != nil {
return false
}
rel = filepath.ToSlash(rel)
ext := strings.ToLower(filepath.Ext(rel))
if slices.Contains(effective.Inventory.AssetExtensions, ext) {
return true
}
for root, exts := range musicSourceRoots {
if !pathIsUnder(root, rel) {
continue
}
_, ok := exts[ext]
return ok
}
return false
})
if err != nil {
if errors.Is(err, os.ErrNotExist) {
@@ -1393,6 +1429,15 @@ func validateOutputFileName(field, name, expectedExt string) error {
return nil
}
func pathIsUnder(root, rel string) bool {
root = strings.Trim(strings.TrimSpace(filepath.ToSlash(root)), "/")
rel = strings.Trim(strings.TrimSpace(filepath.ToSlash(rel)), "/")
if root == "" || rel == "" {
return false
}
return rel == root || strings.HasPrefix(rel, root+"/")
}
func validateRelativePath(field, path string) []error {
var failures []error
trimmed := strings.TrimSpace(path)
internal/project/project_test.go
+50
View File
@@ -81,6 +81,9 @@ paths:
if got := strings.Join(effective.Validation.RequiredFields["ifo"], ","); got != "Mod_Name" {
t.Fatalf("expected default IFO required fields, got %#v", effective.Validation.RequiredFields)
}
if got, want := strings.Join(effective.Music.ConvertExtensions, ","), ".flac,.m4a,.mp3,.ogg,.wav"; got != want {
t.Fatalf("expected default music convert extensions %q, got %q", want, got)
}
if prov := effective.Provenance["paths.build"]; prov.Source != "toolkit default" {
t.Fatalf("expected paths.build toolkit default provenance, got %#v", prov)
}
@@ -120,6 +123,53 @@ validation:
}
}
func TestScanUsesYAMLMusicConvertExtensionsWithinConfiguredDatasetRoots(t *testing.T) {
root := t.TempDir()
if err := os.MkdirAll(filepath.Join(root, "assets", "audio", "westgate"), 0o755); err != nil {
t.Fatalf("mkdir westgate audio: %v", err)
}
if err := os.MkdirAll(filepath.Join(root, "assets", "audio", "other"), 0o755); err != nil {
t.Fatalf("mkdir other audio: %v", err)
}
writeProjectFile(t, filepath.Join(root, ConfigFile), `
module:
name: Test Module
resref: testmod
paths:
source: src
assets: assets
music:
defaults:
convert_extensions:
- .flac
datasets:
westgate_audio:
source: audio/westgate
convert_extensions:
- .aac
`)
writeProjectFile(t, filepath.Join(root, "assets", "audio", "westgate", "theme.aac"), "aac")
writeProjectFile(t, filepath.Join(root, "assets", "audio", "westgate", "theme.flac"), "flac")
writeProjectFile(t, filepath.Join(root, "assets", "audio", "other", "ignored.aac"), "aac")
proj, err := Load(root)
if err != nil {
t.Fatalf("Load returned error: %v", err)
}
if err := proj.Scan(); err != nil {
t.Fatalf("Scan returned error: %v", err)
}
if !slices.Contains(proj.Inventory.AssetFiles, "audio/westgate/theme.aac") {
t.Fatalf("expected configured dataset AAC file to be discovered, got %#v", proj.Inventory.AssetFiles)
}
if !slices.Contains(proj.Inventory.AssetFiles, "audio/westgate/theme.flac") {
t.Fatalf("expected configured dataset FLAC file to be discovered, got %#v", proj.Inventory.AssetFiles)
}
if slices.Contains(proj.Inventory.AssetFiles, "audio/other/ignored.aac") {
t.Fatalf("did not expect non-dataset AAC file to be discovered, got %#v", proj.Inventory.AssetFiles)
}
}
func TestEffectiveConfigHonorsConfiguredOutputAndCacheFields(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `