Phase 5: scaffold Crucible (sow-tools) — dispatcher + builder shims, container, PR-first CI, fail-closed (D11, D19).
This commit is contained in:
@@ -1,70 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
type ApplyManifestResult struct {
|
||||
ManifestPath string
|
||||
ModuleSource string
|
||||
HAKCount int
|
||||
}
|
||||
|
||||
func ApplyHAKManifest(p *project.Project, manifestPath string) (ApplyManifestResult, error) {
|
||||
if manifestPath == "" {
|
||||
manifestPath = p.HAKManifestPath()
|
||||
}
|
||||
if strings.TrimSpace(p.EffectiveConfig().Paths.Source) == "" {
|
||||
return ApplyManifestResult{}, fmt.Errorf("cannot apply hak manifest: paths.source is not configured")
|
||||
}
|
||||
sourceRoot := filepath.Clean(p.SourceDir())
|
||||
if sourceRoot == "." || sourceRoot == string(filepath.Separator) || sourceRoot == filepath.Clean(p.Root) {
|
||||
return ApplyManifestResult{}, fmt.Errorf("cannot apply hak manifest: paths.source resolves to unsafe source root %s", sourceRoot)
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("read hak manifest: %w", err)
|
||||
}
|
||||
|
||||
var manifest BuildManifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("parse hak manifest: %w", err)
|
||||
}
|
||||
|
||||
moduleSource := filepath.Join(sourceRoot, "module", "module.ifo.json")
|
||||
sourceRaw, err := os.ReadFile(moduleSource)
|
||||
if err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("read module ifo source: %w", err)
|
||||
}
|
||||
|
||||
var document gff.Document
|
||||
if err := json.Unmarshal(sourceRaw, &document); err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("parse module ifo source: %w", err)
|
||||
}
|
||||
|
||||
setModuleHAKList(&document, manifest.ModuleHAKs)
|
||||
|
||||
formatted, err := json.MarshalIndent(document, "", " ")
|
||||
if err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("marshal updated module ifo: %w", err)
|
||||
}
|
||||
formatted = append(formatted, '\n')
|
||||
|
||||
if err := os.WriteFile(moduleSource, formatted, 0o644); err != nil {
|
||||
return ApplyManifestResult{}, fmt.Errorf("write module ifo source: %w", err)
|
||||
}
|
||||
|
||||
return ApplyManifestResult{
|
||||
ManifestPath: manifestPath,
|
||||
ModuleSource: moduleSource,
|
||||
HAKCount: len(manifest.ModuleHAKs),
|
||||
}, nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,27 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
)
|
||||
|
||||
func ensureUniqueChunkResources(chunk hakChunk) error {
|
||||
seen := map[string]string{}
|
||||
for _, asset := range chunk.Assets {
|
||||
key := fmt.Sprintf("%s:%04x", asset.Resource.Name, asset.Resource.Type)
|
||||
if previous, exists := seen[key]; exists {
|
||||
return fmt.Errorf("resource %s from %s conflicts with %s inside generated hak %s", chunkResourceLabel(asset), asset.Rel, previous, chunk.Name)
|
||||
}
|
||||
seen[key] = asset.Rel
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func chunkResourceLabel(asset assetResource) string {
|
||||
extension, ok := erf.ExtensionForResourceType(asset.Resource.Type)
|
||||
if !ok {
|
||||
return asset.Resource.Name
|
||||
}
|
||||
return asset.Resource.Name + "." + extension
|
||||
}
|
||||
@@ -1,359 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
type CompareResult struct {
|
||||
ModulePath string
|
||||
HAKPaths []string
|
||||
Checked int
|
||||
}
|
||||
|
||||
type resourceExpectation struct {
|
||||
Key string
|
||||
Kind string
|
||||
Bytes []byte
|
||||
}
|
||||
|
||||
func Compare(p *project.Project) (CompareResult, error) {
|
||||
modulePath := p.ModuleArchivePath()
|
||||
|
||||
var hakPaths []string
|
||||
if len(p.Inventory.AssetFiles) > 0 {
|
||||
var err error
|
||||
hakPaths, err = manifestHAKPaths(p)
|
||||
if err != nil {
|
||||
return CompareResult{}, err
|
||||
}
|
||||
}
|
||||
if err := ensureBuildIsCurrent(p, modulePath, hakPaths); err != nil {
|
||||
return CompareResult{ModulePath: modulePath, HAKPaths: hakPaths}, err
|
||||
}
|
||||
|
||||
moduleArchive, err := readArchive(modulePath)
|
||||
if err != nil {
|
||||
return CompareResult{}, err
|
||||
}
|
||||
|
||||
expected, err := expectedResources(p)
|
||||
if err != nil {
|
||||
return CompareResult{}, err
|
||||
}
|
||||
|
||||
actual := archiveIndex(moduleArchive)
|
||||
for _, hakPath := range hakPaths {
|
||||
hakArchive, err := readArchive(hakPath)
|
||||
if err != nil {
|
||||
return CompareResult{}, err
|
||||
}
|
||||
for key, value := range archiveIndex(hakArchive) {
|
||||
actual[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
var diagnostics []error
|
||||
checked := 0
|
||||
|
||||
expectedKeys := make([]string, 0, len(expected))
|
||||
for key := range expected {
|
||||
expectedKeys = append(expectedKeys, key)
|
||||
}
|
||||
slices.Sort(expectedKeys)
|
||||
|
||||
for _, key := range expectedKeys {
|
||||
want := expected[key]
|
||||
got, ok := actual[key]
|
||||
if !ok {
|
||||
diagnostics = append(diagnostics, fmt.Errorf("missing resource in built archives: %s", key))
|
||||
continue
|
||||
}
|
||||
if !bytes.Equal(want.Bytes, got.Data) {
|
||||
diagnostics = append(diagnostics, fmt.Errorf("resource content mismatch: %s", key))
|
||||
continue
|
||||
}
|
||||
checked++
|
||||
delete(actual, key)
|
||||
}
|
||||
|
||||
if len(actual) > 0 {
|
||||
extraKeys := make([]string, 0, len(actual))
|
||||
for key := range actual {
|
||||
extraKeys = append(extraKeys, key)
|
||||
}
|
||||
slices.Sort(extraKeys)
|
||||
for _, key := range extraKeys {
|
||||
diagnostics = append(diagnostics, fmt.Errorf("unexpected extra resource in built archives: %s", key))
|
||||
}
|
||||
}
|
||||
|
||||
result := CompareResult{
|
||||
ModulePath: modulePath,
|
||||
HAKPaths: hakPaths,
|
||||
Checked: checked,
|
||||
}
|
||||
if len(diagnostics) > 0 {
|
||||
return result, errors.Join(diagnostics...)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func expectedResources(p *project.Project) (map[string]resourceExpectation, error) {
|
||||
out := map[string]resourceExpectation{}
|
||||
moduleHakOrder, err := plannedModuleHAKOrder(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, rel := range p.Inventory.SourceFiles {
|
||||
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
|
||||
name, extension, err := splitSourceName(abs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s: %w", abs, err)
|
||||
}
|
||||
var document gff.Document
|
||||
if err := json.Unmarshal(raw, &document); err != nil {
|
||||
return nil, fmt.Errorf("parse gff json %s: %w", abs, err)
|
||||
}
|
||||
if extension == ".ifo" && strings.EqualFold(name, "module") && len(moduleHakOrder) > 0 {
|
||||
setModuleHAKList(&document, moduleHakOrder)
|
||||
}
|
||||
|
||||
canonical, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal canonical gff json %s: %w", abs, err)
|
||||
}
|
||||
|
||||
out[resourceKey(name, extension)] = resourceExpectation{
|
||||
Key: resourceKey(name, extension),
|
||||
Kind: "gff-json",
|
||||
Bytes: canonical,
|
||||
}
|
||||
}
|
||||
|
||||
for _, rel := range p.Inventory.ScriptFiles {
|
||||
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
|
||||
data, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s: %w", abs, err)
|
||||
}
|
||||
name := strings.ToLower(strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs)))
|
||||
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(abs)), ".")
|
||||
out[resourceKey(name, extension)] = resourceExpectation{
|
||||
Key: resourceKey(name, extension),
|
||||
Kind: "raw",
|
||||
Bytes: data,
|
||||
}
|
||||
}
|
||||
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
|
||||
data, err := os.ReadFile(abs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read %s: %w", abs, err)
|
||||
}
|
||||
name := strings.ToLower(strings.TrimSuffix(filepath.Base(abs), filepath.Ext(abs)))
|
||||
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(abs)), ".")
|
||||
out[resourceKey(name, extension)] = resourceExpectation{
|
||||
Key: resourceKey(name, extension),
|
||||
Kind: "raw",
|
||||
Bytes: data,
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func ensureBuildIsCurrent(p *project.Project, modulePath string, hakPaths []string) error {
|
||||
archivePaths := make([]string, 0, 1+len(hakPaths))
|
||||
archivePaths = append(archivePaths, modulePath)
|
||||
archivePaths = append(archivePaths, hakPaths...)
|
||||
|
||||
oldestArchive := time.Time{}
|
||||
for _, path := range archivePaths {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat archive %s: %w", path, err)
|
||||
}
|
||||
if oldestArchive.IsZero() || info.ModTime().Before(oldestArchive) {
|
||||
oldestArchive = info.ModTime()
|
||||
}
|
||||
}
|
||||
|
||||
sourcePaths := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
|
||||
for _, rel := range p.Inventory.SourceFiles {
|
||||
sourcePaths = append(sourcePaths, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
for _, rel := range p.Inventory.ScriptFiles {
|
||||
sourcePaths = append(sourcePaths, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
sourcePaths = append(sourcePaths, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
|
||||
newestSource := time.Time{}
|
||||
newestPath := ""
|
||||
for _, path := range sourcePaths {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat source %s: %w", path, err)
|
||||
}
|
||||
if newestSource.IsZero() || info.ModTime().After(newestSource) {
|
||||
newestSource = info.ModTime()
|
||||
newestPath = path
|
||||
}
|
||||
}
|
||||
|
||||
if !newestSource.IsZero() && newestSource.After(oldestArchive) {
|
||||
return fmt.Errorf("built archives are older than source file %s; run build-module or build-haks before compare", newestPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readArchive(path string) (erf.Archive, error) {
|
||||
input, err := os.Open(path)
|
||||
if err != nil {
|
||||
return erf.Archive{}, fmt.Errorf("open archive %s: %w", path, err)
|
||||
}
|
||||
defer input.Close()
|
||||
|
||||
archive, err := erf.Read(input)
|
||||
if err != nil {
|
||||
return erf.Archive{}, fmt.Errorf("read archive %s: %w", path, err)
|
||||
}
|
||||
return archive, nil
|
||||
}
|
||||
|
||||
func archiveIndex(archive erf.Archive) map[string]erf.Resource {
|
||||
out := map[string]erf.Resource{}
|
||||
for _, resource := range archive.Resources {
|
||||
extension, ok := erf.ExtensionForResourceType(resource.Type)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
key := resourceKey(strings.ToLower(resource.Name), extension)
|
||||
switch extension {
|
||||
case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
|
||||
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl":
|
||||
document, err := gff.Read(bytes.NewReader(resource.Data))
|
||||
if err != nil {
|
||||
out[key] = resource
|
||||
continue
|
||||
}
|
||||
canonical, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
out[key] = resource
|
||||
continue
|
||||
}
|
||||
resource.Data = canonical
|
||||
}
|
||||
|
||||
out[key] = resource
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func resourceKey(name, extension string) string {
|
||||
return strings.ToLower(name) + "." + strings.TrimPrefix(strings.ToLower(extension), ".")
|
||||
}
|
||||
|
||||
func manifestHAKPaths(p *project.Project) ([]string, error) {
|
||||
manifestPath := p.HAKManifestPath()
|
||||
raw, err := os.ReadFile(manifestPath)
|
||||
if err == nil {
|
||||
var manifest BuildManifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("parse hak manifest: %w", err)
|
||||
}
|
||||
paths := make([]string, 0, len(manifest.HAKs))
|
||||
for _, hak := range manifest.HAKs {
|
||||
paths = append(paths, p.HAKArchivePath(hak.Name))
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("read hak manifest: %w", err)
|
||||
}
|
||||
|
||||
legacy := filepath.Join(p.BuildDir(), "*.hak")
|
||||
paths, err := filepath.Glob(legacy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan hak archives: %w", err)
|
||||
}
|
||||
filtered := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
if filepath.Base(path) == p.TopDataPackageHAKName() {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, path)
|
||||
}
|
||||
slices.Sort(filtered)
|
||||
return filtered, nil
|
||||
}
|
||||
|
||||
func topPackageSourcePaths(p *project.Project) ([]string, error) {
|
||||
sourceDir := p.TopDataSourceDir()
|
||||
generated2DADir := filepath.Join(sourceDir, "assets", "2da")
|
||||
paths := make([]string, 0)
|
||||
for _, candidate := range []string{
|
||||
filepath.Join(sourceDir, ".tlk_state.json"),
|
||||
filepath.Join(sourceDir, "base_dialog.json"),
|
||||
} {
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
paths = append(paths, candidate)
|
||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("stat topdata source %s: %w", candidate, err)
|
||||
}
|
||||
}
|
||||
for _, dir := range []string{
|
||||
filepath.Join(sourceDir, "data"),
|
||||
filepath.Join(sourceDir, "assets"),
|
||||
} {
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("stat topdata path %s: %w", dir, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
continue
|
||||
}
|
||||
err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() && path == generated2DADir {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
paths = append(paths, path)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan topdata path %s: %w", dir, err)
|
||||
}
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
@@ -1,621 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
type ExtractResult struct {
|
||||
ModulePath string
|
||||
HAKPaths []string
|
||||
DeletedArchivePaths []string
|
||||
Written int
|
||||
Overwritten int
|
||||
Removed int
|
||||
Skipped int
|
||||
}
|
||||
|
||||
type extractionArchive struct {
|
||||
Path string
|
||||
Rel string
|
||||
Extension string
|
||||
}
|
||||
|
||||
type extractionScope struct {
|
||||
Source bool
|
||||
Assets bool
|
||||
}
|
||||
|
||||
func Extract(p *project.Project, files ...string) (ExtractResult, error) {
|
||||
var result ExtractResult
|
||||
var failures []error
|
||||
desired := map[string]struct{}{}
|
||||
scope := extractionScope{}
|
||||
|
||||
archives, err := resolveExtractionArchives(p, files)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
for _, archivePath := range archives {
|
||||
input, err := os.Open(archivePath.Path)
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Errorf("open archive %s: %w", archivePath.Path, err))
|
||||
continue
|
||||
}
|
||||
archive, err := erf.Read(input)
|
||||
input.Close()
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Errorf("read archive %s: %w", archivePath.Path, err))
|
||||
continue
|
||||
}
|
||||
|
||||
switch archivePath.Extension {
|
||||
case ".mod":
|
||||
if result.ModulePath == "" {
|
||||
result.ModulePath = archivePath.Path
|
||||
}
|
||||
case ".hak":
|
||||
result.HAKPaths = append(result.HAKPaths, archivePath.Path)
|
||||
}
|
||||
|
||||
written, overwritten, skipped, extractedScope, errs := extractArchiveResources(p, archive, desired)
|
||||
result.Written += written
|
||||
result.Overwritten += overwritten
|
||||
result.Skipped += skipped
|
||||
scope.Source = scope.Source || extractedScope.Source
|
||||
scope.Assets = scope.Assets || extractedScope.Assets
|
||||
failures = append(failures, errs...)
|
||||
}
|
||||
|
||||
if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale {
|
||||
removed, errs := cleanupStaleFiles(p, desired, scope)
|
||||
result.Removed = removed
|
||||
failures = append(failures, errs...)
|
||||
}
|
||||
|
||||
if len(failures) > 0 {
|
||||
return result, errors.Join(failures...)
|
||||
}
|
||||
consumeAll, consumeModules := extractionConsumePolicy(p)
|
||||
if consumeAll || consumeModules {
|
||||
for _, archivePath := range archives {
|
||||
if !consumeAll && archivePath.Extension != ".mod" {
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(archivePath.Path); err != nil {
|
||||
return result, fmt.Errorf("delete consumed archive %s: %w", archivePath.Path, err)
|
||||
}
|
||||
result.DeletedArchivePaths = append(result.DeletedArchivePaths, archivePath.Path)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, extractionScope, []error) {
|
||||
var failures []error
|
||||
writtenCount := 0
|
||||
overwrittenCount := 0
|
||||
skippedCount := 0
|
||||
scope := extractionScope{}
|
||||
|
||||
ignored := make(map[string]bool)
|
||||
for _, ext := range p.Config.Extract.IgnoreExtensions {
|
||||
ext := strings.TrimPrefix(ext, ".")
|
||||
ignored[ext] = true
|
||||
}
|
||||
|
||||
for _, resource := range archive.Resources {
|
||||
ext, ok := erf.ExtensionForResourceType(resource.Type)
|
||||
if !ok {
|
||||
failures = append(failures, fmt.Errorf("unsupported resource type 0x%04X for %s", resource.Type, resource.Name))
|
||||
continue
|
||||
}
|
||||
if ignored[ext] {
|
||||
skippedCount++
|
||||
continue
|
||||
}
|
||||
|
||||
target, data, err := extractedFile(p, resource, ext)
|
||||
if err != nil {
|
||||
failures = append(failures, err)
|
||||
continue
|
||||
}
|
||||
desired[target] = struct{}{}
|
||||
scope.Source = scope.Source || pathWithinRoot(target, p.SourceDir())
|
||||
scope.Assets = scope.Assets || pathWithinRoot(target, p.AssetsDir())
|
||||
|
||||
state, err := writeManagedFile(target, data)
|
||||
if err != nil {
|
||||
failures = append(failures, err)
|
||||
continue
|
||||
}
|
||||
switch state {
|
||||
case writeNew:
|
||||
writtenCount++
|
||||
case writeOverwritten:
|
||||
overwrittenCount++
|
||||
case writeSkipped:
|
||||
skippedCount++
|
||||
}
|
||||
}
|
||||
|
||||
return writtenCount, overwrittenCount, skippedCount, scope, failures
|
||||
}
|
||||
|
||||
func resolveExtractionArchives(p *project.Project, overrides []string) ([]extractionArchive, error) {
|
||||
patterns := p.EffectiveConfig().Extract.Archives
|
||||
if len(overrides) > 0 {
|
||||
patterns = overrides
|
||||
}
|
||||
if len(patterns) == 0 {
|
||||
return nil, fmt.Errorf("extract.archives must contain at least one build-relative archive pattern")
|
||||
}
|
||||
|
||||
buildRoot := filepath.Clean(p.BuildDir())
|
||||
byPath := map[string]extractionArchive{}
|
||||
for _, rawPattern := range patterns {
|
||||
pattern, err := normalizeExtractionArchivePattern(p, rawPattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
matched, err := matchExtractionArchives(buildRoot, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(matched) == 0 {
|
||||
return nil, fmt.Errorf("extract archive pattern %q matched no files under %s", rawPattern, buildRoot)
|
||||
}
|
||||
for _, archive := range matched {
|
||||
byPath[archive.Path] = archive
|
||||
}
|
||||
}
|
||||
|
||||
archives := make([]extractionArchive, 0, len(byPath))
|
||||
for _, archive := range byPath {
|
||||
archives = append(archives, archive)
|
||||
}
|
||||
slices.SortFunc(archives, func(a, b extractionArchive) int {
|
||||
return strings.Compare(a.Rel, b.Rel)
|
||||
})
|
||||
return archives, nil
|
||||
}
|
||||
|
||||
func normalizeExtractionArchivePattern(p *project.Project, raw string) (string, error) {
|
||||
pattern := strings.TrimSpace(raw)
|
||||
if pattern == "" {
|
||||
return "", fmt.Errorf("extract archive pattern must not be empty")
|
||||
}
|
||||
pattern = strings.NewReplacer(
|
||||
"{module.resref}", strings.TrimSpace(p.Config.Module.ResRef),
|
||||
).Replace(pattern)
|
||||
if strings.Contains(pattern, "\x00") {
|
||||
return "", fmt.Errorf("extract archive pattern %q must not contain NUL bytes", raw)
|
||||
}
|
||||
if filepath.IsAbs(pattern) {
|
||||
return "", fmt.Errorf("extract archive pattern %q must be relative to paths.build", raw)
|
||||
}
|
||||
clean := filepath.Clean(filepath.FromSlash(pattern))
|
||||
if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("extract archive pattern %q must not escape paths.build", raw)
|
||||
}
|
||||
return filepath.ToSlash(clean), nil
|
||||
}
|
||||
|
||||
func matchExtractionArchives(buildRoot, pattern string) ([]extractionArchive, error) {
|
||||
var archives []extractionArchive
|
||||
err := filepath.WalkDir(buildRoot, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(buildRoot, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if !matchPathPattern(rel, pattern) {
|
||||
return nil
|
||||
}
|
||||
archive, err := validateExtractionArchive(buildRoot, rel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
archives = append(archives, archive)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("scan build directory %s: %w", buildRoot, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return archives, nil
|
||||
}
|
||||
|
||||
func validateExtractionArchive(buildRoot, rel string) (extractionArchive, error) {
|
||||
cleanRel := filepath.Clean(filepath.FromSlash(rel))
|
||||
if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(filepath.Separator)) {
|
||||
return extractionArchive{}, fmt.Errorf("extract archive %q must not escape paths.build", rel)
|
||||
}
|
||||
path := filepath.Join(buildRoot, cleanRel)
|
||||
if !pathWithinRoot(path, buildRoot) {
|
||||
return extractionArchive{}, fmt.Errorf("extract archive %q resolves outside paths.build", rel)
|
||||
}
|
||||
extension := strings.ToLower(filepath.Ext(cleanRel))
|
||||
switch extension {
|
||||
case ".mod", ".hak":
|
||||
return extractionArchive{Path: path, Rel: filepath.ToSlash(cleanRel), Extension: extension}, nil
|
||||
case ".erf":
|
||||
return extractionArchive{}, fmt.Errorf("extract archive %q is an .erf; ERF extraction is unsafe because ERFs lack module.ifo context", rel)
|
||||
default:
|
||||
return extractionArchive{}, fmt.Errorf("extract archive %q uses unsupported extension %q", rel, extension)
|
||||
}
|
||||
}
|
||||
|
||||
func extractionConsumePolicy(p *project.Project) (consumeAll bool, consumeModules bool) {
|
||||
if configured := p.Config.Extract.ConsumeArchives; configured != nil {
|
||||
return *configured, false
|
||||
}
|
||||
return false, p.Config.Extract.DeleteModuleArchiveAfterSuccess
|
||||
}
|
||||
|
||||
func pathWithinRoot(path, root string) bool {
|
||||
root = filepath.Clean(root)
|
||||
if root == "" || root == "." {
|
||||
return false
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel)
|
||||
}
|
||||
|
||||
func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) {
|
||||
resref := strings.ToLower(resource.Name)
|
||||
effective := p.EffectiveConfig()
|
||||
|
||||
switch extension {
|
||||
case "nss":
|
||||
target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), "scripts", resref+".nss")
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return target, resource.Data, nil
|
||||
case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
|
||||
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl":
|
||||
document, err := gff.Read(bytes.NewReader(resource.Data))
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err)
|
||||
}
|
||||
target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json")
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if err := mergeExtractedGFFJSON(p, target, &document); err != nil {
|
||||
return "", nil, fmt.Errorf("merge extracted gff json %s.%s: %w", resource.Name, extension, err)
|
||||
}
|
||||
formatted, err := json.MarshalIndent(document, "", " ")
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err)
|
||||
}
|
||||
formatted = append(formatted, '\n')
|
||||
return target, formatted, nil
|
||||
default:
|
||||
target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return target, resource.Data, nil
|
||||
}
|
||||
}
|
||||
|
||||
func extractionTarget(p *project.Project, field, configured, root string, parts ...string) (string, error) {
|
||||
if strings.TrimSpace(configured) == "" {
|
||||
return "", fmt.Errorf("cannot extract resource: %s is not configured", field)
|
||||
}
|
||||
|
||||
cleanRoot := filepath.Clean(root)
|
||||
if cleanRoot == "." || cleanRoot == string(filepath.Separator) || cleanRoot == filepath.Clean(p.Root) {
|
||||
return "", fmt.Errorf("cannot extract resource: %s resolves to unsafe extraction root %s", field, cleanRoot)
|
||||
}
|
||||
|
||||
target := filepath.Join(append([]string{cleanRoot}, parts...)...)
|
||||
rel, err := filepath.Rel(cleanRoot, target)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve extraction target %s: %w", target, err)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
|
||||
return "", fmt.Errorf("refusing to extract resource outside %s: %s", cleanRoot, target)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
func mergeExtractedGFFJSON(p *project.Project, target string, extracted *gff.Document) error {
|
||||
rule, ok, err := extractGFFJSONMergeRule(p, target)
|
||||
if err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("read existing source %s: %w", target, err)
|
||||
}
|
||||
var existing gff.Document
|
||||
if err := json.Unmarshal(raw, &existing); err != nil {
|
||||
return fmt.Errorf("parse existing source %s: %w", target, err)
|
||||
}
|
||||
|
||||
for _, label := range rule.PreserveFields {
|
||||
if field, ok := gffField(existing.Root, label); ok {
|
||||
setGFFField(&extracted.Root, field)
|
||||
}
|
||||
}
|
||||
for _, listRule := range rule.MergeLists {
|
||||
if err := mergeGFFListByKey(&extracted.Root, existing.Root, listRule); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractGFFJSONMergeRule(p *project.Project, target string) (project.ExtractGFFJSONMergeRule, bool, error) {
|
||||
rel, err := filepath.Rel(filepath.Clean(p.SourceDir()), filepath.Clean(target))
|
||||
if err != nil {
|
||||
return project.ExtractGFFJSONMergeRule{}, false, fmt.Errorf("resolve source-relative target %s: %w", target, err)
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if rel == ".." || strings.HasPrefix(rel, "../") {
|
||||
return project.ExtractGFFJSONMergeRule{}, false, nil
|
||||
}
|
||||
for _, rule := range p.EffectiveConfig().Extract.Merge.GFFJSON {
|
||||
if rule.Target == rel {
|
||||
return rule, true, nil
|
||||
}
|
||||
}
|
||||
return project.ExtractGFFJSONMergeRule{}, false, nil
|
||||
}
|
||||
|
||||
func gffField(s gff.Struct, label string) (gff.Field, bool) {
|
||||
for _, field := range s.Fields {
|
||||
if field.Label == label {
|
||||
return field, true
|
||||
}
|
||||
}
|
||||
return gff.Field{}, false
|
||||
}
|
||||
|
||||
func setGFFField(s *gff.Struct, replacement gff.Field) {
|
||||
for index, field := range s.Fields {
|
||||
if field.Label == replacement.Label {
|
||||
s.Fields[index] = replacement
|
||||
return
|
||||
}
|
||||
}
|
||||
s.Fields = append(s.Fields, replacement)
|
||||
}
|
||||
|
||||
func mergeGFFListByKey(extracted *gff.Struct, existing gff.Struct, rule project.ExtractListMergeRule) error {
|
||||
extractedField, ok := gffField(*extracted, rule.Field)
|
||||
if !ok {
|
||||
return fmt.Errorf("extracted field %q not found", rule.Field)
|
||||
}
|
||||
extractedList, ok := extractedField.Value.(gff.ListValue)
|
||||
if !ok {
|
||||
return fmt.Errorf("extracted field %q is %s, not List", rule.Field, extractedField.Type)
|
||||
}
|
||||
existingField, ok := gffField(existing, rule.Field)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
existingList, ok := existingField.Value.(gff.ListValue)
|
||||
if !ok {
|
||||
return fmt.Errorf("existing field %q is %s, not List", rule.Field, existingField.Type)
|
||||
}
|
||||
|
||||
extractedByKey := map[string]gff.Struct{}
|
||||
extractedOrder := make([]string, 0, len(extractedList))
|
||||
for _, item := range extractedList {
|
||||
key, err := gffStructKey(item, rule.KeyField)
|
||||
if err != nil {
|
||||
return fmt.Errorf("extracted field %q: %w", rule.Field, err)
|
||||
}
|
||||
if _, exists := extractedByKey[key]; exists {
|
||||
return fmt.Errorf("extracted field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
|
||||
}
|
||||
extractedByKey[key] = item
|
||||
extractedOrder = append(extractedOrder, key)
|
||||
}
|
||||
|
||||
merged := make(gff.ListValue, 0, len(extractedList))
|
||||
seen := map[string]struct{}{}
|
||||
for _, item := range existingList {
|
||||
key, err := gffStructKey(item, rule.KeyField)
|
||||
if err != nil {
|
||||
return fmt.Errorf("existing field %q: %w", rule.Field, err)
|
||||
}
|
||||
if _, duplicate := seen[key]; duplicate {
|
||||
return fmt.Errorf("existing field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
|
||||
}
|
||||
if extractedItem, exists := extractedByKey[key]; exists {
|
||||
merged = append(merged, extractedItem)
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, key := range extractedOrder {
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
merged = append(merged, extractedByKey[key])
|
||||
}
|
||||
|
||||
setGFFField(extracted, gff.NewField(rule.Field, merged))
|
||||
return nil
|
||||
}
|
||||
|
||||
func gffStructKey(s gff.Struct, keyField string) (string, error) {
|
||||
field, ok := gffField(s, keyField)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("key field %q not found", keyField)
|
||||
}
|
||||
switch value := field.Value.(type) {
|
||||
case gff.ResRefValue:
|
||||
return string(value), nil
|
||||
case gff.StringValue:
|
||||
return string(value), nil
|
||||
default:
|
||||
return "", fmt.Errorf("key field %q is %s, not ResRef or CExoString", keyField, field.Type)
|
||||
}
|
||||
}
|
||||
|
||||
type writeState int
|
||||
|
||||
const (
|
||||
writeSkipped writeState = iota
|
||||
writeNew
|
||||
writeOverwritten
|
||||
)
|
||||
|
||||
func writeManagedFile(path string, data []byte) (writeState, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return writeSkipped, fmt.Errorf("create parent directory for %s: %w", path, err)
|
||||
}
|
||||
|
||||
existing, err := os.ReadFile(path)
|
||||
if err == nil {
|
||||
if bytes.Equal(existing, data) {
|
||||
return writeSkipped, nil
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
return writeSkipped, fmt.Errorf("overwrite %s: %w", path, err)
|
||||
}
|
||||
return writeOverwritten, nil
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return writeSkipped, fmt.Errorf("check existing file %s: %w", path, err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
return writeSkipped, fmt.Errorf("write %s: %w", path, err)
|
||||
}
|
||||
return writeNew, nil
|
||||
}
|
||||
|
||||
func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) {
|
||||
candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
|
||||
if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) {
|
||||
for _, rel := range p.Inventory.SourceFiles {
|
||||
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
for _, rel := range p.Inventory.ScriptFiles {
|
||||
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
}
|
||||
if scope.Assets && safeCleanupRoot(p.AssetsDir(), p.Root) {
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
|
||||
}
|
||||
}
|
||||
|
||||
removed := 0
|
||||
var failures []error
|
||||
for _, path := range candidates {
|
||||
if _, keep := desired[path]; keep {
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(path); err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
failures = append(failures, fmt.Errorf("remove stale file %s: %w", path, err))
|
||||
continue
|
||||
}
|
||||
removed++
|
||||
cleanupEmptyParents(filepath.Dir(path), p.SourceDir(), p.AssetsDir())
|
||||
}
|
||||
|
||||
return removed, failures
|
||||
}
|
||||
|
||||
func safeCleanupRoot(root, projectRoot string) bool {
|
||||
cleanRoot := filepath.Clean(root)
|
||||
return cleanRoot != "." && cleanRoot != string(filepath.Separator) && cleanRoot != filepath.Clean(projectRoot)
|
||||
}
|
||||
|
||||
func cleanupEmptyParents(dir string, roots ...string) {
|
||||
for {
|
||||
if dir == "." || dir == string(filepath.Separator) {
|
||||
return
|
||||
}
|
||||
stop := false
|
||||
for _, root := range roots {
|
||||
if dir == root {
|
||||
stop = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if stop {
|
||||
return
|
||||
}
|
||||
if err := os.Remove(dir); err != nil {
|
||||
return
|
||||
}
|
||||
dir = filepath.Dir(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func sourceSubdir(extension string) string {
|
||||
switch strings.ToLower(extension) {
|
||||
case "are":
|
||||
return "areas"
|
||||
case "dlg":
|
||||
return "dialogs"
|
||||
case "fac":
|
||||
return "factions"
|
||||
case "gic":
|
||||
return "instance"
|
||||
case "git":
|
||||
return "instance"
|
||||
case "ifo":
|
||||
return "module"
|
||||
case "itp":
|
||||
return "palettes"
|
||||
case "jrl":
|
||||
return "journal"
|
||||
case "utc":
|
||||
return filepath.Join("blueprints", "creatures")
|
||||
case "utd":
|
||||
return filepath.Join("blueprints", "doors")
|
||||
case "ute":
|
||||
return filepath.Join("blueprints", "encounters")
|
||||
case "uti":
|
||||
return filepath.Join("blueprints", "items")
|
||||
case "utm":
|
||||
return filepath.Join("blueprints", "merchants")
|
||||
case "utp":
|
||||
return filepath.Join("blueprints", "placeables")
|
||||
case "uts":
|
||||
return filepath.Join("blueprints", "sounds")
|
||||
case "utt":
|
||||
return filepath.Join("blueprints", "triggers")
|
||||
case "utw":
|
||||
return filepath.Join("blueprints", "waypoints")
|
||||
default:
|
||||
return extension
|
||||
}
|
||||
}
|
||||
@@ -1,524 +0,0 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/music"
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
type preparedMusicAssets struct {
|
||||
Generated []assetResource
|
||||
SkipSourceRel map[string]struct{}
|
||||
Artifacts []string
|
||||
TempRoots []string
|
||||
Summary CreditsRefreshSummary
|
||||
}
|
||||
|
||||
type MusicBuildOptions struct {
|
||||
DatasetIDs []string
|
||||
Write bool
|
||||
}
|
||||
|
||||
type musicPrepareOptions struct {
|
||||
datasetIDs []string
|
||||
write bool
|
||||
}
|
||||
|
||||
type musicSourceGroup struct {
|
||||
DatasetID string
|
||||
Dataset project.EffectiveMusicDataset
|
||||
SourceDir string
|
||||
OutputDir string
|
||||
Sources []string
|
||||
}
|
||||
|
||||
type musicCreditGroup struct {
|
||||
CreditsRoot string
|
||||
SourceDir string
|
||||
Entries []music.CreditsEntry
|
||||
}
|
||||
|
||||
func musicRootPath(p *project.Project, path string) string {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return p.Root
|
||||
}
|
||||
if filepath.IsAbs(path) {
|
||||
return path
|
||||
}
|
||||
return filepath.Join(p.Root, filepath.FromSlash(path))
|
||||
}
|
||||
|
||||
func effectiveMusicDatasets(p *project.Project, onlyIDs []string) (map[string]project.EffectiveMusicDataset, error) {
|
||||
effective := p.EffectiveConfig()
|
||||
wanted := map[string]struct{}{}
|
||||
for _, id := range onlyIDs {
|
||||
id = strings.TrimSpace(id)
|
||||
if id != "" {
|
||||
wanted[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
datasets := map[string]project.EffectiveMusicDataset{}
|
||||
for id, dataset := range effective.Music.Datasets {
|
||||
if len(wanted) > 0 {
|
||||
if _, ok := wanted[id]; !ok {
|
||||
continue
|
||||
}
|
||||
delete(wanted, id)
|
||||
}
|
||||
datasets[id] = dataset
|
||||
}
|
||||
if len(wanted) > 0 {
|
||||
unknown := make([]string, 0, len(wanted))
|
||||
for id := range wanted {
|
||||
unknown = append(unknown, id)
|
||||
}
|
||||
sort.Strings(unknown)
|
||||
return nil, fmt.Errorf("unknown music dataset(s): %s", strings.Join(unknown, ", "))
|
||||
}
|
||||
if len(datasets) == 0 && len(wanted) == 0 {
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
ext := strings.ToLower(filepath.Ext(rel))
|
||||
if !music.PathIsUnder("envi/music", rel) {
|
||||
continue
|
||||
}
|
||||
if _, ok := extensionSet(effective.Music.ConvertExtensions)[ext]; ok {
|
||||
datasets["legacy_envi_music"] = project.EffectiveMusicDataset{
|
||||
Source: "envi/music",
|
||||
Output: "envi/music",
|
||||
StageRoot: effective.Music.StageRoot,
|
||||
CreditsRoot: effective.Music.CreditsRoot,
|
||||
NamingScheme: effective.Music.NamingScheme,
|
||||
OutputExtension: effective.Music.OutputExtension,
|
||||
MaxStemLength: effective.Music.MaxStemLength,
|
||||
PackageMode: "hak_asset",
|
||||
ConvertExtensions: effective.Music.ConvertExtensions,
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return datasets, nil
|
||||
}
|
||||
|
||||
func extensionSet(values []string) map[string]struct{} {
|
||||
out := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value != "" {
|
||||
out[value] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func datasetForMusicSource(datasets map[string]project.EffectiveMusicDataset, rel, ext string) (string, project.EffectiveMusicDataset, bool) {
|
||||
bestID := ""
|
||||
var best project.EffectiveMusicDataset
|
||||
bestLen := -1
|
||||
for id, dataset := range datasets {
|
||||
if _, ok := extensionSet(dataset.ConvertExtensions)[ext]; !ok {
|
||||
continue
|
||||
}
|
||||
if !music.PathIsUnder(dataset.Source, rel) {
|
||||
continue
|
||||
}
|
||||
if len(dataset.Source) > bestLen {
|
||||
bestID = id
|
||||
best = dataset
|
||||
bestLen = len(dataset.Source)
|
||||
}
|
||||
}
|
||||
return bestID, best, bestID != ""
|
||||
}
|
||||
|
||||
func plannedMusicAsset(outputRel, sourceRel string) (assetResource, error) {
|
||||
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(outputRel)), ".")
|
||||
resourceType, ok := erf.HAKResourceTypeForExtension(extension)
|
||||
if !ok {
|
||||
return assetResource{}, fmt.Errorf("unsupported generated music resource extension %q", filepath.Ext(outputRel))
|
||||
}
|
||||
name := strings.ToLower(strings.TrimSuffix(filepath.Base(outputRel), filepath.Ext(outputRel)))
|
||||
return assetResource{
|
||||
Rel: outputRel,
|
||||
Resource: erf.Resource{
|
||||
Name: name,
|
||||
Type: resourceType,
|
||||
},
|
||||
ContentID: "music-plan:" + filepath.ToSlash(sourceRel),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func BuildMusic(p *project.Project, opts MusicBuildOptions) (BuildResult, error) {
|
||||
prepared, err := prepareMusicAssetsWithOptions(p, musicPrepareOptions{datasetIDs: opts.DatasetIDs, write: opts.Write})
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
if err := cleanupPreparedMusicAssets(prepared); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
return BuildResult{
|
||||
CreditsArtifactPaths: prepared.Artifacts,
|
||||
CreditsSummary: prepared.Summary,
|
||||
HAKAssets: len(prepared.Generated),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func prepareMusicAssets(p *project.Project) (*preparedMusicAssets, error) {
|
||||
return prepareMusicAssetsWithOptions(p, musicPrepareOptions{write: true})
|
||||
}
|
||||
|
||||
func prepareMusicAssetsWithOptions(p *project.Project, opts musicPrepareOptions) (*preparedMusicAssets, error) {
|
||||
usedNames := make(map[string]struct{})
|
||||
groupsByKey := make(map[string]*musicSourceGroup)
|
||||
datasets, err := effectiveMusicDatasets(p, opts.datasetIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
rel = filepath.ToSlash(rel)
|
||||
ext := strings.ToLower(filepath.Ext(rel))
|
||||
name := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
|
||||
if datasetID, dataset, ok := datasetForMusicSource(datasets, rel, ext); ok {
|
||||
sourceDir := filepath.ToSlash(filepath.Dir(rel))
|
||||
relDir, err := filepath.Rel(filepath.FromSlash(dataset.Source), filepath.FromSlash(sourceDir))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve music source directory %s: %w", sourceDir, err)
|
||||
}
|
||||
relDir = filepath.ToSlash(relDir)
|
||||
outputDir := dataset.Output
|
||||
if relDir != "." && relDir != "" {
|
||||
outputDir = filepath.ToSlash(filepath.Join(outputDir, filepath.FromSlash(relDir)))
|
||||
}
|
||||
key := datasetID + "\x00" + sourceDir
|
||||
group := groupsByKey[key]
|
||||
if group == nil {
|
||||
group = &musicSourceGroup{
|
||||
DatasetID: datasetID,
|
||||
Dataset: dataset,
|
||||
SourceDir: sourceDir,
|
||||
OutputDir: outputDir,
|
||||
}
|
||||
groupsByKey[key] = group
|
||||
}
|
||||
group.Sources = append(group.Sources, rel)
|
||||
continue
|
||||
}
|
||||
if name != "" {
|
||||
usedNames[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
result := &preparedMusicAssets{
|
||||
SkipSourceRel: make(map[string]struct{}),
|
||||
}
|
||||
if len(groupsByKey) == 0 {
|
||||
if !opts.write {
|
||||
return result, nil
|
||||
}
|
||||
writeResult, err := writeCreditsArtifacts(p, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Artifacts = writeResult.Artifacts
|
||||
result.Summary = writeResult.Summary
|
||||
return result, nil
|
||||
}
|
||||
|
||||
groups := make([]*musicSourceGroup, 0, len(groupsByKey))
|
||||
for _, group := range groupsByKey {
|
||||
sort.Strings(group.Sources)
|
||||
groups = append(groups, group)
|
||||
}
|
||||
sort.Slice(groups, func(i, j int) bool {
|
||||
if groups[i].SourceDir != groups[j].SourceDir {
|
||||
return groups[i].SourceDir < groups[j].SourceDir
|
||||
}
|
||||
return groups[i].DatasetID < groups[j].DatasetID
|
||||
})
|
||||
|
||||
creditGroups := make([]musicCreditGroup, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
result.Summary.ScannedDirs = append(result.Summary.ScannedDirs, group.SourceDir)
|
||||
result.Summary.TrackCount += len(group.Sources)
|
||||
}
|
||||
ffmpegPath := ""
|
||||
ffprobePath := ""
|
||||
if opts.write {
|
||||
var err error
|
||||
ffmpegPath, err = music.ResolveFFmpegWithConfig(p.MusicFFmpegPath())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ffprobePath, err = music.ResolveFFprobeWithConfig(p.MusicFFprobePath())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
for _, group := range groups {
|
||||
stageRoot := musicRootPath(p, group.Dataset.StageRoot)
|
||||
result.TempRoots = append(result.TempRoots, stageRoot)
|
||||
prefix := music.SanitizePrefix(group.Dataset.Prefix)
|
||||
overlayPath := filepath.Join(p.AssetsDir(), filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay)
|
||||
overlay, err := music.ParseCreditsOverlay(overlayPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse credits overlay %s: %w", overlayPath, err)
|
||||
}
|
||||
|
||||
entries := make([]music.CreditsEntry, 0, len(group.Sources))
|
||||
for _, rel := range group.Sources {
|
||||
base := filepath.Base(rel)
|
||||
manual := overlay.Match(base, "")
|
||||
outputFile := ""
|
||||
switch {
|
||||
case manual != nil && strings.TrimSpace(manual.OutputFile) != "":
|
||||
outputFile = strings.ToLower(strings.TrimSpace(manual.OutputFile))
|
||||
if err := music.ValidateManualOutputFileWithRules(outputFile, group.Dataset.OutputExtension, group.Dataset.MaxStemLength); err != nil {
|
||||
return nil, fmt.Errorf("manual credits output for %s: %w", rel, err)
|
||||
}
|
||||
if err := music.ReserveName(strings.TrimSuffix(outputFile, filepath.Ext(outputFile)), usedNames); err != nil {
|
||||
return nil, fmt.Errorf("manual credits output for %s: %w", rel, err)
|
||||
}
|
||||
default:
|
||||
stem, err := music.GenerateStemWithMax(base, prefix, group.Dataset.MaxStemLength, usedNames)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate music name for %s: %w", rel, err)
|
||||
}
|
||||
outputFile = stem + group.Dataset.OutputExtension
|
||||
}
|
||||
outputRel := filepath.ToSlash(filepath.Join(group.OutputDir, outputFile))
|
||||
result.Summary.Mappings = append(result.Summary.Mappings, MusicFileMapping{
|
||||
Source: rel,
|
||||
Output: outputRel,
|
||||
})
|
||||
result.SkipSourceRel[rel] = struct{}{}
|
||||
|
||||
if !opts.write {
|
||||
if group.Dataset.PackageMode == "hak_asset" {
|
||||
asset, err := plannedMusicAsset(outputRel, rel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.Generated = append(result.Generated, asset)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
metadata, err := music.ProbeMetadata(ffprobePath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("probe music metadata for %s: %w", rel, err)
|
||||
}
|
||||
entry := music.CreditsEntry{
|
||||
Artist: metadata.Artist,
|
||||
Title: music.FirstNonEmpty(metadata.Title, strings.TrimSuffix(base, filepath.Ext(base))),
|
||||
OutputFile: outputFile,
|
||||
OriginalFile: base,
|
||||
Album: metadata.Album,
|
||||
Date: metadata.Date,
|
||||
Rights: music.FirstNonEmpty(metadata.Rights, music.JoinNonEmpty("; ", metadata.License, metadata.Copyright)),
|
||||
Notes: music.FirstNonEmpty(metadata.Notes, metadata.Comment),
|
||||
}
|
||||
music.ApplyCreditsOverlay(&entry, manual)
|
||||
|
||||
stagePath := filepath.Join(stageRoot, filepath.FromSlash(outputRel))
|
||||
if err := os.MkdirAll(filepath.Dir(stagePath), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create music stage dir: %w", err)
|
||||
}
|
||||
if err := music.ConvertSource(ffmpegPath, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)), stagePath); err != nil {
|
||||
return nil, fmt.Errorf("convert music %s: %w", rel, err)
|
||||
}
|
||||
if group.Dataset.PackageMode == "hak_asset" {
|
||||
resourceInfo, err := assetResourceFromPath(stagePath, outputRel, lfsAssetInfo{}, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load converted music %s: %w", outputRel, err)
|
||||
}
|
||||
info, err := os.Stat(stagePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat converted music %s: %w", stagePath, err)
|
||||
}
|
||||
result.Generated = append(result.Generated, assetResource{
|
||||
Rel: outputRel,
|
||||
Resource: resourceInfo.Resource,
|
||||
Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}),
|
||||
CreatedAt: info.ModTime(),
|
||||
ContentID: resourceInfo.ContentID,
|
||||
})
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if opts.write {
|
||||
if err := music.ValidateOverlayEntries(group.SourceDir, overlay, entries); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
creditGroups = append(creditGroups, musicCreditGroup{
|
||||
CreditsRoot: group.Dataset.CreditsRoot,
|
||||
SourceDir: group.SourceDir,
|
||||
Entries: entries,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.write {
|
||||
sort.Slice(result.Generated, func(i, j int) bool {
|
||||
return compareResourceKeys(result.Generated[i].Resource, result.Generated[j].Resource) < 0
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
writeResult, err := writeCreditsArtifacts(p, creditGroups)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writeResult.Summary.ScannedDirs = append(writeResult.Summary.ScannedDirs, result.Summary.ScannedDirs...)
|
||||
writeResult.Summary.TrackCount = result.Summary.TrackCount
|
||||
result.Artifacts = writeResult.Artifacts
|
||||
result.Summary = writeResult.Summary
|
||||
sort.Slice(result.Generated, func(i, j int) bool {
|
||||
return compareResourceKeys(result.Generated[i].Resource, result.Generated[j].Resource) < 0
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type creditsArtifactWriteResult struct {
|
||||
Artifacts []string
|
||||
Summary CreditsRefreshSummary
|
||||
}
|
||||
|
||||
func writeCreditsArtifacts(p *project.Project, generated []musicCreditGroup) (creditsArtifactWriteResult, error) {
|
||||
defaultCreditsRoot := p.CreditsDir()
|
||||
sources := make([]music.CreditsSource, 0)
|
||||
desiredByRoot := map[string]map[string][]byte{}
|
||||
summary := CreditsRefreshSummary{}
|
||||
if generated != nil {
|
||||
sort.Slice(generated, func(i, j int) bool {
|
||||
if generated[i].SourceDir != generated[j].SourceDir {
|
||||
return generated[i].SourceDir < generated[j].SourceDir
|
||||
}
|
||||
return generated[i].CreditsRoot < generated[j].CreditsRoot
|
||||
})
|
||||
for _, group := range generated {
|
||||
creditsRoot := musicRootPath(p, group.CreditsRoot)
|
||||
if desiredByRoot[creditsRoot] == nil {
|
||||
desiredByRoot[creditsRoot] = map[string][]byte{}
|
||||
}
|
||||
entries := append([]music.CreditsEntry(nil), group.Entries...)
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
if strings.ToLower(entries[i].Artist) != strings.ToLower(entries[j].Artist) {
|
||||
return strings.ToLower(entries[i].Artist) < strings.ToLower(entries[j].Artist)
|
||||
}
|
||||
if strings.ToLower(entries[i].Title) != strings.ToLower(entries[j].Title) {
|
||||
return strings.ToLower(entries[i].Title) < strings.ToLower(entries[j].Title)
|
||||
}
|
||||
return strings.ToLower(entries[i].OutputFile) < strings.ToLower(entries[j].OutputFile)
|
||||
})
|
||||
outputPath := filepath.Join(creditsRoot, filepath.FromSlash(group.SourceDir), p.EffectiveConfig().Music.CreditsOverlay)
|
||||
desiredByRoot[creditsRoot][outputPath] = []byte(music.RenderCreditsMarkdown(entries))
|
||||
summary.GeneratedPaths = append(summary.GeneratedPaths, outputPath)
|
||||
for _, entry := range entries {
|
||||
summary.Mappings = append(summary.Mappings, MusicFileMapping{
|
||||
Source: entry.OriginalFile,
|
||||
Output: entry.OutputFile,
|
||||
})
|
||||
}
|
||||
sources = append(sources, music.CreditsSource{
|
||||
Path: filepath.ToSlash(outputPath),
|
||||
Kind: "generated_music",
|
||||
Entries: entries,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
err := filepath.WalkDir(p.AssetsDir(), func(path string, d os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if strings.ToLower(d.Name()) != "credits.md" {
|
||||
return nil
|
||||
}
|
||||
entries, err := music.ParseCreditsMarkdown(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse credits %s: %w", path, err)
|
||||
}
|
||||
rel, err := filepath.Rel(p.Root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sources = append(sources, music.CreditsSource{
|
||||
Path: filepath.ToSlash(rel),
|
||||
Kind: "authored",
|
||||
Entries: entries,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return creditsArtifactWriteResult{}, err
|
||||
}
|
||||
|
||||
sort.Slice(sources, func(i, j int) bool { return sources[i].Path < sources[j].Path })
|
||||
payload, err := json.MarshalIndent(music.CreditsInventory{Sources: sources}, "", " ")
|
||||
if err != nil {
|
||||
return creditsArtifactWriteResult{}, fmt.Errorf("marshal credits inventory: %w", err)
|
||||
}
|
||||
payload = append(payload, '\n')
|
||||
if len(desiredByRoot) == 0 {
|
||||
desiredByRoot[defaultCreditsRoot] = map[string][]byte{}
|
||||
}
|
||||
artifacts := make([]string, 0)
|
||||
changedFiles := 0
|
||||
roots := make([]string, 0, len(desiredByRoot))
|
||||
for root := range desiredByRoot {
|
||||
roots = append(roots, root)
|
||||
}
|
||||
sort.Strings(roots)
|
||||
for _, creditsRoot := range roots {
|
||||
desiredFiles := desiredByRoot[creditsRoot]
|
||||
inventoryPath := filepath.Join(creditsRoot, "credits.json")
|
||||
desiredFiles[inventoryPath] = payload
|
||||
if err := os.MkdirAll(creditsRoot, 0o755); err != nil {
|
||||
return creditsArtifactWriteResult{}, fmt.Errorf("create credits dir: %w", err)
|
||||
}
|
||||
changed, err := music.SyncGeneratedCreditsArtifacts(creditsRoot, desiredFiles)
|
||||
if err != nil {
|
||||
return creditsArtifactWriteResult{}, err
|
||||
}
|
||||
changedFiles += changed
|
||||
for path := range desiredFiles {
|
||||
artifacts = append(artifacts, path)
|
||||
}
|
||||
if summary.InventoryPath == "" {
|
||||
summary.InventoryPath = inventoryPath
|
||||
}
|
||||
}
|
||||
sort.Strings(artifacts)
|
||||
summary.ArtifactPaths = append(summary.ArtifactPaths, artifacts...)
|
||||
summary.ChangedFiles = changedFiles
|
||||
return creditsArtifactWriteResult{
|
||||
Artifacts: artifacts,
|
||||
Summary: summary,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func cleanupPreparedMusicAssets(prepared *preparedMusicAssets) error {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
}
|
||||
for _, root := range prepared.TempRoots {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(root); err != nil {
|
||||
return fmt.Errorf("remove temporary music artifact root %s: %w", root, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user