Extraction Hardening
This commit is contained in:
@@ -49,7 +49,7 @@ The shared CLI exposes these project utilities:
|
|||||||
sow-toolkit build
|
sow-toolkit build
|
||||||
sow-toolkit build-module
|
sow-toolkit build-module
|
||||||
sow-toolkit build-haks [--hak <name>] [--archive <name>] [--source-manifest <path>] [--plan-only] [--skip-music] [--music-dataset <id>]
|
sow-toolkit build-haks [--hak <name>] [--archive <name>] [--source-manifest <path>] [--plan-only] [--skip-music] [--music-dataset <id>]
|
||||||
sow-toolkit extract [<module-or-hak-file> ...]
|
sow-toolkit extract [<build-relative-archive-or-glob> ...]
|
||||||
sow-toolkit validate
|
sow-toolkit validate
|
||||||
sow-toolkit compare
|
sow-toolkit compare
|
||||||
sow-toolkit apply-hak-manifest [<manifest-path>]
|
sow-toolkit apply-hak-manifest [<manifest-path>]
|
||||||
@@ -185,10 +185,12 @@ scripts:
|
|||||||
|
|
||||||
extract:
|
extract:
|
||||||
layout: nwn_canonical_json
|
layout: nwn_canonical_json
|
||||||
hak_discovery: build_glob # or configured_haks
|
archives:
|
||||||
|
- "{module.resref}.mod"
|
||||||
cleanup_stale: true
|
cleanup_stale: true
|
||||||
|
consume_archives: false
|
||||||
ignore_extensions: []
|
ignore_extensions: []
|
||||||
delete_module_archive_after_success: false
|
delete_module_archive_after_success: false # deprecated compatibility alias
|
||||||
|
|
||||||
topdata:
|
topdata:
|
||||||
build: "{paths.build}/topdata"
|
build: "{paths.build}/topdata"
|
||||||
@@ -222,11 +224,29 @@ cache paths must stay relative to the repository root unless a specific setting
|
|||||||
documents otherwise.
|
documents otherwise.
|
||||||
|
|
||||||
Extraction is deliberately guarded because it writes source files from binary
|
Extraction is deliberately guarded because it writes source files from binary
|
||||||
archives. Module resources require a configured `paths.source`; HAK assets
|
archives. `extract.archives` is a list of glob patterns relative to
|
||||||
require a configured `paths.assets`; both roots must resolve to safe subtrees.
|
`paths.build`; the default is only the configured module `.mod`. HAK extraction
|
||||||
If a root is unset or resolves to the repository root, extraction fails before
|
is opt-in, for example:
|
||||||
writing instead of creating extension directories like `mdl/` or `png/` at the
|
|
||||||
repo top level. Stale cleanup uses the same safe-root rule.
|
```yaml
|
||||||
|
extract:
|
||||||
|
archives:
|
||||||
|
- "{module.resref}.mod"
|
||||||
|
- "sow_over*.hak"
|
||||||
|
```
|
||||||
|
|
||||||
|
Module resources require a configured `paths.source`; HAK assets require a
|
||||||
|
configured `paths.assets`; both roots must resolve to safe subtrees. If a root
|
||||||
|
is unset or resolves to the repository root, extraction fails before writing
|
||||||
|
instead of creating extension directories like `mdl/` or `png/` at the repo top
|
||||||
|
level. `.erf` files are rejected by extraction because they do not contain
|
||||||
|
`module.ifo`, so they cannot safely update module-level area membership.
|
||||||
|
|
||||||
|
`consume_archives: true` deletes the selected build archive files only after the
|
||||||
|
entire extraction succeeds. The older `delete_module_archive_after_success`
|
||||||
|
setting remains as a module-only compatibility alias when `consume_archives` is
|
||||||
|
not set. Stale cleanup is scoped to roots actually touched by extracted
|
||||||
|
resources, so the default module-only extraction does not prune assets.
|
||||||
|
|
||||||
Music defaults (used when not explicitly configured):
|
Music defaults (used when not explicitly configured):
|
||||||
|
|
||||||
|
|||||||
+5
-1
@@ -1006,8 +1006,12 @@ func configValidationRules(key string) []string {
|
|||||||
return []string{"generated output names must use the configured extension and must not escape the repository root"}
|
return []string{"generated output names must use the configured extension and must not escape the repository root"}
|
||||||
case key == "extract.layout":
|
case key == "extract.layout":
|
||||||
return []string{"supported value: nwn_canonical_json"}
|
return []string{"supported value: nwn_canonical_json"}
|
||||||
|
case key == "extract.archives":
|
||||||
|
return []string{"build-relative .mod/.hak glob patterns; .erf is intentionally rejected"}
|
||||||
|
case key == "extract.consume_archives":
|
||||||
|
return []string{"when true, selected build archives are deleted only after successful extraction"}
|
||||||
case key == "extract.hak_discovery":
|
case key == "extract.hak_discovery":
|
||||||
return []string{"supported values: build_glob, configured_haks"}
|
return []string{"deprecated; use extract.archives instead"}
|
||||||
case key == "autogen.cache.max_age":
|
case key == "autogen.cache.max_age":
|
||||||
return []string{"Go duration string, for example 1h or 30m"}
|
return []string{"Go duration string, for example 1h or 30m"}
|
||||||
case strings.HasPrefix(key, "topdata.package_hak"):
|
case strings.HasPrefix(key, "topdata.package_hak"):
|
||||||
|
|||||||
+181
-76
@@ -25,71 +25,60 @@ type ExtractResult struct {
|
|||||||
Skipped int
|
Skipped int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type extractionArchive struct {
|
||||||
|
Path string
|
||||||
|
Rel string
|
||||||
|
Extension string
|
||||||
|
}
|
||||||
|
|
||||||
|
type extractionScope struct {
|
||||||
|
Source bool
|
||||||
|
Assets bool
|
||||||
|
}
|
||||||
|
|
||||||
func Extract(p *project.Project, files ...string) (ExtractResult, error) {
|
func Extract(p *project.Project, files ...string) (ExtractResult, error) {
|
||||||
var result ExtractResult
|
var result ExtractResult
|
||||||
var failures []error
|
var failures []error
|
||||||
desired := map[string]struct{}{}
|
desired := map[string]struct{}{}
|
||||||
|
scope := extractionScope{}
|
||||||
|
|
||||||
allowed := make(map[string]bool)
|
archives, err := resolveExtractionArchives(p, files)
|
||||||
for _, f := range files {
|
|
||||||
allowed[f] = true
|
|
||||||
}
|
|
||||||
|
|
||||||
shouldExtractMod := len(allowed) == 0 || allowed[p.Config.Module.ResRef+".mod"]
|
|
||||||
if shouldExtractMod {
|
|
||||||
modulePath := p.ModuleArchivePath()
|
|
||||||
input, err := os.Open(modulePath)
|
|
||||||
if err != nil {
|
|
||||||
return ExtractResult{}, fmt.Errorf("open module archive: %w", err)
|
|
||||||
}
|
|
||||||
defer input.Close()
|
|
||||||
|
|
||||||
archive, err := erf.Read(input)
|
|
||||||
if err != nil {
|
|
||||||
return ExtractResult{}, fmt.Errorf("read module archive: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result.ModulePath = modulePath
|
|
||||||
written, overwritten, skipped, errs := extractArchiveResources(p, archive, desired)
|
|
||||||
result.Written += written
|
|
||||||
result.Overwritten += overwritten
|
|
||||||
result.Skipped += skipped
|
|
||||||
failures = append(failures, errs...)
|
|
||||||
}
|
|
||||||
|
|
||||||
hakPaths, err := extractHAKPaths(p)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return result, fmt.Errorf("scan hak archives: %w", err)
|
return result, err
|
||||||
}
|
}
|
||||||
slices.Sort(hakPaths)
|
for _, archivePath := range archives {
|
||||||
for _, hakPath := range hakPaths {
|
input, err := os.Open(archivePath.Path)
|
||||||
filename := filepath.Base(hakPath)
|
|
||||||
if len(allowed) > 0 && !allowed[filename] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
input, err := os.Open(hakPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
failures = append(failures, fmt.Errorf("open hak archive %s: %w", hakPath, err))
|
failures = append(failures, fmt.Errorf("open archive %s: %w", archivePath.Path, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
hakArchive, err := erf.Read(input)
|
archive, err := erf.Read(input)
|
||||||
input.Close()
|
input.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
failures = append(failures, fmt.Errorf("read hak archive %s: %w", hakPath, err))
|
failures = append(failures, fmt.Errorf("read archive %s: %w", archivePath.Path, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
result.HAKPaths = append(result.HAKPaths, hakPath)
|
switch archivePath.Extension {
|
||||||
written, overwritten, skipped, errs := extractArchiveResources(p, hakArchive, desired)
|
case ".mod":
|
||||||
|
if result.ModulePath == "" {
|
||||||
|
result.ModulePath = archivePath.Path
|
||||||
|
}
|
||||||
|
case ".hak":
|
||||||
|
result.HAKPaths = append(result.HAKPaths, archivePath.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
written, overwritten, skipped, extractedScope, errs := extractArchiveResources(p, archive, desired)
|
||||||
result.Written += written
|
result.Written += written
|
||||||
result.Overwritten += overwritten
|
result.Overwritten += overwritten
|
||||||
result.Skipped += skipped
|
result.Skipped += skipped
|
||||||
|
scope.Source = scope.Source || extractedScope.Source
|
||||||
|
scope.Assets = scope.Assets || extractedScope.Assets
|
||||||
failures = append(failures, errs...)
|
failures = append(failures, errs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale {
|
if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale {
|
||||||
removed, errs := cleanupStaleFiles(p, desired)
|
removed, errs := cleanupStaleFiles(p, desired, scope)
|
||||||
result.Removed = removed
|
result.Removed = removed
|
||||||
failures = append(failures, errs...)
|
failures = append(failures, errs...)
|
||||||
}
|
}
|
||||||
@@ -97,20 +86,27 @@ func Extract(p *project.Project, files ...string) (ExtractResult, error) {
|
|||||||
if len(failures) > 0 {
|
if len(failures) > 0 {
|
||||||
return result, errors.Join(failures...)
|
return result, errors.Join(failures...)
|
||||||
}
|
}
|
||||||
if p.Config.Extract.DeleteModuleArchiveAfterSuccess && result.ModulePath != "" {
|
consumeAll, consumeModules := extractionConsumePolicy(p)
|
||||||
if err := os.Remove(result.ModulePath); err != nil {
|
if consumeAll || consumeModules {
|
||||||
return result, fmt.Errorf("delete consumed module archive %s: %w", result.ModulePath, err)
|
for _, archivePath := range archives {
|
||||||
|
if !consumeAll && archivePath.Extension != ".mod" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := os.Remove(archivePath.Path); err != nil {
|
||||||
|
return result, fmt.Errorf("delete consumed archive %s: %w", archivePath.Path, err)
|
||||||
|
}
|
||||||
|
result.DeletedArchivePaths = append(result.DeletedArchivePaths, archivePath.Path)
|
||||||
}
|
}
|
||||||
result.DeletedArchivePaths = append(result.DeletedArchivePaths, result.ModulePath)
|
|
||||||
}
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, []error) {
|
func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, extractionScope, []error) {
|
||||||
var failures []error
|
var failures []error
|
||||||
writtenCount := 0
|
writtenCount := 0
|
||||||
overwrittenCount := 0
|
overwrittenCount := 0
|
||||||
skippedCount := 0
|
skippedCount := 0
|
||||||
|
scope := extractionScope{}
|
||||||
|
|
||||||
ignored := make(map[string]bool)
|
ignored := make(map[string]bool)
|
||||||
for _, ext := range p.Config.Extract.IgnoreExtensions {
|
for _, ext := range p.Config.Extract.IgnoreExtensions {
|
||||||
@@ -135,6 +131,8 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
desired[target] = struct{}{}
|
desired[target] = struct{}{}
|
||||||
|
scope.Source = scope.Source || pathWithinRoot(target, p.SourceDir())
|
||||||
|
scope.Assets = scope.Assets || pathWithinRoot(target, p.AssetsDir())
|
||||||
|
|
||||||
state, err := writeManagedFile(target, data)
|
state, err := writeManagedFile(target, data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -151,7 +149,138 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return writtenCount, overwrittenCount, skippedCount, failures
|
return writtenCount, overwrittenCount, skippedCount, scope, failures
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveExtractionArchives(p *project.Project, overrides []string) ([]extractionArchive, error) {
|
||||||
|
patterns := p.EffectiveConfig().Extract.Archives
|
||||||
|
if len(overrides) > 0 {
|
||||||
|
patterns = overrides
|
||||||
|
}
|
||||||
|
if len(patterns) == 0 {
|
||||||
|
return nil, fmt.Errorf("extract.archives must contain at least one build-relative archive pattern")
|
||||||
|
}
|
||||||
|
|
||||||
|
buildRoot := filepath.Clean(p.BuildDir())
|
||||||
|
byPath := map[string]extractionArchive{}
|
||||||
|
for _, rawPattern := range patterns {
|
||||||
|
pattern, err := normalizeExtractionArchivePattern(p, rawPattern)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
matched, err := matchExtractionArchives(buildRoot, pattern)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(matched) == 0 {
|
||||||
|
return nil, fmt.Errorf("extract archive pattern %q matched no files under %s", rawPattern, buildRoot)
|
||||||
|
}
|
||||||
|
for _, archive := range matched {
|
||||||
|
byPath[archive.Path] = archive
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
archives := make([]extractionArchive, 0, len(byPath))
|
||||||
|
for _, archive := range byPath {
|
||||||
|
archives = append(archives, archive)
|
||||||
|
}
|
||||||
|
slices.SortFunc(archives, func(a, b extractionArchive) int {
|
||||||
|
return strings.Compare(a.Rel, b.Rel)
|
||||||
|
})
|
||||||
|
return archives, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeExtractionArchivePattern(p *project.Project, raw string) (string, error) {
|
||||||
|
pattern := strings.TrimSpace(raw)
|
||||||
|
if pattern == "" {
|
||||||
|
return "", fmt.Errorf("extract archive pattern must not be empty")
|
||||||
|
}
|
||||||
|
pattern = strings.NewReplacer(
|
||||||
|
"{module.resref}", strings.TrimSpace(p.Config.Module.ResRef),
|
||||||
|
).Replace(pattern)
|
||||||
|
if strings.Contains(pattern, "\x00") {
|
||||||
|
return "", fmt.Errorf("extract archive pattern %q must not contain NUL bytes", raw)
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(pattern) {
|
||||||
|
return "", fmt.Errorf("extract archive pattern %q must be relative to paths.build", raw)
|
||||||
|
}
|
||||||
|
clean := filepath.Clean(filepath.FromSlash(pattern))
|
||||||
|
if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||||
|
return "", fmt.Errorf("extract archive pattern %q must not escape paths.build", raw)
|
||||||
|
}
|
||||||
|
return filepath.ToSlash(clean), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func matchExtractionArchives(buildRoot, pattern string) ([]extractionArchive, error) {
|
||||||
|
var archives []extractionArchive
|
||||||
|
err := filepath.WalkDir(buildRoot, func(path string, entry os.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if entry.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(buildRoot, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rel = filepath.ToSlash(rel)
|
||||||
|
if !matchPathPattern(rel, pattern) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
archive, err := validateExtractionArchive(buildRoot, rel)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
archives = append(archives, archive)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, fmt.Errorf("scan build directory %s: %w", buildRoot, err)
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return archives, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateExtractionArchive(buildRoot, rel string) (extractionArchive, error) {
|
||||||
|
cleanRel := filepath.Clean(filepath.FromSlash(rel))
|
||||||
|
if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(filepath.Separator)) {
|
||||||
|
return extractionArchive{}, fmt.Errorf("extract archive %q must not escape paths.build", rel)
|
||||||
|
}
|
||||||
|
path := filepath.Join(buildRoot, cleanRel)
|
||||||
|
if !pathWithinRoot(path, buildRoot) {
|
||||||
|
return extractionArchive{}, fmt.Errorf("extract archive %q resolves outside paths.build", rel)
|
||||||
|
}
|
||||||
|
extension := strings.ToLower(filepath.Ext(cleanRel))
|
||||||
|
switch extension {
|
||||||
|
case ".mod", ".hak":
|
||||||
|
return extractionArchive{Path: path, Rel: filepath.ToSlash(cleanRel), Extension: extension}, nil
|
||||||
|
case ".erf":
|
||||||
|
return extractionArchive{}, fmt.Errorf("extract archive %q is an .erf; ERF extraction is unsafe because ERFs lack module.ifo context", rel)
|
||||||
|
default:
|
||||||
|
return extractionArchive{}, fmt.Errorf("extract archive %q uses unsupported extension %q", rel, extension)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractionConsumePolicy(p *project.Project) (consumeAll bool, consumeModules bool) {
|
||||||
|
if configured := p.Config.Extract.ConsumeArchives; configured != nil {
|
||||||
|
return *configured, false
|
||||||
|
}
|
||||||
|
return false, p.Config.Extract.DeleteModuleArchiveAfterSuccess
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathWithinRoot(path, root string) bool {
|
||||||
|
root = filepath.Clean(root)
|
||||||
|
if root == "" || root == "." {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(root, path)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel)
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) {
|
func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) {
|
||||||
@@ -211,30 +340,6 @@ func extractionTarget(p *project.Project, field, configured, root string, parts
|
|||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractHAKPaths(p *project.Project) ([]string, error) {
|
|
||||||
switch p.EffectiveConfig().Extract.HAKDiscovery {
|
|
||||||
case "configured_haks":
|
|
||||||
paths := make([]string, 0, len(p.Config.HAKs))
|
|
||||||
for _, hak := range p.Config.HAKs {
|
|
||||||
path := p.HAKArchivePath(hak.Name)
|
|
||||||
if _, err := os.Stat(path); err == nil {
|
|
||||||
paths = append(paths, path)
|
|
||||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
slices.Sort(paths)
|
|
||||||
return paths, nil
|
|
||||||
default:
|
|
||||||
hakPaths, err := filepath.Glob(filepath.Join(p.BuildDir(), "*.hak"))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
slices.Sort(hakPaths)
|
|
||||||
return hakPaths, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type writeState int
|
type writeState int
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -268,9 +373,9 @@ func writeManagedFile(path string, data []byte) (writeState, error) {
|
|||||||
return writeNew, nil
|
return writeNew, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func cleanupStaleFiles(p *project.Project, desired map[string]struct{}) (int, []error) {
|
func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) {
|
||||||
candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
|
candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
|
||||||
if safeCleanupRoot(p.SourceDir(), p.Root) {
|
if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) {
|
||||||
for _, rel := range p.Inventory.SourceFiles {
|
for _, rel := range p.Inventory.SourceFiles {
|
||||||
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
||||||
}
|
}
|
||||||
@@ -278,7 +383,7 @@ func cleanupStaleFiles(p *project.Project, desired map[string]struct{}) (int, []
|
|||||||
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if safeCleanupRoot(p.AssetsDir(), p.Root) {
|
if scope.Assets && safeCleanupRoot(p.AssetsDir(), p.Root) {
|
||||||
for _, rel := range p.Inventory.AssetFiles {
|
for _, rel := range p.Inventory.AssetFiles {
|
||||||
candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
|
candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -120,6 +120,9 @@ func TestExtractReadsHAKAssets(t *testing.T) {
|
|||||||
"source": "src",
|
"source": "src",
|
||||||
"assets": "assets",
|
"assets": "assets",
|
||||||
"build": "build"
|
"build": "build"
|
||||||
|
},
|
||||||
|
"extract": {
|
||||||
|
"archives": ["*.hak"]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
`)
|
`)
|
||||||
@@ -175,6 +178,281 @@ func TestExtractReadsHAKAssets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtractDefaultOnlyReadsConfiguredModuleArchive(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
mustMkdir(t, filepath.Join(root, "src", "module"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "assets", "mdl"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "build"))
|
||||||
|
|
||||||
|
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||||
|
module:
|
||||||
|
name: Test Module
|
||||||
|
resref: testmod
|
||||||
|
paths:
|
||||||
|
source: src
|
||||||
|
assets: assets
|
||||||
|
build: build
|
||||||
|
`)
|
||||||
|
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
|
||||||
|
"file_type": "IFO ",
|
||||||
|
"file_version": "V3.2",
|
||||||
|
"root": {
|
||||||
|
"struct_type": 0,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"label": "Mod_Name",
|
||||||
|
"type": "CExoString",
|
||||||
|
"value": "Original Module"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`)
|
||||||
|
mustWriteFile(t, filepath.Join(root, "assets", "mdl", "stale_asset.mdl"), "keep")
|
||||||
|
|
||||||
|
p, err := project.Load(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load project: %v", err)
|
||||||
|
}
|
||||||
|
if err := p.ValidateLayout(); err != nil {
|
||||||
|
t.Fatalf("validate layout: %v", err)
|
||||||
|
}
|
||||||
|
if err := p.Scan(); err != nil {
|
||||||
|
t.Fatalf("scan: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := BuildModule(p); err != nil {
|
||||||
|
t.Fatalf("build module: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil {
|
||||||
|
t.Fatalf("remove source file: %v", err)
|
||||||
|
}
|
||||||
|
if err := p.Scan(); err != nil {
|
||||||
|
t.Fatalf("rescan before extract: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
hakFile, err := os.Create(filepath.Join(root, "build", "vfx.hak"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create hak: %v", err)
|
||||||
|
}
|
||||||
|
if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{
|
||||||
|
{Name: "test_vfx", Type: 0x07D2, Data: []byte("mdl-data")},
|
||||||
|
})); err != nil {
|
||||||
|
t.Fatalf("write hak: %v", err)
|
||||||
|
}
|
||||||
|
if err := hakFile.Close(); err != nil {
|
||||||
|
t.Fatalf("close hak: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := Extract(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("extract: %v", err)
|
||||||
|
}
|
||||||
|
if len(result.HAKPaths) != 0 {
|
||||||
|
t.Fatalf("expected default extract to ignore haks, got %#v", result.HAKPaths)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil {
|
||||||
|
t.Fatalf("expected module source to be restored: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "stale_asset.mdl")); err != nil {
|
||||||
|
t.Fatalf("expected asset cleanup to be skipped for module-only extract: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "test_vfx.mdl")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected hak asset not to be extracted by default, err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractRejectsERFArchiveTargets(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
mustMkdir(t, filepath.Join(root, "src"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "assets"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "build"))
|
||||||
|
|
||||||
|
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||||
|
module:
|
||||||
|
name: Test Module
|
||||||
|
resref: testmod
|
||||||
|
paths:
|
||||||
|
source: src
|
||||||
|
assets: assets
|
||||||
|
build: build
|
||||||
|
extract:
|
||||||
|
archives:
|
||||||
|
- imported.erf
|
||||||
|
`)
|
||||||
|
|
||||||
|
erfFile, err := os.Create(filepath.Join(root, "build", "imported.erf"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create erf: %v", err)
|
||||||
|
}
|
||||||
|
if err := erf.Write(erfFile, erf.New("ERF ", nil)); err != nil {
|
||||||
|
t.Fatalf("write erf: %v", err)
|
||||||
|
}
|
||||||
|
if err := erfFile.Close(); err != nil {
|
||||||
|
t.Fatalf("close erf: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
p, err := project.Load(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load project: %v", err)
|
||||||
|
}
|
||||||
|
_, err = Extract(p)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected erf extraction to fail")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "ERF extraction is unsafe") {
|
||||||
|
t.Fatalf("expected unsafe erf error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractRejectsEscapingArchiveOverride(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
mustMkdir(t, filepath.Join(root, "src"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "assets"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "build"))
|
||||||
|
|
||||||
|
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||||
|
module:
|
||||||
|
name: Test Module
|
||||||
|
resref: testmod
|
||||||
|
paths:
|
||||||
|
source: src
|
||||||
|
assets: assets
|
||||||
|
build: build
|
||||||
|
`)
|
||||||
|
|
||||||
|
p, err := project.Load(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load project: %v", err)
|
||||||
|
}
|
||||||
|
_, err = Extract(p, "../outside.mod")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected escaping archive override to fail")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "must not escape paths.build") {
|
||||||
|
t.Fatalf("expected escaping archive error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractConsumesConfiguredHAKArchiveAfterSuccess(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
mustMkdir(t, filepath.Join(root, "src"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "assets"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "build"))
|
||||||
|
|
||||||
|
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||||
|
module:
|
||||||
|
name: Test Module
|
||||||
|
resref: testmod
|
||||||
|
paths:
|
||||||
|
source: src
|
||||||
|
assets: assets
|
||||||
|
build: build
|
||||||
|
extract:
|
||||||
|
archives:
|
||||||
|
- vfx.hak
|
||||||
|
consume_archives: true
|
||||||
|
`)
|
||||||
|
|
||||||
|
hakPath := filepath.Join(root, "build", "vfx.hak")
|
||||||
|
hakFile, err := os.Create(hakPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create hak: %v", err)
|
||||||
|
}
|
||||||
|
if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{
|
||||||
|
{Name: "test_vfx", Type: 0x07D2, Data: []byte("mdl-data")},
|
||||||
|
})); err != nil {
|
||||||
|
t.Fatalf("write hak: %v", err)
|
||||||
|
}
|
||||||
|
if err := hakFile.Close(); err != nil {
|
||||||
|
t.Fatalf("close hak: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
p, err := project.Load(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load project: %v", err)
|
||||||
|
}
|
||||||
|
result, err := Extract(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("extract: %v", err)
|
||||||
|
}
|
||||||
|
if len(result.DeletedArchivePaths) != 1 || result.DeletedArchivePaths[0] != hakPath {
|
||||||
|
t.Fatalf("expected consumed hak path, got %#v", result.DeletedArchivePaths)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(hakPath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected hak archive to be deleted, stat err=%v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "test_vfx.mdl")); err != nil {
|
||||||
|
t.Fatalf("expected extracted hak asset: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractConsumeArchivesFalseOverridesLegacyDeleteFlag(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
mustMkdir(t, filepath.Join(root, "src", "module"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "assets"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "build"))
|
||||||
|
|
||||||
|
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||||
|
module:
|
||||||
|
name: Test Module
|
||||||
|
resref: testmod
|
||||||
|
paths:
|
||||||
|
source: src
|
||||||
|
assets: assets
|
||||||
|
build: build
|
||||||
|
extract:
|
||||||
|
consume_archives: false
|
||||||
|
delete_module_archive_after_success: true
|
||||||
|
`)
|
||||||
|
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
|
||||||
|
"file_type": "IFO ",
|
||||||
|
"file_version": "V3.2",
|
||||||
|
"root": {
|
||||||
|
"struct_type": 0,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"label": "Mod_Name",
|
||||||
|
"type": "CExoString",
|
||||||
|
"value": "Original Module"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`)
|
||||||
|
|
||||||
|
p, err := project.Load(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load project: %v", err)
|
||||||
|
}
|
||||||
|
if err := p.ValidateLayout(); err != nil {
|
||||||
|
t.Fatalf("validate layout: %v", err)
|
||||||
|
}
|
||||||
|
if err := p.Scan(); err != nil {
|
||||||
|
t.Fatalf("scan: %v", err)
|
||||||
|
}
|
||||||
|
buildResult, err := BuildModule(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("build module: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil {
|
||||||
|
t.Fatalf("remove source file: %v", err)
|
||||||
|
}
|
||||||
|
if err := p.Scan(); err != nil {
|
||||||
|
t.Fatalf("rescan before extract: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := Extract(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("extract: %v", err)
|
||||||
|
}
|
||||||
|
if len(result.DeletedArchivePaths) != 0 {
|
||||||
|
t.Fatalf("expected no deleted archives, got %#v", result.DeletedArchivePaths)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(buildResult.ModulePath); err != nil {
|
||||||
|
t.Fatalf("expected module archive to remain, stat err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExtractRefusesAssetsWhenAssetsPathIsUnset(t *testing.T) {
|
func TestExtractRefusesAssetsWhenAssetsPathIsUnset(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
t.Chdir(root)
|
t.Chdir(root)
|
||||||
@@ -226,7 +504,7 @@ paths:
|
|||||||
t.Fatalf("close hak: %v", err)
|
t.Fatalf("close hak: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = Extract(p)
|
_, err = Extract(p, "vfx.hak")
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected extract to fail")
|
t.Fatal("expected extract to fail")
|
||||||
}
|
}
|
||||||
@@ -285,7 +563,7 @@ paths:
|
|||||||
t.Fatalf("close hak: %v", err)
|
t.Fatalf("close hak: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = Extract(p)
|
_, err = Extract(p, "vfx.hak")
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected extract to fail")
|
t.Fatal("expected extract to fail")
|
||||||
}
|
}
|
||||||
@@ -357,7 +635,8 @@ paths:
|
|||||||
assets: assets
|
assets: assets
|
||||||
build: build
|
build: build
|
||||||
extract:
|
extract:
|
||||||
hak_discovery: configured_haks
|
archives:
|
||||||
|
- wanted.hak
|
||||||
haks:
|
haks:
|
||||||
- name: wanted
|
- name: wanted
|
||||||
priority: 1
|
priority: 1
|
||||||
|
|||||||
@@ -231,10 +231,17 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
|
|||||||
extract := p.Config.Extract
|
extract := p.Config.Extract
|
||||||
extract.Layout = defaultString(extract.Layout, DefaultExtractLayout)
|
extract.Layout = defaultString(extract.Layout, DefaultExtractLayout)
|
||||||
extract.HAKDiscovery = defaultString(extract.HAKDiscovery, DefaultExtractHAKDiscovery)
|
extract.HAKDiscovery = defaultString(extract.HAKDiscovery, DefaultExtractHAKDiscovery)
|
||||||
|
if len(extract.Archives) == 0 {
|
||||||
|
extract.Archives = []string{outputs.ModuleArchiveName(p.Config.Module)}
|
||||||
|
}
|
||||||
if extract.CleanupStale == nil {
|
if extract.CleanupStale == nil {
|
||||||
defaultCleanup := true
|
defaultCleanup := true
|
||||||
extract.CleanupStale = &defaultCleanup
|
extract.CleanupStale = &defaultCleanup
|
||||||
}
|
}
|
||||||
|
if extract.ConsumeArchives == nil {
|
||||||
|
defaultConsume := false
|
||||||
|
extract.ConsumeArchives = &defaultConsume
|
||||||
|
}
|
||||||
|
|
||||||
musicStageRoot := expandPathTemplate(DefaultMusicStageRoot, paths)
|
musicStageRoot := expandPathTemplate(DefaultMusicStageRoot, paths)
|
||||||
musicCreditsRoot := expandPathTemplate(DefaultCreditsRoot, paths)
|
musicCreditsRoot := expandPathTemplate(DefaultCreditsRoot, paths)
|
||||||
@@ -435,7 +442,9 @@ func markMissingDefaults(provenance ConfigProvenance) {
|
|||||||
"topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary,
|
"topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary,
|
||||||
"extract.layout": DefaultExtractLayout,
|
"extract.layout": DefaultExtractLayout,
|
||||||
"extract.hak_discovery": DefaultExtractHAKDiscovery,
|
"extract.hak_discovery": DefaultExtractHAKDiscovery,
|
||||||
|
"extract.archives": DefaultModuleArchiveTemplate,
|
||||||
"extract.cleanup_stale": "true",
|
"extract.cleanup_stale": "true",
|
||||||
|
"extract.consume_archives": "false",
|
||||||
"autogen.cache.root": DefaultAutogenCacheRoot,
|
"autogen.cache.root": DefaultAutogenCacheRoot,
|
||||||
"autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(),
|
"autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(),
|
||||||
"autogen.cache.refresh_env": DefaultAutogenRefreshEnv,
|
"autogen.cache.refresh_env": DefaultAutogenRefreshEnv,
|
||||||
|
|||||||
@@ -214,9 +214,11 @@ type TopDataWikiConfig struct {
|
|||||||
|
|
||||||
type ExtractConfig struct {
|
type ExtractConfig struct {
|
||||||
IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"`
|
IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"`
|
||||||
|
Archives []string `json:"archives,omitempty" yaml:"archives,omitempty"`
|
||||||
Layout string `json:"layout" yaml:"layout"`
|
Layout string `json:"layout" yaml:"layout"`
|
||||||
HAKDiscovery string `json:"hak_discovery" yaml:"hak_discovery"`
|
HAKDiscovery string `json:"hak_discovery" yaml:"hak_discovery"`
|
||||||
CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"`
|
CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"`
|
||||||
|
ConsumeArchives *bool `json:"consume_archives,omitempty" yaml:"consume_archives,omitempty"`
|
||||||
DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"`
|
DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -506,6 +508,7 @@ func (p *Project) ValidateLayout() error {
|
|||||||
default:
|
default:
|
||||||
failures = append(failures, fmt.Errorf("extract.hak_discovery %q is not supported", effective.Extract.HAKDiscovery))
|
failures = append(failures, fmt.Errorf("extract.hak_discovery %q is not supported", effective.Extract.HAKDiscovery))
|
||||||
}
|
}
|
||||||
|
failures = append(failures, validateGlobList("extract.archives", effective.Extract.Archives)...)
|
||||||
if _, err := time.ParseDuration(effective.Autogen.Cache.MaxAge); err != nil {
|
if _, err := time.ParseDuration(effective.Autogen.Cache.MaxAge); err != nil {
|
||||||
failures = append(failures, fmt.Errorf("autogen.cache.max_age must be a duration: %w", err))
|
failures = append(failures, fmt.Errorf("autogen.cache.max_age must be a duration: %w", err))
|
||||||
}
|
}
|
||||||
@@ -963,6 +966,7 @@ func normalizeConfig(cfg *Config) {
|
|||||||
cfg.HAKs[i].Include = normalizeStringSlice(cfg.HAKs[i].Include)
|
cfg.HAKs[i].Include = normalizeStringSlice(cfg.HAKs[i].Include)
|
||||||
}
|
}
|
||||||
cfg.Extract.IgnoreExtensions = normalizeStringSlice(cfg.Extract.IgnoreExtensions)
|
cfg.Extract.IgnoreExtensions = normalizeStringSlice(cfg.Extract.IgnoreExtensions)
|
||||||
|
cfg.Extract.Archives = normalizeStringSlice(cfg.Extract.Archives)
|
||||||
for i := range cfg.Autogen.Producers {
|
for i := range cfg.Autogen.Producers {
|
||||||
cfg.Autogen.Producers[i].Include = normalizeStringSlice(cfg.Autogen.Producers[i].Include)
|
cfg.Autogen.Producers[i].Include = normalizeStringSlice(cfg.Autogen.Producers[i].Include)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -66,6 +66,12 @@ paths:
|
|||||||
if got, want := effective.Outputs.HAKManifest, "haks.json"; got != want {
|
if got, want := effective.Outputs.HAKManifest, "haks.json"; got != want {
|
||||||
t.Fatalf("expected default HAK manifest %q, got %q", want, got)
|
t.Fatalf("expected default HAK manifest %q, got %q", want, got)
|
||||||
}
|
}
|
||||||
|
if got, want := strings.Join(effective.Extract.Archives, ","), "testmod.mod"; got != want {
|
||||||
|
t.Fatalf("expected default extract archives %q, got %q", want, got)
|
||||||
|
}
|
||||||
|
if effective.Extract.ConsumeArchives == nil || *effective.Extract.ConsumeArchives {
|
||||||
|
t.Fatalf("expected default extract consume_archives false, got %#v", effective.Extract.ConsumeArchives)
|
||||||
|
}
|
||||||
if got, want := effective.TopData.PackageHAK, "sow_top.hak"; got != want {
|
if got, want := effective.TopData.PackageHAK, "sow_top.hak"; got != want {
|
||||||
t.Fatalf("expected default topdata HAK %q, got %q", want, got)
|
t.Fatalf("expected default topdata HAK %q, got %q", want, got)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user