diff --git a/README.md b/README.md index 75188c6..057a754 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The shared CLI exposes these project utilities: sow-toolkit build sow-toolkit build-module sow-toolkit build-haks [--hak ] [--archive ] [--source-manifest ] [--plan-only] [--skip-music] [--music-dataset ] -sow-toolkit extract [ ...] +sow-toolkit extract [ ...] sow-toolkit validate sow-toolkit compare sow-toolkit apply-hak-manifest [] @@ -185,10 +185,12 @@ scripts: extract: layout: nwn_canonical_json - hak_discovery: build_glob # or configured_haks + archives: + - "{module.resref}.mod" cleanup_stale: true + consume_archives: false ignore_extensions: [] - delete_module_archive_after_success: false + delete_module_archive_after_success: false # deprecated compatibility alias topdata: build: "{paths.build}/topdata" @@ -222,11 +224,29 @@ cache paths must stay relative to the repository root unless a specific setting documents otherwise. Extraction is deliberately guarded because it writes source files from binary -archives. Module resources require a configured `paths.source`; HAK assets -require a configured `paths.assets`; both roots must resolve to safe subtrees. -If a root is unset or resolves to the repository root, extraction fails before -writing instead of creating extension directories like `mdl/` or `png/` at the -repo top level. Stale cleanup uses the same safe-root rule. +archives. `extract.archives` is a list of glob patterns relative to +`paths.build`; the default is only the configured module `.mod`. HAK extraction +is opt-in, for example: + +```yaml +extract: + archives: + - "{module.resref}.mod" + - "sow_over*.hak" +``` + +Module resources require a configured `paths.source`; HAK assets require a +configured `paths.assets`; both roots must resolve to safe subtrees. If a root +is unset or resolves to the repository root, extraction fails before writing +instead of creating extension directories like `mdl/` or `png/` at the repo top +level. `.erf` files are rejected by extraction because they do not contain +`module.ifo`, so they cannot safely update module-level area membership. + +`consume_archives: true` deletes the selected build archive files only after the +entire extraction succeeds. The older `delete_module_archive_after_success` +setting remains as a module-only compatibility alias when `consume_archives` is +not set. Stale cleanup is scoped to roots actually touched by extracted +resources, so the default module-only extraction does not prune assets. Music defaults (used when not explicitly configured): diff --git a/internal/app/app.go b/internal/app/app.go index 04cf1d3..87658b0 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1006,8 +1006,12 @@ func configValidationRules(key string) []string { return []string{"generated output names must use the configured extension and must not escape the repository root"} case key == "extract.layout": return []string{"supported value: nwn_canonical_json"} + case key == "extract.archives": + return []string{"build-relative .mod/.hak glob patterns; .erf is intentionally rejected"} + case key == "extract.consume_archives": + return []string{"when true, selected build archives are deleted only after successful extraction"} case key == "extract.hak_discovery": - return []string{"supported values: build_glob, configured_haks"} + return []string{"deprecated; use extract.archives instead"} case key == "autogen.cache.max_age": return []string{"Go duration string, for example 1h or 30m"} case strings.HasPrefix(key, "topdata.package_hak"): diff --git a/internal/pipeline/extract.go b/internal/pipeline/extract.go index e52225d..e49e9d9 100644 --- a/internal/pipeline/extract.go +++ b/internal/pipeline/extract.go @@ -25,71 +25,60 @@ type ExtractResult struct { Skipped int } +type extractionArchive struct { + Path string + Rel string + Extension string +} + +type extractionScope struct { + Source bool + Assets bool +} + func Extract(p *project.Project, files ...string) (ExtractResult, error) { var result ExtractResult var failures []error desired := map[string]struct{}{} + scope := extractionScope{} - allowed := make(map[string]bool) - for _, f := range files { - allowed[f] = true - } - - shouldExtractMod := len(allowed) == 0 || allowed[p.Config.Module.ResRef+".mod"] - if shouldExtractMod { - modulePath := p.ModuleArchivePath() - input, err := os.Open(modulePath) - if err != nil { - return ExtractResult{}, fmt.Errorf("open module archive: %w", err) - } - defer input.Close() - - archive, err := erf.Read(input) - if err != nil { - return ExtractResult{}, fmt.Errorf("read module archive: %w", err) - } - - result.ModulePath = modulePath - written, overwritten, skipped, errs := extractArchiveResources(p, archive, desired) - result.Written += written - result.Overwritten += overwritten - result.Skipped += skipped - failures = append(failures, errs...) - } - - hakPaths, err := extractHAKPaths(p) + archives, err := resolveExtractionArchives(p, files) if err != nil { - return result, fmt.Errorf("scan hak archives: %w", err) + return result, err } - slices.Sort(hakPaths) - for _, hakPath := range hakPaths { - filename := filepath.Base(hakPath) - if len(allowed) > 0 && !allowed[filename] { - continue - } - - input, err := os.Open(hakPath) + for _, archivePath := range archives { + input, err := os.Open(archivePath.Path) if err != nil { - failures = append(failures, fmt.Errorf("open hak archive %s: %w", hakPath, err)) + failures = append(failures, fmt.Errorf("open archive %s: %w", archivePath.Path, err)) continue } - hakArchive, err := erf.Read(input) + archive, err := erf.Read(input) input.Close() if err != nil { - failures = append(failures, fmt.Errorf("read hak archive %s: %w", hakPath, err)) + failures = append(failures, fmt.Errorf("read archive %s: %w", archivePath.Path, err)) continue } - result.HAKPaths = append(result.HAKPaths, hakPath) - written, overwritten, skipped, errs := extractArchiveResources(p, hakArchive, desired) + switch archivePath.Extension { + case ".mod": + if result.ModulePath == "" { + result.ModulePath = archivePath.Path + } + case ".hak": + result.HAKPaths = append(result.HAKPaths, archivePath.Path) + } + + written, overwritten, skipped, extractedScope, errs := extractArchiveResources(p, archive, desired) result.Written += written result.Overwritten += overwritten result.Skipped += skipped + scope.Source = scope.Source || extractedScope.Source + scope.Assets = scope.Assets || extractedScope.Assets failures = append(failures, errs...) } if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale { - removed, errs := cleanupStaleFiles(p, desired) + removed, errs := cleanupStaleFiles(p, desired, scope) result.Removed = removed failures = append(failures, errs...) } @@ -97,20 +86,27 @@ func Extract(p *project.Project, files ...string) (ExtractResult, error) { if len(failures) > 0 { return result, errors.Join(failures...) } - if p.Config.Extract.DeleteModuleArchiveAfterSuccess && result.ModulePath != "" { - if err := os.Remove(result.ModulePath); err != nil { - return result, fmt.Errorf("delete consumed module archive %s: %w", result.ModulePath, err) + consumeAll, consumeModules := extractionConsumePolicy(p) + if consumeAll || consumeModules { + for _, archivePath := range archives { + if !consumeAll && archivePath.Extension != ".mod" { + continue + } + if err := os.Remove(archivePath.Path); err != nil { + return result, fmt.Errorf("delete consumed archive %s: %w", archivePath.Path, err) + } + result.DeletedArchivePaths = append(result.DeletedArchivePaths, archivePath.Path) } - result.DeletedArchivePaths = append(result.DeletedArchivePaths, result.ModulePath) } return result, nil } -func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, []error) { +func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, extractionScope, []error) { var failures []error writtenCount := 0 overwrittenCount := 0 skippedCount := 0 + scope := extractionScope{} ignored := make(map[string]bool) for _, ext := range p.Config.Extract.IgnoreExtensions { @@ -135,6 +131,8 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma continue } desired[target] = struct{}{} + scope.Source = scope.Source || pathWithinRoot(target, p.SourceDir()) + scope.Assets = scope.Assets || pathWithinRoot(target, p.AssetsDir()) state, err := writeManagedFile(target, data) if err != nil { @@ -151,7 +149,138 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma } } - return writtenCount, overwrittenCount, skippedCount, failures + return writtenCount, overwrittenCount, skippedCount, scope, failures +} + +func resolveExtractionArchives(p *project.Project, overrides []string) ([]extractionArchive, error) { + patterns := p.EffectiveConfig().Extract.Archives + if len(overrides) > 0 { + patterns = overrides + } + if len(patterns) == 0 { + return nil, fmt.Errorf("extract.archives must contain at least one build-relative archive pattern") + } + + buildRoot := filepath.Clean(p.BuildDir()) + byPath := map[string]extractionArchive{} + for _, rawPattern := range patterns { + pattern, err := normalizeExtractionArchivePattern(p, rawPattern) + if err != nil { + return nil, err + } + matched, err := matchExtractionArchives(buildRoot, pattern) + if err != nil { + return nil, err + } + if len(matched) == 0 { + return nil, fmt.Errorf("extract archive pattern %q matched no files under %s", rawPattern, buildRoot) + } + for _, archive := range matched { + byPath[archive.Path] = archive + } + } + + archives := make([]extractionArchive, 0, len(byPath)) + for _, archive := range byPath { + archives = append(archives, archive) + } + slices.SortFunc(archives, func(a, b extractionArchive) int { + return strings.Compare(a.Rel, b.Rel) + }) + return archives, nil +} + +func normalizeExtractionArchivePattern(p *project.Project, raw string) (string, error) { + pattern := strings.TrimSpace(raw) + if pattern == "" { + return "", fmt.Errorf("extract archive pattern must not be empty") + } + pattern = strings.NewReplacer( + "{module.resref}", strings.TrimSpace(p.Config.Module.ResRef), + ).Replace(pattern) + if strings.Contains(pattern, "\x00") { + return "", fmt.Errorf("extract archive pattern %q must not contain NUL bytes", raw) + } + if filepath.IsAbs(pattern) { + return "", fmt.Errorf("extract archive pattern %q must be relative to paths.build", raw) + } + clean := filepath.Clean(filepath.FromSlash(pattern)) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("extract archive pattern %q must not escape paths.build", raw) + } + return filepath.ToSlash(clean), nil +} + +func matchExtractionArchives(buildRoot, pattern string) ([]extractionArchive, error) { + var archives []extractionArchive + err := filepath.WalkDir(buildRoot, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + rel, err := filepath.Rel(buildRoot, path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if !matchPathPattern(rel, pattern) { + return nil + } + archive, err := validateExtractionArchive(buildRoot, rel) + if err != nil { + return err + } + archives = append(archives, archive) + return nil + }) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("scan build directory %s: %w", buildRoot, err) + } + return nil, err + } + return archives, nil +} + +func validateExtractionArchive(buildRoot, rel string) (extractionArchive, error) { + cleanRel := filepath.Clean(filepath.FromSlash(rel)) + if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(filepath.Separator)) { + return extractionArchive{}, fmt.Errorf("extract archive %q must not escape paths.build", rel) + } + path := filepath.Join(buildRoot, cleanRel) + if !pathWithinRoot(path, buildRoot) { + return extractionArchive{}, fmt.Errorf("extract archive %q resolves outside paths.build", rel) + } + extension := strings.ToLower(filepath.Ext(cleanRel)) + switch extension { + case ".mod", ".hak": + return extractionArchive{Path: path, Rel: filepath.ToSlash(cleanRel), Extension: extension}, nil + case ".erf": + return extractionArchive{}, fmt.Errorf("extract archive %q is an .erf; ERF extraction is unsafe because ERFs lack module.ifo context", rel) + default: + return extractionArchive{}, fmt.Errorf("extract archive %q uses unsupported extension %q", rel, extension) + } +} + +func extractionConsumePolicy(p *project.Project) (consumeAll bool, consumeModules bool) { + if configured := p.Config.Extract.ConsumeArchives; configured != nil { + return *configured, false + } + return false, p.Config.Extract.DeleteModuleArchiveAfterSuccess +} + +func pathWithinRoot(path, root string) bool { + root = filepath.Clean(root) + if root == "" || root == "." { + return false + } + rel, err := filepath.Rel(root, path) + if err != nil { + return false + } + return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel) } func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) { @@ -211,30 +340,6 @@ func extractionTarget(p *project.Project, field, configured, root string, parts return target, nil } -func extractHAKPaths(p *project.Project) ([]string, error) { - switch p.EffectiveConfig().Extract.HAKDiscovery { - case "configured_haks": - paths := make([]string, 0, len(p.Config.HAKs)) - for _, hak := range p.Config.HAKs { - path := p.HAKArchivePath(hak.Name) - if _, err := os.Stat(path); err == nil { - paths = append(paths, path) - } else if err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, err - } - } - slices.Sort(paths) - return paths, nil - default: - hakPaths, err := filepath.Glob(filepath.Join(p.BuildDir(), "*.hak")) - if err != nil { - return nil, err - } - slices.Sort(hakPaths) - return hakPaths, nil - } -} - type writeState int const ( @@ -268,9 +373,9 @@ func writeManagedFile(path string, data []byte) (writeState, error) { return writeNew, nil } -func cleanupStaleFiles(p *project.Project, desired map[string]struct{}) (int, []error) { +func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) { candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles)) - if safeCleanupRoot(p.SourceDir(), p.Root) { + if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) { for _, rel := range p.Inventory.SourceFiles { candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) } @@ -278,7 +383,7 @@ func cleanupStaleFiles(p *project.Project, desired map[string]struct{}) (int, [] candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) } } - if safeCleanupRoot(p.AssetsDir(), p.Root) { + if scope.Assets && safeCleanupRoot(p.AssetsDir(), p.Root) { for _, rel := range p.Inventory.AssetFiles { candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) } diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 0766120..e8524cd 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -120,6 +120,9 @@ func TestExtractReadsHAKAssets(t *testing.T) { "source": "src", "assets": "assets", "build": "build" + }, + "extract": { + "archives": ["*.hak"] } } `) @@ -175,6 +178,281 @@ func TestExtractReadsHAKAssets(t *testing.T) { } } +func TestExtractDefaultOnlyReadsConfiguredModuleArchive(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "mdl")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Original Module" + } + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "mdl", "stale_asset.mdl"), "keep") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + if err := os.Remove(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil { + t.Fatalf("remove source file: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("rescan before extract: %v", err) + } + + hakFile, err := os.Create(filepath.Join(root, "build", "vfx.hak")) + if err != nil { + t.Fatalf("create hak: %v", err) + } + if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{ + {Name: "test_vfx", Type: 0x07D2, Data: []byte("mdl-data")}, + })); err != nil { + t.Fatalf("write hak: %v", err) + } + if err := hakFile.Close(); err != nil { + t.Fatalf("close hak: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(result.HAKPaths) != 0 { + t.Fatalf("expected default extract to ignore haks, got %#v", result.HAKPaths) + } + if _, err := os.Stat(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil { + t.Fatalf("expected module source to be restored: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "stale_asset.mdl")); err != nil { + t.Fatalf("expected asset cleanup to be skipped for module-only extract: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "test_vfx.mdl")); !os.IsNotExist(err) { + t.Fatalf("expected hak asset not to be extracted by default, err=%v", err) + } +} + +func TestExtractRejectsERFArchiveTargets(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +extract: + archives: + - imported.erf +`) + + erfFile, err := os.Create(filepath.Join(root, "build", "imported.erf")) + if err != nil { + t.Fatalf("create erf: %v", err) + } + if err := erf.Write(erfFile, erf.New("ERF ", nil)); err != nil { + t.Fatalf("write erf: %v", err) + } + if err := erfFile.Close(); err != nil { + t.Fatalf("close erf: %v", err) + } + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + _, err = Extract(p) + if err == nil { + t.Fatal("expected erf extraction to fail") + } + if !strings.Contains(err.Error(), "ERF extraction is unsafe") { + t.Fatalf("expected unsafe erf error, got %v", err) + } +} + +func TestExtractRejectsEscapingArchiveOverride(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + _, err = Extract(p, "../outside.mod") + if err == nil { + t.Fatal("expected escaping archive override to fail") + } + if !strings.Contains(err.Error(), "must not escape paths.build") { + t.Fatalf("expected escaping archive error, got %v", err) + } +} + +func TestExtractConsumesConfiguredHAKArchiveAfterSuccess(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +extract: + archives: + - vfx.hak + consume_archives: true +`) + + hakPath := filepath.Join(root, "build", "vfx.hak") + hakFile, err := os.Create(hakPath) + if err != nil { + t.Fatalf("create hak: %v", err) + } + if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{ + {Name: "test_vfx", Type: 0x07D2, Data: []byte("mdl-data")}, + })); err != nil { + t.Fatalf("write hak: %v", err) + } + if err := hakFile.Close(); err != nil { + t.Fatalf("close hak: %v", err) + } + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(result.DeletedArchivePaths) != 1 || result.DeletedArchivePaths[0] != hakPath { + t.Fatalf("expected consumed hak path, got %#v", result.DeletedArchivePaths) + } + if _, err := os.Stat(hakPath); !os.IsNotExist(err) { + t.Fatalf("expected hak archive to be deleted, stat err=%v", err) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "test_vfx.mdl")); err != nil { + t.Fatalf("expected extracted hak asset: %v", err) + } +} + +func TestExtractConsumeArchivesFalseOverridesLegacyDeleteFlag(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +extract: + consume_archives: false + delete_module_archive_after_success: true +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Original Module" + } + ] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + buildResult, err := BuildModule(p) + if err != nil { + t.Fatalf("build module: %v", err) + } + if err := os.Remove(filepath.Join(root, "src", "module", "module.ifo.json")); err != nil { + t.Fatalf("remove source file: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("rescan before extract: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(result.DeletedArchivePaths) != 0 { + t.Fatalf("expected no deleted archives, got %#v", result.DeletedArchivePaths) + } + if _, err := os.Stat(buildResult.ModulePath); err != nil { + t.Fatalf("expected module archive to remain, stat err=%v", err) + } +} + func TestExtractRefusesAssetsWhenAssetsPathIsUnset(t *testing.T) { root := t.TempDir() t.Chdir(root) @@ -226,7 +504,7 @@ paths: t.Fatalf("close hak: %v", err) } - _, err = Extract(p) + _, err = Extract(p, "vfx.hak") if err == nil { t.Fatal("expected extract to fail") } @@ -285,7 +563,7 @@ paths: t.Fatalf("close hak: %v", err) } - _, err = Extract(p) + _, err = Extract(p, "vfx.hak") if err == nil { t.Fatal("expected extract to fail") } @@ -357,7 +635,8 @@ paths: assets: assets build: build extract: - hak_discovery: configured_haks + archives: + - wanted.hak haks: - name: wanted priority: 1 diff --git a/internal/project/effective.go b/internal/project/effective.go index 671a014..ec8c565 100644 --- a/internal/project/effective.go +++ b/internal/project/effective.go @@ -231,10 +231,17 @@ func (p *Project) EffectiveConfig() EffectiveConfig { extract := p.Config.Extract extract.Layout = defaultString(extract.Layout, DefaultExtractLayout) extract.HAKDiscovery = defaultString(extract.HAKDiscovery, DefaultExtractHAKDiscovery) + if len(extract.Archives) == 0 { + extract.Archives = []string{outputs.ModuleArchiveName(p.Config.Module)} + } if extract.CleanupStale == nil { defaultCleanup := true extract.CleanupStale = &defaultCleanup } + if extract.ConsumeArchives == nil { + defaultConsume := false + extract.ConsumeArchives = &defaultConsume + } musicStageRoot := expandPathTemplate(DefaultMusicStageRoot, paths) musicCreditsRoot := expandPathTemplate(DefaultCreditsRoot, paths) @@ -435,7 +442,9 @@ func markMissingDefaults(provenance ConfigProvenance) { "topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary, "extract.layout": DefaultExtractLayout, "extract.hak_discovery": DefaultExtractHAKDiscovery, + "extract.archives": DefaultModuleArchiveTemplate, "extract.cleanup_stale": "true", + "extract.consume_archives": "false", "autogen.cache.root": DefaultAutogenCacheRoot, "autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(), "autogen.cache.refresh_env": DefaultAutogenRefreshEnv, diff --git a/internal/project/project.go b/internal/project/project.go index 43275a1..af818b5 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -214,9 +214,11 @@ type TopDataWikiConfig struct { type ExtractConfig struct { IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"` + Archives []string `json:"archives,omitempty" yaml:"archives,omitempty"` Layout string `json:"layout" yaml:"layout"` HAKDiscovery string `json:"hak_discovery" yaml:"hak_discovery"` CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"` + ConsumeArchives *bool `json:"consume_archives,omitempty" yaml:"consume_archives,omitempty"` DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"` } @@ -506,6 +508,7 @@ func (p *Project) ValidateLayout() error { default: failures = append(failures, fmt.Errorf("extract.hak_discovery %q is not supported", effective.Extract.HAKDiscovery)) } + failures = append(failures, validateGlobList("extract.archives", effective.Extract.Archives)...) if _, err := time.ParseDuration(effective.Autogen.Cache.MaxAge); err != nil { failures = append(failures, fmt.Errorf("autogen.cache.max_age must be a duration: %w", err)) } @@ -963,6 +966,7 @@ func normalizeConfig(cfg *Config) { cfg.HAKs[i].Include = normalizeStringSlice(cfg.HAKs[i].Include) } cfg.Extract.IgnoreExtensions = normalizeStringSlice(cfg.Extract.IgnoreExtensions) + cfg.Extract.Archives = normalizeStringSlice(cfg.Extract.Archives) for i := range cfg.Autogen.Producers { cfg.Autogen.Producers[i].Include = normalizeStringSlice(cfg.Autogen.Producers[i].Include) } diff --git a/internal/project/project_test.go b/internal/project/project_test.go index 85dc0dc..90fd119 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -66,6 +66,12 @@ paths: if got, want := effective.Outputs.HAKManifest, "haks.json"; got != want { t.Fatalf("expected default HAK manifest %q, got %q", want, got) } + if got, want := strings.Join(effective.Extract.Archives, ","), "testmod.mod"; got != want { + t.Fatalf("expected default extract archives %q, got %q", want, got) + } + if effective.Extract.ConsumeArchives == nil || *effective.Extract.ConsumeArchives { + t.Fatalf("expected default extract consume_archives false, got %#v", effective.Extract.ConsumeArchives) + } if got, want := effective.TopData.PackageHAK, "sow_top.hak"; got != want { t.Fatalf("expected default topdata HAK %q, got %q", want, got) }