package pipeline import ( "bytes" "encoding/json" "errors" "fmt" "os" "path/filepath" "slices" "strings" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) type ExtractResult struct { ModulePath string HAKPaths []string DeletedArchivePaths []string Written int Overwritten int Removed int Skipped int } type extractionArchive struct { Path string Rel string Extension string } type extractionScope struct { Source bool Assets bool } func Extract(p *project.Project, files ...string) (ExtractResult, error) { var result ExtractResult var failures []error desired := map[string]struct{}{} scope := extractionScope{} archives, err := resolveExtractionArchives(p, files) if err != nil { return result, err } for _, archivePath := range archives { input, err := os.Open(archivePath.Path) if err != nil { failures = append(failures, fmt.Errorf("open archive %s: %w", archivePath.Path, err)) continue } archive, err := erf.Read(input) input.Close() if err != nil { failures = append(failures, fmt.Errorf("read archive %s: %w", archivePath.Path, err)) continue } switch archivePath.Extension { case ".mod": if result.ModulePath == "" { result.ModulePath = archivePath.Path } case ".hak": result.HAKPaths = append(result.HAKPaths, archivePath.Path) } written, overwritten, skipped, extractedScope, errs := extractArchiveResources(p, archive, desired) result.Written += written result.Overwritten += overwritten result.Skipped += skipped scope.Source = scope.Source || extractedScope.Source scope.Assets = scope.Assets || extractedScope.Assets failures = append(failures, errs...) } if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale { removed, errs := cleanupStaleFiles(p, desired, scope) result.Removed = removed failures = append(failures, errs...) } if len(failures) > 0 { return result, errors.Join(failures...) } consumeAll, consumeModules := extractionConsumePolicy(p) if consumeAll || consumeModules { for _, archivePath := range archives { if !consumeAll && archivePath.Extension != ".mod" { continue } if err := os.Remove(archivePath.Path); err != nil { return result, fmt.Errorf("delete consumed archive %s: %w", archivePath.Path, err) } result.DeletedArchivePaths = append(result.DeletedArchivePaths, archivePath.Path) } } return result, nil } func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, extractionScope, []error) { var failures []error writtenCount := 0 overwrittenCount := 0 skippedCount := 0 scope := extractionScope{} ignored := make(map[string]bool) for _, ext := range p.Config.Extract.IgnoreExtensions { ext := strings.TrimPrefix(ext, ".") ignored[ext] = true } for _, resource := range archive.Resources { ext, ok := erf.ExtensionForResourceType(resource.Type) if !ok { failures = append(failures, fmt.Errorf("unsupported resource type 0x%04X for %s", resource.Type, resource.Name)) continue } if ignored[ext] { skippedCount++ continue } target, data, err := extractedFile(p, resource, ext) if err != nil { failures = append(failures, err) continue } desired[target] = struct{}{} scope.Source = scope.Source || pathWithinRoot(target, p.SourceDir()) scope.Assets = scope.Assets || pathWithinRoot(target, p.AssetsDir()) state, err := writeManagedFile(target, data) if err != nil { failures = append(failures, err) continue } switch state { case writeNew: writtenCount++ case writeOverwritten: overwrittenCount++ case writeSkipped: skippedCount++ } } return writtenCount, overwrittenCount, skippedCount, scope, failures } func resolveExtractionArchives(p *project.Project, overrides []string) ([]extractionArchive, error) { patterns := p.EffectiveConfig().Extract.Archives if len(overrides) > 0 { patterns = overrides } if len(patterns) == 0 { return nil, fmt.Errorf("extract.archives must contain at least one build-relative archive pattern") } buildRoot := filepath.Clean(p.BuildDir()) byPath := map[string]extractionArchive{} for _, rawPattern := range patterns { pattern, err := normalizeExtractionArchivePattern(p, rawPattern) if err != nil { return nil, err } matched, err := matchExtractionArchives(buildRoot, pattern) if err != nil { return nil, err } if len(matched) == 0 { return nil, fmt.Errorf("extract archive pattern %q matched no files under %s", rawPattern, buildRoot) } for _, archive := range matched { byPath[archive.Path] = archive } } archives := make([]extractionArchive, 0, len(byPath)) for _, archive := range byPath { archives = append(archives, archive) } slices.SortFunc(archives, func(a, b extractionArchive) int { return strings.Compare(a.Rel, b.Rel) }) return archives, nil } func normalizeExtractionArchivePattern(p *project.Project, raw string) (string, error) { pattern := strings.TrimSpace(raw) if pattern == "" { return "", fmt.Errorf("extract archive pattern must not be empty") } pattern = strings.NewReplacer( "{module.resref}", strings.TrimSpace(p.Config.Module.ResRef), ).Replace(pattern) if strings.Contains(pattern, "\x00") { return "", fmt.Errorf("extract archive pattern %q must not contain NUL bytes", raw) } if filepath.IsAbs(pattern) { return "", fmt.Errorf("extract archive pattern %q must be relative to paths.build", raw) } clean := filepath.Clean(filepath.FromSlash(pattern)) if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { return "", fmt.Errorf("extract archive pattern %q must not escape paths.build", raw) } return filepath.ToSlash(clean), nil } func matchExtractionArchives(buildRoot, pattern string) ([]extractionArchive, error) { var archives []extractionArchive err := filepath.WalkDir(buildRoot, func(path string, entry os.DirEntry, err error) error { if err != nil { return err } if entry.IsDir() { return nil } rel, err := filepath.Rel(buildRoot, path) if err != nil { return err } rel = filepath.ToSlash(rel) if !matchPathPattern(rel, pattern) { return nil } archive, err := validateExtractionArchive(buildRoot, rel) if err != nil { return err } archives = append(archives, archive) return nil }) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("scan build directory %s: %w", buildRoot, err) } return nil, err } return archives, nil } func validateExtractionArchive(buildRoot, rel string) (extractionArchive, error) { cleanRel := filepath.Clean(filepath.FromSlash(rel)) if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(filepath.Separator)) { return extractionArchive{}, fmt.Errorf("extract archive %q must not escape paths.build", rel) } path := filepath.Join(buildRoot, cleanRel) if !pathWithinRoot(path, buildRoot) { return extractionArchive{}, fmt.Errorf("extract archive %q resolves outside paths.build", rel) } extension := strings.ToLower(filepath.Ext(cleanRel)) switch extension { case ".mod", ".hak": return extractionArchive{Path: path, Rel: filepath.ToSlash(cleanRel), Extension: extension}, nil case ".erf": return extractionArchive{}, fmt.Errorf("extract archive %q is an .erf; ERF extraction is unsafe because ERFs lack module.ifo context", rel) default: return extractionArchive{}, fmt.Errorf("extract archive %q uses unsupported extension %q", rel, extension) } } func extractionConsumePolicy(p *project.Project) (consumeAll bool, consumeModules bool) { if configured := p.Config.Extract.ConsumeArchives; configured != nil { return *configured, false } return false, p.Config.Extract.DeleteModuleArchiveAfterSuccess } func pathWithinRoot(path, root string) bool { root = filepath.Clean(root) if root == "" || root == "." { return false } rel, err := filepath.Rel(root, path) if err != nil { return false } return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel) } func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) { resref := strings.ToLower(resource.Name) effective := p.EffectiveConfig() switch extension { case "nss": target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), filepath.FromSlash(effective.Scripts.SourceDir), resref+".nss") if err != nil { return "", nil, err } return target, resource.Data, nil case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw", "are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl": document, err := gff.Read(bytes.NewReader(resource.Data)) if err != nil { return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err) } formatted, err := json.MarshalIndent(document, "", " ") if err != nil { return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err) } formatted = append(formatted, '\n') target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json") if err != nil { return "", nil, err } return target, formatted, nil default: target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension) if err != nil { return "", nil, err } return target, resource.Data, nil } } func extractionTarget(p *project.Project, field, configured, root string, parts ...string) (string, error) { if strings.TrimSpace(configured) == "" { return "", fmt.Errorf("cannot extract resource: %s is not configured", field) } cleanRoot := filepath.Clean(root) if cleanRoot == "." || cleanRoot == string(filepath.Separator) || cleanRoot == filepath.Clean(p.Root) { return "", fmt.Errorf("cannot extract resource: %s resolves to unsafe extraction root %s", field, cleanRoot) } target := filepath.Join(append([]string{cleanRoot}, parts...)...) rel, err := filepath.Rel(cleanRoot, target) if err != nil { return "", fmt.Errorf("resolve extraction target %s: %w", target, err) } if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { return "", fmt.Errorf("refusing to extract resource outside %s: %s", cleanRoot, target) } return target, nil } type writeState int const ( writeSkipped writeState = iota writeNew writeOverwritten ) func writeManagedFile(path string, data []byte) (writeState, error) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return writeSkipped, fmt.Errorf("create parent directory for %s: %w", path, err) } existing, err := os.ReadFile(path) if err == nil { if bytes.Equal(existing, data) { return writeSkipped, nil } if err := os.WriteFile(path, data, 0o644); err != nil { return writeSkipped, fmt.Errorf("overwrite %s: %w", path, err) } return writeOverwritten, nil } if !errors.Is(err, os.ErrNotExist) { return writeSkipped, fmt.Errorf("check existing file %s: %w", path, err) } if err := os.WriteFile(path, data, 0o644); err != nil { return writeSkipped, fmt.Errorf("write %s: %w", path, err) } return writeNew, nil } func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) { candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles)) if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) { for _, rel := range p.Inventory.SourceFiles { candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) } for _, rel := range p.Inventory.ScriptFiles { candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) } } if scope.Assets && safeCleanupRoot(p.AssetsDir(), p.Root) { for _, rel := range p.Inventory.AssetFiles { candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) } } removed := 0 var failures []error for _, path := range candidates { if _, keep := desired[path]; keep { continue } if err := os.Remove(path); err != nil { if errors.Is(err, os.ErrNotExist) { continue } failures = append(failures, fmt.Errorf("remove stale file %s: %w", path, err)) continue } removed++ cleanupEmptyParents(filepath.Dir(path), p.SourceDir(), p.AssetsDir()) } return removed, failures } func safeCleanupRoot(root, projectRoot string) bool { cleanRoot := filepath.Clean(root) return cleanRoot != "." && cleanRoot != string(filepath.Separator) && cleanRoot != filepath.Clean(projectRoot) } func cleanupEmptyParents(dir string, roots ...string) { for { if dir == "." || dir == string(filepath.Separator) { return } stop := false for _, root := range roots { if dir == root { stop = true break } } if stop { return } if err := os.Remove(dir); err != nil { return } dir = filepath.Dir(dir) } } func sourceSubdir(extension string) string { switch strings.ToLower(extension) { case "are": return "areas" case "dlg": return "dialogs" case "fac": return "factions" case "gic": return "instance" case "git": return "instance" case "ifo": return "module" case "itp": return "palettes" case "jrl": return "journal" case "utc": return filepath.Join("blueprints", "creatures") case "utd": return filepath.Join("blueprints", "doors") case "ute": return filepath.Join("blueprints", "encounters") case "uti": return filepath.Join("blueprints", "items") case "utm": return filepath.Join("blueprints", "merchants") case "utp": return filepath.Join("blueprints", "placeables") case "uts": return filepath.Join("blueprints", "sounds") case "utt": return filepath.Join("blueprints", "triggers") case "utw": return filepath.Join("blueprints", "waypoints") default: return extension } }