package pipeline import ( "bytes" "encoding/json" "errors" "fmt" "os" "path/filepath" "slices" "strings" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) type ExtractResult struct { ModulePath string HAKPaths []string DeletedArchivePaths []string Written int Overwritten int Removed int Skipped int } type extractionArchive struct { Path string Rel string Extension string } type extractionScope struct { Source bool Assets bool } func Extract(p *project.Project, files ...string) (ExtractResult, error) { var result ExtractResult var failures []error desired := map[string]struct{}{} scope := extractionScope{} archives, err := resolveExtractionArchives(p, files) if err != nil { return result, err } for _, archivePath := range archives { input, err := os.Open(archivePath.Path) if err != nil { failures = append(failures, fmt.Errorf("open archive %s: %w", archivePath.Path, err)) continue } archive, err := erf.Read(input) input.Close() if err != nil { failures = append(failures, fmt.Errorf("read archive %s: %w", archivePath.Path, err)) continue } switch archivePath.Extension { case ".mod": if result.ModulePath == "" { result.ModulePath = archivePath.Path } case ".hak": result.HAKPaths = append(result.HAKPaths, archivePath.Path) } written, overwritten, skipped, extractedScope, errs := extractArchiveResources(p, archive, desired) result.Written += written result.Overwritten += overwritten result.Skipped += skipped scope.Source = scope.Source || extractedScope.Source scope.Assets = scope.Assets || extractedScope.Assets failures = append(failures, errs...) } if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale { removed, errs := cleanupStaleFiles(p, desired, scope) result.Removed = removed failures = append(failures, errs...) } if len(failures) > 0 { return result, errors.Join(failures...) } consumeAll, consumeModules := extractionConsumePolicy(p) if consumeAll || consumeModules { for _, archivePath := range archives { if !consumeAll && archivePath.Extension != ".mod" { continue } if err := os.Remove(archivePath.Path); err != nil { return result, fmt.Errorf("delete consumed archive %s: %w", archivePath.Path, err) } result.DeletedArchivePaths = append(result.DeletedArchivePaths, archivePath.Path) } } return result, nil } func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, extractionScope, []error) { var failures []error writtenCount := 0 overwrittenCount := 0 skippedCount := 0 scope := extractionScope{} ignored := make(map[string]bool) for _, ext := range p.Config.Extract.IgnoreExtensions { ext := strings.TrimPrefix(ext, ".") ignored[ext] = true } for _, resource := range archive.Resources { ext, ok := erf.ExtensionForResourceType(resource.Type) if !ok { failures = append(failures, fmt.Errorf("unsupported resource type 0x%04X for %s", resource.Type, resource.Name)) continue } if ignored[ext] { skippedCount++ continue } target, data, err := extractedFile(p, resource, ext) if err != nil { failures = append(failures, err) continue } desired[target] = struct{}{} scope.Source = scope.Source || pathWithinRoot(target, p.SourceDir()) scope.Assets = scope.Assets || pathWithinRoot(target, p.AssetsDir()) state, err := writeManagedFile(target, data) if err != nil { failures = append(failures, err) continue } switch state { case writeNew: writtenCount++ case writeOverwritten: overwrittenCount++ case writeSkipped: skippedCount++ } } return writtenCount, overwrittenCount, skippedCount, scope, failures } func resolveExtractionArchives(p *project.Project, overrides []string) ([]extractionArchive, error) { patterns := p.EffectiveConfig().Extract.Archives if len(overrides) > 0 { patterns = overrides } if len(patterns) == 0 { return nil, fmt.Errorf("extract.archives must contain at least one build-relative archive pattern") } buildRoot := filepath.Clean(p.BuildDir()) byPath := map[string]extractionArchive{} for _, rawPattern := range patterns { pattern, err := normalizeExtractionArchivePattern(p, rawPattern) if err != nil { return nil, err } matched, err := matchExtractionArchives(buildRoot, pattern) if err != nil { return nil, err } if len(matched) == 0 { return nil, fmt.Errorf("extract archive pattern %q matched no files under %s", rawPattern, buildRoot) } for _, archive := range matched { byPath[archive.Path] = archive } } archives := make([]extractionArchive, 0, len(byPath)) for _, archive := range byPath { archives = append(archives, archive) } slices.SortFunc(archives, func(a, b extractionArchive) int { return strings.Compare(a.Rel, b.Rel) }) return archives, nil } func normalizeExtractionArchivePattern(p *project.Project, raw string) (string, error) { pattern := strings.TrimSpace(raw) if pattern == "" { return "", fmt.Errorf("extract archive pattern must not be empty") } pattern = strings.NewReplacer( "{module.resref}", strings.TrimSpace(p.Config.Module.ResRef), ).Replace(pattern) if strings.Contains(pattern, "\x00") { return "", fmt.Errorf("extract archive pattern %q must not contain NUL bytes", raw) } if filepath.IsAbs(pattern) { return "", fmt.Errorf("extract archive pattern %q must be relative to paths.build", raw) } clean := filepath.Clean(filepath.FromSlash(pattern)) if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { return "", fmt.Errorf("extract archive pattern %q must not escape paths.build", raw) } return filepath.ToSlash(clean), nil } func matchExtractionArchives(buildRoot, pattern string) ([]extractionArchive, error) { var archives []extractionArchive err := filepath.WalkDir(buildRoot, func(path string, entry os.DirEntry, err error) error { if err != nil { return err } if entry.IsDir() { return nil } rel, err := filepath.Rel(buildRoot, path) if err != nil { return err } rel = filepath.ToSlash(rel) if !matchPathPattern(rel, pattern) { return nil } archive, err := validateExtractionArchive(buildRoot, rel) if err != nil { return err } archives = append(archives, archive) return nil }) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("scan build directory %s: %w", buildRoot, err) } return nil, err } return archives, nil } func validateExtractionArchive(buildRoot, rel string) (extractionArchive, error) { cleanRel := filepath.Clean(filepath.FromSlash(rel)) if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(filepath.Separator)) { return extractionArchive{}, fmt.Errorf("extract archive %q must not escape paths.build", rel) } path := filepath.Join(buildRoot, cleanRel) if !pathWithinRoot(path, buildRoot) { return extractionArchive{}, fmt.Errorf("extract archive %q resolves outside paths.build", rel) } extension := strings.ToLower(filepath.Ext(cleanRel)) switch extension { case ".mod", ".hak": return extractionArchive{Path: path, Rel: filepath.ToSlash(cleanRel), Extension: extension}, nil case ".erf": return extractionArchive{}, fmt.Errorf("extract archive %q is an .erf; ERF extraction is unsafe because ERFs lack module.ifo context", rel) default: return extractionArchive{}, fmt.Errorf("extract archive %q uses unsupported extension %q", rel, extension) } } func extractionConsumePolicy(p *project.Project) (consumeAll bool, consumeModules bool) { if configured := p.Config.Extract.ConsumeArchives; configured != nil { return *configured, false } return false, p.Config.Extract.DeleteModuleArchiveAfterSuccess } func pathWithinRoot(path, root string) bool { root = filepath.Clean(root) if root == "" || root == "." { return false } rel, err := filepath.Rel(root, path) if err != nil { return false } return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel) } func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) { resref := strings.ToLower(resource.Name) effective := p.EffectiveConfig() switch extension { case "nss": target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), filepath.FromSlash(effective.Scripts.SourceDir), resref+".nss") if err != nil { return "", nil, err } return target, resource.Data, nil case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw", "are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl": document, err := gff.Read(bytes.NewReader(resource.Data)) if err != nil { return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err) } target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json") if err != nil { return "", nil, err } if err := mergeExtractedGFFJSON(p, target, &document); err != nil { return "", nil, fmt.Errorf("merge extracted gff json %s.%s: %w", resource.Name, extension, err) } formatted, err := json.MarshalIndent(document, "", " ") if err != nil { return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err) } formatted = append(formatted, '\n') return target, formatted, nil default: target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension) if err != nil { return "", nil, err } return target, resource.Data, nil } } func extractionTarget(p *project.Project, field, configured, root string, parts ...string) (string, error) { if strings.TrimSpace(configured) == "" { return "", fmt.Errorf("cannot extract resource: %s is not configured", field) } cleanRoot := filepath.Clean(root) if cleanRoot == "." || cleanRoot == string(filepath.Separator) || cleanRoot == filepath.Clean(p.Root) { return "", fmt.Errorf("cannot extract resource: %s resolves to unsafe extraction root %s", field, cleanRoot) } target := filepath.Join(append([]string{cleanRoot}, parts...)...) rel, err := filepath.Rel(cleanRoot, target) if err != nil { return "", fmt.Errorf("resolve extraction target %s: %w", target, err) } if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { return "", fmt.Errorf("refusing to extract resource outside %s: %s", cleanRoot, target) } return target, nil } func mergeExtractedGFFJSON(p *project.Project, target string, extracted *gff.Document) error { rule, ok, err := extractGFFJSONMergeRule(p, target) if err != nil || !ok { return err } raw, err := os.ReadFile(target) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil } return fmt.Errorf("read existing source %s: %w", target, err) } var existing gff.Document if err := json.Unmarshal(raw, &existing); err != nil { return fmt.Errorf("parse existing source %s: %w", target, err) } for _, label := range rule.PreserveFields { if field, ok := gffField(existing.Root, label); ok { setGFFField(&extracted.Root, field) } } for _, listRule := range rule.MergeLists { if err := mergeGFFListByKey(&extracted.Root, existing.Root, listRule); err != nil { return err } } return nil } func extractGFFJSONMergeRule(p *project.Project, target string) (project.ExtractGFFJSONMergeRule, bool, error) { rel, err := filepath.Rel(filepath.Clean(p.SourceDir()), filepath.Clean(target)) if err != nil { return project.ExtractGFFJSONMergeRule{}, false, fmt.Errorf("resolve source-relative target %s: %w", target, err) } rel = filepath.ToSlash(rel) if rel == ".." || strings.HasPrefix(rel, "../") { return project.ExtractGFFJSONMergeRule{}, false, nil } for _, rule := range p.EffectiveConfig().Extract.Merge.GFFJSON { if rule.Target == rel { return rule, true, nil } } return project.ExtractGFFJSONMergeRule{}, false, nil } func gffField(s gff.Struct, label string) (gff.Field, bool) { for _, field := range s.Fields { if field.Label == label { return field, true } } return gff.Field{}, false } func setGFFField(s *gff.Struct, replacement gff.Field) { for index, field := range s.Fields { if field.Label == replacement.Label { s.Fields[index] = replacement return } } s.Fields = append(s.Fields, replacement) } func mergeGFFListByKey(extracted *gff.Struct, existing gff.Struct, rule project.ExtractListMergeRule) error { extractedField, ok := gffField(*extracted, rule.Field) if !ok { return fmt.Errorf("extracted field %q not found", rule.Field) } extractedList, ok := extractedField.Value.(gff.ListValue) if !ok { return fmt.Errorf("extracted field %q is %s, not List", rule.Field, extractedField.Type) } existingField, ok := gffField(existing, rule.Field) if !ok { return nil } existingList, ok := existingField.Value.(gff.ListValue) if !ok { return fmt.Errorf("existing field %q is %s, not List", rule.Field, existingField.Type) } extractedByKey := map[string]gff.Struct{} extractedOrder := make([]string, 0, len(extractedList)) for _, item := range extractedList { key, err := gffStructKey(item, rule.KeyField) if err != nil { return fmt.Errorf("extracted field %q: %w", rule.Field, err) } if _, exists := extractedByKey[key]; exists { return fmt.Errorf("extracted field %q has duplicate %s key %q", rule.Field, rule.KeyField, key) } extractedByKey[key] = item extractedOrder = append(extractedOrder, key) } merged := make(gff.ListValue, 0, len(extractedList)) seen := map[string]struct{}{} for _, item := range existingList { key, err := gffStructKey(item, rule.KeyField) if err != nil { return fmt.Errorf("existing field %q: %w", rule.Field, err) } if _, duplicate := seen[key]; duplicate { return fmt.Errorf("existing field %q has duplicate %s key %q", rule.Field, rule.KeyField, key) } if extractedItem, exists := extractedByKey[key]; exists { merged = append(merged, extractedItem) seen[key] = struct{}{} } } for _, key := range extractedOrder { if _, exists := seen[key]; exists { continue } merged = append(merged, extractedByKey[key]) } setGFFField(extracted, gff.NewField(rule.Field, merged)) return nil } func gffStructKey(s gff.Struct, keyField string) (string, error) { field, ok := gffField(s, keyField) if !ok { return "", fmt.Errorf("key field %q not found", keyField) } switch value := field.Value.(type) { case gff.ResRefValue: return string(value), nil case gff.StringValue: return string(value), nil default: return "", fmt.Errorf("key field %q is %s, not ResRef or CExoString", keyField, field.Type) } } type writeState int const ( writeSkipped writeState = iota writeNew writeOverwritten ) func writeManagedFile(path string, data []byte) (writeState, error) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return writeSkipped, fmt.Errorf("create parent directory for %s: %w", path, err) } existing, err := os.ReadFile(path) if err == nil { if bytes.Equal(existing, data) { return writeSkipped, nil } if err := os.WriteFile(path, data, 0o644); err != nil { return writeSkipped, fmt.Errorf("overwrite %s: %w", path, err) } return writeOverwritten, nil } if !errors.Is(err, os.ErrNotExist) { return writeSkipped, fmt.Errorf("check existing file %s: %w", path, err) } if err := os.WriteFile(path, data, 0o644); err != nil { return writeSkipped, fmt.Errorf("write %s: %w", path, err) } return writeNew, nil } func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) { candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles)) if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) { for _, rel := range p.Inventory.SourceFiles { candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) } for _, rel := range p.Inventory.ScriptFiles { candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) } } if scope.Assets && safeCleanupRoot(p.AssetsDir(), p.Root) { for _, rel := range p.Inventory.AssetFiles { candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) } } removed := 0 var failures []error for _, path := range candidates { if _, keep := desired[path]; keep { continue } if err := os.Remove(path); err != nil { if errors.Is(err, os.ErrNotExist) { continue } failures = append(failures, fmt.Errorf("remove stale file %s: %w", path, err)) continue } removed++ cleanupEmptyParents(filepath.Dir(path), p.SourceDir(), p.AssetsDir()) } return removed, failures } func safeCleanupRoot(root, projectRoot string) bool { cleanRoot := filepath.Clean(root) return cleanRoot != "." && cleanRoot != string(filepath.Separator) && cleanRoot != filepath.Clean(projectRoot) } func cleanupEmptyParents(dir string, roots ...string) { for { if dir == "." || dir == string(filepath.Separator) { return } stop := false for _, root := range roots { if dir == root { stop = true break } } if stop { return } if err := os.Remove(dir); err != nil { return } dir = filepath.Dir(dir) } } func sourceSubdir(extension string) string { switch strings.ToLower(extension) { case "are": return "areas" case "dlg": return "dialogs" case "fac": return "factions" case "gic": return "instance" case "git": return "instance" case "ifo": return "module" case "itp": return "palettes" case "jrl": return "journal" case "utc": return filepath.Join("blueprints", "creatures") case "utd": return filepath.Join("blueprints", "doors") case "ute": return filepath.Join("blueprints", "encounters") case "uti": return filepath.Join("blueprints", "items") case "utm": return filepath.Join("blueprints", "merchants") case "utp": return filepath.Join("blueprints", "placeables") case "uts": return filepath.Join("blueprints", "sounds") case "utt": return filepath.Join("blueprints", "triggers") case "utw": return filepath.Join("blueprints", "waypoints") default: return extension } }