claude: fold in old sow-tools

internal/pipeline/extract.go

@@ -0,0 +1,621 @@
+package pipeline
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+type ExtractResult struct {
+	ModulePath          string
+	HAKPaths            []string
+	DeletedArchivePaths []string
+	Written             int
+	Overwritten         int
+	Removed             int
+	Skipped             int
+}
+
+type extractionArchive struct {
+	Path      string
+	Rel       string
+	Extension string
+}
+
+type extractionScope struct {
+	Source bool
+	Assets bool
+}
+
+func Extract(p *project.Project, files ...string) (ExtractResult, error) {
+	var result ExtractResult
+	var failures []error
+	desired := map[string]struct{}{}
+	scope := extractionScope{}
+
+	archives, err := resolveExtractionArchives(p, files)
+	if err != nil {
+		return result, err
+	}
+	for _, archivePath := range archives {
+		input, err := os.Open(archivePath.Path)
+		if err != nil {
+			failures = append(failures, fmt.Errorf("open archive %s: %w", archivePath.Path, err))
+			continue
+		}
+		archive, err := erf.Read(input)
+		input.Close()
+		if err != nil {
+			failures = append(failures, fmt.Errorf("read archive %s: %w", archivePath.Path, err))
+			continue
+		}
+
+		switch archivePath.Extension {
+		case ".mod":
+			if result.ModulePath == "" {
+				result.ModulePath = archivePath.Path
+			}
+		case ".hak":
+			result.HAKPaths = append(result.HAKPaths, archivePath.Path)
+		}
+
+		written, overwritten, skipped, extractedScope, errs := extractArchiveResources(p, archive, desired)
+		result.Written += written
+		result.Overwritten += overwritten
+		result.Skipped += skipped
+		scope.Source = scope.Source || extractedScope.Source
+		scope.Assets = scope.Assets || extractedScope.Assets
+		failures = append(failures, errs...)
+	}
+
+	if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale {
+		removed, errs := cleanupStaleFiles(p, desired, scope)
+		result.Removed = removed
+		failures = append(failures, errs...)
+	}
+
+	if len(failures) > 0 {
+		return result, errors.Join(failures...)
+	}
+	consumeAll, consumeModules := extractionConsumePolicy(p)
+	if consumeAll || consumeModules {
+		for _, archivePath := range archives {
+			if !consumeAll && archivePath.Extension != ".mod" {
+				continue
+			}
+			if err := os.Remove(archivePath.Path); err != nil {
+				return result, fmt.Errorf("delete consumed archive %s: %w", archivePath.Path, err)
+			}
+			result.DeletedArchivePaths = append(result.DeletedArchivePaths, archivePath.Path)
+		}
+	}
+	return result, nil
+}
+
+func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, extractionScope, []error) {
+	var failures []error
+	writtenCount := 0
+	overwrittenCount := 0
+	skippedCount := 0
+	scope := extractionScope{}
+
+	ignored := make(map[string]bool)
+	for _, ext := range p.Config.Extract.IgnoreExtensions {
+		ext := strings.TrimPrefix(ext, ".")
+		ignored[ext] = true
+	}
+
+	for _, resource := range archive.Resources {
+		ext, ok := erf.ExtensionForResourceType(resource.Type)
+		if !ok {
+			failures = append(failures, fmt.Errorf("unsupported resource type 0x%04X for %s", resource.Type, resource.Name))
+			continue
+		}
+		if ignored[ext] {
+			skippedCount++
+			continue
+		}
+
+		target, data, err := extractedFile(p, resource, ext)
+		if err != nil {
+			failures = append(failures, err)
+			continue
+		}
+		desired[target] = struct{}{}
+		scope.Source = scope.Source || pathWithinRoot(target, p.SourceDir())
+		scope.Assets = scope.Assets || pathWithinRoot(target, p.AssetsDir())
+
+		state, err := writeManagedFile(target, data)
+		if err != nil {
+			failures = append(failures, err)
+			continue
+		}
+		switch state {
+		case writeNew:
+			writtenCount++
+		case writeOverwritten:
+			overwrittenCount++
+		case writeSkipped:
+			skippedCount++
+		}
+	}
+
+	return writtenCount, overwrittenCount, skippedCount, scope, failures
+}
+
+func resolveExtractionArchives(p *project.Project, overrides []string) ([]extractionArchive, error) {
+	patterns := p.EffectiveConfig().Extract.Archives
+	if len(overrides) > 0 {
+		patterns = overrides
+	}
+	if len(patterns) == 0 {
+		return nil, fmt.Errorf("extract.archives must contain at least one build-relative archive pattern")
+	}
+
+	buildRoot := filepath.Clean(p.BuildDir())
+	byPath := map[string]extractionArchive{}
+	for _, rawPattern := range patterns {
+		pattern, err := normalizeExtractionArchivePattern(p, rawPattern)
+		if err != nil {
+			return nil, err
+		}
+		matched, err := matchExtractionArchives(buildRoot, pattern)
+		if err != nil {
+			return nil, err
+		}
+		if len(matched) == 0 {
+			return nil, fmt.Errorf("extract archive pattern %q matched no files under %s", rawPattern, buildRoot)
+		}
+		for _, archive := range matched {
+			byPath[archive.Path] = archive
+		}
+	}
+
+	archives := make([]extractionArchive, 0, len(byPath))
+	for _, archive := range byPath {
+		archives = append(archives, archive)
+	}
+	slices.SortFunc(archives, func(a, b extractionArchive) int {
+		return strings.Compare(a.Rel, b.Rel)
+	})
+	return archives, nil
+}
+
+func normalizeExtractionArchivePattern(p *project.Project, raw string) (string, error) {
+	pattern := strings.TrimSpace(raw)
+	if pattern == "" {
+		return "", fmt.Errorf("extract archive pattern must not be empty")
+	}
+	pattern = strings.NewReplacer(
+		"{module.resref}", strings.TrimSpace(p.Config.Module.ResRef),
+	).Replace(pattern)
+	if strings.Contains(pattern, "\x00") {
+		return "", fmt.Errorf("extract archive pattern %q must not contain NUL bytes", raw)
+	}
+	if filepath.IsAbs(pattern) {
+		return "", fmt.Errorf("extract archive pattern %q must be relative to paths.build", raw)
+	}
+	clean := filepath.Clean(filepath.FromSlash(pattern))
+	if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
+		return "", fmt.Errorf("extract archive pattern %q must not escape paths.build", raw)
+	}
+	return filepath.ToSlash(clean), nil
+}
+
+func matchExtractionArchives(buildRoot, pattern string) ([]extractionArchive, error) {
+	var archives []extractionArchive
+	err := filepath.WalkDir(buildRoot, func(path string, entry os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() {
+			return nil
+		}
+		rel, err := filepath.Rel(buildRoot, path)
+		if err != nil {
+			return err
+		}
+		rel = filepath.ToSlash(rel)
+		if !matchPathPattern(rel, pattern) {
+			return nil
+		}
+		archive, err := validateExtractionArchive(buildRoot, rel)
+		if err != nil {
+			return err
+		}
+		archives = append(archives, archive)
+		return nil
+	})
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, fmt.Errorf("scan build directory %s: %w", buildRoot, err)
+		}
+		return nil, err
+	}
+	return archives, nil
+}
+
+func validateExtractionArchive(buildRoot, rel string) (extractionArchive, error) {
+	cleanRel := filepath.Clean(filepath.FromSlash(rel))
+	if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(filepath.Separator)) {
+		return extractionArchive{}, fmt.Errorf("extract archive %q must not escape paths.build", rel)
+	}
+	path := filepath.Join(buildRoot, cleanRel)
+	if !pathWithinRoot(path, buildRoot) {
+		return extractionArchive{}, fmt.Errorf("extract archive %q resolves outside paths.build", rel)
+	}
+	extension := strings.ToLower(filepath.Ext(cleanRel))
+	switch extension {
+	case ".mod", ".hak":
+		return extractionArchive{Path: path, Rel: filepath.ToSlash(cleanRel), Extension: extension}, nil
+	case ".erf":
+		return extractionArchive{}, fmt.Errorf("extract archive %q is an .erf; ERF extraction is unsafe because ERFs lack module.ifo context", rel)
+	default:
+		return extractionArchive{}, fmt.Errorf("extract archive %q uses unsupported extension %q", rel, extension)
+	}
+}
+
+func extractionConsumePolicy(p *project.Project) (consumeAll bool, consumeModules bool) {
+	if configured := p.Config.Extract.ConsumeArchives; configured != nil {
+		return *configured, false
+	}
+	return false, p.Config.Extract.DeleteModuleArchiveAfterSuccess
+}
+
+func pathWithinRoot(path, root string) bool {
+	root = filepath.Clean(root)
+	if root == "" || root == "." {
+		return false
+	}
+	rel, err := filepath.Rel(root, path)
+	if err != nil {
+		return false
+	}
+	return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel)
+}
+
+func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) {
+	resref := strings.ToLower(resource.Name)
+	effective := p.EffectiveConfig()
+
+	switch extension {
+	case "nss":
+		target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), "scripts", resref+".nss")
+		if err != nil {
+			return "", nil, err
+		}
+		return target, resource.Data, nil
+	case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
+		"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl":
+		document, err := gff.Read(bytes.NewReader(resource.Data))
+		if err != nil {
+			return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err)
+		}
+		target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json")
+		if err != nil {
+			return "", nil, err
+		}
+		if err := mergeExtractedGFFJSON(p, target, &document); err != nil {
+			return "", nil, fmt.Errorf("merge extracted gff json %s.%s: %w", resource.Name, extension, err)
+		}
+		formatted, err := json.MarshalIndent(document, "", "  ")
+		if err != nil {
+			return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err)
+		}
+		formatted = append(formatted, '\n')
+		return target, formatted, nil
+	default:
+		target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension)
+		if err != nil {
+			return "", nil, err
+		}
+		return target, resource.Data, nil
+	}
+}
+
+func extractionTarget(p *project.Project, field, configured, root string, parts ...string) (string, error) {
+	if strings.TrimSpace(configured) == "" {
+		return "", fmt.Errorf("cannot extract resource: %s is not configured", field)
+	}
+
+	cleanRoot := filepath.Clean(root)
+	if cleanRoot == "." || cleanRoot == string(filepath.Separator) || cleanRoot == filepath.Clean(p.Root) {
+		return "", fmt.Errorf("cannot extract resource: %s resolves to unsafe extraction root %s", field, cleanRoot)
+	}
+
+	target := filepath.Join(append([]string{cleanRoot}, parts...)...)
+	rel, err := filepath.Rel(cleanRoot, target)
+	if err != nil {
+		return "", fmt.Errorf("resolve extraction target %s: %w", target, err)
+	}
+	if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
+		return "", fmt.Errorf("refusing to extract resource outside %s: %s", cleanRoot, target)
+	}
+	return target, nil
+}
+
+func mergeExtractedGFFJSON(p *project.Project, target string, extracted *gff.Document) error {
+	rule, ok, err := extractGFFJSONMergeRule(p, target)
+	if err != nil || !ok {
+		return err
+	}
+
+	raw, err := os.ReadFile(target)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil
+		}
+		return fmt.Errorf("read existing source %s: %w", target, err)
+	}
+	var existing gff.Document
+	if err := json.Unmarshal(raw, &existing); err != nil {
+		return fmt.Errorf("parse existing source %s: %w", target, err)
+	}
+
+	for _, label := range rule.PreserveFields {
+		if field, ok := gffField(existing.Root, label); ok {
+			setGFFField(&extracted.Root, field)
+		}
+	}
+	for _, listRule := range rule.MergeLists {
+		if err := mergeGFFListByKey(&extracted.Root, existing.Root, listRule); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func extractGFFJSONMergeRule(p *project.Project, target string) (project.ExtractGFFJSONMergeRule, bool, error) {
+	rel, err := filepath.Rel(filepath.Clean(p.SourceDir()), filepath.Clean(target))
+	if err != nil {
+		return project.ExtractGFFJSONMergeRule{}, false, fmt.Errorf("resolve source-relative target %s: %w", target, err)
+	}
+	rel = filepath.ToSlash(rel)
+	if rel == ".." || strings.HasPrefix(rel, "../") {
+		return project.ExtractGFFJSONMergeRule{}, false, nil
+	}
+	for _, rule := range p.EffectiveConfig().Extract.Merge.GFFJSON {
+		if rule.Target == rel {
+			return rule, true, nil
+		}
+	}
+	return project.ExtractGFFJSONMergeRule{}, false, nil
+}
+
+func gffField(s gff.Struct, label string) (gff.Field, bool) {
+	for _, field := range s.Fields {
+		if field.Label == label {
+			return field, true
+		}
+	}
+	return gff.Field{}, false
+}
+
+func setGFFField(s *gff.Struct, replacement gff.Field) {
+	for index, field := range s.Fields {
+		if field.Label == replacement.Label {
+			s.Fields[index] = replacement
+			return
+		}
+	}
+	s.Fields = append(s.Fields, replacement)
+}
+
+func mergeGFFListByKey(extracted *gff.Struct, existing gff.Struct, rule project.ExtractListMergeRule) error {
+	extractedField, ok := gffField(*extracted, rule.Field)
+	if !ok {
+		return fmt.Errorf("extracted field %q not found", rule.Field)
+	}
+	extractedList, ok := extractedField.Value.(gff.ListValue)
+	if !ok {
+		return fmt.Errorf("extracted field %q is %s, not List", rule.Field, extractedField.Type)
+	}
+	existingField, ok := gffField(existing, rule.Field)
+	if !ok {
+		return nil
+	}
+	existingList, ok := existingField.Value.(gff.ListValue)
+	if !ok {
+		return fmt.Errorf("existing field %q is %s, not List", rule.Field, existingField.Type)
+	}
+
+	extractedByKey := map[string]gff.Struct{}
+	extractedOrder := make([]string, 0, len(extractedList))
+	for _, item := range extractedList {
+		key, err := gffStructKey(item, rule.KeyField)
+		if err != nil {
+			return fmt.Errorf("extracted field %q: %w", rule.Field, err)
+		}
+		if _, exists := extractedByKey[key]; exists {
+			return fmt.Errorf("extracted field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
+		}
+		extractedByKey[key] = item
+		extractedOrder = append(extractedOrder, key)
+	}
+
+	merged := make(gff.ListValue, 0, len(extractedList))
+	seen := map[string]struct{}{}
+	for _, item := range existingList {
+		key, err := gffStructKey(item, rule.KeyField)
+		if err != nil {
+			return fmt.Errorf("existing field %q: %w", rule.Field, err)
+		}
+		if _, duplicate := seen[key]; duplicate {
+			return fmt.Errorf("existing field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
+		}
+		if extractedItem, exists := extractedByKey[key]; exists {
+			merged = append(merged, extractedItem)
+			seen[key] = struct{}{}
+		}
+	}
+	for _, key := range extractedOrder {
+		if _, exists := seen[key]; exists {
+			continue
+		}
+		merged = append(merged, extractedByKey[key])
+	}
+
+	setGFFField(extracted, gff.NewField(rule.Field, merged))
+	return nil
+}
+
+func gffStructKey(s gff.Struct, keyField string) (string, error) {
+	field, ok := gffField(s, keyField)
+	if !ok {
+		return "", fmt.Errorf("key field %q not found", keyField)
+	}
+	switch value := field.Value.(type) {
+	case gff.ResRefValue:
+		return string(value), nil
+	case gff.StringValue:
+		return string(value), nil
+	default:
+		return "", fmt.Errorf("key field %q is %s, not ResRef or CExoString", keyField, field.Type)
+	}
+}
+
+type writeState int
+
+const (
+	writeSkipped writeState = iota
+	writeNew
+	writeOverwritten
+)
+
+func writeManagedFile(path string, data []byte) (writeState, error) {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return writeSkipped, fmt.Errorf("create parent directory for %s: %w", path, err)
+	}
+
+	existing, err := os.ReadFile(path)
+	if err == nil {
+		if bytes.Equal(existing, data) {
+			return writeSkipped, nil
+		}
+		if err := os.WriteFile(path, data, 0o644); err != nil {
+			return writeSkipped, fmt.Errorf("overwrite %s: %w", path, err)
+		}
+		return writeOverwritten, nil
+	}
+	if !errors.Is(err, os.ErrNotExist) {
+		return writeSkipped, fmt.Errorf("check existing file %s: %w", path, err)
+	}
+
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		return writeSkipped, fmt.Errorf("write %s: %w", path, err)
+	}
+	return writeNew, nil
+}
+
+func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) {
+	candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
+	if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) {
+		for _, rel := range p.Inventory.SourceFiles {
+			candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
+		}
+		for _, rel := range p.Inventory.ScriptFiles {
+			candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
+		}
+	}
+	if scope.Assets && safeCleanupRoot(p.AssetsDir(), p.Root) {
+		for _, rel := range p.Inventory.AssetFiles {
+			candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
+		}
+	}
+
+	removed := 0
+	var failures []error
+	for _, path := range candidates {
+		if _, keep := desired[path]; keep {
+			continue
+		}
+		if err := os.Remove(path); err != nil {
+			if errors.Is(err, os.ErrNotExist) {
+				continue
+			}
+			failures = append(failures, fmt.Errorf("remove stale file %s: %w", path, err))
+			continue
+		}
+		removed++
+		cleanupEmptyParents(filepath.Dir(path), p.SourceDir(), p.AssetsDir())
+	}
+
+	return removed, failures
+}
+
+func safeCleanupRoot(root, projectRoot string) bool {
+	cleanRoot := filepath.Clean(root)
+	return cleanRoot != "." && cleanRoot != string(filepath.Separator) && cleanRoot != filepath.Clean(projectRoot)
+}
+
+func cleanupEmptyParents(dir string, roots ...string) {
+	for {
+		if dir == "." || dir == string(filepath.Separator) {
+			return
+		}
+		stop := false
+		for _, root := range roots {
+			if dir == root {
+				stop = true
+				break
+			}
+		}
+		if stop {
+			return
+		}
+		if err := os.Remove(dir); err != nil {
+			return
+		}
+		dir = filepath.Dir(dir)
+	}
+}
+
+func sourceSubdir(extension string) string {
+	switch strings.ToLower(extension) {
+	case "are":
+		return "areas"
+	case "dlg":
+		return "dialogs"
+	case "fac":
+		return "factions"
+	case "gic":
+		return "instance"
+	case "git":
+		return "instance"
+	case "ifo":
+		return "module"
+	case "itp":
+		return "palettes"
+	case "jrl":
+		return "journal"
+	case "utc":
+		return filepath.Join("blueprints", "creatures")
+	case "utd":
+		return filepath.Join("blueprints", "doors")
+	case "ute":
+		return filepath.Join("blueprints", "encounters")
+	case "uti":
+		return filepath.Join("blueprints", "items")
+	case "utm":
+		return filepath.Join("blueprints", "merchants")
+	case "utp":
+		return filepath.Join("blueprints", "placeables")
+	case "uts":
+		return filepath.Join("blueprints", "sounds")
+	case "utt":
+		return filepath.Join("blueprints", "triggers")
+	case "utw":
+		return filepath.Join("blueprints", "waypoints")
+	default:
+		return extension
+	}
+}