Extraction Hardening

internal/pipeline/extract.go

@@ -25,71 +25,60 @@ type ExtractResult struct {
 	Skipped             int
 }
 
+type extractionArchive struct {
+	Path      string
+	Rel       string
+	Extension string
+}
+
+type extractionScope struct {
+	Source bool
+	Assets bool
+}
+
 func Extract(p *project.Project, files ...string) (ExtractResult, error) {
 	var result ExtractResult
 	var failures []error
 	desired := map[string]struct{}{}
+	scope := extractionScope{}
 
-	allowed := make(map[string]bool)
-	for _, f := range files {
-		allowed[f] = true
-	}
-
-	shouldExtractMod := len(allowed) == 0 || allowed[p.Config.Module.ResRef+".mod"]
-	if shouldExtractMod {
-		modulePath := p.ModuleArchivePath()
-		input, err := os.Open(modulePath)
-		if err != nil {
-			return ExtractResult{}, fmt.Errorf("open module archive: %w", err)
-		}
-		defer input.Close()
-
-		archive, err := erf.Read(input)
-		if err != nil {
-			return ExtractResult{}, fmt.Errorf("read module archive: %w", err)
-		}
-
-		result.ModulePath = modulePath
-		written, overwritten, skipped, errs := extractArchiveResources(p, archive, desired)
-		result.Written += written
-		result.Overwritten += overwritten
-		result.Skipped += skipped
-		failures = append(failures, errs...)
-	}
-
-	hakPaths, err := extractHAKPaths(p)
+	archives, err := resolveExtractionArchives(p, files)
 	if err != nil {
-		return result, fmt.Errorf("scan hak archives: %w", err)
+		return result, err
 	}
-	slices.Sort(hakPaths)
-	for _, hakPath := range hakPaths {
-		filename := filepath.Base(hakPath)
-		if len(allowed) > 0 && !allowed[filename] {
-			continue
-		}
-
-		input, err := os.Open(hakPath)
+	for _, archivePath := range archives {
+		input, err := os.Open(archivePath.Path)
 		if err != nil {
-			failures = append(failures, fmt.Errorf("open hak archive %s: %w", hakPath, err))
+			failures = append(failures, fmt.Errorf("open archive %s: %w", archivePath.Path, err))
 			continue
 		}
-		hakArchive, err := erf.Read(input)
+		archive, err := erf.Read(input)
 		input.Close()
 		if err != nil {
-			failures = append(failures, fmt.Errorf("read hak archive %s: %w", hakPath, err))
+			failures = append(failures, fmt.Errorf("read archive %s: %w", archivePath.Path, err))
 			continue
 		}
 
-		result.HAKPaths = append(result.HAKPaths, hakPath)
-		written, overwritten, skipped, errs := extractArchiveResources(p, hakArchive, desired)
+		switch archivePath.Extension {
+		case ".mod":
+			if result.ModulePath == "" {
+				result.ModulePath = archivePath.Path
+			}
+		case ".hak":
+			result.HAKPaths = append(result.HAKPaths, archivePath.Path)
+		}
+
+		written, overwritten, skipped, extractedScope, errs := extractArchiveResources(p, archive, desired)
 		result.Written += written
 		result.Overwritten += overwritten
 		result.Skipped += skipped
+		scope.Source = scope.Source || extractedScope.Source
+		scope.Assets = scope.Assets || extractedScope.Assets
 		failures = append(failures, errs...)
 	}
 
 	if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale {
-		removed, errs := cleanupStaleFiles(p, desired)
+		removed, errs := cleanupStaleFiles(p, desired, scope)
 		result.Removed = removed
 		failures = append(failures, errs...)
 	}
@@ -97,20 +86,27 @@ func Extract(p *project.Project, files ...string) (ExtractResult, error) {
 	if len(failures) > 0 {
 		return result, errors.Join(failures...)
 	}
-	if p.Config.Extract.DeleteModuleArchiveAfterSuccess && result.ModulePath != "" {
-		if err := os.Remove(result.ModulePath); err != nil {
-			return result, fmt.Errorf("delete consumed module archive %s: %w", result.ModulePath, err)
+	consumeAll, consumeModules := extractionConsumePolicy(p)
+	if consumeAll || consumeModules {
+		for _, archivePath := range archives {
+			if !consumeAll && archivePath.Extension != ".mod" {
+				continue
+			}
+			if err := os.Remove(archivePath.Path); err != nil {
+				return result, fmt.Errorf("delete consumed archive %s: %w", archivePath.Path, err)
+			}
+			result.DeletedArchivePaths = append(result.DeletedArchivePaths, archivePath.Path)
 		}
-		result.DeletedArchivePaths = append(result.DeletedArchivePaths, result.ModulePath)
 	}
 	return result, nil
 }
 
-func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, []error) {
+func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, extractionScope, []error) {
 	var failures []error
 	writtenCount := 0
 	overwrittenCount := 0
 	skippedCount := 0
+	scope := extractionScope{}
 
 	ignored := make(map[string]bool)
 	for _, ext := range p.Config.Extract.IgnoreExtensions {
@@ -135,6 +131,8 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma
 			continue
 		}
 		desired[target] = struct{}{}
+		scope.Source = scope.Source || pathWithinRoot(target, p.SourceDir())
+		scope.Assets = scope.Assets || pathWithinRoot(target, p.AssetsDir())
 
 		state, err := writeManagedFile(target, data)
 		if err != nil {
@@ -151,7 +149,138 @@ func extractArchiveResources(p *project.Project, archive erf.Archive, desired ma
 		}
 	}
 
-	return writtenCount, overwrittenCount, skippedCount, failures
+	return writtenCount, overwrittenCount, skippedCount, scope, failures
+}
+
+func resolveExtractionArchives(p *project.Project, overrides []string) ([]extractionArchive, error) {
+	patterns := p.EffectiveConfig().Extract.Archives
+	if len(overrides) > 0 {
+		patterns = overrides
+	}
+	if len(patterns) == 0 {
+		return nil, fmt.Errorf("extract.archives must contain at least one build-relative archive pattern")
+	}
+
+	buildRoot := filepath.Clean(p.BuildDir())
+	byPath := map[string]extractionArchive{}
+	for _, rawPattern := range patterns {
+		pattern, err := normalizeExtractionArchivePattern(p, rawPattern)
+		if err != nil {
+			return nil, err
+		}
+		matched, err := matchExtractionArchives(buildRoot, pattern)
+		if err != nil {
+			return nil, err
+		}
+		if len(matched) == 0 {
+			return nil, fmt.Errorf("extract archive pattern %q matched no files under %s", rawPattern, buildRoot)
+		}
+		for _, archive := range matched {
+			byPath[archive.Path] = archive
+		}
+	}
+
+	archives := make([]extractionArchive, 0, len(byPath))
+	for _, archive := range byPath {
+		archives = append(archives, archive)
+	}
+	slices.SortFunc(archives, func(a, b extractionArchive) int {
+		return strings.Compare(a.Rel, b.Rel)
+	})
+	return archives, nil
+}
+
+func normalizeExtractionArchivePattern(p *project.Project, raw string) (string, error) {
+	pattern := strings.TrimSpace(raw)
+	if pattern == "" {
+		return "", fmt.Errorf("extract archive pattern must not be empty")
+	}
+	pattern = strings.NewReplacer(
+		"{module.resref}", strings.TrimSpace(p.Config.Module.ResRef),
+	).Replace(pattern)
+	if strings.Contains(pattern, "\x00") {
+		return "", fmt.Errorf("extract archive pattern %q must not contain NUL bytes", raw)
+	}
+	if filepath.IsAbs(pattern) {
+		return "", fmt.Errorf("extract archive pattern %q must be relative to paths.build", raw)
+	}
+	clean := filepath.Clean(filepath.FromSlash(pattern))
+	if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
+		return "", fmt.Errorf("extract archive pattern %q must not escape paths.build", raw)
+	}
+	return filepath.ToSlash(clean), nil
+}
+
+func matchExtractionArchives(buildRoot, pattern string) ([]extractionArchive, error) {
+	var archives []extractionArchive
+	err := filepath.WalkDir(buildRoot, func(path string, entry os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() {
+			return nil
+		}
+		rel, err := filepath.Rel(buildRoot, path)
+		if err != nil {
+			return err
+		}
+		rel = filepath.ToSlash(rel)
+		if !matchPathPattern(rel, pattern) {
+			return nil
+		}
+		archive, err := validateExtractionArchive(buildRoot, rel)
+		if err != nil {
+			return err
+		}
+		archives = append(archives, archive)
+		return nil
+	})
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, fmt.Errorf("scan build directory %s: %w", buildRoot, err)
+		}
+		return nil, err
+	}
+	return archives, nil
+}
+
+func validateExtractionArchive(buildRoot, rel string) (extractionArchive, error) {
+	cleanRel := filepath.Clean(filepath.FromSlash(rel))
+	if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(filepath.Separator)) {
+		return extractionArchive{}, fmt.Errorf("extract archive %q must not escape paths.build", rel)
+	}
+	path := filepath.Join(buildRoot, cleanRel)
+	if !pathWithinRoot(path, buildRoot) {
+		return extractionArchive{}, fmt.Errorf("extract archive %q resolves outside paths.build", rel)
+	}
+	extension := strings.ToLower(filepath.Ext(cleanRel))
+	switch extension {
+	case ".mod", ".hak":
+		return extractionArchive{Path: path, Rel: filepath.ToSlash(cleanRel), Extension: extension}, nil
+	case ".erf":
+		return extractionArchive{}, fmt.Errorf("extract archive %q is an .erf; ERF extraction is unsafe because ERFs lack module.ifo context", rel)
+	default:
+		return extractionArchive{}, fmt.Errorf("extract archive %q uses unsupported extension %q", rel, extension)
+	}
+}
+
+func extractionConsumePolicy(p *project.Project) (consumeAll bool, consumeModules bool) {
+	if configured := p.Config.Extract.ConsumeArchives; configured != nil {
+		return *configured, false
+	}
+	return false, p.Config.Extract.DeleteModuleArchiveAfterSuccess
+}
+
+func pathWithinRoot(path, root string) bool {
+	root = filepath.Clean(root)
+	if root == "" || root == "." {
+		return false
+	}
+	rel, err := filepath.Rel(root, path)
+	if err != nil {
+		return false
+	}
+	return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel)
 }
 
 func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) {
@@ -211,30 +340,6 @@ func extractionTarget(p *project.Project, field, configured, root string, parts
 	return target, nil
 }
 
-func extractHAKPaths(p *project.Project) ([]string, error) {
-	switch p.EffectiveConfig().Extract.HAKDiscovery {
-	case "configured_haks":
-		paths := make([]string, 0, len(p.Config.HAKs))
-		for _, hak := range p.Config.HAKs {
-			path := p.HAKArchivePath(hak.Name)
-			if _, err := os.Stat(path); err == nil {
-				paths = append(paths, path)
-			} else if err != nil && !errors.Is(err, os.ErrNotExist) {
-				return nil, err
-			}
-		}
-		slices.Sort(paths)
-		return paths, nil
-	default:
-		hakPaths, err := filepath.Glob(filepath.Join(p.BuildDir(), "*.hak"))
-		if err != nil {
-			return nil, err
-		}
-		slices.Sort(hakPaths)
-		return hakPaths, nil
-	}
-}
-
 type writeState int
 
 const (
@@ -268,9 +373,9 @@ func writeManagedFile(path string, data []byte) (writeState, error) {
 	return writeNew, nil
 }
 
-func cleanupStaleFiles(p *project.Project, desired map[string]struct{}) (int, []error) {
+func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) {
 	candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
-	if safeCleanupRoot(p.SourceDir(), p.Root) {
+	if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) {
 		for _, rel := range p.Inventory.SourceFiles {
 			candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
 		}
@@ -278,7 +383,7 @@ func cleanupStaleFiles(p *project.Project, desired map[string]struct{}) (int, []
 			candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
 		}
 	}
-	if safeCleanupRoot(p.AssetsDir(), p.Root) {
+	if scope.Assets && safeCleanupRoot(p.AssetsDir(), p.Root) {
 		for _, rel := range p.Inventory.AssetFiles {
 			candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
 		}