claude: fold in old sow-tools

This commit is contained in:
2026-06-13 09:49:29 +02:00
parent d4a435883b
commit 56c67132f3
112 changed files with 70812 additions and 74 deletions
+621
View File
@@ -0,0 +1,621 @@
package pipeline
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
type ExtractResult struct {
ModulePath string
HAKPaths []string
DeletedArchivePaths []string
Written int
Overwritten int
Removed int
Skipped int
}
type extractionArchive struct {
Path string
Rel string
Extension string
}
type extractionScope struct {
Source bool
Assets bool
}
func Extract(p *project.Project, files ...string) (ExtractResult, error) {
var result ExtractResult
var failures []error
desired := map[string]struct{}{}
scope := extractionScope{}
archives, err := resolveExtractionArchives(p, files)
if err != nil {
return result, err
}
for _, archivePath := range archives {
input, err := os.Open(archivePath.Path)
if err != nil {
failures = append(failures, fmt.Errorf("open archive %s: %w", archivePath.Path, err))
continue
}
archive, err := erf.Read(input)
input.Close()
if err != nil {
failures = append(failures, fmt.Errorf("read archive %s: %w", archivePath.Path, err))
continue
}
switch archivePath.Extension {
case ".mod":
if result.ModulePath == "" {
result.ModulePath = archivePath.Path
}
case ".hak":
result.HAKPaths = append(result.HAKPaths, archivePath.Path)
}
written, overwritten, skipped, extractedScope, errs := extractArchiveResources(p, archive, desired)
result.Written += written
result.Overwritten += overwritten
result.Skipped += skipped
scope.Source = scope.Source || extractedScope.Source
scope.Assets = scope.Assets || extractedScope.Assets
failures = append(failures, errs...)
}
if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale {
removed, errs := cleanupStaleFiles(p, desired, scope)
result.Removed = removed
failures = append(failures, errs...)
}
if len(failures) > 0 {
return result, errors.Join(failures...)
}
consumeAll, consumeModules := extractionConsumePolicy(p)
if consumeAll || consumeModules {
for _, archivePath := range archives {
if !consumeAll && archivePath.Extension != ".mod" {
continue
}
if err := os.Remove(archivePath.Path); err != nil {
return result, fmt.Errorf("delete consumed archive %s: %w", archivePath.Path, err)
}
result.DeletedArchivePaths = append(result.DeletedArchivePaths, archivePath.Path)
}
}
return result, nil
}
func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, extractionScope, []error) {
var failures []error
writtenCount := 0
overwrittenCount := 0
skippedCount := 0
scope := extractionScope{}
ignored := make(map[string]bool)
for _, ext := range p.Config.Extract.IgnoreExtensions {
ext := strings.TrimPrefix(ext, ".")
ignored[ext] = true
}
for _, resource := range archive.Resources {
ext, ok := erf.ExtensionForResourceType(resource.Type)
if !ok {
failures = append(failures, fmt.Errorf("unsupported resource type 0x%04X for %s", resource.Type, resource.Name))
continue
}
if ignored[ext] {
skippedCount++
continue
}
target, data, err := extractedFile(p, resource, ext)
if err != nil {
failures = append(failures, err)
continue
}
desired[target] = struct{}{}
scope.Source = scope.Source || pathWithinRoot(target, p.SourceDir())
scope.Assets = scope.Assets || pathWithinRoot(target, p.AssetsDir())
state, err := writeManagedFile(target, data)
if err != nil {
failures = append(failures, err)
continue
}
switch state {
case writeNew:
writtenCount++
case writeOverwritten:
overwrittenCount++
case writeSkipped:
skippedCount++
}
}
return writtenCount, overwrittenCount, skippedCount, scope, failures
}
func resolveExtractionArchives(p *project.Project, overrides []string) ([]extractionArchive, error) {
patterns := p.EffectiveConfig().Extract.Archives
if len(overrides) > 0 {
patterns = overrides
}
if len(patterns) == 0 {
return nil, fmt.Errorf("extract.archives must contain at least one build-relative archive pattern")
}
buildRoot := filepath.Clean(p.BuildDir())
byPath := map[string]extractionArchive{}
for _, rawPattern := range patterns {
pattern, err := normalizeExtractionArchivePattern(p, rawPattern)
if err != nil {
return nil, err
}
matched, err := matchExtractionArchives(buildRoot, pattern)
if err != nil {
return nil, err
}
if len(matched) == 0 {
return nil, fmt.Errorf("extract archive pattern %q matched no files under %s", rawPattern, buildRoot)
}
for _, archive := range matched {
byPath[archive.Path] = archive
}
}
archives := make([]extractionArchive, 0, len(byPath))
for _, archive := range byPath {
archives = append(archives, archive)
}
slices.SortFunc(archives, func(a, b extractionArchive) int {
return strings.Compare(a.Rel, b.Rel)
})
return archives, nil
}
func normalizeExtractionArchivePattern(p *project.Project, raw string) (string, error) {
pattern := strings.TrimSpace(raw)
if pattern == "" {
return "", fmt.Errorf("extract archive pattern must not be empty")
}
pattern = strings.NewReplacer(
"{module.resref}", strings.TrimSpace(p.Config.Module.ResRef),
).Replace(pattern)
if strings.Contains(pattern, "\x00") {
return "", fmt.Errorf("extract archive pattern %q must not contain NUL bytes", raw)
}
if filepath.IsAbs(pattern) {
return "", fmt.Errorf("extract archive pattern %q must be relative to paths.build", raw)
}
clean := filepath.Clean(filepath.FromSlash(pattern))
if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
return "", fmt.Errorf("extract archive pattern %q must not escape paths.build", raw)
}
return filepath.ToSlash(clean), nil
}
func matchExtractionArchives(buildRoot, pattern string) ([]extractionArchive, error) {
var archives []extractionArchive
err := filepath.WalkDir(buildRoot, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() {
return nil
}
rel, err := filepath.Rel(buildRoot, path)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
if !matchPathPattern(rel, pattern) {
return nil
}
archive, err := validateExtractionArchive(buildRoot, rel)
if err != nil {
return err
}
archives = append(archives, archive)
return nil
})
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("scan build directory %s: %w", buildRoot, err)
}
return nil, err
}
return archives, nil
}
func validateExtractionArchive(buildRoot, rel string) (extractionArchive, error) {
cleanRel := filepath.Clean(filepath.FromSlash(rel))
if cleanRel == "." || cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(filepath.Separator)) {
return extractionArchive{}, fmt.Errorf("extract archive %q must not escape paths.build", rel)
}
path := filepath.Join(buildRoot, cleanRel)
if !pathWithinRoot(path, buildRoot) {
return extractionArchive{}, fmt.Errorf("extract archive %q resolves outside paths.build", rel)
}
extension := strings.ToLower(filepath.Ext(cleanRel))
switch extension {
case ".mod", ".hak":
return extractionArchive{Path: path, Rel: filepath.ToSlash(cleanRel), Extension: extension}, nil
case ".erf":
return extractionArchive{}, fmt.Errorf("extract archive %q is an .erf; ERF extraction is unsafe because ERFs lack module.ifo context", rel)
default:
return extractionArchive{}, fmt.Errorf("extract archive %q uses unsupported extension %q", rel, extension)
}
}
func extractionConsumePolicy(p *project.Project) (consumeAll bool, consumeModules bool) {
if configured := p.Config.Extract.ConsumeArchives; configured != nil {
return *configured, false
}
return false, p.Config.Extract.DeleteModuleArchiveAfterSuccess
}
func pathWithinRoot(path, root string) bool {
root = filepath.Clean(root)
if root == "" || root == "." {
return false
}
rel, err := filepath.Rel(root, path)
if err != nil {
return false
}
return rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) && !filepath.IsAbs(rel)
}
func extractedFile(p *project.Project, resource erf.Resource, extension string) (string, []byte, error) {
resref := strings.ToLower(resource.Name)
effective := p.EffectiveConfig()
switch extension {
case "nss":
target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), "scripts", resref+".nss")
if err != nil {
return "", nil, err
}
return target, resource.Data, nil
case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl":
document, err := gff.Read(bytes.NewReader(resource.Data))
if err != nil {
return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err)
}
target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json")
if err != nil {
return "", nil, err
}
if err := mergeExtractedGFFJSON(p, target, &document); err != nil {
return "", nil, fmt.Errorf("merge extracted gff json %s.%s: %w", resource.Name, extension, err)
}
formatted, err := json.MarshalIndent(document, "", " ")
if err != nil {
return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err)
}
formatted = append(formatted, '\n')
return target, formatted, nil
default:
target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension)
if err != nil {
return "", nil, err
}
return target, resource.Data, nil
}
}
func extractionTarget(p *project.Project, field, configured, root string, parts ...string) (string, error) {
if strings.TrimSpace(configured) == "" {
return "", fmt.Errorf("cannot extract resource: %s is not configured", field)
}
cleanRoot := filepath.Clean(root)
if cleanRoot == "." || cleanRoot == string(filepath.Separator) || cleanRoot == filepath.Clean(p.Root) {
return "", fmt.Errorf("cannot extract resource: %s resolves to unsafe extraction root %s", field, cleanRoot)
}
target := filepath.Join(append([]string{cleanRoot}, parts...)...)
rel, err := filepath.Rel(cleanRoot, target)
if err != nil {
return "", fmt.Errorf("resolve extraction target %s: %w", target, err)
}
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
return "", fmt.Errorf("refusing to extract resource outside %s: %s", cleanRoot, target)
}
return target, nil
}
func mergeExtractedGFFJSON(p *project.Project, target string, extracted *gff.Document) error {
rule, ok, err := extractGFFJSONMergeRule(p, target)
if err != nil || !ok {
return err
}
raw, err := os.ReadFile(target)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return fmt.Errorf("read existing source %s: %w", target, err)
}
var existing gff.Document
if err := json.Unmarshal(raw, &existing); err != nil {
return fmt.Errorf("parse existing source %s: %w", target, err)
}
for _, label := range rule.PreserveFields {
if field, ok := gffField(existing.Root, label); ok {
setGFFField(&extracted.Root, field)
}
}
for _, listRule := range rule.MergeLists {
if err := mergeGFFListByKey(&extracted.Root, existing.Root, listRule); err != nil {
return err
}
}
return nil
}
func extractGFFJSONMergeRule(p *project.Project, target string) (project.ExtractGFFJSONMergeRule, bool, error) {
rel, err := filepath.Rel(filepath.Clean(p.SourceDir()), filepath.Clean(target))
if err != nil {
return project.ExtractGFFJSONMergeRule{}, false, fmt.Errorf("resolve source-relative target %s: %w", target, err)
}
rel = filepath.ToSlash(rel)
if rel == ".." || strings.HasPrefix(rel, "../") {
return project.ExtractGFFJSONMergeRule{}, false, nil
}
for _, rule := range p.EffectiveConfig().Extract.Merge.GFFJSON {
if rule.Target == rel {
return rule, true, nil
}
}
return project.ExtractGFFJSONMergeRule{}, false, nil
}
func gffField(s gff.Struct, label string) (gff.Field, bool) {
for _, field := range s.Fields {
if field.Label == label {
return field, true
}
}
return gff.Field{}, false
}
func setGFFField(s *gff.Struct, replacement gff.Field) {
for index, field := range s.Fields {
if field.Label == replacement.Label {
s.Fields[index] = replacement
return
}
}
s.Fields = append(s.Fields, replacement)
}
func mergeGFFListByKey(extracted *gff.Struct, existing gff.Struct, rule project.ExtractListMergeRule) error {
extractedField, ok := gffField(*extracted, rule.Field)
if !ok {
return fmt.Errorf("extracted field %q not found", rule.Field)
}
extractedList, ok := extractedField.Value.(gff.ListValue)
if !ok {
return fmt.Errorf("extracted field %q is %s, not List", rule.Field, extractedField.Type)
}
existingField, ok := gffField(existing, rule.Field)
if !ok {
return nil
}
existingList, ok := existingField.Value.(gff.ListValue)
if !ok {
return fmt.Errorf("existing field %q is %s, not List", rule.Field, existingField.Type)
}
extractedByKey := map[string]gff.Struct{}
extractedOrder := make([]string, 0, len(extractedList))
for _, item := range extractedList {
key, err := gffStructKey(item, rule.KeyField)
if err != nil {
return fmt.Errorf("extracted field %q: %w", rule.Field, err)
}
if _, exists := extractedByKey[key]; exists {
return fmt.Errorf("extracted field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
}
extractedByKey[key] = item
extractedOrder = append(extractedOrder, key)
}
merged := make(gff.ListValue, 0, len(extractedList))
seen := map[string]struct{}{}
for _, item := range existingList {
key, err := gffStructKey(item, rule.KeyField)
if err != nil {
return fmt.Errorf("existing field %q: %w", rule.Field, err)
}
if _, duplicate := seen[key]; duplicate {
return fmt.Errorf("existing field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
}
if extractedItem, exists := extractedByKey[key]; exists {
merged = append(merged, extractedItem)
seen[key] = struct{}{}
}
}
for _, key := range extractedOrder {
if _, exists := seen[key]; exists {
continue
}
merged = append(merged, extractedByKey[key])
}
setGFFField(extracted, gff.NewField(rule.Field, merged))
return nil
}
func gffStructKey(s gff.Struct, keyField string) (string, error) {
field, ok := gffField(s, keyField)
if !ok {
return "", fmt.Errorf("key field %q not found", keyField)
}
switch value := field.Value.(type) {
case gff.ResRefValue:
return string(value), nil
case gff.StringValue:
return string(value), nil
default:
return "", fmt.Errorf("key field %q is %s, not ResRef or CExoString", keyField, field.Type)
}
}
type writeState int
const (
writeSkipped writeState = iota
writeNew
writeOverwritten
)
func writeManagedFile(path string, data []byte) (writeState, error) {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return writeSkipped, fmt.Errorf("create parent directory for %s: %w", path, err)
}
existing, err := os.ReadFile(path)
if err == nil {
if bytes.Equal(existing, data) {
return writeSkipped, nil
}
if err := os.WriteFile(path, data, 0o644); err != nil {
return writeSkipped, fmt.Errorf("overwrite %s: %w", path, err)
}
return writeOverwritten, nil
}
if !errors.Is(err, os.ErrNotExist) {
return writeSkipped, fmt.Errorf("check existing file %s: %w", path, err)
}
if err := os.WriteFile(path, data, 0o644); err != nil {
return writeSkipped, fmt.Errorf("write %s: %w", path, err)
}
return writeNew, nil
}
func cleanupStaleFiles(p *project.Project, desired map[string]struct{}, scope extractionScope) (int, []error) {
candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
if scope.Source && safeCleanupRoot(p.SourceDir(), p.Root) {
for _, rel := range p.Inventory.SourceFiles {
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
}
for _, rel := range p.Inventory.ScriptFiles {
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
}
}
if scope.Assets && safeCleanupRoot(p.AssetsDir(), p.Root) {
for _, rel := range p.Inventory.AssetFiles {
candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
}
}
removed := 0
var failures []error
for _, path := range candidates {
if _, keep := desired[path]; keep {
continue
}
if err := os.Remove(path); err != nil {
if errors.Is(err, os.ErrNotExist) {
continue
}
failures = append(failures, fmt.Errorf("remove stale file %s: %w", path, err))
continue
}
removed++
cleanupEmptyParents(filepath.Dir(path), p.SourceDir(), p.AssetsDir())
}
return removed, failures
}
func safeCleanupRoot(root, projectRoot string) bool {
cleanRoot := filepath.Clean(root)
return cleanRoot != "." && cleanRoot != string(filepath.Separator) && cleanRoot != filepath.Clean(projectRoot)
}
func cleanupEmptyParents(dir string, roots ...string) {
for {
if dir == "." || dir == string(filepath.Separator) {
return
}
stop := false
for _, root := range roots {
if dir == root {
stop = true
break
}
}
if stop {
return
}
if err := os.Remove(dir); err != nil {
return
}
dir = filepath.Dir(dir)
}
}
func sourceSubdir(extension string) string {
switch strings.ToLower(extension) {
case "are":
return "areas"
case "dlg":
return "dialogs"
case "fac":
return "factions"
case "gic":
return "instance"
case "git":
return "instance"
case "ifo":
return "module"
case "itp":
return "palettes"
case "jrl":
return "journal"
case "utc":
return filepath.Join("blueprints", "creatures")
case "utd":
return filepath.Join("blueprints", "doors")
case "ute":
return filepath.Join("blueprints", "encounters")
case "uti":
return filepath.Join("blueprints", "items")
case "utm":
return filepath.Join("blueprints", "merchants")
case "utp":
return filepath.Join("blueprints", "placeables")
case "uts":
return filepath.Join("blueprints", "sounds")
case "utt":
return filepath.Join("blueprints", "triggers")
case "utw":
return filepath.Join("blueprints", "waypoints")
default:
return extension
}
}