diff --git a/internal/app/app.go b/internal/app/app.go index a5bb5c5..f5467be 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -181,6 +181,8 @@ func runExtract(ctx context) error { fmt.Fprintf(ctx.stdout, "hak archives: %d\n", len(result.HAKPaths)) } fmt.Fprintf(ctx.stdout, "written: %d\n", result.Written) + fmt.Fprintf(ctx.stdout, "overwritten: %d\n", result.Overwritten) + fmt.Fprintf(ctx.stdout, "removed: %d\n", result.Removed) fmt.Fprintf(ctx.stdout, "skipped: %d\n", result.Skipped) return nil } diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index d9246ff..b951427 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -269,7 +269,7 @@ func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error } return erf.Resource{ - Name: name, + Name: strings.ToLower(name), Type: resourceType, Data: buf.Bytes(), }, nil @@ -281,7 +281,7 @@ func pathResource(path string) (erf.Resource, error) { if !ok { return erf.Resource{}, fmt.Errorf("unsupported resource extension %q", filepath.Ext(path)) } - name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) info, err := os.Stat(path) if err != nil { return erf.Resource{}, fmt.Errorf("stat %s: %w", path, err) @@ -300,13 +300,13 @@ func rawResource(path string) (erf.Resource, error) { if !ok { return erf.Resource{}, fmt.Errorf("unsupported resource extension %q", filepath.Ext(path)) } - name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) data, err := os.ReadFile(path) if err != nil { return erf.Resource{}, fmt.Errorf("read %s: %w", path, err) } return erf.Resource{ - Name: name, + Name: strings.ToLower(name), Type: resourceType, Data: data, }, nil @@ -322,7 +322,7 @@ func splitSourceName(path string) (string, string, error) { if extension == "" { return "", "", fmt.Errorf("source file must include target resource extension before .json: %s", path) } - return strings.TrimSuffix(stem, extension), strings.ToLower(extension), nil + return strings.ToLower(strings.TrimSuffix(stem, extension)), strings.ToLower(extension), nil } func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, error) { diff --git a/internal/pipeline/extract.go b/internal/pipeline/extract.go index 55e82f3..a752e68 100644 --- a/internal/pipeline/extract.go +++ b/internal/pipeline/extract.go @@ -16,10 +16,12 @@ import ( ) type ExtractResult struct { - ModulePath string - HAKPaths []string - Written int - Skipped int + ModulePath string + HAKPaths []string + Written int + Overwritten int + Removed int + Skipped int } func Extract(p *project.Project) (ExtractResult, error) { @@ -38,9 +40,11 @@ func Extract(p *project.Project) (ExtractResult, error) { var result ExtractResult result.ModulePath = modulePath var failures []error + desired := map[string]struct{}{} - written, skipped, errs := extractArchiveResources(p, archive) + written, overwritten, skipped, errs := extractArchiveResources(p, archive, desired) result.Written += written + result.Overwritten += overwritten result.Skipped += skipped failures = append(failures, errs...) @@ -63,21 +67,27 @@ func Extract(p *project.Project) (ExtractResult, error) { } result.HAKPaths = append(result.HAKPaths, hakPath) - written, skipped, errs := extractArchiveResources(p, hakArchive) + written, overwritten, skipped, errs := extractArchiveResources(p, hakArchive, desired) result.Written += written + result.Overwritten += overwritten result.Skipped += skipped failures = append(failures, errs...) } + removed, errs := cleanupStaleFiles(p, desired) + result.Removed = removed + failures = append(failures, errs...) + if len(failures) > 0 { return result, errors.Join(failures...) } return result, nil } -func extractArchiveResources(p *project.Project, archive erf.Archive) (int, int, []error) { +func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, []error) { var failures []error writtenCount := 0 + overwrittenCount := 0 skippedCount := 0 for _, resource := range archive.Resources { @@ -86,20 +96,24 @@ func extractArchiveResources(p *project.Project, archive erf.Archive) (int, int, failures = append(failures, err) continue } + desired[target] = struct{}{} - written, err := writeSafely(target, data) + state, err := writeManagedFile(target, data) if err != nil { failures = append(failures, err) continue } - if written { + switch state { + case writeNew: writtenCount++ - } else { + case writeOverwritten: + overwrittenCount++ + case writeSkipped: skippedCount++ } } - return writtenCount, skippedCount, failures + return writtenCount, overwrittenCount, skippedCount, failures } func extractedFile(p *project.Project, resource erf.Resource) (string, []byte, error) { @@ -107,10 +121,11 @@ func extractedFile(p *project.Project, resource erf.Resource) (string, []byte, e if !ok { return "", nil, fmt.Errorf("unsupported resource type 0x%04X for %s", resource.Type, resource.Name) } + resref := strings.ToLower(resource.Name) switch extension { case "nss": - return filepath.Join(p.SourceDir(), "scripts", resource.Name+".nss"), resource.Data, nil + return filepath.Join(p.SourceDir(), "scripts", resref+".nss"), resource.Data, nil case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw", "are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl": document, err := gff.Read(bytes.NewReader(resource.Data)) @@ -122,32 +137,96 @@ func extractedFile(p *project.Project, resource erf.Resource) (string, []byte, e return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err) } formatted = append(formatted, '\n') - return filepath.Join(p.SourceDir(), sourceSubdir(extension), resource.Name+"."+extension+".json"), formatted, nil + return filepath.Join(p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json"), formatted, nil default: - return filepath.Join(p.AssetsDir(), extension, resource.Name+"."+extension), resource.Data, nil + return filepath.Join(p.AssetsDir(), extension, resref+"."+extension), resource.Data, nil } } -func writeSafely(path string, data []byte) (bool, error) { +type writeState int + +const ( + writeSkipped writeState = iota + writeNew + writeOverwritten +) + +func writeManagedFile(path string, data []byte) (writeState, error) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return false, fmt.Errorf("create parent directory for %s: %w", path, err) + return writeSkipped, fmt.Errorf("create parent directory for %s: %w", path, err) } existing, err := os.ReadFile(path) if err == nil { if bytes.Equal(existing, data) { - return false, nil + return writeSkipped, nil } - return false, fmt.Errorf("refusing to overwrite existing file with different contents: %s", path) + if err := os.WriteFile(path, data, 0o644); err != nil { + return writeSkipped, fmt.Errorf("overwrite %s: %w", path, err) + } + return writeOverwritten, nil } if !errors.Is(err, os.ErrNotExist) { - return false, fmt.Errorf("check existing file %s: %w", path, err) + return writeSkipped, fmt.Errorf("check existing file %s: %w", path, err) } if err := os.WriteFile(path, data, 0o644); err != nil { - return false, fmt.Errorf("write %s: %w", path, err) + return writeSkipped, fmt.Errorf("write %s: %w", path, err) + } + return writeNew, nil +} + +func cleanupStaleFiles(p *project.Project, desired map[string]struct{}) (int, []error) { + candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles)) + for _, rel := range p.Inventory.SourceFiles { + candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) + } + for _, rel := range p.Inventory.ScriptFiles { + candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel))) + } + for _, rel := range p.Inventory.AssetFiles { + candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))) + } + + removed := 0 + var failures []error + for _, path := range candidates { + if _, keep := desired[path]; keep { + continue + } + if err := os.Remove(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + failures = append(failures, fmt.Errorf("remove stale file %s: %w", path, err)) + continue + } + removed++ + cleanupEmptyParents(filepath.Dir(path), p.SourceDir(), p.AssetsDir()) + } + return removed, failures +} + +func cleanupEmptyParents(dir string, roots ...string) { + for { + if dir == "." || dir == string(filepath.Separator) { + return + } + stop := false + for _, root := range roots { + if dir == root { + stop = true + break + } + } + if stop { + return + } + if err := os.Remove(dir); err != nil { + return + } + dir = filepath.Dir(dir) } - return true, nil } func sourceSubdir(extension string) string { diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 1b8b47d..cac2964 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -607,6 +607,172 @@ func TestBuildSkipsEmptyHAKGroupsInModuleOrder(t *testing.T) { } } +func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "src", "areas")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Original Module" + } + ] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Changed In Toolset" + } + ] + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "areas", "area001.are.json"), `{"stale": true}`) + + if err := p.Scan(); err != nil { + t.Fatalf("rescan before extract: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if result.Overwritten != 1 { + t.Fatalf("expected 1 overwritten file, got %d", result.Overwritten) + } + if result.Removed != 1 { + t.Fatalf("expected 1 removed stale file, got %d", result.Removed) + } + + raw, err := os.ReadFile(filepath.Join(root, "src", "module", "module.ifo.json")) + if err != nil { + t.Fatalf("read extracted module file: %v", err) + } + if !strings.Contains(string(raw), "Original Module") { + t.Fatalf("expected extracted module file to be overwritten with archive contents:\n%s", string(raw)) + } + if _, err := os.Stat(filepath.Join(root, "src", "areas", "area001.are.json")); !os.IsNotExist(err) { + t.Fatalf("expected stale area file to be removed, stat err=%v", err) + } +} + +func TestExtractNormalizesResourceNamesToLowercase(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "blueprints", "items")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + + mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json"), `{ + "file_type": "UTI ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "LocalizedName", + "type": "CExoString", + "value": "Elven Chain" + } + ] + } +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + if err := os.Remove(filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json")); err != nil { + t.Fatalf("remove uppercase source file: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("rescan before extract: %v", err) + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if result.Written != 1 { + t.Fatalf("expected 1 written file, got %d", result.Written) + } + if _, err := os.Stat(filepath.Join(root, "src", "blueprints", "items", "i_elvenchain.uti.json")); err != nil { + t.Fatalf("expected lowercase extracted resource: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json")); !os.IsNotExist(err) { + t.Fatalf("expected uppercase extracted resource path to be absent, stat err=%v", err) + } +} + func mustMkdir(t *testing.T, path string) { t.Helper() if err := os.MkdirAll(path, 0o755); err != nil { diff --git a/internal/validator/validator.go b/internal/validator/validator.go index b6c303c..ca08515 100644 --- a/internal/validator/validator.go +++ b/internal/validator/validator.go @@ -76,6 +76,9 @@ func ValidateProject(p *project.Project) Report { resolver := newAssetGroupResolver(p) for _, rel := range p.Inventory.SourceFiles { + if hasUppercaseResourceName(rel) { + report.add(rel, "resource filenames should be lowercase", SeverityWarning) + } abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel)) document, resref, extension, err := loadDocument(abs) if err != nil { @@ -90,7 +93,7 @@ func ValidateProject(p *project.Project) Report { Document: document, }) - key := resref + "." + extension + key := strings.ToLower(resref) + "." + extension if previous, exists := resourceIndex[key]; exists { report.add(rel, fmt.Sprintf("duplicate resource %s also defined by %s", key, previous), SeverityError) } else { @@ -101,7 +104,10 @@ func ValidateProject(p *project.Project) Report { } for _, rel := range p.Inventory.ScriptFiles { - base := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)) + if hasUppercaseResourceName(rel) { + report.add(rel, "resource filenames should be lowercase", SeverityWarning) + } + base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))) if previous, exists := scriptIndex[base]; exists { report.add(rel, fmt.Sprintf("duplicate script resource %s also defined by %s", base, previous), SeverityError) } else { @@ -110,9 +116,12 @@ func ValidateProject(p *project.Project) Report { } for _, rel := range p.Inventory.AssetFiles { - base := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)) + if hasUppercaseResourceName(rel) { + report.add(rel, "resource filenames should be lowercase", SeverityWarning) + } + base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))) extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(rel)), ".") - key := base + "." + extension + key := strings.ToLower(base) + "." + extension if _, exists := assetIndex[key]; !exists { assetIndex[key] = rel } @@ -187,7 +196,7 @@ func loadDocument(path string) (gff.Document, string, string, error) { if extension == "" { return gff.Document{}, "", "", fmt.Errorf("missing resource extension before .json") } - resref := strings.TrimSuffix(stem, "."+extension) + resref := strings.ToLower(strings.TrimSuffix(stem, "."+extension)) var document gff.Document if err := json.Unmarshal(raw, &document); err != nil { @@ -250,25 +259,25 @@ func validateReferences(report *Report, document loadedDocument, resources, scri if isBuiltinScript(value) { return } - if _, exists := scripts[value]; !exists { + if _, exists := scripts[strings.ToLower(value)]; !exists { report.add(document.Path, fmt.Sprintf("missing script reference %q from field %q", value, field.Label), SeverityError) } case field.Label == "Conversation": - key := value + ".dlg" + key := strings.ToLower(value) + ".dlg" if _, exists := resources[key]; !exists { report.add(document.Path, fmt.Sprintf("missing dialog reference %q", key), SeverityError) } case field.Label == "Mod_Entry_Area": - key := value + ".are" + key := strings.ToLower(value) + ".are" if _, exists := resources[key]; !exists { report.add(document.Path, fmt.Sprintf("missing area reference %q", key), SeverityError) } case field.Label == "Model": - if !hasAsset(value, []string{"mdl"}, assets) { + if !hasAsset(strings.ToLower(value), []string{"mdl"}, assets) { report.add(document.Path, fmt.Sprintf("missing model asset for %q", value), SeverityError) } case field.Label == "Sound": - if !hasAsset(value, []string{"wav"}, assets) { + if !hasAsset(strings.ToLower(value), []string{"wav"}, assets) { report.add(document.Path, fmt.Sprintf("missing sound asset for %q", value), SeverityError) } } @@ -390,6 +399,11 @@ func walkFields(s gff.Struct, visit func(gff.Field)) { } } +func hasUppercaseResourceName(rel string) bool { + base := filepath.Base(rel) + return base != strings.ToLower(base) +} + func fieldStringValue(value gff.Value) (string, bool) { switch typed := value.(type) { case gff.StringValue: diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index 76bf94b..e4e6b3c 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -54,6 +54,49 @@ func TestValidateProjectWarnsForCrossHAKDuplicateAssets(t *testing.T) { } } +func TestValidateProjectWarnsForUppercaseResourceNames(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "blueprints", "items")) + mustMkdir(t, filepath.Join(root, "assets", "vfx")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod" + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +`) + mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json"), `{ + "file_type": "UTI ", + "file_version": "V3.2", + "root": {"struct_type": 0, "fields": []} +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "vfx", "SPELL_FIRE.tga"), "fire") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + report := ValidateProject(p) + if report.WarningCount() < 2 { + t.Fatalf("expected lowercase warnings, got %#v", report.Diagnostics) + } +} + func mustMkdir(t *testing.T, path string) { t.Helper() if err := os.MkdirAll(path, 0o755); err != nil {