ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Tool: Polish Pass 1

Browse Source 
In sow-tools, extract now behaves like a real sync by default: it
overwrites changed extracted files, removes stale extracted files that
no longer exist in the built archives, and normalizes extracted resource
filenames to lowercase. I also made validation warn on uppercase
resource filenames so mixed-case names like I_ELVENCHAIN are surfaced
instead of quietly lingering. The extract command output now includes
overwritten and removed counts too.
This commit is contained in:
archvillainette
2026-04-02 20:04:41 +02:00
parent 97465e451e
commit 77b21081d7
6 changed files with 340 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/app/app.go
+2
View File
@@ -181,6 +181,8 @@ func runExtract(ctx context) error {
fmt.Fprintf(ctx.stdout, "hak archives: %d\n", len(result.HAKPaths))
}
fmt.Fprintf(ctx.stdout, "written: %d\n", result.Written)
fmt.Fprintf(ctx.stdout, "overwritten: %d\n", result.Overwritten)
fmt.Fprintf(ctx.stdout, "removed: %d\n", result.Removed)
fmt.Fprintf(ctx.stdout, "skipped: %d\n", result.Skipped)
return nil
}
internal/pipeline/build.go
+5 -5
View File
@@ -269,7 +269,7 @@ func resourceFromJSON(path string, moduleHakOrder []string) (erf.Resource, error
}
return erf.Resource{
Name: name,
Name: strings.ToLower(name),
Type: resourceType,
Data: buf.Bytes(),
}, nil
@@ -281,7 +281,7 @@ func pathResource(path string) (erf.Resource, error) {
if !ok {
return erf.Resource{}, fmt.Errorf("unsupported resource extension %q", filepath.Ext(path))
}
name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)))
info, err := os.Stat(path)
if err != nil {
return erf.Resource{}, fmt.Errorf("stat %s: %w", path, err)
@@ -300,13 +300,13 @@ func rawResource(path string) (erf.Resource, error) {
if !ok {
return erf.Resource{}, fmt.Errorf("unsupported resource extension %q", filepath.Ext(path))
}
name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)))
data, err := os.ReadFile(path)
if err != nil {
return erf.Resource{}, fmt.Errorf("read %s: %w", path, err)
}
return erf.Resource{
Name: name,
Name: strings.ToLower(name),
Type: resourceType,
Data: data,
}, nil
@@ -322,7 +322,7 @@ func splitSourceName(path string) (string, string, error) {
if extension == "" {
return "", "", fmt.Errorf("source file must include target resource extension before .json: %s", path)
}
return strings.TrimSuffix(stem, extension), strings.ToLower(extension), nil
return strings.ToLower(strings.TrimSuffix(stem, extension)), strings.ToLower(extension), nil
}
func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, error) {
internal/pipeline/extract.go
+100 -21
View File
@@ -16,10 +16,12 @@ import (
)
type ExtractResult struct {
ModulePath string
HAKPaths []string
Written int
Skipped int
ModulePath string
HAKPaths []string
Written int
Overwritten int
Removed int
Skipped int
}
func Extract(p *project.Project) (ExtractResult, error) {
@@ -38,9 +40,11 @@ func Extract(p *project.Project) (ExtractResult, error) {
var result ExtractResult
result.ModulePath = modulePath
var failures []error
desired := map[string]struct{}{}
written, skipped, errs := extractArchiveResources(p, archive)
written, overwritten, skipped, errs := extractArchiveResources(p, archive, desired)
result.Written += written
result.Overwritten += overwritten
result.Skipped += skipped
failures = append(failures, errs...)
@@ -63,21 +67,27 @@ func Extract(p *project.Project) (ExtractResult, error) {
}
result.HAKPaths = append(result.HAKPaths, hakPath)
written, skipped, errs := extractArchiveResources(p, hakArchive)
written, overwritten, skipped, errs := extractArchiveResources(p, hakArchive, desired)
result.Written += written
result.Overwritten += overwritten
result.Skipped += skipped
failures = append(failures, errs...)
}
removed, errs := cleanupStaleFiles(p, desired)
result.Removed = removed
failures = append(failures, errs...)
if len(failures) > 0 {
return result, errors.Join(failures...)
}
return result, nil
}
func extractArchiveResources(p *project.Project, archive erf.Archive) (int, int, []error) {
func extractArchiveResources(p *project.Project, archive erf.Archive, desired map[string]struct{}) (int, int, int, []error) {
var failures []error
writtenCount := 0
overwrittenCount := 0
skippedCount := 0
for _, resource := range archive.Resources {
@@ -86,20 +96,24 @@ func extractArchiveResources(p *project.Project, archive erf.Archive) (int, int,
failures = append(failures, err)
continue
}
desired[target] = struct{}{}
written, err := writeSafely(target, data)
state, err := writeManagedFile(target, data)
if err != nil {
failures = append(failures, err)
continue
}
if written {
switch state {
case writeNew:
writtenCount++
} else {
case writeOverwritten:
overwrittenCount++
case writeSkipped:
skippedCount++
}
}
return writtenCount, skippedCount, failures
return writtenCount, overwrittenCount, skippedCount, failures
}
func extractedFile(p *project.Project, resource erf.Resource) (string, []byte, error) {
@@ -107,10 +121,11 @@ func extractedFile(p *project.Project, resource erf.Resource) (string, []byte, e
if !ok {
return "", nil, fmt.Errorf("unsupported resource type 0x%04X for %s", resource.Type, resource.Name)
}
resref := strings.ToLower(resource.Name)
switch extension {
case "nss":
return filepath.Join(p.SourceDir(), "scripts", resource.Name+".nss"), resource.Data, nil
return filepath.Join(p.SourceDir(), "scripts", resref+".nss"), resource.Data, nil
case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl":
document, err := gff.Read(bytes.NewReader(resource.Data))
@@ -122,32 +137,96 @@ func extractedFile(p *project.Project, resource erf.Resource) (string, []byte, e
return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err)
}
formatted = append(formatted, '\n')
return filepath.Join(p.SourceDir(), sourceSubdir(extension), resource.Name+"."+extension+".json"), formatted, nil
return filepath.Join(p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json"), formatted, nil
default:
return filepath.Join(p.AssetsDir(), extension, resource.Name+"."+extension), resource.Data, nil
return filepath.Join(p.AssetsDir(), extension, resref+"."+extension), resource.Data, nil
}
}
func writeSafely(path string, data []byte) (bool, error) {
type writeState int
const (
writeSkipped writeState = iota
writeNew
writeOverwritten
)
func writeManagedFile(path string, data []byte) (writeState, error) {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return false, fmt.Errorf("create parent directory for %s: %w", path, err)
return writeSkipped, fmt.Errorf("create parent directory for %s: %w", path, err)
}
existing, err := os.ReadFile(path)
if err == nil {
if bytes.Equal(existing, data) {
return false, nil
return writeSkipped, nil
}
return false, fmt.Errorf("refusing to overwrite existing file with different contents: %s", path)
if err := os.WriteFile(path, data, 0o644); err != nil {
return writeSkipped, fmt.Errorf("overwrite %s: %w", path, err)
}
return writeOverwritten, nil
}
if !errors.Is(err, os.ErrNotExist) {
return false, fmt.Errorf("check existing file %s: %w", path, err)
return writeSkipped, fmt.Errorf("check existing file %s: %w", path, err)
}
if err := os.WriteFile(path, data, 0o644); err != nil {
return false, fmt.Errorf("write %s: %w", path, err)
return writeSkipped, fmt.Errorf("write %s: %w", path, err)
}
return writeNew, nil
}
func cleanupStaleFiles(p *project.Project, desired map[string]struct{}) (int, []error) {
candidates := make([]string, 0, len(p.Inventory.SourceFiles)+len(p.Inventory.ScriptFiles)+len(p.Inventory.AssetFiles))
for _, rel := range p.Inventory.SourceFiles {
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
}
for _, rel := range p.Inventory.ScriptFiles {
candidates = append(candidates, filepath.Join(p.SourceDir(), filepath.FromSlash(rel)))
}
for _, rel := range p.Inventory.AssetFiles {
candidates = append(candidates, filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)))
}
removed := 0
var failures []error
for _, path := range candidates {
if _, keep := desired[path]; keep {
continue
}
if err := os.Remove(path); err != nil {
if errors.Is(err, os.ErrNotExist) {
continue
}
failures = append(failures, fmt.Errorf("remove stale file %s: %w", path, err))
continue
}
removed++
cleanupEmptyParents(filepath.Dir(path), p.SourceDir(), p.AssetsDir())
}
return removed, failures
}
func cleanupEmptyParents(dir string, roots ...string) {
for {
if dir == "." || dir == string(filepath.Separator) {
return
}
stop := false
for _, root := range roots {
if dir == root {
stop = true
break
}
}
if stop {
return
}
if err := os.Remove(dir); err != nil {
return
}
dir = filepath.Dir(dir)
}
return true, nil
}
func sourceSubdir(extension string) string {
internal/pipeline/pipeline_test.go
+166
View File
@@ -607,6 +607,172 @@ func TestBuildSkipsEmptyHAKGroupsInModuleOrder(t *testing.T) {
}
}
func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "src", "areas"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod"
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
}
}
`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Original Module"
}
]
}
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
if _, err := BuildModule(p); err != nil {
t.Fatalf("build module: %v", err)
}
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Changed In Toolset"
}
]
}
}
`)
mustWriteFile(t, filepath.Join(root, "src", "areas", "area001.are.json"), `{"stale": true}`)
if err := p.Scan(); err != nil {
t.Fatalf("rescan before extract: %v", err)
}
result, err := Extract(p)
if err != nil {
t.Fatalf("extract: %v", err)
}
if result.Overwritten != 1 {
t.Fatalf("expected 1 overwritten file, got %d", result.Overwritten)
}
if result.Removed != 1 {
t.Fatalf("expected 1 removed stale file, got %d", result.Removed)
}
raw, err := os.ReadFile(filepath.Join(root, "src", "module", "module.ifo.json"))
if err != nil {
t.Fatalf("read extracted module file: %v", err)
}
if !strings.Contains(string(raw), "Original Module") {
t.Fatalf("expected extracted module file to be overwritten with archive contents:\n%s", string(raw))
}
if _, err := os.Stat(filepath.Join(root, "src", "areas", "area001.are.json")); !os.IsNotExist(err) {
t.Fatalf("expected stale area file to be removed, stat err=%v", err)
}
}
func TestExtractNormalizesResourceNamesToLowercase(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "blueprints", "items"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod"
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
}
}
`)
mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json"), `{
"file_type": "UTI ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "LocalizedName",
"type": "CExoString",
"value": "Elven Chain"
}
]
}
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
if _, err := BuildModule(p); err != nil {
t.Fatalf("build module: %v", err)
}
if err := os.Remove(filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json")); err != nil {
t.Fatalf("remove uppercase source file: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("rescan before extract: %v", err)
}
result, err := Extract(p)
if err != nil {
t.Fatalf("extract: %v", err)
}
if result.Written != 1 {
t.Fatalf("expected 1 written file, got %d", result.Written)
}
if _, err := os.Stat(filepath.Join(root, "src", "blueprints", "items", "i_elvenchain.uti.json")); err != nil {
t.Fatalf("expected lowercase extracted resource: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json")); !os.IsNotExist(err) {
t.Fatalf("expected uppercase extracted resource path to be absent, stat err=%v", err)
}
}
func mustMkdir(t *testing.T, path string) {
t.Helper()
if err := os.MkdirAll(path, 0o755); err != nil {
internal/validator/validator.go
+24 -10
View File
@@ -76,6 +76,9 @@ func ValidateProject(p *project.Project) Report {
resolver := newAssetGroupResolver(p)
for _, rel := range p.Inventory.SourceFiles {
if hasUppercaseResourceName(rel) {
report.add(rel, "resource filenames should be lowercase", SeverityWarning)
}
abs := filepath.Join(p.SourceDir(), filepath.FromSlash(rel))
document, resref, extension, err := loadDocument(abs)
if err != nil {
@@ -90,7 +93,7 @@ func ValidateProject(p *project.Project) Report {
Document: document,
})
key := resref + "." + extension
key := strings.ToLower(resref) + "." + extension
if previous, exists := resourceIndex[key]; exists {
report.add(rel, fmt.Sprintf("duplicate resource %s also defined by %s", key, previous), SeverityError)
} else {
@@ -101,7 +104,10 @@ func ValidateProject(p *project.Project) Report {
}
for _, rel := range p.Inventory.ScriptFiles {
base := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))
if hasUppercaseResourceName(rel) {
report.add(rel, "resource filenames should be lowercase", SeverityWarning)
}
base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
if previous, exists := scriptIndex[base]; exists {
report.add(rel, fmt.Sprintf("duplicate script resource %s also defined by %s", base, previous), SeverityError)
} else {
@@ -110,9 +116,12 @@ func ValidateProject(p *project.Project) Report {
}
for _, rel := range p.Inventory.AssetFiles {
base := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))
if hasUppercaseResourceName(rel) {
report.add(rel, "resource filenames should be lowercase", SeverityWarning)
}
base := strings.ToLower(strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel)))
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(rel)), ".")
key := base + "." + extension
key := strings.ToLower(base) + "." + extension
if _, exists := assetIndex[key]; !exists {
assetIndex[key] = rel
}
@@ -187,7 +196,7 @@ func loadDocument(path string) (gff.Document, string, string, error) {
if extension == "" {
return gff.Document{}, "", "", fmt.Errorf("missing resource extension before .json")
}
resref := strings.TrimSuffix(stem, "."+extension)
resref := strings.ToLower(strings.TrimSuffix(stem, "."+extension))
var document gff.Document
if err := json.Unmarshal(raw, &document); err != nil {
@@ -250,25 +259,25 @@ func validateReferences(report *Report, document loadedDocument, resources, scri
if isBuiltinScript(value) {
return
}
if _, exists := scripts[value]; !exists {
if _, exists := scripts[strings.ToLower(value)]; !exists {
report.add(document.Path, fmt.Sprintf("missing script reference %q from field %q", value, field.Label), SeverityError)
}
case field.Label == "Conversation":
key := value + ".dlg"
key := strings.ToLower(value) + ".dlg"
if _, exists := resources[key]; !exists {
report.add(document.Path, fmt.Sprintf("missing dialog reference %q", key), SeverityError)
}
case field.Label == "Mod_Entry_Area":
key := value + ".are"
key := strings.ToLower(value) + ".are"
if _, exists := resources[key]; !exists {
report.add(document.Path, fmt.Sprintf("missing area reference %q", key), SeverityError)
}
case field.Label == "Model":
if !hasAsset(value, []string{"mdl"}, assets) {
if !hasAsset(strings.ToLower(value), []string{"mdl"}, assets) {
report.add(document.Path, fmt.Sprintf("missing model asset for %q", value), SeverityError)
}
case field.Label == "Sound":
if !hasAsset(value, []string{"wav"}, assets) {
if !hasAsset(strings.ToLower(value), []string{"wav"}, assets) {
report.add(document.Path, fmt.Sprintf("missing sound asset for %q", value), SeverityError)
}
}
@@ -390,6 +399,11 @@ func walkFields(s gff.Struct, visit func(gff.Field)) {
}
}
func hasUppercaseResourceName(rel string) bool {
base := filepath.Base(rel)
return base != strings.ToLower(base)
}
func fieldStringValue(value gff.Value) (string, bool) {
switch typed := value.(type) {
case gff.StringValue:
internal/validator/validator_test.go
+43
View File
@@ -54,6 +54,49 @@ func TestValidateProjectWarnsForCrossHAKDuplicateAssets(t *testing.T) {
}
}
func TestValidateProjectWarnsForUppercaseResourceNames(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "blueprints", "items"))
mustMkdir(t, filepath.Join(root, "assets", "vfx"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod"
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
}
}
`)
mustWriteFile(t, filepath.Join(root, "src", "blueprints", "items", "I_ELVENCHAIN.uti.json"), `{
"file_type": "UTI ",
"file_version": "V3.2",
"root": {"struct_type": 0, "fields": []}
}
`)
mustWriteFile(t, filepath.Join(root, "assets", "vfx", "SPELL_FIRE.tga"), "fire")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
report := ValidateProject(p)
if report.WarningCount() < 2 {
t.Fatalf("expected lowercase warnings, got %#v", report.Diagnostics)
}
}
func mustMkdir(t *testing.T, path string) {
t.Helper()
if err := os.MkdirAll(path, 0o755); err != nil {