diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index ea1af83..9410e96 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -2,6 +2,7 @@ package pipeline import ( "bytes" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -32,12 +33,13 @@ type BuildManifest struct { } type BuildManifestHAK struct { - Name string `json:"name"` - Group string `json:"group"` - Priority int `json:"priority"` - MaxBytes int64 `json:"max_bytes"` - SizeBytes int64 `json:"size_bytes"` - Assets []string `json:"assets"` + Name string `json:"name"` + Group string `json:"group"` + Priority int `json:"priority"` + MaxBytes int64 `json:"max_bytes"` + SizeBytes int64 `json:"size_bytes"` + Assets []string `json:"assets"` + ContentHash string `json:"content_hash,omitempty"` } type assetResource struct { @@ -128,13 +130,13 @@ func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { if err != nil { return BuildResult{}, err } - progressf(progress, "Cleaning previous generated HAKs...") - if err := cleanupGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef); err != nil { - return BuildResult{}, err - } result := BuildResult{HAKAssets: len(assetResources)} if len(assetResources) == 0 { + progressf(progress, "Cleaning previous generated HAKs...") + if err := cleanupGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef); err != nil { + return BuildResult{}, err + } return result, nil } @@ -148,39 +150,48 @@ func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { if err != nil { return BuildResult{}, err } + previousManifest, err := loadPreviousBuildManifest(p.BuildDir()) + if err != nil { + return BuildResult{}, err + } + progressf(progress, "Reusing unchanged generated HAKs where possible...") manifest := BuildManifest{ ModuleHAKs: moduleHakOrder, HAKs: make([]BuildManifestHAK, 0, len(chunks)), } + currentNames := make(map[string]struct{}, len(chunks)) for index, chunk := range chunks { - progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets))) hakPath := filepath.Join(p.BuildDir(), chunk.Name+".hak") - hakOutput, err := os.Create(hakPath) + manifestEntry := buildManifestEntry(chunk) + currentNames[chunk.Name] = struct{}{} + reused, err := reuseExistingHAK(previousManifest, manifestEntry, hakPath) if err != nil { - return BuildResult{}, fmt.Errorf("create hak archive: %w", err) + return BuildResult{}, err } - if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil { - hakOutput.Close() - return BuildResult{}, fmt.Errorf("write hak archive: %w", err) - } - if err := hakOutput.Close(); err != nil { - return BuildResult{}, fmt.Errorf("close hak archive: %w", err) + if reused { + progressf(progress, fmt.Sprintf("Reusing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets))) + } else { + progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets))) + hakOutput, err := os.Create(hakPath) + if err != nil { + return BuildResult{}, fmt.Errorf("create hak archive: %w", err) + } + if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil { + hakOutput.Close() + return BuildResult{}, fmt.Errorf("write hak archive: %w", err) + } + if err := hakOutput.Close(); err != nil { + return BuildResult{}, fmt.Errorf("close hak archive: %w", err) + } } result.HAKPaths = append(result.HAKPaths, hakPath) - assets := make([]string, 0, len(chunk.Assets)) - for _, asset := range chunk.Assets { - assets = append(assets, asset.Rel) - } - manifest.HAKs = append(manifest.HAKs, BuildManifestHAK{ - Name: chunk.Name, - Group: chunk.Config.Name, - Priority: chunk.Config.Priority, - MaxBytes: chunk.Config.MaxBytes, - SizeBytes: chunk.Size, - Assets: assets, - }) + manifest.HAKs = append(manifest.HAKs, manifestEntry) + } + progressf(progress, "Cleaning stale generated HAKs...") + if err := cleanupStaleGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef, currentNames); err != nil { + return BuildResult{}, err } progressf(progress, "Writing HAK manifest...") @@ -686,6 +697,112 @@ func resourceSlice(assets []assetResource) []erf.Resource { return out } +func buildManifestEntry(chunk hakChunk) BuildManifestHAK { + assets := make([]string, 0, len(chunk.Assets)) + for _, asset := range chunk.Assets { + assets = append(assets, asset.Rel) + } + return BuildManifestHAK{ + Name: chunk.Name, + Group: chunk.Config.Name, + Priority: chunk.Config.Priority, + MaxBytes: chunk.Config.MaxBytes, + SizeBytes: chunk.Size, + Assets: assets, + ContentHash: chunkContentHash(chunk), + } +} + +func chunkContentHash(chunk hakChunk) string { + sum := sha256.New() + fmt.Fprintf(sum, "name:%s\n", chunk.Name) + fmt.Fprintf(sum, "group:%s\n", chunk.Config.Name) + fmt.Fprintf(sum, "priority:%d\n", chunk.Config.Priority) + fmt.Fprintf(sum, "max_bytes:%d\n", chunk.Config.MaxBytes) + fmt.Fprintf(sum, "size_bytes:%d\n", chunk.Size) + for _, asset := range chunk.Assets { + fmt.Fprintf(sum, "asset:%s\n", asset.Rel) + fmt.Fprintf(sum, "resref:%s\n", asset.Resource.Name) + fmt.Fprintf(sum, "restype:%d\n", asset.Resource.Type) + fmt.Fprintf(sum, "datasize:%d\n", len(asset.Resource.Data)) + _, _ = sum.Write(asset.Resource.Data) + _, _ = sum.Write([]byte{'\n'}) + } + return fmt.Sprintf("%x", sum.Sum(nil)) +} + +func loadPreviousBuildManifest(buildDir string) (*BuildManifest, error) { + manifestPath := filepath.Join(buildDir, "haks.json") + raw, err := os.ReadFile(manifestPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("read previous hak manifest: %w", err) + } + var manifest BuildManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("parse previous hak manifest: %w", err) + } + return &manifest, nil +} + +func reuseExistingHAK(previousManifest *BuildManifest, current BuildManifestHAK, hakPath string) (bool, error) { + if previousManifest == nil { + return false, nil + } + var previous *BuildManifestHAK + for index := range previousManifest.HAKs { + entry := &previousManifest.HAKs[index] + if entry.Name == current.Name { + previous = entry + break + } + } + if previous == nil { + return false, nil + } + if previous.Group != current.Group || + previous.Priority != current.Priority || + previous.MaxBytes != current.MaxBytes || + previous.SizeBytes != current.SizeBytes || + previous.ContentHash != current.ContentHash || + !slices.Equal(previous.Assets, current.Assets) { + return false, nil + } + if _, err := os.Stat(hakPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, fmt.Errorf("stat existing hak archive %s: %w", hakPath, err) + } + return true, nil +} + +func cleanupStaleGeneratedHAKs(buildDir, moduleResRef string, keepNames map[string]struct{}) error { + paths, err := manifestHAKPaths(buildDir) + if err != nil { + return err + } + for _, path := range paths { + name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + if _, keep := keepNames[name]; keep { + continue + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove stale generated hak %s: %w", path, err) + } + } + + legacy := filepath.Join(buildDir, moduleResRef+".hak") + if _, keep := keepNames[moduleResRef]; !keep { + if err := os.Remove(legacy); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove legacy generated hak %s: %w", legacy, err) + } + } + return nil +} + func matchesAnyPattern(path string, patterns []string) bool { for _, pattern := range patterns { if matchPathPattern(filepath.ToSlash(path), filepath.ToSlash(pattern)) { diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 0d82aff..6603292 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "testing" + "time" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" @@ -31,7 +32,7 @@ func TestBuildThenExtract(t *testing.T) { `) mustMkdir(t, filepath.Join(root, "src", "module")) - mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ +mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ "file_type": "IFO ", "file_version": "V3.2", "root": { @@ -46,7 +47,6 @@ func TestBuildThenExtract(t *testing.T) { } } `) - p, err := project.Load(root) if err != nil { t.Fatalf("load project: %v", err) @@ -750,6 +750,231 @@ func TestBuildSkipsEmptyHAKGroupsInModuleOrder(t *testing.T) { } } +func TestBuildHAKsReusesUnchangedChunks(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "assets", "vfx")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core", "group:vfx"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 1024, + "split": false, + "include": ["core/**"] + }, + { + "name": "vfx", + "priority": 2, + "max_bytes": 1024, + "split": false, + "include": ["vfx/**"] + } + ] +}`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +}`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a") + mustWriteFile(t, filepath.Join(root, "assets", "vfx", "vfx_a.tga"), "vfx-a") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildHAKs(p); err != nil { + t.Fatalf("first build haks: %v", err) + } + corePath := filepath.Join(root, "build", "core.hak") + vfxPath := filepath.Join(root, "build", "vfx.hak") + coreInfoBefore, err := os.Stat(corePath) + if err != nil { + t.Fatalf("stat core before: %v", err) + } + vfxInfoBefore, err := os.Stat(vfxPath) + if err != nil { + t.Fatalf("stat vfx before: %v", err) + } + + time.Sleep(1100 * time.Millisecond) + + if _, err := BuildHAKs(p); err != nil { + t.Fatalf("second build haks: %v", err) + } + coreInfoAfter, err := os.Stat(corePath) + if err != nil { + t.Fatalf("stat core after: %v", err) + } + vfxInfoAfter, err := os.Stat(vfxPath) + if err != nil { + t.Fatalf("stat vfx after: %v", err) + } + + if !coreInfoAfter.ModTime().Equal(coreInfoBefore.ModTime()) { + t.Fatalf("expected unchanged core.hak to be reused") + } + if !vfxInfoAfter.ModTime().Equal(vfxInfoBefore.ModTime()) { + t.Fatalf("expected unchanged vfx.hak to be reused") + } + + rawManifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + var manifest BuildManifest + if err := json.Unmarshal(rawManifest, &manifest); err != nil { + t.Fatalf("parse manifest: %v", err) + } + for _, hak := range manifest.HAKs { + if hak.ContentHash == "" { + t.Fatalf("expected content hash for %s", hak.Name) + } + } +} + +func TestBuildHAKsRebuildsOnlyChangedChunks(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "assets", "vfx")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core", "group:vfx"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 1024, + "split": false, + "include": ["core/**"] + }, + { + "name": "vfx", + "priority": 2, + "max_bytes": 1024, + "split": false, + "include": ["vfx/**"] + } + ] +}`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +}`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a") + mustWriteFile(t, filepath.Join(root, "assets", "vfx", "vfx_a.tga"), "vfx-a") + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + if _, err := BuildHAKs(p); err != nil { + t.Fatalf("first build haks: %v", err) + } + corePath := filepath.Join(root, "build", "core.hak") + vfxPath := filepath.Join(root, "build", "vfx.hak") + coreInfoBefore, err := os.Stat(corePath) + if err != nil { + t.Fatalf("stat core before: %v", err) + } + vfxInfoBefore, err := os.Stat(vfxPath) + if err != nil { + t.Fatalf("stat vfx before: %v", err) + } + + time.Sleep(1100 * time.Millisecond) + mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a-updated") + if err := p.Scan(); err != nil { + t.Fatalf("rescan after change: %v", err) + } + + if _, err := BuildHAKs(p); err != nil { + t.Fatalf("second build haks: %v", err) + } + coreInfoAfter, err := os.Stat(corePath) + if err != nil { + t.Fatalf("stat core after: %v", err) + } + vfxInfoAfter, err := os.Stat(vfxPath) + if err != nil { + t.Fatalf("stat vfx after: %v", err) + } + + if !coreInfoAfter.ModTime().After(coreInfoBefore.ModTime()) { + t.Fatalf("expected changed core.hak to be rebuilt") + } + if !vfxInfoAfter.ModTime().Equal(vfxInfoBefore.ModTime()) { + t.Fatalf("expected unchanged vfx.hak to be reused") + } +} func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src", "module"))