From 29ddf8e3c59cee5d11ab1a6cc67bfc0b3ca83c6d Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Fri, 10 Apr 2026 22:08:54 +0200 Subject: [PATCH] (Potentially Breaking Change) Git-Based Hak Chunk Ordering --- internal/pipeline/build.go | 120 ++++++++++++- internal/pipeline/pipeline_test.go | 266 +++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+), 6 deletions(-) diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index fbda22c..de06c14 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -11,7 +11,9 @@ import ( "path/filepath" "runtime" "slices" + "strconv" "strings" + "time" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff" @@ -47,9 +49,10 @@ type BuildManifestHAK struct { } type assetResource struct { - Rel string - Resource erf.Resource - Size int64 + Rel string + Resource erf.Resource + Size int64 + ChangedAt time.Time } type hakChunk struct { @@ -447,6 +450,15 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) { func collectAssetResources(p *project.Project) ([]assetResource, error) { var hakResources []assetResource + assetChangedAt, err := collectGitAssetChangeTimes(p) + if err != nil { + return nil, err + } + assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir()) + if err != nil { + return nil, fmt.Errorf("resolve assets dir relative path: %w", err) + } + assetsRelPath = filepath.ToSlash(assetsRelPath) for _, rel := range p.Inventory.AssetFiles { abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)) @@ -454,10 +466,20 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) { if err != nil { return nil, err } + info, err := os.Stat(abs) + if err != nil { + return nil, fmt.Errorf("stat %s: %w", abs, err) + } + repoRel := filepath.ToSlash(filepath.Join(assetsRelPath, filepath.FromSlash(rel))) + changedAt := assetChangedAt[repoRel] + if changedAt.IsZero() { + changedAt = info.ModTime() + } hakResources = append(hakResources, assetResource{ - Rel: rel, - Resource: resource, - Size: erf.ArchiveSize([]erf.Resource{resource}), + Rel: rel, + Resource: resource, + Size: erf.ArchiveSize([]erf.Resource{resource}), + ChangedAt: changedAt, }) } @@ -467,6 +489,73 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) { return hakResources, nil } +func collectGitAssetChangeTimes(p *project.Project) (map[string]time.Time, error) { + result := make(map[string]time.Time, len(p.Inventory.AssetFiles)) + if len(p.Inventory.AssetFiles) == 0 { + return result, nil + } + + assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir()) + if err != nil { + return result, fmt.Errorf("resolve assets dir relative path: %w", err) + } + assetsRelPath = filepath.ToSlash(assetsRelPath) + + output, err := gitOutput(p.Root, "log", "--format=commit:%ct", "--name-only", "--", assetsRelPath) + if err != nil { + return result, nil + } + + wanted := make(map[string]struct{}, len(p.Inventory.AssetFiles)) + for _, rel := range p.Inventory.AssetFiles { + wanted[filepath.ToSlash(filepath.Join(assetsRelPath, filepath.FromSlash(rel)))] = struct{}{} + } + + var currentCommitTime time.Time + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if strings.HasPrefix(line, "commit:") { + sec, err := strconv.ParseInt(strings.TrimPrefix(line, "commit:"), 10, 64) + if err != nil { + currentCommitTime = time.Time{} + continue + } + currentCommitTime = time.Unix(sec, 0).UTC() + continue + } + if currentCommitTime.IsZero() { + continue + } + if _, ok := wanted[line]; !ok { + continue + } + if _, exists := result[line]; exists { + continue + } + result[line] = currentCommitTime + if len(result) == len(wanted) { + break + } + } + + return result, nil +} + +func gitOutput(dir string, args ...string) (string, error) { + cmd := exec.Command("git", args...) + if dir != "" { + cmd.Dir = dir + } + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output))) + } + return string(output), nil +} + func sortResources(resources []erf.Resource) { slices.SortFunc(resources, func(a, b erf.Resource) int { return compareResourceKeys(a, b) @@ -629,6 +718,7 @@ func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, erro if len(groupAssets) == 0 { continue } + slices.SortFunc(groupAssets, compareAssetBuildOrder) groupChunks, err := splitHAKGroup(cfg, groupAssets) if err != nil { @@ -708,6 +798,16 @@ func splitHAKGroup(cfg project.HAKConfig, assets []assetResource) ([]hakChunk, e return chunks, nil } +func compareAssetBuildOrder(a, b assetResource) int { + if cmp := a.ChangedAt.Compare(b.ChangedAt); cmp != 0 { + return cmp + } + if cmp := compareResourceKeys(a.Resource, b.Resource); cmp != 0 { + return cmp + } + return strings.Compare(a.Rel, b.Rel) +} + func chunkName(base string, index int, split bool) string { if !split { return base @@ -868,6 +968,14 @@ func cleanupGeneratedHAKs(buildDir, moduleResRef string) error { } func plannedModuleHAKOrder(p *project.Project) ([]string, error) { + manifest, err := loadPreviousBuildManifest(p.BuildDir()) + if err != nil { + return nil, err + } + if manifest != nil && len(manifest.ModuleHAKs) > 0 { + return append([]string(nil), manifest.ModuleHAKs...), nil + } + assetResources, err := collectAssetResources(p) if err != nil { return nil, err diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 97a34d7..b1b69c8 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -4,7 +4,9 @@ import ( "bytes" "encoding/json" "os" + "os/exec" "path/filepath" + "slices" "strings" "testing" "time" @@ -1251,6 +1253,123 @@ func TestBuildHAKsReusesUnchangedChunks(t *testing.T) { } } +func TestBuildModulePrefersManifestModuleHAKOrder(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["manual_top", "group:core"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 1024, + "split": false, + "include": ["core/**"] + } + ] +}`) + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_HakList", + "type": "List", + "value": [] + } + ] + } +} +`) + + mustWriteFile(t, filepath.Join(root, "build", "haks.json"), `{ + "module_haks": ["sow_top", "sow_core_01", "sow_appr_01"], + "haks": [] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildModule(p) + if err != nil { + t.Fatalf("build module: %v", err) + } + + archive, err := readArchive(result.ModulePath) + if err != nil { + t.Fatalf("read module archive: %v", err) + } + + var ifo erf.Resource + found := false + for _, resource := range archive.Resources { + if resource.Name == "module" { + ifo = resource + found = true + break + } + } + if !found { + t.Fatalf("module.ifo not found in built archive") + } + + document, err := gff.Read(bytes.NewReader(ifo.Data)) + if err != nil { + t.Fatalf("decode module ifo: %v", err) + } + + var got []string + for _, field := range document.Root.Fields { + if field.Label != "Mod_HakList" { + continue + } + list, ok := field.Value.(gff.ListValue) + if !ok { + t.Fatalf("expected Mod_HakList list, got %T", field.Value) + } + for _, item := range list { + for _, nested := range item.Fields { + if nested.Label == "Mod_Hak" { + got = append(got, string(nested.Value.(gff.StringValue))) + } + } + } + } + + if want := "sow_top,sow_core_01,sow_appr_01"; strings.Join(got, ",") != want { + t.Fatalf("unexpected module hak order: got %q want %q", strings.Join(got, ","), want) + } +} + func TestBuildHAKsRebuildsOnlyChangedChunks(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src", "module")) @@ -1358,6 +1477,153 @@ func TestBuildHAKsRebuildsOnlyChangedChunks(t *testing.T) { t.Fatalf("expected unchanged vfx.hak to be reused") } } + +func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["group:core"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + }, + "haks": [ + { + "name": "core", + "priority": 1, + "max_bytes": 400, + "split": true, + "include": ["core/**"] + } + ] +}`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + } + ] + } +} +`) + + runGitTest(t, root, "init") + runGitTest(t, root, "config", "user.name", "Test User") + runGitTest(t, root, "config", "user.email", "test@example.com") + + mustWriteFile(t, filepath.Join(root, "assets", "core", "old_a.tga"), strings.Repeat("a", 120)) + runGitCommitTest(t, root, "2001-01-01T00:00:00Z", "add old_a") + + mustWriteFile(t, filepath.Join(root, "assets", "core", "old_b.tga"), strings.Repeat("b", 120)) + runGitCommitTest(t, root, "2002-01-01T00:00:00Z", "add old_b") + + mustWriteFile(t, filepath.Join(root, "assets", "core", "new_c.tga"), strings.Repeat("c", 120)) + runGitCommitTest(t, root, "2003-01-01T00:00:00Z", "add new_c") + + now := time.Now() + if err := os.Chtimes(filepath.Join(root, "assets", "core", "old_a.tga"), now, now); err != nil { + t.Fatalf("chtimes old_a: %v", err) + } + if err := os.Chtimes(filepath.Join(root, "assets", "core", "old_b.tga"), now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil { + t.Fatalf("chtimes old_b: %v", err) + } + if err := os.Chtimes(filepath.Join(root, "assets", "core", "new_c.tga"), now.Add(-4*time.Hour), now.Add(-4*time.Hour)); err != nil { + t.Fatalf("chtimes new_c: %v", err) + } + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + assets, err := collectAssetResources(p) + if err != nil { + t.Fatalf("collect asset resources: %v", err) + } + slices.SortFunc(assets, compareAssetBuildOrder) + twoAssetSize := erf.ArchiveSize(resourceSlice(assets[:2])) + threeAssetSize := erf.ArchiveSize(resourceSlice(assets)) + if threeAssetSize <= twoAssetSize { + t.Fatalf("expected three-asset archive to be larger than two-asset archive") + } + p.Config.HAKs[0].MaxBytes = twoAssetSize + + chunks, err := planHAKChunks(p, assets) + if err != nil { + t.Fatalf("plan hak chunks: %v", err) + } + if len(chunks) != 2 { + t.Fatalf("expected 2 chunks, got %d", len(chunks)) + } + + firstChunkAssets := make([]string, 0, len(chunks[0].Assets)) + for _, asset := range chunks[0].Assets { + firstChunkAssets = append(firstChunkAssets, asset.Rel) + } + lastChunkAssets := make([]string, 0, len(chunks[1].Assets)) + for _, asset := range chunks[1].Assets { + lastChunkAssets = append(lastChunkAssets, asset.Rel) + } + + if got, want := strings.Join(firstChunkAssets, ","), "core/old_a.tga,core/old_b.tga"; got != want { + t.Fatalf("unexpected first chunk assets: got %q want %q", got, want) + } + if got, want := strings.Join(lastChunkAssets, ","), "core/new_c.tga"; got != want { + t.Fatalf("unexpected last chunk assets: got %q want %q", got, want) + } +} + +func runGitCommitTest(t *testing.T, dir, timestamp, message string) { + t.Helper() + runGitTest(t, dir, "add", ".") + cmd := exec.Command("git", "commit", "-m", message) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_DATE="+timestamp, + "GIT_COMMITTER_DATE="+timestamp, + ) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git commit failed: %v\n%s", err, string(output)) + } +} + +func runGitTest(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + if dir != "" { + cmd.Dir = dir + } + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v failed: %v\n%s", args, err, string(output)) + } +} + func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src", "module"))