(Potentially Breaking Change) Git-Based Hak Chunk Ordering

internal/pipeline/build.go

@@ -11,7 +11,9 @@ import (
 	"path/filepath"
 	"runtime"
 	"slices"
+	"strconv"
 	"strings"
+	"time"
 
 	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
 	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
@@ -47,9 +49,10 @@ type BuildManifestHAK struct {
 }
 
 type assetResource struct {
-	Rel      string
+	Rel        string
-	Resource erf.Resource
+	Resource   erf.Resource
-	Size     int64
+	Size       int64
+	ChangedAt  time.Time
 }
 
 type hakChunk struct {
@@ -447,6 +450,15 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) {
 
 func collectAssetResources(p *project.Project) ([]assetResource, error) {
 	var hakResources []assetResource
+	assetChangedAt, err := collectGitAssetChangeTimes(p)
+	if err != nil {
+		return nil, err
+	}
+	assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir())
+	if err != nil {
+		return nil, fmt.Errorf("resolve assets dir relative path: %w", err)
+	}
+	assetsRelPath = filepath.ToSlash(assetsRelPath)
 
 	for _, rel := range p.Inventory.AssetFiles {
 		abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
@@ -454,10 +466,20 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) {
 		if err != nil {
 			return nil, err
 		}
+		info, err := os.Stat(abs)
+		if err != nil {
+			return nil, fmt.Errorf("stat %s: %w", abs, err)
+		}
+		repoRel := filepath.ToSlash(filepath.Join(assetsRelPath, filepath.FromSlash(rel)))
+		changedAt := assetChangedAt[repoRel]
+		if changedAt.IsZero() {
+			changedAt = info.ModTime()
+		}
 		hakResources = append(hakResources, assetResource{
-			Rel:      rel,
+			Rel:        rel,
-			Resource: resource,
+			Resource:   resource,
-			Size:     erf.ArchiveSize([]erf.Resource{resource}),
+			Size:       erf.ArchiveSize([]erf.Resource{resource}),
+			ChangedAt:  changedAt,
 		})
 	}
 
@@ -467,6 +489,73 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) {
 	return hakResources, nil
 }
 
+func collectGitAssetChangeTimes(p *project.Project) (map[string]time.Time, error) {
+	result := make(map[string]time.Time, len(p.Inventory.AssetFiles))
+	if len(p.Inventory.AssetFiles) == 0 {
+		return result, nil
+	}
+
+	assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir())
+	if err != nil {
+		return result, fmt.Errorf("resolve assets dir relative path: %w", err)
+	}
+	assetsRelPath = filepath.ToSlash(assetsRelPath)
+
+	output, err := gitOutput(p.Root, "log", "--format=commit:%ct", "--name-only", "--", assetsRelPath)
+	if err != nil {
+		return result, nil
+	}
+
+	wanted := make(map[string]struct{}, len(p.Inventory.AssetFiles))
+	for _, rel := range p.Inventory.AssetFiles {
+		wanted[filepath.ToSlash(filepath.Join(assetsRelPath, filepath.FromSlash(rel)))] = struct{}{}
+	}
+
+	var currentCommitTime time.Time
+	for _, line := range strings.Split(output, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		if strings.HasPrefix(line, "commit:") {
+			sec, err := strconv.ParseInt(strings.TrimPrefix(line, "commit:"), 10, 64)
+			if err != nil {
+				currentCommitTime = time.Time{}
+				continue
+			}
+			currentCommitTime = time.Unix(sec, 0).UTC()
+			continue
+		}
+		if currentCommitTime.IsZero() {
+			continue
+		}
+		if _, ok := wanted[line]; !ok {
+			continue
+		}
+		if _, exists := result[line]; exists {
+			continue
+		}
+		result[line] = currentCommitTime
+		if len(result) == len(wanted) {
+			break
+		}
+	}
+
+	return result, nil
+}
+
+func gitOutput(dir string, args ...string) (string, error) {
+	cmd := exec.Command("git", args...)
+	if dir != "" {
+		cmd.Dir = dir
+	}
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output)))
+	}
+	return string(output), nil
+}
+
 func sortResources(resources []erf.Resource) {
 	slices.SortFunc(resources, func(a, b erf.Resource) int {
 		return compareResourceKeys(a, b)
@@ -629,6 +718,7 @@ func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, erro
 		if len(groupAssets) == 0 {
 			continue
 		}
+		slices.SortFunc(groupAssets, compareAssetBuildOrder)
 
 		groupChunks, err := splitHAKGroup(cfg, groupAssets)
 		if err != nil {
@@ -708,6 +798,16 @@ func splitHAKGroup(cfg project.HAKConfig, assets []assetResource) ([]hakChunk, e
 	return chunks, nil
 }
 
+func compareAssetBuildOrder(a, b assetResource) int {
+	if cmp := a.ChangedAt.Compare(b.ChangedAt); cmp != 0 {
+		return cmp
+	}
+	if cmp := compareResourceKeys(a.Resource, b.Resource); cmp != 0 {
+		return cmp
+	}
+	return strings.Compare(a.Rel, b.Rel)
+}
+
 func chunkName(base string, index int, split bool) string {
 	if !split {
 		return base
@@ -868,6 +968,14 @@ func cleanupGeneratedHAKs(buildDir, moduleResRef string) error {
 }
 
 func plannedModuleHAKOrder(p *project.Project) ([]string, error) {
+	manifest, err := loadPreviousBuildManifest(p.BuildDir())
+	if err != nil {
+		return nil, err
+	}
+	if manifest != nil && len(manifest.ModuleHAKs) > 0 {
+		return append([]string(nil), manifest.ModuleHAKs...), nil
+	}
+
 	assetResources, err := collectAssetResources(p)
 	if err != nil {
 		return nil, err

internal/pipeline/pipeline_test.go

@@ -4,7 +4,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -1251,6 +1253,123 @@ func TestBuildHAKsReusesUnchangedChunks(t *testing.T) {
 	}
 }
 
+func TestBuildModulePrefersManifestModuleHAKOrder(t *testing.T) {
+	root := t.TempDir()
+	mustMkdir(t, filepath.Join(root, "src", "module"))
+	mustMkdir(t, filepath.Join(root, "assets"))
+	mustMkdir(t, filepath.Join(root, "build"))
+
+	mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
+  "module": {
+    "name": "Test Module",
+    "resref": "testmod",
+    "hak_order": ["manual_top", "group:core"]
+  },
+  "paths": {
+    "source": "src",
+    "assets": "assets",
+    "build": "build"
+  },
+  "haks": [
+    {
+      "name": "core",
+      "priority": 1,
+      "max_bytes": 1024,
+      "split": false,
+      "include": ["core/**"]
+    }
+  ]
+}`)
+
+	mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
+  "file_type": "IFO ",
+  "file_version": "V3.2",
+  "root": {
+    "struct_type": 0,
+    "fields": [
+      {
+        "label": "Mod_Name",
+        "type": "CExoString",
+        "value": "Test Module"
+      },
+      {
+        "label": "Mod_HakList",
+        "type": "List",
+        "value": []
+      }
+    ]
+  }
+}
+`)
+
+	mustWriteFile(t, filepath.Join(root, "build", "haks.json"), `{
+  "module_haks": ["sow_top", "sow_core_01", "sow_appr_01"],
+  "haks": []
+}
+`)
+
+	p, err := project.Load(root)
+	if err != nil {
+		t.Fatalf("load project: %v", err)
+	}
+	if err := p.ValidateLayout(); err != nil {
+		t.Fatalf("validate layout: %v", err)
+	}
+	if err := p.Scan(); err != nil {
+		t.Fatalf("scan: %v", err)
+	}
+
+	result, err := BuildModule(p)
+	if err != nil {
+		t.Fatalf("build module: %v", err)
+	}
+
+	archive, err := readArchive(result.ModulePath)
+	if err != nil {
+		t.Fatalf("read module archive: %v", err)
+	}
+
+	var ifo erf.Resource
+	found := false
+	for _, resource := range archive.Resources {
+		if resource.Name == "module" {
+			ifo = resource
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("module.ifo not found in built archive")
+	}
+
+	document, err := gff.Read(bytes.NewReader(ifo.Data))
+	if err != nil {
+		t.Fatalf("decode module ifo: %v", err)
+	}
+
+	var got []string
+	for _, field := range document.Root.Fields {
+		if field.Label != "Mod_HakList" {
+			continue
+		}
+		list, ok := field.Value.(gff.ListValue)
+		if !ok {
+			t.Fatalf("expected Mod_HakList list, got %T", field.Value)
+		}
+		for _, item := range list {
+			for _, nested := range item.Fields {
+				if nested.Label == "Mod_Hak" {
+					got = append(got, string(nested.Value.(gff.StringValue)))
+				}
+			}
+		}
+	}
+
+	if want := "sow_top,sow_core_01,sow_appr_01"; strings.Join(got, ",") != want {
+		t.Fatalf("unexpected module hak order: got %q want %q", strings.Join(got, ","), want)
+	}
+}
+
 func TestBuildHAKsRebuildsOnlyChangedChunks(t *testing.T) {
 	root := t.TempDir()
 	mustMkdir(t, filepath.Join(root, "src", "module"))
@@ -1358,6 +1477,153 @@ func TestBuildHAKsRebuildsOnlyChangedChunks(t *testing.T) {
 		t.Fatalf("expected unchanged vfx.hak to be reused")
 	}
 }
+
+func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) {
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git not available")
+	}
+
+	root := t.TempDir()
+	mustMkdir(t, filepath.Join(root, "src", "module"))
+	mustMkdir(t, filepath.Join(root, "assets", "core"))
+	mustMkdir(t, filepath.Join(root, "build"))
+
+	mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
+  "module": {
+    "name": "Test Module",
+    "resref": "testmod",
+    "hak_order": ["group:core"]
+  },
+  "paths": {
+    "source": "src",
+    "assets": "assets",
+    "build": "build"
+  },
+  "haks": [
+    {
+      "name": "core",
+      "priority": 1,
+      "max_bytes": 400,
+      "split": true,
+      "include": ["core/**"]
+    }
+  ]
+}`)
+	mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
+  "file_type": "IFO ",
+  "file_version": "V3.2",
+  "root": {
+    "struct_type": 0,
+    "fields": [
+      {
+        "label": "Mod_Name",
+        "type": "CExoString",
+        "value": "Test Module"
+      }
+    ]
+  }
+}
+`)
+
+	runGitTest(t, root, "init")
+	runGitTest(t, root, "config", "user.name", "Test User")
+	runGitTest(t, root, "config", "user.email", "test@example.com")
+
+	mustWriteFile(t, filepath.Join(root, "assets", "core", "old_a.tga"), strings.Repeat("a", 120))
+	runGitCommitTest(t, root, "2001-01-01T00:00:00Z", "add old_a")
+
+	mustWriteFile(t, filepath.Join(root, "assets", "core", "old_b.tga"), strings.Repeat("b", 120))
+	runGitCommitTest(t, root, "2002-01-01T00:00:00Z", "add old_b")
+
+	mustWriteFile(t, filepath.Join(root, "assets", "core", "new_c.tga"), strings.Repeat("c", 120))
+	runGitCommitTest(t, root, "2003-01-01T00:00:00Z", "add new_c")
+
+	now := time.Now()
+	if err := os.Chtimes(filepath.Join(root, "assets", "core", "old_a.tga"), now, now); err != nil {
+		t.Fatalf("chtimes old_a: %v", err)
+	}
+	if err := os.Chtimes(filepath.Join(root, "assets", "core", "old_b.tga"), now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil {
+		t.Fatalf("chtimes old_b: %v", err)
+	}
+	if err := os.Chtimes(filepath.Join(root, "assets", "core", "new_c.tga"), now.Add(-4*time.Hour), now.Add(-4*time.Hour)); err != nil {
+		t.Fatalf("chtimes new_c: %v", err)
+	}
+
+	p, err := project.Load(root)
+	if err != nil {
+		t.Fatalf("load project: %v", err)
+	}
+	if err := p.ValidateLayout(); err != nil {
+		t.Fatalf("validate layout: %v", err)
+	}
+	if err := p.Scan(); err != nil {
+		t.Fatalf("scan: %v", err)
+	}
+
+	assets, err := collectAssetResources(p)
+	if err != nil {
+		t.Fatalf("collect asset resources: %v", err)
+	}
+	slices.SortFunc(assets, compareAssetBuildOrder)
+	twoAssetSize := erf.ArchiveSize(resourceSlice(assets[:2]))
+	threeAssetSize := erf.ArchiveSize(resourceSlice(assets))
+	if threeAssetSize <= twoAssetSize {
+		t.Fatalf("expected three-asset archive to be larger than two-asset archive")
+	}
+	p.Config.HAKs[0].MaxBytes = twoAssetSize
+
+	chunks, err := planHAKChunks(p, assets)
+	if err != nil {
+		t.Fatalf("plan hak chunks: %v", err)
+	}
+	if len(chunks) != 2 {
+		t.Fatalf("expected 2 chunks, got %d", len(chunks))
+	}
+
+	firstChunkAssets := make([]string, 0, len(chunks[0].Assets))
+	for _, asset := range chunks[0].Assets {
+		firstChunkAssets = append(firstChunkAssets, asset.Rel)
+	}
+	lastChunkAssets := make([]string, 0, len(chunks[1].Assets))
+	for _, asset := range chunks[1].Assets {
+		lastChunkAssets = append(lastChunkAssets, asset.Rel)
+	}
+
+	if got, want := strings.Join(firstChunkAssets, ","), "core/old_a.tga,core/old_b.tga"; got != want {
+		t.Fatalf("unexpected first chunk assets: got %q want %q", got, want)
+	}
+	if got, want := strings.Join(lastChunkAssets, ","), "core/new_c.tga"; got != want {
+		t.Fatalf("unexpected last chunk assets: got %q want %q", got, want)
+	}
+}
+
+func runGitCommitTest(t *testing.T, dir, timestamp, message string) {
+	t.Helper()
+	runGitTest(t, dir, "add", ".")
+	cmd := exec.Command("git", "commit", "-m", message)
+	cmd.Dir = dir
+	cmd.Env = append(os.Environ(),
+		"GIT_AUTHOR_DATE="+timestamp,
+		"GIT_COMMITTER_DATE="+timestamp,
+	)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("git commit failed: %v\n%s", err, string(output))
+	}
+}
+
+func runGitTest(t *testing.T, dir string, args ...string) {
+	t.Helper()
+	cmd := exec.Command("git", args...)
+	if dir != "" {
+		cmd.Dir = dir
+	}
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("git %v failed: %v\n%s", args, err, string(output))
+	}
+}
+
 func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) {
 	root := t.TempDir()
 	mustMkdir(t, filepath.Join(root, "src", "module"))