ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

(Potentially Breaking Change) Git-Based Hak Chunk Ordering

Browse Source
This commit is contained in:
archvillainette
2026-04-10 22:08:54 +02:00
parent e05620d285
commit 29ddf8e3c5
2 changed files with 380 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/pipeline/build.go
+114 -6
View File
@@ -11,7 +11,9 @@ import (
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"
"time"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/gff"
@@ -47,9 +49,10 @@ type BuildManifestHAK struct {
}
type assetResource struct {
Rel string
Resource erf.Resource
Size int64
Rel string
Resource erf.Resource
Size int64
ChangedAt time.Time
}
type hakChunk struct {
@@ -447,6 +450,15 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) {
func collectAssetResources(p *project.Project) ([]assetResource, error) {
var hakResources []assetResource
assetChangedAt, err := collectGitAssetChangeTimes(p)
if err != nil {
return nil, err
}
assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir())
if err != nil {
return nil, fmt.Errorf("resolve assets dir relative path: %w", err)
}
assetsRelPath = filepath.ToSlash(assetsRelPath)
for _, rel := range p.Inventory.AssetFiles {
abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
@@ -454,10 +466,20 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) {
if err != nil {
return nil, err
}
info, err := os.Stat(abs)
if err != nil {
return nil, fmt.Errorf("stat %s: %w", abs, err)
}
repoRel := filepath.ToSlash(filepath.Join(assetsRelPath, filepath.FromSlash(rel)))
changedAt := assetChangedAt[repoRel]
if changedAt.IsZero() {
changedAt = info.ModTime()
}
hakResources = append(hakResources, assetResource{
Rel: rel,
Resource: resource,
Size: erf.ArchiveSize([]erf.Resource{resource}),
Rel: rel,
Resource: resource,
Size: erf.ArchiveSize([]erf.Resource{resource}),
ChangedAt: changedAt,
})
}
@@ -467,6 +489,73 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) {
return hakResources, nil
}
func collectGitAssetChangeTimes(p *project.Project) (map[string]time.Time, error) {
result := make(map[string]time.Time, len(p.Inventory.AssetFiles))
if len(p.Inventory.AssetFiles) == 0 {
return result, nil
}
assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir())
if err != nil {
return result, fmt.Errorf("resolve assets dir relative path: %w", err)
}
assetsRelPath = filepath.ToSlash(assetsRelPath)
output, err := gitOutput(p.Root, "log", "--format=commit:%ct", "--name-only", "--", assetsRelPath)
if err != nil {
return result, nil
}
wanted := make(map[string]struct{}, len(p.Inventory.AssetFiles))
for _, rel := range p.Inventory.AssetFiles {
wanted[filepath.ToSlash(filepath.Join(assetsRelPath, filepath.FromSlash(rel)))] = struct{}{}
}
var currentCommitTime time.Time
for _, line := range strings.Split(output, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if strings.HasPrefix(line, "commit:") {
sec, err := strconv.ParseInt(strings.TrimPrefix(line, "commit:"), 10, 64)
if err != nil {
currentCommitTime = time.Time{}
continue
}
currentCommitTime = time.Unix(sec, 0).UTC()
continue
}
if currentCommitTime.IsZero() {
continue
}
if _, ok := wanted[line]; !ok {
continue
}
if _, exists := result[line]; exists {
continue
}
result[line] = currentCommitTime
if len(result) == len(wanted) {
break
}
}
return result, nil
}
func gitOutput(dir string, args ...string) (string, error) {
cmd := exec.Command("git", args...)
if dir != "" {
cmd.Dir = dir
}
output, err := cmd.CombinedOutput()
if err != nil {
return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output)))
}
return string(output), nil
}
func sortResources(resources []erf.Resource) {
slices.SortFunc(resources, func(a, b erf.Resource) int {
return compareResourceKeys(a, b)
@@ -629,6 +718,7 @@ func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, erro
if len(groupAssets) == 0 {
continue
}
slices.SortFunc(groupAssets, compareAssetBuildOrder)
groupChunks, err := splitHAKGroup(cfg, groupAssets)
if err != nil {
@@ -708,6 +798,16 @@ func splitHAKGroup(cfg project.HAKConfig, assets []assetResource) ([]hakChunk, e
return chunks, nil
}
func compareAssetBuildOrder(a, b assetResource) int {
if cmp := a.ChangedAt.Compare(b.ChangedAt); cmp != 0 {
return cmp
}
if cmp := compareResourceKeys(a.Resource, b.Resource); cmp != 0 {
return cmp
}
return strings.Compare(a.Rel, b.Rel)
}
func chunkName(base string, index int, split bool) string {
if !split {
return base
@@ -868,6 +968,14 @@ func cleanupGeneratedHAKs(buildDir, moduleResRef string) error {
}
func plannedModuleHAKOrder(p *project.Project) ([]string, error) {
manifest, err := loadPreviousBuildManifest(p.BuildDir())
if err != nil {
return nil, err
}
if manifest != nil && len(manifest.ModuleHAKs) > 0 {
return append([]string(nil), manifest.ModuleHAKs...), nil
}
assetResources, err := collectAssetResources(p)
if err != nil {
return nil, err
internal/pipeline/pipeline_test.go
+266
View File
@@ -4,7 +4,9 @@ import (
"bytes"
"encoding/json"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -1251,6 +1253,123 @@ func TestBuildHAKsReusesUnchangedChunks(t *testing.T) {
}
}
func TestBuildModulePrefersManifestModuleHAKOrder(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["manual_top", "group:core"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"haks": [
{
"name": "core",
"priority": 1,
"max_bytes": 1024,
"split": false,
"include": ["core/**"]
}
]
}`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
},
{
"label": "Mod_HakList",
"type": "List",
"value": []
}
]
}
}
`)
mustWriteFile(t, filepath.Join(root, "build", "haks.json"), `{
"module_haks": ["sow_top", "sow_core_01", "sow_appr_01"],
"haks": []
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildModule(p)
if err != nil {
t.Fatalf("build module: %v", err)
}
archive, err := readArchive(result.ModulePath)
if err != nil {
t.Fatalf("read module archive: %v", err)
}
var ifo erf.Resource
found := false
for _, resource := range archive.Resources {
if resource.Name == "module" {
ifo = resource
found = true
break
}
}
if !found {
t.Fatalf("module.ifo not found in built archive")
}
document, err := gff.Read(bytes.NewReader(ifo.Data))
if err != nil {
t.Fatalf("decode module ifo: %v", err)
}
var got []string
for _, field := range document.Root.Fields {
if field.Label != "Mod_HakList" {
continue
}
list, ok := field.Value.(gff.ListValue)
if !ok {
t.Fatalf("expected Mod_HakList list, got %T", field.Value)
}
for _, item := range list {
for _, nested := range item.Fields {
if nested.Label == "Mod_Hak" {
got = append(got, string(nested.Value.(gff.StringValue)))
}
}
}
}
if want := "sow_top,sow_core_01,sow_appr_01"; strings.Join(got, ",") != want {
t.Fatalf("unexpected module hak order: got %q want %q", strings.Join(got, ","), want)
}
}
func TestBuildHAKsRebuildsOnlyChangedChunks(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
@@ -1358,6 +1477,153 @@ func TestBuildHAKsRebuildsOnlyChangedChunks(t *testing.T) {
t.Fatalf("expected unchanged vfx.hak to be reused")
}
}
func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) {
if _, err := exec.LookPath("git"); err != nil {
t.Skip("git not available")
}
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets", "core"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["group:core"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"haks": [
{
"name": "core",
"priority": 1,
"max_bytes": 400,
"split": true,
"include": ["core/**"]
}
]
}`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
}
]
}
}
`)
runGitTest(t, root, "init")
runGitTest(t, root, "config", "user.name", "Test User")
runGitTest(t, root, "config", "user.email", "test@example.com")
mustWriteFile(t, filepath.Join(root, "assets", "core", "old_a.tga"), strings.Repeat("a", 120))
runGitCommitTest(t, root, "2001-01-01T00:00:00Z", "add old_a")
mustWriteFile(t, filepath.Join(root, "assets", "core", "old_b.tga"), strings.Repeat("b", 120))
runGitCommitTest(t, root, "2002-01-01T00:00:00Z", "add old_b")
mustWriteFile(t, filepath.Join(root, "assets", "core", "new_c.tga"), strings.Repeat("c", 120))
runGitCommitTest(t, root, "2003-01-01T00:00:00Z", "add new_c")
now := time.Now()
if err := os.Chtimes(filepath.Join(root, "assets", "core", "old_a.tga"), now, now); err != nil {
t.Fatalf("chtimes old_a: %v", err)
}
if err := os.Chtimes(filepath.Join(root, "assets", "core", "old_b.tga"), now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil {
t.Fatalf("chtimes old_b: %v", err)
}
if err := os.Chtimes(filepath.Join(root, "assets", "core", "new_c.tga"), now.Add(-4*time.Hour), now.Add(-4*time.Hour)); err != nil {
t.Fatalf("chtimes new_c: %v", err)
}
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
assets, err := collectAssetResources(p)
if err != nil {
t.Fatalf("collect asset resources: %v", err)
}
slices.SortFunc(assets, compareAssetBuildOrder)
twoAssetSize := erf.ArchiveSize(resourceSlice(assets[:2]))
threeAssetSize := erf.ArchiveSize(resourceSlice(assets))
if threeAssetSize <= twoAssetSize {
t.Fatalf("expected three-asset archive to be larger than two-asset archive")
}
p.Config.HAKs[0].MaxBytes = twoAssetSize
chunks, err := planHAKChunks(p, assets)
if err != nil {
t.Fatalf("plan hak chunks: %v", err)
}
if len(chunks) != 2 {
t.Fatalf("expected 2 chunks, got %d", len(chunks))
}
firstChunkAssets := make([]string, 0, len(chunks[0].Assets))
for _, asset := range chunks[0].Assets {
firstChunkAssets = append(firstChunkAssets, asset.Rel)
}
lastChunkAssets := make([]string, 0, len(chunks[1].Assets))
for _, asset := range chunks[1].Assets {
lastChunkAssets = append(lastChunkAssets, asset.Rel)
}
if got, want := strings.Join(firstChunkAssets, ","), "core/old_a.tga,core/old_b.tga"; got != want {
t.Fatalf("unexpected first chunk assets: got %q want %q", got, want)
}
if got, want := strings.Join(lastChunkAssets, ","), "core/new_c.tga"; got != want {
t.Fatalf("unexpected last chunk assets: got %q want %q", got, want)
}
}
func runGitCommitTest(t *testing.T, dir, timestamp, message string) {
t.Helper()
runGitTest(t, dir, "add", ".")
cmd := exec.Command("git", "commit", "-m", message)
cmd.Dir = dir
cmd.Env = append(os.Environ(),
"GIT_AUTHOR_DATE="+timestamp,
"GIT_COMMITTER_DATE="+timestamp,
)
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git commit failed: %v\n%s", err, string(output))
}
}
func runGitTest(t *testing.T, dir string, args ...string) {
t.Helper()
cmd := exec.Command("git", args...)
if dir != "" {
cmd.Dir = dir
}
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v failed: %v\n%s", args, err, string(output))
}
}
func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))