ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Conservative build-haks

Browse Source 
In build.go, build-haks now does conservative chunk reuse instead of
wiping all generated HAKs up front. It plans chunks as before, computes
a deterministic per-chunk content_hash, reuses an existing .hak only
when the prior manifest entry and on-disk archive still match, rebuilds
only changed chunks, and deletes only stale generated HAKs that fall out
of the new plan. I added coverage for unchanged reuse and selective
rebuilds in pipeline_test.go, and go test ./... passes in sow-tools.
This commit is contained in:
archvillainette
2026-04-04 11:51:53 +02:00
parent cd96a3ceee
commit bf87144688
2 changed files with 375 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/pipeline/build.go
+148 -31
View File
@@ -2,6 +2,7 @@ package pipeline
import (
"bytes"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
@@ -32,12 +33,13 @@ type BuildManifest struct {
}
type BuildManifestHAK struct {
Name string `json:"name"`
Group string `json:"group"`
Priority int `json:"priority"`
MaxBytes int64 `json:"max_bytes"`
SizeBytes int64 `json:"size_bytes"`
Assets []string `json:"assets"`
Name string `json:"name"`
Group string `json:"group"`
Priority int `json:"priority"`
MaxBytes int64 `json:"max_bytes"`
SizeBytes int64 `json:"size_bytes"`
Assets []string `json:"assets"`
ContentHash string `json:"content_hash,omitempty"`
}
type assetResource struct {
@@ -128,13 +130,13 @@ func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
if err != nil {
return BuildResult{}, err
}
progressf(progress, "Cleaning previous generated HAKs...")
if err := cleanupGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef); err != nil {
return BuildResult{}, err
}
result := BuildResult{HAKAssets: len(assetResources)}
if len(assetResources) == 0 {
progressf(progress, "Cleaning previous generated HAKs...")
if err := cleanupGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef); err != nil {
return BuildResult{}, err
}
return result, nil
}
@@ -148,39 +150,48 @@ func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
if err != nil {
return BuildResult{}, err
}
previousManifest, err := loadPreviousBuildManifest(p.BuildDir())
if err != nil {
return BuildResult{}, err
}
progressf(progress, "Reusing unchanged generated HAKs where possible...")
manifest := BuildManifest{
ModuleHAKs: moduleHakOrder,
HAKs: make([]BuildManifestHAK, 0, len(chunks)),
}
currentNames := make(map[string]struct{}, len(chunks))
for index, chunk := range chunks {
progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets)))
hakPath := filepath.Join(p.BuildDir(), chunk.Name+".hak")
hakOutput, err := os.Create(hakPath)
manifestEntry := buildManifestEntry(chunk)
currentNames[chunk.Name] = struct{}{}
reused, err := reuseExistingHAK(previousManifest, manifestEntry, hakPath)
if err != nil {
return BuildResult{}, fmt.Errorf("create hak archive: %w", err)
return BuildResult{}, err
}
if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil {
hakOutput.Close()
return BuildResult{}, fmt.Errorf("write hak archive: %w", err)
}
if err := hakOutput.Close(); err != nil {
return BuildResult{}, fmt.Errorf("close hak archive: %w", err)
if reused {
progressf(progress, fmt.Sprintf("Reusing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets)))
} else {
progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets)))
hakOutput, err := os.Create(hakPath)
if err != nil {
return BuildResult{}, fmt.Errorf("create hak archive: %w", err)
}
if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil {
hakOutput.Close()
return BuildResult{}, fmt.Errorf("write hak archive: %w", err)
}
if err := hakOutput.Close(); err != nil {
return BuildResult{}, fmt.Errorf("close hak archive: %w", err)
}
}
result.HAKPaths = append(result.HAKPaths, hakPath)
assets := make([]string, 0, len(chunk.Assets))
for _, asset := range chunk.Assets {
assets = append(assets, asset.Rel)
}
manifest.HAKs = append(manifest.HAKs, BuildManifestHAK{
Name: chunk.Name,
Group: chunk.Config.Name,
Priority: chunk.Config.Priority,
MaxBytes: chunk.Config.MaxBytes,
SizeBytes: chunk.Size,
Assets: assets,
})
manifest.HAKs = append(manifest.HAKs, manifestEntry)
}
progressf(progress, "Cleaning stale generated HAKs...")
if err := cleanupStaleGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef, currentNames); err != nil {
return BuildResult{}, err
}
progressf(progress, "Writing HAK manifest...")
@@ -686,6 +697,112 @@ func resourceSlice(assets []assetResource) []erf.Resource {
return out
}
func buildManifestEntry(chunk hakChunk) BuildManifestHAK {
assets := make([]string, 0, len(chunk.Assets))
for _, asset := range chunk.Assets {
assets = append(assets, asset.Rel)
}
return BuildManifestHAK{
Name: chunk.Name,
Group: chunk.Config.Name,
Priority: chunk.Config.Priority,
MaxBytes: chunk.Config.MaxBytes,
SizeBytes: chunk.Size,
Assets: assets,
ContentHash: chunkContentHash(chunk),
}
}
func chunkContentHash(chunk hakChunk) string {
sum := sha256.New()
fmt.Fprintf(sum, "name:%s\n", chunk.Name)
fmt.Fprintf(sum, "group:%s\n", chunk.Config.Name)
fmt.Fprintf(sum, "priority:%d\n", chunk.Config.Priority)
fmt.Fprintf(sum, "max_bytes:%d\n", chunk.Config.MaxBytes)
fmt.Fprintf(sum, "size_bytes:%d\n", chunk.Size)
for _, asset := range chunk.Assets {
fmt.Fprintf(sum, "asset:%s\n", asset.Rel)
fmt.Fprintf(sum, "resref:%s\n", asset.Resource.Name)
fmt.Fprintf(sum, "restype:%d\n", asset.Resource.Type)
fmt.Fprintf(sum, "datasize:%d\n", len(asset.Resource.Data))
_, _ = sum.Write(asset.Resource.Data)
_, _ = sum.Write([]byte{'\n'})
}
return fmt.Sprintf("%x", sum.Sum(nil))
}
func loadPreviousBuildManifest(buildDir string) (*BuildManifest, error) {
manifestPath := filepath.Join(buildDir, "haks.json")
raw, err := os.ReadFile(manifestPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("read previous hak manifest: %w", err)
}
var manifest BuildManifest
if err := json.Unmarshal(raw, &manifest); err != nil {
return nil, fmt.Errorf("parse previous hak manifest: %w", err)
}
return &manifest, nil
}
func reuseExistingHAK(previousManifest *BuildManifest, current BuildManifestHAK, hakPath string) (bool, error) {
if previousManifest == nil {
return false, nil
}
var previous *BuildManifestHAK
for index := range previousManifest.HAKs {
entry := &previousManifest.HAKs[index]
if entry.Name == current.Name {
previous = entry
break
}
}
if previous == nil {
return false, nil
}
if previous.Group != current.Group ||
previous.Priority != current.Priority ||
previous.MaxBytes != current.MaxBytes ||
previous.SizeBytes != current.SizeBytes ||
previous.ContentHash != current.ContentHash ||
!slices.Equal(previous.Assets, current.Assets) {
return false, nil
}
if _, err := os.Stat(hakPath); err != nil {
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
return false, fmt.Errorf("stat existing hak archive %s: %w", hakPath, err)
}
return true, nil
}
func cleanupStaleGeneratedHAKs(buildDir, moduleResRef string, keepNames map[string]struct{}) error {
paths, err := manifestHAKPaths(buildDir)
if err != nil {
return err
}
for _, path := range paths {
name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
if _, keep := keepNames[name]; keep {
continue
}
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("remove stale generated hak %s: %w", path, err)
}
}
legacy := filepath.Join(buildDir, moduleResRef+".hak")
if _, keep := keepNames[moduleResRef]; !keep {
if err := os.Remove(legacy); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("remove legacy generated hak %s: %w", legacy, err)
}
}
return nil
}
func matchesAnyPattern(path string, patterns []string) bool {
for _, pattern := range patterns {
if matchPathPattern(filepath.ToSlash(path), filepath.ToSlash(pattern)) {
internal/pipeline/pipeline_test.go
+227 -2
View File
@@ -6,6 +6,7 @@ import (
"path/filepath"
"strings"
"testing"
"time"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
@@ -31,7 +32,7 @@ func TestBuildThenExtract(t *testing.T) {
`)
mustMkdir(t, filepath.Join(root, "src", "module"))
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
@@ -46,7 +47,6 @@ func TestBuildThenExtract(t *testing.T) {
}
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
@@ -750,6 +750,231 @@ func TestBuildSkipsEmptyHAKGroupsInModuleOrder(t *testing.T) {
}
}
func TestBuildHAKsReusesUnchangedChunks(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets", "core"))
mustMkdir(t, filepath.Join(root, "assets", "vfx"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["group:core", "group:vfx"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"haks": [
{
"name": "core",
"priority": 1,
"max_bytes": 1024,
"split": false,
"include": ["core/**"]
},
{
"name": "vfx",
"priority": 2,
"max_bytes": 1024,
"split": false,
"include": ["vfx/**"]
}
]
}`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
},
{
"label": "Mod_HakList",
"type": "List",
"value": []
}
]
}
}`)
mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a")
mustWriteFile(t, filepath.Join(root, "assets", "vfx", "vfx_a.tga"), "vfx-a")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
if _, err := BuildHAKs(p); err != nil {
t.Fatalf("first build haks: %v", err)
}
corePath := filepath.Join(root, "build", "core.hak")
vfxPath := filepath.Join(root, "build", "vfx.hak")
coreInfoBefore, err := os.Stat(corePath)
if err != nil {
t.Fatalf("stat core before: %v", err)
}
vfxInfoBefore, err := os.Stat(vfxPath)
if err != nil {
t.Fatalf("stat vfx before: %v", err)
}
time.Sleep(1100 * time.Millisecond)
if _, err := BuildHAKs(p); err != nil {
t.Fatalf("second build haks: %v", err)
}
coreInfoAfter, err := os.Stat(corePath)
if err != nil {
t.Fatalf("stat core after: %v", err)
}
vfxInfoAfter, err := os.Stat(vfxPath)
if err != nil {
t.Fatalf("stat vfx after: %v", err)
}
if !coreInfoAfter.ModTime().Equal(coreInfoBefore.ModTime()) {
t.Fatalf("expected unchanged core.hak to be reused")
}
if !vfxInfoAfter.ModTime().Equal(vfxInfoBefore.ModTime()) {
t.Fatalf("expected unchanged vfx.hak to be reused")
}
rawManifest, err := os.ReadFile(filepath.Join(root, "build", "haks.json"))
if err != nil {
t.Fatalf("read manifest: %v", err)
}
var manifest BuildManifest
if err := json.Unmarshal(rawManifest, &manifest); err != nil {
t.Fatalf("parse manifest: %v", err)
}
for _, hak := range manifest.HAKs {
if hak.ContentHash == "" {
t.Fatalf("expected content hash for %s", hak.Name)
}
}
}
func TestBuildHAKsRebuildsOnlyChangedChunks(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets", "core"))
mustMkdir(t, filepath.Join(root, "assets", "vfx"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["group:core", "group:vfx"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
},
"haks": [
{
"name": "core",
"priority": 1,
"max_bytes": 1024,
"split": false,
"include": ["core/**"]
},
{
"name": "vfx",
"priority": 2,
"max_bytes": 1024,
"split": false,
"include": ["vfx/**"]
}
]
}`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
},
{
"label": "Mod_HakList",
"type": "List",
"value": []
}
]
}
}`)
mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a")
mustWriteFile(t, filepath.Join(root, "assets", "vfx", "vfx_a.tga"), "vfx-a")
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
if _, err := BuildHAKs(p); err != nil {
t.Fatalf("first build haks: %v", err)
}
corePath := filepath.Join(root, "build", "core.hak")
vfxPath := filepath.Join(root, "build", "vfx.hak")
coreInfoBefore, err := os.Stat(corePath)
if err != nil {
t.Fatalf("stat core before: %v", err)
}
vfxInfoBefore, err := os.Stat(vfxPath)
if err != nil {
t.Fatalf("stat vfx before: %v", err)
}
time.Sleep(1100 * time.Millisecond)
mustWriteFile(t, filepath.Join(root, "assets", "core", "core_a.tga"), "core-a-updated")
if err := p.Scan(); err != nil {
t.Fatalf("rescan after change: %v", err)
}
if _, err := BuildHAKs(p); err != nil {
t.Fatalf("second build haks: %v", err)
}
coreInfoAfter, err := os.Stat(corePath)
if err != nil {
t.Fatalf("stat core after: %v", err)
}
vfxInfoAfter, err := os.Stat(vfxPath)
if err != nil {
t.Fatalf("stat vfx after: %v", err)
}
if !coreInfoAfter.ModTime().After(coreInfoBefore.ModTime()) {
t.Fatalf("expected changed core.hak to be rebuilt")
}
if !vfxInfoAfter.ModTime().Equal(vfxInfoBefore.ModTime()) {
t.Fatalf("expected unchanged vfx.hak to be reused")
}
}
func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))