Conservative build-haks
In build.go, build-haks now does conservative chunk reuse instead of wiping all generated HAKs up front. It plans chunks as before, computes a deterministic per-chunk content_hash, reuses an existing .hak only when the prior manifest entry and on-disk archive still match, rebuilds only changed chunks, and deletes only stale generated HAKs that fall out of the new plan. I added coverage for unchanged reuse and selective rebuilds in pipeline_test.go, and go test ./... passes in sow-tools.
This commit is contained in:
+148
-31
@@ -2,6 +2,7 @@ package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -32,12 +33,13 @@ type BuildManifest struct {
|
||||
}
|
||||
|
||||
type BuildManifestHAK struct {
|
||||
Name string `json:"name"`
|
||||
Group string `json:"group"`
|
||||
Priority int `json:"priority"`
|
||||
MaxBytes int64 `json:"max_bytes"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Assets []string `json:"assets"`
|
||||
Name string `json:"name"`
|
||||
Group string `json:"group"`
|
||||
Priority int `json:"priority"`
|
||||
MaxBytes int64 `json:"max_bytes"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Assets []string `json:"assets"`
|
||||
ContentHash string `json:"content_hash,omitempty"`
|
||||
}
|
||||
|
||||
type assetResource struct {
|
||||
@@ -128,13 +130,13 @@ func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
progressf(progress, "Cleaning previous generated HAKs...")
|
||||
if err := cleanupGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
|
||||
result := BuildResult{HAKAssets: len(assetResources)}
|
||||
if len(assetResources) == 0 {
|
||||
progressf(progress, "Cleaning previous generated HAKs...")
|
||||
if err := cleanupGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -148,39 +150,48 @@ func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
previousManifest, err := loadPreviousBuildManifest(p.BuildDir())
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
progressf(progress, "Reusing unchanged generated HAKs where possible...")
|
||||
|
||||
manifest := BuildManifest{
|
||||
ModuleHAKs: moduleHakOrder,
|
||||
HAKs: make([]BuildManifestHAK, 0, len(chunks)),
|
||||
}
|
||||
currentNames := make(map[string]struct{}, len(chunks))
|
||||
for index, chunk := range chunks {
|
||||
progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets)))
|
||||
hakPath := filepath.Join(p.BuildDir(), chunk.Name+".hak")
|
||||
hakOutput, err := os.Create(hakPath)
|
||||
manifestEntry := buildManifestEntry(chunk)
|
||||
currentNames[chunk.Name] = struct{}{}
|
||||
reused, err := reuseExistingHAK(previousManifest, manifestEntry, hakPath)
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("create hak archive: %w", err)
|
||||
return BuildResult{}, err
|
||||
}
|
||||
if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil {
|
||||
hakOutput.Close()
|
||||
return BuildResult{}, fmt.Errorf("write hak archive: %w", err)
|
||||
}
|
||||
if err := hakOutput.Close(); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("close hak archive: %w", err)
|
||||
if reused {
|
||||
progressf(progress, fmt.Sprintf("Reusing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets)))
|
||||
} else {
|
||||
progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets)))
|
||||
hakOutput, err := os.Create(hakPath)
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("create hak archive: %w", err)
|
||||
}
|
||||
if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil {
|
||||
hakOutput.Close()
|
||||
return BuildResult{}, fmt.Errorf("write hak archive: %w", err)
|
||||
}
|
||||
if err := hakOutput.Close(); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("close hak archive: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
result.HAKPaths = append(result.HAKPaths, hakPath)
|
||||
assets := make([]string, 0, len(chunk.Assets))
|
||||
for _, asset := range chunk.Assets {
|
||||
assets = append(assets, asset.Rel)
|
||||
}
|
||||
manifest.HAKs = append(manifest.HAKs, BuildManifestHAK{
|
||||
Name: chunk.Name,
|
||||
Group: chunk.Config.Name,
|
||||
Priority: chunk.Config.Priority,
|
||||
MaxBytes: chunk.Config.MaxBytes,
|
||||
SizeBytes: chunk.Size,
|
||||
Assets: assets,
|
||||
})
|
||||
manifest.HAKs = append(manifest.HAKs, manifestEntry)
|
||||
}
|
||||
progressf(progress, "Cleaning stale generated HAKs...")
|
||||
if err := cleanupStaleGeneratedHAKs(p.BuildDir(), p.Config.Module.ResRef, currentNames); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
|
||||
progressf(progress, "Writing HAK manifest...")
|
||||
@@ -686,6 +697,112 @@ func resourceSlice(assets []assetResource) []erf.Resource {
|
||||
return out
|
||||
}
|
||||
|
||||
func buildManifestEntry(chunk hakChunk) BuildManifestHAK {
|
||||
assets := make([]string, 0, len(chunk.Assets))
|
||||
for _, asset := range chunk.Assets {
|
||||
assets = append(assets, asset.Rel)
|
||||
}
|
||||
return BuildManifestHAK{
|
||||
Name: chunk.Name,
|
||||
Group: chunk.Config.Name,
|
||||
Priority: chunk.Config.Priority,
|
||||
MaxBytes: chunk.Config.MaxBytes,
|
||||
SizeBytes: chunk.Size,
|
||||
Assets: assets,
|
||||
ContentHash: chunkContentHash(chunk),
|
||||
}
|
||||
}
|
||||
|
||||
func chunkContentHash(chunk hakChunk) string {
|
||||
sum := sha256.New()
|
||||
fmt.Fprintf(sum, "name:%s\n", chunk.Name)
|
||||
fmt.Fprintf(sum, "group:%s\n", chunk.Config.Name)
|
||||
fmt.Fprintf(sum, "priority:%d\n", chunk.Config.Priority)
|
||||
fmt.Fprintf(sum, "max_bytes:%d\n", chunk.Config.MaxBytes)
|
||||
fmt.Fprintf(sum, "size_bytes:%d\n", chunk.Size)
|
||||
for _, asset := range chunk.Assets {
|
||||
fmt.Fprintf(sum, "asset:%s\n", asset.Rel)
|
||||
fmt.Fprintf(sum, "resref:%s\n", asset.Resource.Name)
|
||||
fmt.Fprintf(sum, "restype:%d\n", asset.Resource.Type)
|
||||
fmt.Fprintf(sum, "datasize:%d\n", len(asset.Resource.Data))
|
||||
_, _ = sum.Write(asset.Resource.Data)
|
||||
_, _ = sum.Write([]byte{'\n'})
|
||||
}
|
||||
return fmt.Sprintf("%x", sum.Sum(nil))
|
||||
}
|
||||
|
||||
func loadPreviousBuildManifest(buildDir string) (*BuildManifest, error) {
|
||||
manifestPath := filepath.Join(buildDir, "haks.json")
|
||||
raw, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("read previous hak manifest: %w", err)
|
||||
}
|
||||
var manifest BuildManifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("parse previous hak manifest: %w", err)
|
||||
}
|
||||
return &manifest, nil
|
||||
}
|
||||
|
||||
func reuseExistingHAK(previousManifest *BuildManifest, current BuildManifestHAK, hakPath string) (bool, error) {
|
||||
if previousManifest == nil {
|
||||
return false, nil
|
||||
}
|
||||
var previous *BuildManifestHAK
|
||||
for index := range previousManifest.HAKs {
|
||||
entry := &previousManifest.HAKs[index]
|
||||
if entry.Name == current.Name {
|
||||
previous = entry
|
||||
break
|
||||
}
|
||||
}
|
||||
if previous == nil {
|
||||
return false, nil
|
||||
}
|
||||
if previous.Group != current.Group ||
|
||||
previous.Priority != current.Priority ||
|
||||
previous.MaxBytes != current.MaxBytes ||
|
||||
previous.SizeBytes != current.SizeBytes ||
|
||||
previous.ContentHash != current.ContentHash ||
|
||||
!slices.Equal(previous.Assets, current.Assets) {
|
||||
return false, nil
|
||||
}
|
||||
if _, err := os.Stat(hakPath); err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("stat existing hak archive %s: %w", hakPath, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func cleanupStaleGeneratedHAKs(buildDir, moduleResRef string, keepNames map[string]struct{}) error {
|
||||
paths, err := manifestHAKPaths(buildDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, path := range paths {
|
||||
name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
||||
if _, keep := keepNames[name]; keep {
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale generated hak %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
legacy := filepath.Join(buildDir, moduleResRef+".hak")
|
||||
if _, keep := keepNames[moduleResRef]; !keep {
|
||||
if err := os.Remove(legacy); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("remove legacy generated hak %s: %w", legacy, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func matchesAnyPattern(path string, patterns []string) bool {
|
||||
for _, pattern := range patterns {
|
||||
if matchPathPattern(filepath.ToSlash(path), filepath.ToSlash(pattern)) {
|
||||
|
||||
Reference in New Issue
Block a user