From f249108cf0f2695980c2f7d85d897b40063d169a Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Sat, 18 Apr 2026 18:39:15 +0200 Subject: [PATCH] Changed sow-tools HAK split planning to preserve previous split membership --- internal/pipeline/build.go | 150 +++++++++++++++++++++++++++-- internal/pipeline/pipeline_test.go | 64 ++++++++++++ 2 files changed, 208 insertions(+), 6 deletions(-) diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index 04c91b8..4ad76a2 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -195,6 +195,13 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo if err != nil { return BuildResult{}, err } + var previousManifest *BuildManifest + if sourceManifest == nil || writeArchives { + previousManifest, err = loadPreviousBuildManifest(p.BuildDir()) + if err != nil { + return BuildResult{}, err + } + } result := BuildResult{HAKAssets: len(assetResources)} if len(assetResources) == 0 { @@ -210,7 +217,7 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo if sourceManifest != nil { chunks, err = chunksFromManifest(assetResources, sourceManifest.HAKs) } else { - chunks, err = planHAKChunks(p, assetResources) + chunks, err = planHAKChunksWithPrevious(p, assetResources, previousManifest) } if err != nil { return BuildResult{}, err @@ -267,10 +274,6 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo return result, nil } - previousManifest, err := loadPreviousBuildManifest(p.BuildDir()) - if err != nil { - return BuildResult{}, err - } progressf(progress, "Reusing unchanged generated HAKs where possible...") result.HAKPaths = make([]string, 0, len(chunks)) @@ -1099,6 +1102,10 @@ func splitSourceName(path string) (string, string, error) { } func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, error) { + return planHAKChunksWithPrevious(p, assets, nil) +} + +func planHAKChunksWithPrevious(p *project.Project, assets []assetResource, previousManifest *BuildManifest) ([]hakChunk, error) { if len(assets) == 0 { return nil, nil } @@ -1151,7 +1158,7 @@ func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, erro } slices.SortFunc(groupAssets, compareAssetBuildOrder) - groupChunks, err := splitHAKGroup(cfg, groupAssets) + groupChunks, err := splitHAKGroupWithPrevious(cfg, groupAssets, previousManifest) if err != nil { return nil, err } @@ -1160,6 +1167,137 @@ func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, erro return chunks, nil } +func splitHAKGroupWithPrevious(cfg project.HAKConfig, assets []assetResource, previousManifest *BuildManifest) ([]hakChunk, error) { + if previousManifest == nil || cfg.MaxBytes == 0 || !cfg.Split { + return splitHAKGroup(cfg, assets) + } + + previousEntries := make([]BuildManifestHAK, 0) + for _, entry := range previousManifest.HAKs { + if entry.Group != cfg.Name { + continue + } + if entry.Priority != cfg.Priority || entry.MaxBytes != cfg.MaxBytes || entry.Optional != cfg.Optional { + return splitHAKGroup(cfg, assets) + } + previousEntries = append(previousEntries, entry) + } + if len(previousEntries) == 0 { + return splitHAKGroup(cfg, assets) + } + + assetByRel := make(map[string]assetResource, len(assets)) + for _, asset := range assets { + assetByRel[filepath.ToSlash(asset.Rel)] = asset + } + + assigned := make(map[string]struct{}, len(assets)) + chunks := make([]hakChunk, 0, len(previousEntries)) + for _, entry := range previousEntries { + chunkAssets := make([]assetResource, 0, len(entry.Assets)) + for _, rel := range entry.Assets { + key := filepath.ToSlash(rel) + asset, ok := assetByRel[key] + if !ok { + continue + } + if _, exists := assigned[key]; exists { + return splitHAKGroup(cfg, assets) + } + chunkAssets = append(chunkAssets, asset) + assigned[key] = struct{}{} + } + if len(chunkAssets) == 0 { + continue + } + + chunk := hakChunk{ + Config: cfg, + Index: len(chunks) + 1, + Name: entry.Name, + Assets: chunkAssets, + Size: erf.ArchiveSize(resourceSlice(chunkAssets)), + } + if chunk.Size > cfg.MaxBytes { + return splitHAKGroup(cfg, assets) + } + if err := ensureUniqueChunkResources(chunk); err != nil { + return splitHAKGroup(cfg, assets) + } + chunks = append(chunks, chunk) + } + + newAssets := make([]assetResource, 0) + for _, asset := range assets { + key := filepath.ToSlash(asset.Rel) + if _, exists := assigned[key]; exists { + continue + } + newAssets = append(newAssets, asset) + } + slices.SortFunc(newAssets, compareAssetBuildOrder) + + for _, asset := range newAssets { + inserted := false + for index := len(chunks) - 1; index >= 0; index-- { + candidate, ok := chunkWithAppendedAsset(chunks[index], asset, cfg.MaxBytes) + if !ok { + continue + } + chunks[index] = candidate + inserted = true + break + } + if inserted { + continue + } + + chunk := hakChunk{ + Config: cfg, + Index: len(chunks) + 1, + Name: nextChunkName(cfg.Name, chunks), + Assets: []assetResource{asset}, + Size: erf.ArchiveSize([]erf.Resource{asset.Resource}), + } + if chunk.Size > cfg.MaxBytes { + return nil, fmt.Errorf("asset %s alone exceeds max_bytes for hak %s", asset.Rel, cfg.Name) + } + if err := ensureUniqueChunkResources(chunk); err != nil { + return nil, err + } + chunks = append(chunks, chunk) + } + + return chunks, nil +} + +func nextChunkName(base string, chunks []hakChunk) string { + used := make(map[string]struct{}, len(chunks)) + for _, chunk := range chunks { + used[chunk.Name] = struct{}{} + } + for index := len(chunks) + 1; ; index++ { + name := chunkName(base, index, true) + if _, exists := used[name]; !exists { + return name + } + } +} + +func chunkWithAppendedAsset(chunk hakChunk, asset assetResource, maxBytes int64) (hakChunk, bool) { + candidateAssets := append(append([]assetResource{}, chunk.Assets...), asset) + candidate := chunk + candidate.Assets = candidateAssets + candidate.Size = erf.ArchiveSize(resourceSlice(candidateAssets)) + if candidate.Size > maxBytes { + return hakChunk{}, false + } + if err := ensureUniqueChunkResources(candidate); err != nil { + return hakChunk{}, false + } + return candidate, true +} + func splitHAKGroup(cfg project.HAKConfig, assets []assetResource) ([]hakChunk, error) { maxBytes := cfg.MaxBytes if maxBytes == 0 { diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 177792d..d1b3504 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -1874,6 +1874,70 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) { } } +func TestPlanHAKChunksKeepsPreviousSplitsAndPacksNewAssetsAtTail(t *testing.T) { + oldB := testAssetResource("core/old_b.tga", "old_b", "b", time.Date(2002, 1, 1, 0, 0, 0, 0, time.UTC)) + oldC := testAssetResource("core/old_c.tga", "old_c", "c", time.Date(2003, 1, 1, 0, 0, 0, 0, time.UTC)) + newA := testAssetResource("core/new_a.tga", "new_a", "a", time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC)) + + twoAssetSize := erf.ArchiveSize([]erf.Resource{oldB.Resource, oldC.Resource}) + p := &project.Project{ + Config: project.Config{ + HAKs: []project.HAKConfig{ + { + Name: "core", + Priority: 1, + MaxBytes: twoAssetSize, + Split: true, + Include: []string{"core/**"}, + }, + }, + }, + } + previous := &BuildManifest{ + HAKs: []BuildManifestHAK{ + {Name: "core_01", Group: "core", Priority: 1, MaxBytes: twoAssetSize, Assets: []string{"core/old_b.tga"}}, + {Name: "core_02", Group: "core", Priority: 1, MaxBytes: twoAssetSize, Assets: []string{"core/old_c.tga"}}, + }, + } + + chunks, err := planHAKChunksWithPrevious(p, []assetResource{oldB, oldC, newA}, previous) + if err != nil { + t.Fatalf("plan hak chunks: %v", err) + } + if len(chunks) != 2 { + t.Fatalf("expected 2 chunks, got %d", len(chunks)) + } + if got, want := strings.Join(chunkAssetRels(chunks[0]), ","), "core/old_b.tga"; got != want { + t.Fatalf("unexpected first chunk assets: got %q want %q", got, want) + } + if got, want := strings.Join(chunkAssetRels(chunks[1]), ","), "core/old_c.tga,core/new_a.tga"; got != want { + t.Fatalf("unexpected tail chunk assets: got %q want %q", got, want) + } +} + +func testAssetResource(rel, name, payload string, createdAt time.Time) assetResource { + resource := erf.Resource{ + Name: strings.ToLower(name), + Type: 0x0003, + Data: []byte(strings.Repeat(payload, 80)), + } + return assetResource{ + Rel: rel, + Resource: resource, + Size: erf.ArchiveSize([]erf.Resource{resource}), + CreatedAt: createdAt, + ContentID: "test:" + name, + } +} + +func chunkAssetRels(chunk hakChunk) []string { + rels := make([]string, 0, len(chunk.Assets)) + for _, asset := range chunk.Assets { + rels = append(rels, asset.Rel) + } + return rels +} + func TestCollectLFSAssetInfoUsesGitCommonDirForWorktree(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("git not available")