Changed sow-tools HAK split planning to preserve previous split membership

internal/pipeline/build.go

@@ -195,6 +195,13 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
 	if err != nil {
 		return BuildResult{}, err
 	}
+	var previousManifest *BuildManifest
+	if sourceManifest == nil || writeArchives {
+		previousManifest, err = loadPreviousBuildManifest(p.BuildDir())
+		if err != nil {
+			return BuildResult{}, err
+		}
+	}
 
 	result := BuildResult{HAKAssets: len(assetResources)}
 	if len(assetResources) == 0 {
@@ -210,7 +217,7 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
 	if sourceManifest != nil {
 		chunks, err = chunksFromManifest(assetResources, sourceManifest.HAKs)
 	} else {
-		chunks, err = planHAKChunks(p, assetResources)
+		chunks, err = planHAKChunksWithPrevious(p, assetResources, previousManifest)
 	}
 	if err != nil {
 		return BuildResult{}, err
@@ -267,10 +274,6 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
 		return result, nil
 	}
 
-	previousManifest, err := loadPreviousBuildManifest(p.BuildDir())
-	if err != nil {
-		return BuildResult{}, err
-	}
 	progressf(progress, "Reusing unchanged generated HAKs where possible...")
 
 	result.HAKPaths = make([]string, 0, len(chunks))
@@ -1099,6 +1102,10 @@ func splitSourceName(path string) (string, string, error) {
 }
 
 func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, error) {
+	return planHAKChunksWithPrevious(p, assets, nil)
+}
+
+func planHAKChunksWithPrevious(p *project.Project, assets []assetResource, previousManifest *BuildManifest) ([]hakChunk, error) {
 	if len(assets) == 0 {
 		return nil, nil
 	}
@@ -1151,7 +1158,7 @@ func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, erro
 		}
 		slices.SortFunc(groupAssets, compareAssetBuildOrder)
 
-		groupChunks, err := splitHAKGroup(cfg, groupAssets)
+		groupChunks, err := splitHAKGroupWithPrevious(cfg, groupAssets, previousManifest)
 		if err != nil {
 			return nil, err
 		}
@@ -1160,6 +1167,137 @@ func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, erro
 	return chunks, nil
 }
 
+func splitHAKGroupWithPrevious(cfg project.HAKConfig, assets []assetResource, previousManifest *BuildManifest) ([]hakChunk, error) {
+	if previousManifest == nil || cfg.MaxBytes == 0 || !cfg.Split {
+		return splitHAKGroup(cfg, assets)
+	}
+
+	previousEntries := make([]BuildManifestHAK, 0)
+	for _, entry := range previousManifest.HAKs {
+		if entry.Group != cfg.Name {
+			continue
+		}
+		if entry.Priority != cfg.Priority || entry.MaxBytes != cfg.MaxBytes || entry.Optional != cfg.Optional {
+			return splitHAKGroup(cfg, assets)
+		}
+		previousEntries = append(previousEntries, entry)
+	}
+	if len(previousEntries) == 0 {
+		return splitHAKGroup(cfg, assets)
+	}
+
+	assetByRel := make(map[string]assetResource, len(assets))
+	for _, asset := range assets {
+		assetByRel[filepath.ToSlash(asset.Rel)] = asset
+	}
+
+	assigned := make(map[string]struct{}, len(assets))
+	chunks := make([]hakChunk, 0, len(previousEntries))
+	for _, entry := range previousEntries {
+		chunkAssets := make([]assetResource, 0, len(entry.Assets))
+		for _, rel := range entry.Assets {
+			key := filepath.ToSlash(rel)
+			asset, ok := assetByRel[key]
+			if !ok {
+				continue
+			}
+			if _, exists := assigned[key]; exists {
+				return splitHAKGroup(cfg, assets)
+			}
+			chunkAssets = append(chunkAssets, asset)
+			assigned[key] = struct{}{}
+		}
+		if len(chunkAssets) == 0 {
+			continue
+		}
+
+		chunk := hakChunk{
+			Config: cfg,
+			Index:  len(chunks) + 1,
+			Name:   entry.Name,
+			Assets: chunkAssets,
+			Size:   erf.ArchiveSize(resourceSlice(chunkAssets)),
+		}
+		if chunk.Size > cfg.MaxBytes {
+			return splitHAKGroup(cfg, assets)
+		}
+		if err := ensureUniqueChunkResources(chunk); err != nil {
+			return splitHAKGroup(cfg, assets)
+		}
+		chunks = append(chunks, chunk)
+	}
+
+	newAssets := make([]assetResource, 0)
+	for _, asset := range assets {
+		key := filepath.ToSlash(asset.Rel)
+		if _, exists := assigned[key]; exists {
+			continue
+		}
+		newAssets = append(newAssets, asset)
+	}
+	slices.SortFunc(newAssets, compareAssetBuildOrder)
+
+	for _, asset := range newAssets {
+		inserted := false
+		for index := len(chunks) - 1; index >= 0; index-- {
+			candidate, ok := chunkWithAppendedAsset(chunks[index], asset, cfg.MaxBytes)
+			if !ok {
+				continue
+			}
+			chunks[index] = candidate
+			inserted = true
+			break
+		}
+		if inserted {
+			continue
+		}
+
+		chunk := hakChunk{
+			Config: cfg,
+			Index:  len(chunks) + 1,
+			Name:   nextChunkName(cfg.Name, chunks),
+			Assets: []assetResource{asset},
+			Size:   erf.ArchiveSize([]erf.Resource{asset.Resource}),
+		}
+		if chunk.Size > cfg.MaxBytes {
+			return nil, fmt.Errorf("asset %s alone exceeds max_bytes for hak %s", asset.Rel, cfg.Name)
+		}
+		if err := ensureUniqueChunkResources(chunk); err != nil {
+			return nil, err
+		}
+		chunks = append(chunks, chunk)
+	}
+
+	return chunks, nil
+}
+
+func nextChunkName(base string, chunks []hakChunk) string {
+	used := make(map[string]struct{}, len(chunks))
+	for _, chunk := range chunks {
+		used[chunk.Name] = struct{}{}
+	}
+	for index := len(chunks) + 1; ; index++ {
+		name := chunkName(base, index, true)
+		if _, exists := used[name]; !exists {
+			return name
+		}
+	}
+}
+
+func chunkWithAppendedAsset(chunk hakChunk, asset assetResource, maxBytes int64) (hakChunk, bool) {
+	candidateAssets := append(append([]assetResource{}, chunk.Assets...), asset)
+	candidate := chunk
+	candidate.Assets = candidateAssets
+	candidate.Size = erf.ArchiveSize(resourceSlice(candidateAssets))
+	if candidate.Size > maxBytes {
+		return hakChunk{}, false
+	}
+	if err := ensureUniqueChunkResources(candidate); err != nil {
+		return hakChunk{}, false
+	}
+	return candidate, true
+}
+
 func splitHAKGroup(cfg project.HAKConfig, assets []assetResource) ([]hakChunk, error) {
 	maxBytes := cfg.MaxBytes
 	if maxBytes == 0 {

internal/pipeline/pipeline_test.go

@@ -1874,6 +1874,70 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) {
 	}
 }
 
+func TestPlanHAKChunksKeepsPreviousSplitsAndPacksNewAssetsAtTail(t *testing.T) {
+	oldB := testAssetResource("core/old_b.tga", "old_b", "b", time.Date(2002, 1, 1, 0, 0, 0, 0, time.UTC))
+	oldC := testAssetResource("core/old_c.tga", "old_c", "c", time.Date(2003, 1, 1, 0, 0, 0, 0, time.UTC))
+	newA := testAssetResource("core/new_a.tga", "new_a", "a", time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC))
+
+	twoAssetSize := erf.ArchiveSize([]erf.Resource{oldB.Resource, oldC.Resource})
+	p := &project.Project{
+		Config: project.Config{
+			HAKs: []project.HAKConfig{
+				{
+					Name:     "core",
+					Priority: 1,
+					MaxBytes: twoAssetSize,
+					Split:    true,
+					Include:  []string{"core/**"},
+				},
+			},
+		},
+	}
+	previous := &BuildManifest{
+		HAKs: []BuildManifestHAK{
+			{Name: "core_01", Group: "core", Priority: 1, MaxBytes: twoAssetSize, Assets: []string{"core/old_b.tga"}},
+			{Name: "core_02", Group: "core", Priority: 1, MaxBytes: twoAssetSize, Assets: []string{"core/old_c.tga"}},
+		},
+	}
+
+	chunks, err := planHAKChunksWithPrevious(p, []assetResource{oldB, oldC, newA}, previous)
+	if err != nil {
+		t.Fatalf("plan hak chunks: %v", err)
+	}
+	if len(chunks) != 2 {
+		t.Fatalf("expected 2 chunks, got %d", len(chunks))
+	}
+	if got, want := strings.Join(chunkAssetRels(chunks[0]), ","), "core/old_b.tga"; got != want {
+		t.Fatalf("unexpected first chunk assets: got %q want %q", got, want)
+	}
+	if got, want := strings.Join(chunkAssetRels(chunks[1]), ","), "core/old_c.tga,core/new_a.tga"; got != want {
+		t.Fatalf("unexpected tail chunk assets: got %q want %q", got, want)
+	}
+}
+
+func testAssetResource(rel, name, payload string, createdAt time.Time) assetResource {
+	resource := erf.Resource{
+		Name: strings.ToLower(name),
+		Type: 0x0003,
+		Data: []byte(strings.Repeat(payload, 80)),
+	}
+	return assetResource{
+		Rel:       rel,
+		Resource:  resource,
+		Size:      erf.ArchiveSize([]erf.Resource{resource}),
+		CreatedAt: createdAt,
+		ContentID: "test:" + name,
+	}
+}
+
+func chunkAssetRels(chunk hakChunk) []string {
+	rels := make([]string, 0, len(chunk.Assets))
+	for _, asset := range chunk.Assets {
+		rels = append(rels, asset.Rel)
+	}
+	return rels
+}
+
 func TestCollectLFSAssetInfoUsesGitCommonDirForWorktree(t *testing.T) {
 	if _, err := exec.LookPath("git"); err != nil {
 		t.Skip("git not available")