ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Changed sow-tools HAK split planning to preserve previous split membership

Browse Source
This commit is contained in:
archvillainette
2026-04-18 18:39:15 +02:00
parent e0b3164427
commit f249108cf0
2 changed files with 208 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/pipeline/build.go
+144 -6
View File
@@ -195,6 +195,13 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
if err != nil {
return BuildResult{}, err
}
var previousManifest *BuildManifest
if sourceManifest == nil || writeArchives {
previousManifest, err = loadPreviousBuildManifest(p.BuildDir())
if err != nil {
return BuildResult{}, err
}
}
result := BuildResult{HAKAssets: len(assetResources)}
if len(assetResources) == 0 {
@@ -210,7 +217,7 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
if sourceManifest != nil {
chunks, err = chunksFromManifest(assetResources, sourceManifest.HAKs)
} else {
chunks, err = planHAKChunks(p, assetResources)
chunks, err = planHAKChunksWithPrevious(p, assetResources, previousManifest)
}
if err != nil {
return BuildResult{}, err
@@ -267,10 +274,6 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
return result, nil
}
previousManifest, err := loadPreviousBuildManifest(p.BuildDir())
if err != nil {
return BuildResult{}, err
}
progressf(progress, "Reusing unchanged generated HAKs where possible...")
result.HAKPaths = make([]string, 0, len(chunks))
@@ -1099,6 +1102,10 @@ func splitSourceName(path string) (string, string, error) {
}
func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, error) {
return planHAKChunksWithPrevious(p, assets, nil)
}
func planHAKChunksWithPrevious(p *project.Project, assets []assetResource, previousManifest *BuildManifest) ([]hakChunk, error) {
if len(assets) == 0 {
return nil, nil
}
@@ -1151,7 +1158,7 @@ func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, erro
}
slices.SortFunc(groupAssets, compareAssetBuildOrder)
groupChunks, err := splitHAKGroup(cfg, groupAssets)
groupChunks, err := splitHAKGroupWithPrevious(cfg, groupAssets, previousManifest)
if err != nil {
return nil, err
}
@@ -1160,6 +1167,137 @@ func planHAKChunks(p *project.Project, assets []assetResource) ([]hakChunk, erro
return chunks, nil
}
func splitHAKGroupWithPrevious(cfg project.HAKConfig, assets []assetResource, previousManifest *BuildManifest) ([]hakChunk, error) {
if previousManifest == nil || cfg.MaxBytes == 0 || !cfg.Split {
return splitHAKGroup(cfg, assets)
}
previousEntries := make([]BuildManifestHAK, 0)
for _, entry := range previousManifest.HAKs {
if entry.Group != cfg.Name {
continue
}
if entry.Priority != cfg.Priority || entry.MaxBytes != cfg.MaxBytes || entry.Optional != cfg.Optional {
return splitHAKGroup(cfg, assets)
}
previousEntries = append(previousEntries, entry)
}
if len(previousEntries) == 0 {
return splitHAKGroup(cfg, assets)
}
assetByRel := make(map[string]assetResource, len(assets))
for _, asset := range assets {
assetByRel[filepath.ToSlash(asset.Rel)] = asset
}
assigned := make(map[string]struct{}, len(assets))
chunks := make([]hakChunk, 0, len(previousEntries))
for _, entry := range previousEntries {
chunkAssets := make([]assetResource, 0, len(entry.Assets))
for _, rel := range entry.Assets {
key := filepath.ToSlash(rel)
asset, ok := assetByRel[key]
if !ok {
continue
}
if _, exists := assigned[key]; exists {
return splitHAKGroup(cfg, assets)
}
chunkAssets = append(chunkAssets, asset)
assigned[key] = struct{}{}
}
if len(chunkAssets) == 0 {
continue
}
chunk := hakChunk{
Config: cfg,
Index: len(chunks) + 1,
Name: entry.Name,
Assets: chunkAssets,
Size: erf.ArchiveSize(resourceSlice(chunkAssets)),
}
if chunk.Size > cfg.MaxBytes {
return splitHAKGroup(cfg, assets)
}
if err := ensureUniqueChunkResources(chunk); err != nil {
return splitHAKGroup(cfg, assets)
}
chunks = append(chunks, chunk)
}
newAssets := make([]assetResource, 0)
for _, asset := range assets {
key := filepath.ToSlash(asset.Rel)
if _, exists := assigned[key]; exists {
continue
}
newAssets = append(newAssets, asset)
}
slices.SortFunc(newAssets, compareAssetBuildOrder)
for _, asset := range newAssets {
inserted := false
for index := len(chunks) - 1; index >= 0; index-- {
candidate, ok := chunkWithAppendedAsset(chunks[index], asset, cfg.MaxBytes)
if !ok {
continue
}
chunks[index] = candidate
inserted = true
break
}
if inserted {
continue
}
chunk := hakChunk{
Config: cfg,
Index: len(chunks) + 1,
Name: nextChunkName(cfg.Name, chunks),
Assets: []assetResource{asset},
Size: erf.ArchiveSize([]erf.Resource{asset.Resource}),
}
if chunk.Size > cfg.MaxBytes {
return nil, fmt.Errorf("asset %s alone exceeds max_bytes for hak %s", asset.Rel, cfg.Name)
}
if err := ensureUniqueChunkResources(chunk); err != nil {
return nil, err
}
chunks = append(chunks, chunk)
}
return chunks, nil
}
func nextChunkName(base string, chunks []hakChunk) string {
used := make(map[string]struct{}, len(chunks))
for _, chunk := range chunks {
used[chunk.Name] = struct{}{}
}
for index := len(chunks) + 1; ; index++ {
name := chunkName(base, index, true)
if _, exists := used[name]; !exists {
return name
}
}
}
func chunkWithAppendedAsset(chunk hakChunk, asset assetResource, maxBytes int64) (hakChunk, bool) {
candidateAssets := append(append([]assetResource{}, chunk.Assets...), asset)
candidate := chunk
candidate.Assets = candidateAssets
candidate.Size = erf.ArchiveSize(resourceSlice(candidateAssets))
if candidate.Size > maxBytes {
return hakChunk{}, false
}
if err := ensureUniqueChunkResources(candidate); err != nil {
return hakChunk{}, false
}
return candidate, true
}
func splitHAKGroup(cfg project.HAKConfig, assets []assetResource) ([]hakChunk, error) {
maxBytes := cfg.MaxBytes
if maxBytes == 0 {
internal/pipeline/pipeline_test.go
+64
View File
@@ -1874,6 +1874,70 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) {
}
}
func TestPlanHAKChunksKeepsPreviousSplitsAndPacksNewAssetsAtTail(t *testing.T) {
oldB := testAssetResource("core/old_b.tga", "old_b", "b", time.Date(2002, 1, 1, 0, 0, 0, 0, time.UTC))
oldC := testAssetResource("core/old_c.tga", "old_c", "c", time.Date(2003, 1, 1, 0, 0, 0, 0, time.UTC))
newA := testAssetResource("core/new_a.tga", "new_a", "a", time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC))
twoAssetSize := erf.ArchiveSize([]erf.Resource{oldB.Resource, oldC.Resource})
p := &project.Project{
Config: project.Config{
HAKs: []project.HAKConfig{
{
Name: "core",
Priority: 1,
MaxBytes: twoAssetSize,
Split: true,
Include: []string{"core/**"},
},
},
},
}
previous := &BuildManifest{
HAKs: []BuildManifestHAK{
{Name: "core_01", Group: "core", Priority: 1, MaxBytes: twoAssetSize, Assets: []string{"core/old_b.tga"}},
{Name: "core_02", Group: "core", Priority: 1, MaxBytes: twoAssetSize, Assets: []string{"core/old_c.tga"}},
},
}
chunks, err := planHAKChunksWithPrevious(p, []assetResource{oldB, oldC, newA}, previous)
if err != nil {
t.Fatalf("plan hak chunks: %v", err)
}
if len(chunks) != 2 {
t.Fatalf("expected 2 chunks, got %d", len(chunks))
}
if got, want := strings.Join(chunkAssetRels(chunks[0]), ","), "core/old_b.tga"; got != want {
t.Fatalf("unexpected first chunk assets: got %q want %q", got, want)
}
if got, want := strings.Join(chunkAssetRels(chunks[1]), ","), "core/old_c.tga,core/new_a.tga"; got != want {
t.Fatalf("unexpected tail chunk assets: got %q want %q", got, want)
}
}
func testAssetResource(rel, name, payload string, createdAt time.Time) assetResource {
resource := erf.Resource{
Name: strings.ToLower(name),
Type: 0x0003,
Data: []byte(strings.Repeat(payload, 80)),
}
return assetResource{
Rel: rel,
Resource: resource,
Size: erf.ArchiveSize([]erf.Resource{resource}),
CreatedAt: createdAt,
ContentID: "test:" + name,
}
}
func chunkAssetRels(chunk hakChunk) []string {
rels := make([]string, 0, len(chunk.Assets))
for _, asset := range chunk.Assets {
rels = append(rels, asset.Rel)
}
return rels
}
func TestCollectLFSAssetInfoUsesGitCommonDirForWorktree(t *testing.T) {
if _, err := exec.LookPath("git"); err != nil {
t.Skip("git not available")