From ab18baf0679391358aa357ceae7939b796530e57 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Tue, 14 Apr 2026 12:12:42 +0200 Subject: [PATCH] build-haks now accepts --source-manifest When that flag is used, sow-tools: - collects only the assets referenced by that manifest subset - skips full-project chunk planning for the per-archive run - reconstructs just the requested archive chunks from the manifest --- internal/app/app.go | 22 +++++-- internal/pipeline/build.go | 99 ++++++++++++++++++++++++++---- internal/pipeline/pipeline_test.go | 2 +- 3 files changed, 104 insertions(+), 19 deletions(-) diff --git a/internal/app/app.go b/internal/app/app.go index 37066fc..4fabd0d 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -179,9 +179,10 @@ func runBuildModule(ctx context) error { } type buildHAKOptions struct { - filteredHAKs []string + filteredHAKs []string filteredArchives []string - planOnly bool + sourceManifest string + planOnly bool } func runBuildHAKs(ctx context) error { @@ -207,8 +208,9 @@ func runBuildHAKs(ctx context) error { result, err = pipeline.PlanHAKsWithProgress(p, progress) } else { result, err = pipeline.BuildHAKsWithOptions(p, pipeline.BuildHAKOptions{ - Progress: progress, - ArchiveNames: opts.filteredArchives, + Progress: progress, + ArchiveNames: opts.filteredArchives, + SourceManifestPath: opts.sourceManifest, }) } if err != nil { @@ -234,7 +236,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) { arg := args[index] switch arg { case "-h", "--help": - return opts, errors.New("usage: build-haks [--hak ...] [--archive ...] [--plan-only]") + return opts, errors.New("usage: build-haks [--hak ...] [--archive ...] [--source-manifest ] [--plan-only]") case "--hak": index++ if index >= len(args) { @@ -249,6 +251,12 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) { opts.filteredArchives = append(opts.filteredArchives, args[index]) case "--plan-only": opts.planOnly = true + case "--source-manifest": + index++ + if index >= len(args) { + return opts, errors.New("--source-manifest requires a value") + } + opts.sourceManifest = args[index] default: if value, ok := parseInlineFlagValue(arg, "--hak"); ok { opts.filteredHAKs = append(opts.filteredHAKs, value) @@ -258,6 +266,10 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) { opts.filteredArchives = append(opts.filteredArchives, value) continue } + if value, ok := parseInlineFlagValue(arg, "--source-manifest"); ok { + opts.sourceManifest = value + continue + } return opts, fmt.Errorf("unknown build-haks argument %q", arg) } } diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index 59525b6..257438b 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -74,8 +74,9 @@ type hakChunk struct { type ProgressFunc func(string) type BuildHAKOptions struct { - Progress ProgressFunc - ArchiveNames []string + Progress ProgressFunc + ArchiveNames []string + SourceManifestPath string } func Build(p *project.Project) (BuildResult, error) { @@ -158,7 +159,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu } func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) { - return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames) + return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath) } func PlanHAKs(p *project.Project) (BuildResult, error) { @@ -170,14 +171,14 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul } func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { - return planOrBuildHAKs(p, progress, true, nil) + return planOrBuildHAKs(p, progress, true, nil, "") } func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { - return planOrBuildHAKs(p, progress, false, nil) + return planOrBuildHAKs(p, progress, false, nil, "") } -func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string) (BuildResult, error) { +func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string) (BuildResult, error) { preserveExistingHAKs := envBool("SOW_BUILD_HAKS_KEEP_EXISTING") progressf(progress, "Validating project...") @@ -186,7 +187,11 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo } progressf(progress, "Collecting asset resources...") - assetResources, err := collectAssetResources(p, false) + allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(sourceManifestPath) + if err != nil { + return BuildResult{}, err + } + assetResources, err := collectAssetResources(p, false, allowedAssets) if err != nil { return BuildResult{}, err } @@ -201,7 +206,12 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo } progressf(progress, "Planning HAK chunks...") - chunks, err := planHAKChunks(p, assetResources) + var chunks []hakChunk + if sourceManifest != nil { + chunks, err = chunksFromManifest(assetResources, sourceManifest.HAKs) + } else { + chunks, err = planHAKChunks(p, assetResources) + } if err != nil { return BuildResult{}, err } @@ -216,9 +226,14 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo } } progressf(progress, "Resolving module HAK order...") - moduleHakOrder, err := resolveModuleHAKOrder(p, chunks) - if err != nil { - return BuildResult{}, err + moduleHakOrder := []string{} + if sourceManifest != nil { + moduleHakOrder = append(moduleHakOrder, sourceManifest.ModuleHAKs...) + } else { + moduleHakOrder, err = resolveModuleHAKOrder(p, chunks) + if err != nil { + return BuildResult{}, err + } } manifest := BuildManifest{ @@ -606,7 +621,7 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) { } } -func collectAssetResources(p *project.Project, requireContent bool) ([]assetResource, error) { +func collectAssetResources(p *project.Project, requireContent bool, allowed map[string]struct{}) ([]assetResource, error) { var hakResources []assetResource assetChangedAt, err := collectGitAssetChangeTimes(p) if err != nil { @@ -623,6 +638,11 @@ func collectAssetResources(p *project.Project, requireContent bool) ([]assetReso assetsRelPath = filepath.ToSlash(assetsRelPath) for _, rel := range p.Inventory.AssetFiles { + if len(allowed) > 0 { + if _, ok := allowed[filepath.ToSlash(rel)]; !ok { + continue + } + } abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)) resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent) if err != nil { @@ -652,6 +672,59 @@ func collectAssetResources(p *project.Project, requireContent bool) ([]assetReso return hakResources, nil } +func loadSourceManifestAssetSet(path string) (map[string]struct{}, *BuildManifest, error) { + if strings.TrimSpace(path) == "" { + return nil, nil, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, nil, fmt.Errorf("read source manifest %s: %w", path, err) + } + var manifest BuildManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, nil, fmt.Errorf("parse source manifest %s: %w", path, err) + } + allowed := make(map[string]struct{}) + for _, hak := range manifest.HAKs { + for _, rel := range hak.Assets { + allowed[filepath.ToSlash(rel)] = struct{}{} + } + } + return allowed, &manifest, nil +} + +func chunksFromManifest(assets []assetResource, entries []BuildManifestHAK) ([]hakChunk, error) { + assetByRel := make(map[string]assetResource, len(assets)) + for _, asset := range assets { + assetByRel[filepath.ToSlash(asset.Rel)] = asset + } + + chunks := make([]hakChunk, 0, len(entries)) + for index, entry := range entries { + chunkAssets := make([]assetResource, 0, len(entry.Assets)) + for _, rel := range entry.Assets { + asset, ok := assetByRel[filepath.ToSlash(rel)] + if !ok { + return nil, fmt.Errorf("source manifest references missing asset %s in archive %s", rel, entry.Name) + } + chunkAssets = append(chunkAssets, asset) + } + chunks = append(chunks, hakChunk{ + Config: project.HAKConfig{ + Name: entry.Group, + Priority: entry.Priority, + MaxBytes: entry.MaxBytes, + Optional: entry.Optional, + }, + Index: index, + Name: entry.Name, + Assets: chunkAssets, + Size: erf.ArchiveSize(resourceSlice(chunkAssets)), + }) + } + return chunks, nil +} + func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) { result := make(map[string]lfsAssetInfo) for _, rel := range p.Inventory.AssetFiles { @@ -1260,7 +1333,7 @@ func plannedModuleHAKOrder(p *project.Project) ([]string, error) { return append([]string(nil), manifest.ModuleHAKs...), nil } - assetResources, err := collectAssetResources(p, false) + assetResources, err := collectAssetResources(p, false, nil) if err != nil { return nil, err } diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index aa0ea9e..238aa16 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -1831,7 +1831,7 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) { t.Fatalf("scan: %v", err) } - assets, err := collectAssetResources(p, false) + assets, err := collectAssetResources(p, false, nil) if err != nil { t.Fatalf("collect asset resources: %v", err) }