build-haks now accepts --source-manifest <path>

When that flag is used, sow-tools:
- collects only the assets referenced by that manifest subset
- skips full-project chunk planning for the per-archive run
- reconstructs just the requested archive chunks from the manifest
This commit is contained in:
2026-04-14 12:12:42 +02:00
parent 9577bc853b
commit ab18baf067
3 changed files with 104 additions and 19 deletions
+13 -1
View File
@@ -181,6 +181,7 @@ func runBuildModule(ctx context) error {
type buildHAKOptions struct { type buildHAKOptions struct {
filteredHAKs []string filteredHAKs []string
filteredArchives []string filteredArchives []string
sourceManifest string
planOnly bool planOnly bool
} }
@@ -209,6 +210,7 @@ func runBuildHAKs(ctx context) error {
result, err = pipeline.BuildHAKsWithOptions(p, pipeline.BuildHAKOptions{ result, err = pipeline.BuildHAKsWithOptions(p, pipeline.BuildHAKOptions{
Progress: progress, Progress: progress,
ArchiveNames: opts.filteredArchives, ArchiveNames: opts.filteredArchives,
SourceManifestPath: opts.sourceManifest,
}) })
} }
if err != nil { if err != nil {
@@ -234,7 +236,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
arg := args[index] arg := args[index]
switch arg { switch arg {
case "-h", "--help": case "-h", "--help":
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--plan-only]") return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--plan-only]")
case "--hak": case "--hak":
index++ index++
if index >= len(args) { if index >= len(args) {
@@ -249,6 +251,12 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
opts.filteredArchives = append(opts.filteredArchives, args[index]) opts.filteredArchives = append(opts.filteredArchives, args[index])
case "--plan-only": case "--plan-only":
opts.planOnly = true opts.planOnly = true
case "--source-manifest":
index++
if index >= len(args) {
return opts, errors.New("--source-manifest requires a value")
}
opts.sourceManifest = args[index]
default: default:
if value, ok := parseInlineFlagValue(arg, "--hak"); ok { if value, ok := parseInlineFlagValue(arg, "--hak"); ok {
opts.filteredHAKs = append(opts.filteredHAKs, value) opts.filteredHAKs = append(opts.filteredHAKs, value)
@@ -258,6 +266,10 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
opts.filteredArchives = append(opts.filteredArchives, value) opts.filteredArchives = append(opts.filteredArchives, value)
continue continue
} }
if value, ok := parseInlineFlagValue(arg, "--source-manifest"); ok {
opts.sourceManifest = value
continue
}
return opts, fmt.Errorf("unknown build-haks argument %q", arg) return opts, fmt.Errorf("unknown build-haks argument %q", arg)
} }
} }
+82 -9
View File
@@ -76,6 +76,7 @@ type ProgressFunc func(string)
type BuildHAKOptions struct { type BuildHAKOptions struct {
Progress ProgressFunc Progress ProgressFunc
ArchiveNames []string ArchiveNames []string
SourceManifestPath string
} }
func Build(p *project.Project) (BuildResult, error) { func Build(p *project.Project) (BuildResult, error) {
@@ -158,7 +159,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu
} }
func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) { func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames) return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath)
} }
func PlanHAKs(p *project.Project) (BuildResult, error) { func PlanHAKs(p *project.Project) (BuildResult, error) {
@@ -170,14 +171,14 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul
} }
func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
return planOrBuildHAKs(p, progress, true, nil) return planOrBuildHAKs(p, progress, true, nil, "")
} }
func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
return planOrBuildHAKs(p, progress, false, nil) return planOrBuildHAKs(p, progress, false, nil, "")
} }
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string) (BuildResult, error) { func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string) (BuildResult, error) {
preserveExistingHAKs := envBool("SOW_BUILD_HAKS_KEEP_EXISTING") preserveExistingHAKs := envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
progressf(progress, "Validating project...") progressf(progress, "Validating project...")
@@ -186,7 +187,11 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
} }
progressf(progress, "Collecting asset resources...") progressf(progress, "Collecting asset resources...")
assetResources, err := collectAssetResources(p, false) allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(sourceManifestPath)
if err != nil {
return BuildResult{}, err
}
assetResources, err := collectAssetResources(p, false, allowedAssets)
if err != nil { if err != nil {
return BuildResult{}, err return BuildResult{}, err
} }
@@ -201,7 +206,12 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
} }
progressf(progress, "Planning HAK chunks...") progressf(progress, "Planning HAK chunks...")
chunks, err := planHAKChunks(p, assetResources) var chunks []hakChunk
if sourceManifest != nil {
chunks, err = chunksFromManifest(assetResources, sourceManifest.HAKs)
} else {
chunks, err = planHAKChunks(p, assetResources)
}
if err != nil { if err != nil {
return BuildResult{}, err return BuildResult{}, err
} }
@@ -216,10 +226,15 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
} }
} }
progressf(progress, "Resolving module HAK order...") progressf(progress, "Resolving module HAK order...")
moduleHakOrder, err := resolveModuleHAKOrder(p, chunks) moduleHakOrder := []string{}
if sourceManifest != nil {
moduleHakOrder = append(moduleHakOrder, sourceManifest.ModuleHAKs...)
} else {
moduleHakOrder, err = resolveModuleHAKOrder(p, chunks)
if err != nil { if err != nil {
return BuildResult{}, err return BuildResult{}, err
} }
}
manifest := BuildManifest{ manifest := BuildManifest{
ModuleHAKs: moduleHakOrder, ModuleHAKs: moduleHakOrder,
@@ -606,7 +621,7 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) {
} }
} }
func collectAssetResources(p *project.Project, requireContent bool) ([]assetResource, error) { func collectAssetResources(p *project.Project, requireContent bool, allowed map[string]struct{}) ([]assetResource, error) {
var hakResources []assetResource var hakResources []assetResource
assetChangedAt, err := collectGitAssetChangeTimes(p) assetChangedAt, err := collectGitAssetChangeTimes(p)
if err != nil { if err != nil {
@@ -623,6 +638,11 @@ func collectAssetResources(p *project.Project, requireContent bool) ([]assetReso
assetsRelPath = filepath.ToSlash(assetsRelPath) assetsRelPath = filepath.ToSlash(assetsRelPath)
for _, rel := range p.Inventory.AssetFiles { for _, rel := range p.Inventory.AssetFiles {
if len(allowed) > 0 {
if _, ok := allowed[filepath.ToSlash(rel)]; !ok {
continue
}
}
abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)) abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent) resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent)
if err != nil { if err != nil {
@@ -652,6 +672,59 @@ func collectAssetResources(p *project.Project, requireContent bool) ([]assetReso
return hakResources, nil return hakResources, nil
} }
func loadSourceManifestAssetSet(path string) (map[string]struct{}, *BuildManifest, error) {
if strings.TrimSpace(path) == "" {
return nil, nil, nil
}
raw, err := os.ReadFile(path)
if err != nil {
return nil, nil, fmt.Errorf("read source manifest %s: %w", path, err)
}
var manifest BuildManifest
if err := json.Unmarshal(raw, &manifest); err != nil {
return nil, nil, fmt.Errorf("parse source manifest %s: %w", path, err)
}
allowed := make(map[string]struct{})
for _, hak := range manifest.HAKs {
for _, rel := range hak.Assets {
allowed[filepath.ToSlash(rel)] = struct{}{}
}
}
return allowed, &manifest, nil
}
func chunksFromManifest(assets []assetResource, entries []BuildManifestHAK) ([]hakChunk, error) {
assetByRel := make(map[string]assetResource, len(assets))
for _, asset := range assets {
assetByRel[filepath.ToSlash(asset.Rel)] = asset
}
chunks := make([]hakChunk, 0, len(entries))
for index, entry := range entries {
chunkAssets := make([]assetResource, 0, len(entry.Assets))
for _, rel := range entry.Assets {
asset, ok := assetByRel[filepath.ToSlash(rel)]
if !ok {
return nil, fmt.Errorf("source manifest references missing asset %s in archive %s", rel, entry.Name)
}
chunkAssets = append(chunkAssets, asset)
}
chunks = append(chunks, hakChunk{
Config: project.HAKConfig{
Name: entry.Group,
Priority: entry.Priority,
MaxBytes: entry.MaxBytes,
Optional: entry.Optional,
},
Index: index,
Name: entry.Name,
Assets: chunkAssets,
Size: erf.ArchiveSize(resourceSlice(chunkAssets)),
})
}
return chunks, nil
}
func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) { func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) {
result := make(map[string]lfsAssetInfo) result := make(map[string]lfsAssetInfo)
for _, rel := range p.Inventory.AssetFiles { for _, rel := range p.Inventory.AssetFiles {
@@ -1260,7 +1333,7 @@ func plannedModuleHAKOrder(p *project.Project) ([]string, error) {
return append([]string(nil), manifest.ModuleHAKs...), nil return append([]string(nil), manifest.ModuleHAKs...), nil
} }
assetResources, err := collectAssetResources(p, false) assetResources, err := collectAssetResources(p, false, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+1 -1
View File
@@ -1831,7 +1831,7 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) {
t.Fatalf("scan: %v", err) t.Fatalf("scan: %v", err)
} }
assets, err := collectAssetResources(p, false) assets, err := collectAssetResources(p, false, nil)
if err != nil { if err != nil {
t.Fatalf("collect asset resources: %v", err) t.Fatalf("collect asset resources: %v", err)
} }