ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

build-haks now accepts --source-manifest <path>

Browse Source 
When that flag is used, sow-tools:
- collects only the assets referenced by that manifest subset
- skips full-project chunk planning for the per-archive run
- reconstructs just the requested archive chunks from the manifest
This commit is contained in:
archvillainette
2026-04-14 12:12:42 +02:00
parent 9577bc853b
commit ab18baf067
3 changed files with 104 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/app/app.go
+17 -5
View File
@@ -179,9 +179,10 @@ func runBuildModule(ctx context) error {
}
type buildHAKOptions struct {
filteredHAKs []string
filteredHAKs []string
filteredArchives []string
planOnly bool
sourceManifest string
planOnly bool
}
func runBuildHAKs(ctx context) error {
@@ -207,8 +208,9 @@ func runBuildHAKs(ctx context) error {
result, err = pipeline.PlanHAKsWithProgress(p, progress)
} else {
result, err = pipeline.BuildHAKsWithOptions(p, pipeline.BuildHAKOptions{
Progress: progress,
ArchiveNames: opts.filteredArchives,
Progress: progress,
ArchiveNames: opts.filteredArchives,
SourceManifestPath: opts.sourceManifest,
})
}
if err != nil {
@@ -234,7 +236,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
arg := args[index]
switch arg {
case "-h", "--help":
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--plan-only]")
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--plan-only]")
case "--hak":
index++
if index >= len(args) {
@@ -249,6 +251,12 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
opts.filteredArchives = append(opts.filteredArchives, args[index])
case "--plan-only":
opts.planOnly = true
case "--source-manifest":
index++
if index >= len(args) {
return opts, errors.New("--source-manifest requires a value")
}
opts.sourceManifest = args[index]
default:
if value, ok := parseInlineFlagValue(arg, "--hak"); ok {
opts.filteredHAKs = append(opts.filteredHAKs, value)
@@ -258,6 +266,10 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
opts.filteredArchives = append(opts.filteredArchives, value)
continue
}
if value, ok := parseInlineFlagValue(arg, "--source-manifest"); ok {
opts.sourceManifest = value
continue
}
return opts, fmt.Errorf("unknown build-haks argument %q", arg)
}
}
internal/pipeline/build.go
+86 -13
View File
@@ -74,8 +74,9 @@ type hakChunk struct {
type ProgressFunc func(string)
type BuildHAKOptions struct {
Progress ProgressFunc
ArchiveNames []string
Progress ProgressFunc
ArchiveNames []string
SourceManifestPath string
}
func Build(p *project.Project) (BuildResult, error) {
@@ -158,7 +159,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu
}
func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames)
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath)
}
func PlanHAKs(p *project.Project) (BuildResult, error) {
@@ -170,14 +171,14 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul
}
func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
return planOrBuildHAKs(p, progress, true, nil)
return planOrBuildHAKs(p, progress, true, nil, "")
}
func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
return planOrBuildHAKs(p, progress, false, nil)
return planOrBuildHAKs(p, progress, false, nil, "")
}
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string) (BuildResult, error) {
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string) (BuildResult, error) {
preserveExistingHAKs := envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
progressf(progress, "Validating project...")
@@ -186,7 +187,11 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
}
progressf(progress, "Collecting asset resources...")
assetResources, err := collectAssetResources(p, false)
allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(sourceManifestPath)
if err != nil {
return BuildResult{}, err
}
assetResources, err := collectAssetResources(p, false, allowedAssets)
if err != nil {
return BuildResult{}, err
}
@@ -201,7 +206,12 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
}
progressf(progress, "Planning HAK chunks...")
chunks, err := planHAKChunks(p, assetResources)
var chunks []hakChunk
if sourceManifest != nil {
chunks, err = chunksFromManifest(assetResources, sourceManifest.HAKs)
} else {
chunks, err = planHAKChunks(p, assetResources)
}
if err != nil {
return BuildResult{}, err
}
@@ -216,9 +226,14 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
}
}
progressf(progress, "Resolving module HAK order...")
moduleHakOrder, err := resolveModuleHAKOrder(p, chunks)
if err != nil {
return BuildResult{}, err
moduleHakOrder := []string{}
if sourceManifest != nil {
moduleHakOrder = append(moduleHakOrder, sourceManifest.ModuleHAKs...)
} else {
moduleHakOrder, err = resolveModuleHAKOrder(p, chunks)
if err != nil {
return BuildResult{}, err
}
}
manifest := BuildManifest{
@@ -606,7 +621,7 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) {
}
}
func collectAssetResources(p *project.Project, requireContent bool) ([]assetResource, error) {
func collectAssetResources(p *project.Project, requireContent bool, allowed map[string]struct{}) ([]assetResource, error) {
var hakResources []assetResource
assetChangedAt, err := collectGitAssetChangeTimes(p)
if err != nil {
@@ -623,6 +638,11 @@ func collectAssetResources(p *project.Project, requireContent bool) ([]assetReso
assetsRelPath = filepath.ToSlash(assetsRelPath)
for _, rel := range p.Inventory.AssetFiles {
if len(allowed) > 0 {
if _, ok := allowed[filepath.ToSlash(rel)]; !ok {
continue
}
}
abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent)
if err != nil {
@@ -652,6 +672,59 @@ func collectAssetResources(p *project.Project, requireContent bool) ([]assetReso
return hakResources, nil
}
func loadSourceManifestAssetSet(path string) (map[string]struct{}, *BuildManifest, error) {
if strings.TrimSpace(path) == "" {
return nil, nil, nil
}
raw, err := os.ReadFile(path)
if err != nil {
return nil, nil, fmt.Errorf("read source manifest %s: %w", path, err)
}
var manifest BuildManifest
if err := json.Unmarshal(raw, &manifest); err != nil {
return nil, nil, fmt.Errorf("parse source manifest %s: %w", path, err)
}
allowed := make(map[string]struct{})
for _, hak := range manifest.HAKs {
for _, rel := range hak.Assets {
allowed[filepath.ToSlash(rel)] = struct{}{}
}
}
return allowed, &manifest, nil
}
func chunksFromManifest(assets []assetResource, entries []BuildManifestHAK) ([]hakChunk, error) {
assetByRel := make(map[string]assetResource, len(assets))
for _, asset := range assets {
assetByRel[filepath.ToSlash(asset.Rel)] = asset
}
chunks := make([]hakChunk, 0, len(entries))
for index, entry := range entries {
chunkAssets := make([]assetResource, 0, len(entry.Assets))
for _, rel := range entry.Assets {
asset, ok := assetByRel[filepath.ToSlash(rel)]
if !ok {
return nil, fmt.Errorf("source manifest references missing asset %s in archive %s", rel, entry.Name)
}
chunkAssets = append(chunkAssets, asset)
}
chunks = append(chunks, hakChunk{
Config: project.HAKConfig{
Name: entry.Group,
Priority: entry.Priority,
MaxBytes: entry.MaxBytes,
Optional: entry.Optional,
},
Index: index,
Name: entry.Name,
Assets: chunkAssets,
Size: erf.ArchiveSize(resourceSlice(chunkAssets)),
})
}
return chunks, nil
}
func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) {
result := make(map[string]lfsAssetInfo)
for _, rel := range p.Inventory.AssetFiles {
@@ -1260,7 +1333,7 @@ func plannedModuleHAKOrder(p *project.Project) ([]string, error) {
return append([]string(nil), manifest.ModuleHAKs...), nil
}
assetResources, err := collectAssetResources(p, false)
assetResources, err := collectAssetResources(p, false, nil)
if err != nil {
return nil, err
}
internal/pipeline/pipeline_test.go
+1 -1
View File
@@ -1831,7 +1831,7 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) {
t.Fatalf("scan: %v", err)
}
assets, err := collectAssetResources(p, false)
assets, err := collectAssetResources(p, false, nil)
if err != nil {
t.Fatalf("collect asset resources: %v", err)
}