build-haks now accepts --source-manifest <path>
When that flag is used, sow-tools: - collects only the assets referenced by that manifest subset - skips full-project chunk planning for the per-archive run - reconstructs just the requested archive chunks from the manifest
This commit is contained in:
+13
-1
@@ -181,6 +181,7 @@ func runBuildModule(ctx context) error {
|
|||||||
type buildHAKOptions struct {
|
type buildHAKOptions struct {
|
||||||
filteredHAKs []string
|
filteredHAKs []string
|
||||||
filteredArchives []string
|
filteredArchives []string
|
||||||
|
sourceManifest string
|
||||||
planOnly bool
|
planOnly bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,6 +210,7 @@ func runBuildHAKs(ctx context) error {
|
|||||||
result, err = pipeline.BuildHAKsWithOptions(p, pipeline.BuildHAKOptions{
|
result, err = pipeline.BuildHAKsWithOptions(p, pipeline.BuildHAKOptions{
|
||||||
Progress: progress,
|
Progress: progress,
|
||||||
ArchiveNames: opts.filteredArchives,
|
ArchiveNames: opts.filteredArchives,
|
||||||
|
SourceManifestPath: opts.sourceManifest,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -234,7 +236,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
|||||||
arg := args[index]
|
arg := args[index]
|
||||||
switch arg {
|
switch arg {
|
||||||
case "-h", "--help":
|
case "-h", "--help":
|
||||||
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--plan-only]")
|
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--plan-only]")
|
||||||
case "--hak":
|
case "--hak":
|
||||||
index++
|
index++
|
||||||
if index >= len(args) {
|
if index >= len(args) {
|
||||||
@@ -249,6 +251,12 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
|||||||
opts.filteredArchives = append(opts.filteredArchives, args[index])
|
opts.filteredArchives = append(opts.filteredArchives, args[index])
|
||||||
case "--plan-only":
|
case "--plan-only":
|
||||||
opts.planOnly = true
|
opts.planOnly = true
|
||||||
|
case "--source-manifest":
|
||||||
|
index++
|
||||||
|
if index >= len(args) {
|
||||||
|
return opts, errors.New("--source-manifest requires a value")
|
||||||
|
}
|
||||||
|
opts.sourceManifest = args[index]
|
||||||
default:
|
default:
|
||||||
if value, ok := parseInlineFlagValue(arg, "--hak"); ok {
|
if value, ok := parseInlineFlagValue(arg, "--hak"); ok {
|
||||||
opts.filteredHAKs = append(opts.filteredHAKs, value)
|
opts.filteredHAKs = append(opts.filteredHAKs, value)
|
||||||
@@ -258,6 +266,10 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
|||||||
opts.filteredArchives = append(opts.filteredArchives, value)
|
opts.filteredArchives = append(opts.filteredArchives, value)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if value, ok := parseInlineFlagValue(arg, "--source-manifest"); ok {
|
||||||
|
opts.sourceManifest = value
|
||||||
|
continue
|
||||||
|
}
|
||||||
return opts, fmt.Errorf("unknown build-haks argument %q", arg)
|
return opts, fmt.Errorf("unknown build-haks argument %q", arg)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -76,6 +76,7 @@ type ProgressFunc func(string)
|
|||||||
type BuildHAKOptions struct {
|
type BuildHAKOptions struct {
|
||||||
Progress ProgressFunc
|
Progress ProgressFunc
|
||||||
ArchiveNames []string
|
ArchiveNames []string
|
||||||
|
SourceManifestPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
func Build(p *project.Project) (BuildResult, error) {
|
func Build(p *project.Project) (BuildResult, error) {
|
||||||
@@ -158,7 +159,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu
|
|||||||
}
|
}
|
||||||
|
|
||||||
func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
|
func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
|
||||||
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames)
|
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
func PlanHAKs(p *project.Project) (BuildResult, error) {
|
func PlanHAKs(p *project.Project) (BuildResult, error) {
|
||||||
@@ -170,14 +171,14 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul
|
|||||||
}
|
}
|
||||||
|
|
||||||
func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
|
func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
|
||||||
return planOrBuildHAKs(p, progress, true, nil)
|
return planOrBuildHAKs(p, progress, true, nil, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
|
func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
|
||||||
return planOrBuildHAKs(p, progress, false, nil)
|
return planOrBuildHAKs(p, progress, false, nil, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string) (BuildResult, error) {
|
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string) (BuildResult, error) {
|
||||||
preserveExistingHAKs := envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
|
preserveExistingHAKs := envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
|
||||||
|
|
||||||
progressf(progress, "Validating project...")
|
progressf(progress, "Validating project...")
|
||||||
@@ -186,7 +187,11 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
|||||||
}
|
}
|
||||||
|
|
||||||
progressf(progress, "Collecting asset resources...")
|
progressf(progress, "Collecting asset resources...")
|
||||||
assetResources, err := collectAssetResources(p, false)
|
allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(sourceManifestPath)
|
||||||
|
if err != nil {
|
||||||
|
return BuildResult{}, err
|
||||||
|
}
|
||||||
|
assetResources, err := collectAssetResources(p, false, allowedAssets)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return BuildResult{}, err
|
return BuildResult{}, err
|
||||||
}
|
}
|
||||||
@@ -201,7 +206,12 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
|||||||
}
|
}
|
||||||
|
|
||||||
progressf(progress, "Planning HAK chunks...")
|
progressf(progress, "Planning HAK chunks...")
|
||||||
chunks, err := planHAKChunks(p, assetResources)
|
var chunks []hakChunk
|
||||||
|
if sourceManifest != nil {
|
||||||
|
chunks, err = chunksFromManifest(assetResources, sourceManifest.HAKs)
|
||||||
|
} else {
|
||||||
|
chunks, err = planHAKChunks(p, assetResources)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return BuildResult{}, err
|
return BuildResult{}, err
|
||||||
}
|
}
|
||||||
@@ -216,10 +226,15 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
progressf(progress, "Resolving module HAK order...")
|
progressf(progress, "Resolving module HAK order...")
|
||||||
moduleHakOrder, err := resolveModuleHAKOrder(p, chunks)
|
moduleHakOrder := []string{}
|
||||||
|
if sourceManifest != nil {
|
||||||
|
moduleHakOrder = append(moduleHakOrder, sourceManifest.ModuleHAKs...)
|
||||||
|
} else {
|
||||||
|
moduleHakOrder, err = resolveModuleHAKOrder(p, chunks)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return BuildResult{}, err
|
return BuildResult{}, err
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
manifest := BuildManifest{
|
manifest := BuildManifest{
|
||||||
ModuleHAKs: moduleHakOrder,
|
ModuleHAKs: moduleHakOrder,
|
||||||
@@ -606,7 +621,7 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectAssetResources(p *project.Project, requireContent bool) ([]assetResource, error) {
|
func collectAssetResources(p *project.Project, requireContent bool, allowed map[string]struct{}) ([]assetResource, error) {
|
||||||
var hakResources []assetResource
|
var hakResources []assetResource
|
||||||
assetChangedAt, err := collectGitAssetChangeTimes(p)
|
assetChangedAt, err := collectGitAssetChangeTimes(p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -623,6 +638,11 @@ func collectAssetResources(p *project.Project, requireContent bool) ([]assetReso
|
|||||||
assetsRelPath = filepath.ToSlash(assetsRelPath)
|
assetsRelPath = filepath.ToSlash(assetsRelPath)
|
||||||
|
|
||||||
for _, rel := range p.Inventory.AssetFiles {
|
for _, rel := range p.Inventory.AssetFiles {
|
||||||
|
if len(allowed) > 0 {
|
||||||
|
if _, ok := allowed[filepath.ToSlash(rel)]; !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
|
abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
|
||||||
resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent)
|
resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -652,6 +672,59 @@ func collectAssetResources(p *project.Project, requireContent bool) ([]assetReso
|
|||||||
return hakResources, nil
|
return hakResources, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func loadSourceManifestAssetSet(path string) (map[string]struct{}, *BuildManifest, error) {
|
||||||
|
if strings.TrimSpace(path) == "" {
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("read source manifest %s: %w", path, err)
|
||||||
|
}
|
||||||
|
var manifest BuildManifest
|
||||||
|
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("parse source manifest %s: %w", path, err)
|
||||||
|
}
|
||||||
|
allowed := make(map[string]struct{})
|
||||||
|
for _, hak := range manifest.HAKs {
|
||||||
|
for _, rel := range hak.Assets {
|
||||||
|
allowed[filepath.ToSlash(rel)] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return allowed, &manifest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func chunksFromManifest(assets []assetResource, entries []BuildManifestHAK) ([]hakChunk, error) {
|
||||||
|
assetByRel := make(map[string]assetResource, len(assets))
|
||||||
|
for _, asset := range assets {
|
||||||
|
assetByRel[filepath.ToSlash(asset.Rel)] = asset
|
||||||
|
}
|
||||||
|
|
||||||
|
chunks := make([]hakChunk, 0, len(entries))
|
||||||
|
for index, entry := range entries {
|
||||||
|
chunkAssets := make([]assetResource, 0, len(entry.Assets))
|
||||||
|
for _, rel := range entry.Assets {
|
||||||
|
asset, ok := assetByRel[filepath.ToSlash(rel)]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("source manifest references missing asset %s in archive %s", rel, entry.Name)
|
||||||
|
}
|
||||||
|
chunkAssets = append(chunkAssets, asset)
|
||||||
|
}
|
||||||
|
chunks = append(chunks, hakChunk{
|
||||||
|
Config: project.HAKConfig{
|
||||||
|
Name: entry.Group,
|
||||||
|
Priority: entry.Priority,
|
||||||
|
MaxBytes: entry.MaxBytes,
|
||||||
|
Optional: entry.Optional,
|
||||||
|
},
|
||||||
|
Index: index,
|
||||||
|
Name: entry.Name,
|
||||||
|
Assets: chunkAssets,
|
||||||
|
Size: erf.ArchiveSize(resourceSlice(chunkAssets)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return chunks, nil
|
||||||
|
}
|
||||||
|
|
||||||
func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) {
|
func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) {
|
||||||
result := make(map[string]lfsAssetInfo)
|
result := make(map[string]lfsAssetInfo)
|
||||||
for _, rel := range p.Inventory.AssetFiles {
|
for _, rel := range p.Inventory.AssetFiles {
|
||||||
@@ -1260,7 +1333,7 @@ func plannedModuleHAKOrder(p *project.Project) ([]string, error) {
|
|||||||
return append([]string(nil), manifest.ModuleHAKs...), nil
|
return append([]string(nil), manifest.ModuleHAKs...), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
assetResources, err := collectAssetResources(p, false)
|
assetResources, err := collectAssetResources(p, false, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1831,7 +1831,7 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) {
|
|||||||
t.Fatalf("scan: %v", err)
|
t.Fatalf("scan: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
assets, err := collectAssetResources(p, false)
|
assets, err := collectAssetResources(p, false, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("collect asset resources: %v", err)
|
t.Fatalf("collect asset resources: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user