build-haks now accepts --source-manifest <path>
When that flag is used, sow-tools: - collects only the assets referenced by that manifest subset - skips full-project chunk planning for the per-archive run - reconstructs just the requested archive chunks from the manifest
This commit is contained in:
+86
-13
@@ -74,8 +74,9 @@ type hakChunk struct {
|
||||
type ProgressFunc func(string)
|
||||
|
||||
type BuildHAKOptions struct {
|
||||
Progress ProgressFunc
|
||||
ArchiveNames []string
|
||||
Progress ProgressFunc
|
||||
ArchiveNames []string
|
||||
SourceManifestPath string
|
||||
}
|
||||
|
||||
func Build(p *project.Project) (BuildResult, error) {
|
||||
@@ -158,7 +159,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu
|
||||
}
|
||||
|
||||
func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
|
||||
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames)
|
||||
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath)
|
||||
}
|
||||
|
||||
func PlanHAKs(p *project.Project) (BuildResult, error) {
|
||||
@@ -170,14 +171,14 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul
|
||||
}
|
||||
|
||||
func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
|
||||
return planOrBuildHAKs(p, progress, true, nil)
|
||||
return planOrBuildHAKs(p, progress, true, nil, "")
|
||||
}
|
||||
|
||||
func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
|
||||
return planOrBuildHAKs(p, progress, false, nil)
|
||||
return planOrBuildHAKs(p, progress, false, nil, "")
|
||||
}
|
||||
|
||||
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string) (BuildResult, error) {
|
||||
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string) (BuildResult, error) {
|
||||
preserveExistingHAKs := envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
|
||||
|
||||
progressf(progress, "Validating project...")
|
||||
@@ -186,7 +187,11 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
||||
}
|
||||
|
||||
progressf(progress, "Collecting asset resources...")
|
||||
assetResources, err := collectAssetResources(p, false)
|
||||
allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(sourceManifestPath)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
assetResources, err := collectAssetResources(p, false, allowedAssets)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
@@ -201,7 +206,12 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
||||
}
|
||||
|
||||
progressf(progress, "Planning HAK chunks...")
|
||||
chunks, err := planHAKChunks(p, assetResources)
|
||||
var chunks []hakChunk
|
||||
if sourceManifest != nil {
|
||||
chunks, err = chunksFromManifest(assetResources, sourceManifest.HAKs)
|
||||
} else {
|
||||
chunks, err = planHAKChunks(p, assetResources)
|
||||
}
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
@@ -216,9 +226,14 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
||||
}
|
||||
}
|
||||
progressf(progress, "Resolving module HAK order...")
|
||||
moduleHakOrder, err := resolveModuleHAKOrder(p, chunks)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
moduleHakOrder := []string{}
|
||||
if sourceManifest != nil {
|
||||
moduleHakOrder = append(moduleHakOrder, sourceManifest.ModuleHAKs...)
|
||||
} else {
|
||||
moduleHakOrder, err = resolveModuleHAKOrder(p, chunks)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
}
|
||||
|
||||
manifest := BuildManifest{
|
||||
@@ -606,7 +621,7 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) {
|
||||
}
|
||||
}
|
||||
|
||||
func collectAssetResources(p *project.Project, requireContent bool) ([]assetResource, error) {
|
||||
func collectAssetResources(p *project.Project, requireContent bool, allowed map[string]struct{}) ([]assetResource, error) {
|
||||
var hakResources []assetResource
|
||||
assetChangedAt, err := collectGitAssetChangeTimes(p)
|
||||
if err != nil {
|
||||
@@ -623,6 +638,11 @@ func collectAssetResources(p *project.Project, requireContent bool) ([]assetReso
|
||||
assetsRelPath = filepath.ToSlash(assetsRelPath)
|
||||
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
if len(allowed) > 0 {
|
||||
if _, ok := allowed[filepath.ToSlash(rel)]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
|
||||
resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent)
|
||||
if err != nil {
|
||||
@@ -652,6 +672,59 @@ func collectAssetResources(p *project.Project, requireContent bool) ([]assetReso
|
||||
return hakResources, nil
|
||||
}
|
||||
|
||||
func loadSourceManifestAssetSet(path string) (map[string]struct{}, *BuildManifest, error) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return nil, nil, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("read source manifest %s: %w", path, err)
|
||||
}
|
||||
var manifest BuildManifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, nil, fmt.Errorf("parse source manifest %s: %w", path, err)
|
||||
}
|
||||
allowed := make(map[string]struct{})
|
||||
for _, hak := range manifest.HAKs {
|
||||
for _, rel := range hak.Assets {
|
||||
allowed[filepath.ToSlash(rel)] = struct{}{}
|
||||
}
|
||||
}
|
||||
return allowed, &manifest, nil
|
||||
}
|
||||
|
||||
func chunksFromManifest(assets []assetResource, entries []BuildManifestHAK) ([]hakChunk, error) {
|
||||
assetByRel := make(map[string]assetResource, len(assets))
|
||||
for _, asset := range assets {
|
||||
assetByRel[filepath.ToSlash(asset.Rel)] = asset
|
||||
}
|
||||
|
||||
chunks := make([]hakChunk, 0, len(entries))
|
||||
for index, entry := range entries {
|
||||
chunkAssets := make([]assetResource, 0, len(entry.Assets))
|
||||
for _, rel := range entry.Assets {
|
||||
asset, ok := assetByRel[filepath.ToSlash(rel)]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("source manifest references missing asset %s in archive %s", rel, entry.Name)
|
||||
}
|
||||
chunkAssets = append(chunkAssets, asset)
|
||||
}
|
||||
chunks = append(chunks, hakChunk{
|
||||
Config: project.HAKConfig{
|
||||
Name: entry.Group,
|
||||
Priority: entry.Priority,
|
||||
MaxBytes: entry.MaxBytes,
|
||||
Optional: entry.Optional,
|
||||
},
|
||||
Index: index,
|
||||
Name: entry.Name,
|
||||
Assets: chunkAssets,
|
||||
Size: erf.ArchiveSize(resourceSlice(chunkAssets)),
|
||||
})
|
||||
}
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) {
|
||||
result := make(map[string]lfsAssetInfo)
|
||||
for _, rel := range p.Inventory.AssetFiles {
|
||||
@@ -1260,7 +1333,7 @@ func plannedModuleHAKOrder(p *project.Project) ([]string, error) {
|
||||
return append([]string(nil), manifest.ModuleHAKs...), nil
|
||||
}
|
||||
|
||||
assetResources, err := collectAssetResources(p, false)
|
||||
assetResources, err := collectAssetResources(p, false, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1831,7 +1831,7 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
assets, err := collectAssetResources(p, false)
|
||||
assets, err := collectAssetResources(p, false, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("collect asset resources: %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user