add content-addressed hak source mode (#7)

Reviewed-on: #7
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>

internal/pipeline/build.go

@@ -106,11 +106,12 @@ type hakChunk struct {
 type ProgressFunc func(string)
 
 type BuildHAKOptions struct {
-	Progress           ProgressFunc
-	ArchiveNames       []string
-	SourceManifestPath string
-	SkipMusic          bool
-	MusicDatasetIDs    []string
+	Progress             ProgressFunc
+	ArchiveNames         []string
+	SourceManifestPath   string
+	ContentAddressedRoot string
+	SkipMusic            bool
+	MusicDatasetIDs      []string
 }
 
 func Build(p *project.Project) (BuildResult, error) {
@@ -196,7 +197,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu
 }
 
 func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
-	return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs)
+	return planOrBuildHAKs(p, opts.Progress, true, opts)
 }
 
 func PlanHAKs(p *project.Project) (BuildResult, error) {
@@ -208,18 +209,18 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul
 }
 
 func PlanHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
-	return planOrBuildHAKs(p, opts.Progress, false, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs)
+	return planOrBuildHAKs(p, opts.Progress, false, opts)
 }
 
 func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
-	return planOrBuildHAKs(p, progress, true, nil, "", false, nil)
+	return planOrBuildHAKs(p, progress, true, BuildHAKOptions{})
 }
 
 func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
-	return planOrBuildHAKs(p, progress, false, nil, "", false, nil)
+	return planOrBuildHAKs(p, progress, false, BuildHAKOptions{})
 }
 
-func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string, skipMusic bool, musicDatasetIDs []string) (result BuildResult, err error) {
+func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, opts BuildHAKOptions) (result BuildResult, err error) {
 	preserveExistingHAKs := p.EffectiveConfig().Build.KeepExistingHAKs || envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
 
 	progressf(progress, "Validating project...")
@@ -227,15 +228,23 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
 		return BuildResult{}, err
 	}
 
+	directManifest, err := loadDirectSourceManifest(opts.SourceManifestPath)
+	if err != nil {
+		return BuildResult{}, err
+	}
+	if directManifest != nil {
+		return buildDirectHAKs(p, progress, writeArchives, directManifest, opts)
+	}
+
 	progressf(progress, "Collecting asset resources...")
-	allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(sourceManifestPath)
+	allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(opts.SourceManifestPath)
 	if err != nil {
 		return BuildResult{}, err
 	}
 	musicAssets := &preparedMusicAssets{SkipSourceRel: map[string]struct{}{}}
-	if !skipMusic {
+	if !opts.SkipMusic {
 		musicAssets, err = prepareMusicAssetsWithOptions(p, musicPrepareOptions{
-			datasetIDs: musicDatasetIDs,
+			datasetIDs: opts.MusicDatasetIDs,
 			write:      writeArchives,
 		})
 		if err != nil {
@@ -294,7 +303,7 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
 	if err != nil {
 		return BuildResult{}, err
 	}
-	chunks, err = filterHAKChunksByName(chunks, archiveNames)
+	chunks, err = filterHAKChunksByName(chunks, opts.ArchiveNames)
 	if err != nil {
 		return BuildResult{}, err
 	}
@@ -369,16 +378,8 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
 			if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil {
 				return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err)
 			}
-			hakOutput, err := os.Create(hakPath)
-			if err != nil {
-				return BuildResult{}, fmt.Errorf("create hak archive: %w", err)
-			}
-			if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil {
-				hakOutput.Close()
-				return BuildResult{}, fmt.Errorf("write hak archive: %w", err)
-			}
-			if err := hakOutput.Close(); err != nil {
-				return BuildResult{}, fmt.Errorf("close hak archive: %w", err)
+			if err := writeHAKArchive(hakPath, resourceSlice(chunk.Assets)); err != nil {
+				return BuildResult{}, err
 			}
 		}
 
@@ -408,6 +409,110 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
 	return result, nil
 }
 
+// buildDirectHAKs packages HAKs directly from the content-addressed source blob
+// cache. It skips music preparation, generated 2da discovery, git creation-time
+// lookups, LFS discovery, and any materialized asset tree scan.
+func buildDirectHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, manifest *SourceBuildManifest, opts BuildHAKOptions) (BuildResult, error) {
+	progressf(progress, "Collecting asset resources...")
+	assetResources, err := collectDirectSourceResources(manifest, opts.ContentAddressedRoot)
+	if err != nil {
+		return BuildResult{}, err
+	}
+
+	result := BuildResult{HAKAssets: len(assetResources)}
+
+	progressf(progress, "Planning HAK chunks...")
+	chunks, err := chunksFromSourceManifest(assetResources, manifest.HAKs)
+	if err != nil {
+		return BuildResult{}, err
+	}
+	chunks, err = filterHAKChunksByName(chunks, opts.ArchiveNames)
+	if err != nil {
+		return BuildResult{}, err
+	}
+
+	buildManifest := BuildManifest{
+		ModuleHAKs: append([]string(nil), manifest.ModuleHAKs...),
+		HAKs:       make([]BuildManifestHAK, 0, len(chunks)),
+	}
+	for _, chunk := range chunks {
+		buildManifest.HAKs = append(buildManifest.HAKs, buildManifestEntry(chunk))
+	}
+
+	manifestPath := p.HAKManifestPath()
+	manifestBytes, err := json.MarshalIndent(buildManifest, "", "  ")
+	if err != nil {
+		return BuildResult{}, fmt.Errorf("marshal hak manifest: %w", err)
+	}
+	manifestBytes = append(manifestBytes, '\n')
+
+	if writeArchives {
+		progressf(progress, "Writing HAK archives...")
+		result.HAKSummary.Total = len(chunks)
+		result.HAKPaths = make([]string, 0, len(chunks))
+		for index, chunk := range chunks {
+			hakPath := p.HAKArchivePath(chunk.Name)
+			progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets)))
+			if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil {
+				return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err)
+			}
+			if err := writeHAKArchive(hakPath, resourceSlice(chunk.Assets)); err != nil {
+				return BuildResult{}, err
+			}
+			result.HAKSummary.Written++
+			result.HAKSummary.Actions = append(result.HAKSummary.Actions, HAKArchiveAction{
+				Name:       chunk.Name,
+				AssetCount: len(chunk.Assets),
+				OutputPath: hakPath,
+			})
+			result.HAKPaths = append(result.HAKPaths, hakPath)
+		}
+	}
+
+	progressf(progress, "Writing HAK manifest...")
+	if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
+		return BuildResult{}, fmt.Errorf("create hak manifest dir: %w", err)
+	}
+	if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil {
+		return BuildResult{}, fmt.Errorf("write hak manifest: %w", err)
+	}
+	result.Manifest = manifestPath
+	return result, nil
+}
+
+// writeHAKArchive publishes a completed HAK atomically, so a failed write never
+// leaves a partial archive at the final path.
+func writeHAKArchive(hakPath string, resources []erf.Resource) error {
+	tmpPath := hakPath + ".tmp"
+	if err := os.Remove(tmpPath); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("remove stale temporary hak %s: %w", tmpPath, err)
+	}
+
+	output, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
+	if err != nil {
+		return fmt.Errorf("create temporary hak archive %s: %w", tmpPath, err)
+	}
+	if err := erf.Write(output, erf.New("HAK ", resources)); err != nil {
+		output.Close()
+		os.Remove(tmpPath)
+		return fmt.Errorf("write hak archive: %w", err)
+	}
+	if err := output.Sync(); err != nil {
+		output.Close()
+		os.Remove(tmpPath)
+		return fmt.Errorf("sync hak archive: %w", err)
+	}
+	if err := output.Close(); err != nil {
+		os.Remove(tmpPath)
+		return fmt.Errorf("close hak archive: %w", err)
+	}
+	if err := os.Rename(tmpPath, hakPath); err != nil {
+		os.Remove(tmpPath)
+		return fmt.Errorf("publish hak archive %s: %w", hakPath, err)
+	}
+	return nil
+}
+
 func writeAutogenManifestOutputs(progress ProgressFunc, manifests []topdata.ProducedAutogenManifest) error {
 	if len(manifests) == 0 {
 		return nil