add content-addressed hak source mode (#7)

Reviewed-on: #7
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>

internal/pipeline/source_manifest.go

@@ -0,0 +1,232 @@
+package pipeline
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	pathpkg "path"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
+)
+
+// SourceAsset is one content-addressed source blob referenced by a direct source
+// manifest. The blob is resolved by sha256 under the content-addressed root.
+type SourceAsset struct {
+	SHA256    string `json:"sha256"`
+	SizeBytes int64  `json:"size_bytes"`
+}
+
+// SourceManifestHAK describes one output HAK and the logical asset paths it packs.
+type SourceManifestHAK struct {
+	Name     string   `json:"name"`
+	Group    string   `json:"group"`
+	Priority int      `json:"priority"`
+	MaxBytes int64    `json:"max_bytes"`
+	Optional bool     `json:"optional,omitempty"`
+	Assets   []string `json:"assets"`
+	BuildKey string   `json:"build_key"`
+}
+
+// SourceBuildManifest is the generated, content-addressed input to a HAK build.
+// It carries asset_sources, so Crucible reads source bytes directly from the blob
+// cache without scanning a materialized asset tree.
+type SourceBuildManifest struct {
+	Schema       int                    `json:"schema"`
+	BuilderID    string                 `json:"builder_id"`
+	ModuleHAKs   []string               `json:"module_haks"`
+	HAKs         []SourceManifestHAK    `json:"haks"`
+	AssetSources map[string]SourceAsset `json:"asset_sources"`
+}
+
+var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+// hasAssetSources reports whether raw JSON carries the direct content-addressed
+// schema. A legacy BuildManifest has no asset_sources object.
+func hasAssetSources(raw []byte) bool {
+	var header struct {
+		AssetSources map[string]json.RawMessage `json:"asset_sources"`
+	}
+	if err := json.Unmarshal(raw, &header); err != nil {
+		return false
+	}
+	return header.AssetSources != nil
+}
+
+func loadSourceBuildManifest(path string) (*SourceBuildManifest, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read source manifest %s: %w", path, err)
+	}
+	var manifest SourceBuildManifest
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		return nil, fmt.Errorf("parse source manifest %s: %w", path, err)
+	}
+	return &manifest, nil
+}
+
+func validateDirectSourceManifest(manifest *SourceBuildManifest) error {
+	if manifest.Schema != 1 {
+		return fmt.Errorf("unsupported source manifest schema %d", manifest.Schema)
+	}
+	if manifest.BuilderID != buildinfo.String() {
+		return fmt.Errorf("source manifest builder_id %q does not match builder %q", manifest.BuilderID, buildinfo.String())
+	}
+	if len(manifest.ModuleHAKs) == 0 {
+		return fmt.Errorf("source manifest module_haks is empty")
+	}
+
+	referenced := make(map[string]struct{}, len(manifest.AssetSources))
+	for _, hak := range manifest.HAKs {
+		if strings.TrimSpace(hak.Name) == "" {
+			return fmt.Errorf("source manifest hak has empty name")
+		}
+		for _, rel := range hak.Assets {
+			if !validLogicalAssetPath(rel) {
+				return fmt.Errorf("source manifest hak %s has invalid asset path %q", hak.Name, rel)
+			}
+			if _, dup := referenced[rel]; dup {
+				return fmt.Errorf("source manifest asset %q referenced by more than one hak", rel)
+			}
+			referenced[rel] = struct{}{}
+			source, ok := manifest.AssetSources[rel]
+			if !ok {
+				return fmt.Errorf("source manifest hak %s asset %q has no asset_sources entry", hak.Name, rel)
+			}
+			if !sha256Pattern.MatchString(source.SHA256) {
+				return fmt.Errorf("source manifest asset %q has invalid sha256 %q", rel, source.SHA256)
+			}
+			if source.SizeBytes < 0 {
+				return fmt.Errorf("source manifest asset %q has negative size %d", rel, source.SizeBytes)
+			}
+		}
+	}
+
+	for rel := range manifest.AssetSources {
+		if !validLogicalAssetPath(rel) {
+			return fmt.Errorf("source manifest asset_sources has invalid path %q", rel)
+		}
+		if _, ok := referenced[rel]; !ok {
+			return fmt.Errorf("source manifest asset_sources entry %q is not referenced by any hak", rel)
+		}
+	}
+	return nil
+}
+
+func validLogicalAssetPath(path string) bool {
+	if path == "" || filepath.IsAbs(path) || strings.Contains(path, `\`) {
+		return false
+	}
+	clean := pathpkg.Clean(path)
+	return clean == path && clean != "." && clean != ".." &&
+		!strings.HasPrefix(clean, "../")
+}
+
+func contentAddressedBlobPath(root, sha string) (string, error) {
+	if strings.TrimSpace(root) == "" {
+		return "", fmt.Errorf("content-addressed root is required")
+	}
+	if !sha256Pattern.MatchString(sha) {
+		return "", fmt.Errorf("invalid content-addressed sha256 %q", sha)
+	}
+	return filepath.Join(root, "sha256", sha[0:2], sha[2:4], sha), nil
+}
+
+// loadDirectSourceManifest returns a parsed direct source manifest when path
+// points at a content-addressed manifest. It returns (nil, nil) for an empty
+// path or a legacy manifest without asset_sources, so the caller falls through
+// to the legacy path-backed build.
+func loadDirectSourceManifest(path string) (*SourceBuildManifest, error) {
+	if strings.TrimSpace(path) == "" {
+		return nil, nil
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read source manifest %s: %w", path, err)
+	}
+	if !hasAssetSources(raw) {
+		return nil, nil
+	}
+	var manifest SourceBuildManifest
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		return nil, fmt.Errorf("parse source manifest %s: %w", path, err)
+	}
+	return &manifest, nil
+}
+
+// collectDirectSourceResources resolves each manifest-named blob under contentRoot
+// and produces sorted asset resources that stream from the blob cache with their
+// expected SHA-256. It never scans a materialized asset tree.
+func collectDirectSourceResources(manifest *SourceBuildManifest, contentRoot string) ([]assetResource, error) {
+	if err := validateDirectSourceManifest(manifest); err != nil {
+		return nil, err
+	}
+	if strings.TrimSpace(contentRoot) == "" {
+		return nil, fmt.Errorf("content-addressed source manifest requires --content-addressed-root")
+	}
+
+	resources := make([]assetResource, 0, len(manifest.AssetSources))
+	for _, hak := range manifest.HAKs {
+		for _, logicalPath := range hak.Assets {
+			source := manifest.AssetSources[logicalPath]
+			extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(logicalPath)), ".")
+			resourceType, ok := erf.HAKResourceTypeForExtension(extension)
+			if !ok {
+				return nil, fmt.Errorf("unsupported HAK resource extension %q for %s", filepath.Ext(logicalPath), logicalPath)
+			}
+			resref := strings.ToLower(strings.TrimSuffix(filepath.Base(logicalPath), filepath.Ext(logicalPath)))
+
+			blobPath, err := contentAddressedBlobPath(contentRoot, source.SHA256)
+			if err != nil {
+				return nil, err
+			}
+			info, err := os.Stat(blobPath)
+			if err != nil {
+				return nil, fmt.Errorf("missing source blob for %s: %w", logicalPath, err)
+			}
+			if info.Size() != source.SizeBytes {
+				return nil, fmt.Errorf("source blob size mismatch for %s: expected %d, got %d", logicalPath, source.SizeBytes, info.Size())
+			}
+
+			resource := erf.Resource{
+				Name:           resref,
+				Type:           resourceType,
+				SourcePath:     blobPath,
+				Size:           source.SizeBytes,
+				ExpectedSHA256: source.SHA256,
+			}
+			resources = append(resources, assetResource{
+				Rel:       logicalPath,
+				Resource:  resource,
+				Size:      erf.ArchiveSize([]erf.Resource{resource}),
+				ContentID: "sha256:" + source.SHA256,
+			})
+		}
+	}
+
+	slices.SortFunc(resources, func(a, b assetResource) int {
+		return compareResourceKeys(a.Resource, b.Resource)
+	})
+	return resources, nil
+}
+
+// chunksFromSourceManifest converts direct manifest HAK entries into build chunks
+// reusing the legacy manifest chunker.
+func chunksFromSourceManifest(assets []assetResource, entries []SourceManifestHAK) ([]hakChunk, error) {
+	converted := make([]BuildManifestHAK, len(entries))
+	for index, entry := range entries {
+		converted[index] = BuildManifestHAK{
+			Name:     entry.Name,
+			Group:    entry.Group,
+			Priority: entry.Priority,
+			MaxBytes: entry.MaxBytes,
+			Optional: entry.Optional,
+			Assets:   entry.Assets,
+		}
+	}
+	return chunksFromManifest(assets, converted)
+}