package pipeline import ( "encoding/json" "fmt" "os" pathpkg "path" "path/filepath" "regexp" "slices" "strings" "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo" "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" ) // SourceAsset is one content-addressed source blob referenced by a direct source // manifest. The blob is resolved by sha256 under the content-addressed root. type SourceAsset struct { SHA256 string `json:"sha256"` SizeBytes int64 `json:"size_bytes"` } // SourceManifestHAK describes one output HAK and the logical asset paths it packs. type SourceManifestHAK struct { Name string `json:"name"` Group string `json:"group"` Priority int `json:"priority"` MaxBytes int64 `json:"max_bytes"` Optional bool `json:"optional,omitempty"` Assets []string `json:"assets"` BuildKey string `json:"build_key"` } // SourceBuildManifest is the generated, content-addressed input to a HAK build. // It carries asset_sources, so Crucible reads source bytes directly from the blob // cache without scanning a materialized asset tree. type SourceBuildManifest struct { Schema int `json:"schema"` BuilderID string `json:"builder_id"` ModuleHAKs []string `json:"module_haks"` HAKs []SourceManifestHAK `json:"haks"` AssetSources map[string]SourceAsset `json:"asset_sources"` } var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) // hasAssetSources reports whether raw JSON carries the direct content-addressed // schema. A legacy BuildManifest has no asset_sources object. func hasAssetSources(raw []byte) bool { var header struct { AssetSources map[string]json.RawMessage `json:"asset_sources"` } if err := json.Unmarshal(raw, &header); err != nil { return false } return header.AssetSources != nil } func loadSourceBuildManifest(path string) (*SourceBuildManifest, error) { raw, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("read source manifest %s: %w", path, err) } var manifest SourceBuildManifest if err := json.Unmarshal(raw, &manifest); err != nil { return nil, fmt.Errorf("parse source manifest %s: %w", path, err) } return &manifest, nil } func validateDirectSourceManifest(manifest *SourceBuildManifest) error { if manifest.Schema != 1 { return fmt.Errorf("unsupported source manifest schema %d", manifest.Schema) } if manifest.BuilderID != buildinfo.String() { return fmt.Errorf("source manifest builder_id %q does not match builder %q", manifest.BuilderID, buildinfo.String()) } if len(manifest.ModuleHAKs) == 0 { return fmt.Errorf("source manifest module_haks is empty") } referenced := make(map[string]struct{}, len(manifest.AssetSources)) for _, hak := range manifest.HAKs { if strings.TrimSpace(hak.Name) == "" { return fmt.Errorf("source manifest hak has empty name") } for _, rel := range hak.Assets { if !validLogicalAssetPath(rel) { return fmt.Errorf("source manifest hak %s has invalid asset path %q", hak.Name, rel) } if _, dup := referenced[rel]; dup { return fmt.Errorf("source manifest asset %q referenced by more than one hak", rel) } referenced[rel] = struct{}{} source, ok := manifest.AssetSources[rel] if !ok { return fmt.Errorf("source manifest hak %s asset %q has no asset_sources entry", hak.Name, rel) } if !sha256Pattern.MatchString(source.SHA256) { return fmt.Errorf("source manifest asset %q has invalid sha256 %q", rel, source.SHA256) } if source.SizeBytes < 0 { return fmt.Errorf("source manifest asset %q has negative size %d", rel, source.SizeBytes) } } } for rel := range manifest.AssetSources { if !validLogicalAssetPath(rel) { return fmt.Errorf("source manifest asset_sources has invalid path %q", rel) } if _, ok := referenced[rel]; !ok { return fmt.Errorf("source manifest asset_sources entry %q is not referenced by any hak", rel) } } return nil } func validLogicalAssetPath(path string) bool { if path == "" || filepath.IsAbs(path) || strings.Contains(path, `\`) { return false } clean := pathpkg.Clean(path) return clean == path && clean != "." && clean != ".." && !strings.HasPrefix(clean, "../") } func contentAddressedBlobPath(root, sha string) (string, error) { if strings.TrimSpace(root) == "" { return "", fmt.Errorf("content-addressed root is required") } if !sha256Pattern.MatchString(sha) { return "", fmt.Errorf("invalid content-addressed sha256 %q", sha) } return filepath.Join(root, "sha256", sha[0:2], sha[2:4], sha), nil } // loadDirectSourceManifest returns a parsed direct source manifest when path // points at a content-addressed manifest. It returns (nil, nil) for an empty // path or a legacy manifest without asset_sources, so the caller falls through // to the legacy path-backed build. func loadDirectSourceManifest(path string) (*SourceBuildManifest, error) { if strings.TrimSpace(path) == "" { return nil, nil } raw, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("read source manifest %s: %w", path, err) } if !hasAssetSources(raw) { return nil, nil } var manifest SourceBuildManifest if err := json.Unmarshal(raw, &manifest); err != nil { return nil, fmt.Errorf("parse source manifest %s: %w", path, err) } return &manifest, nil } // collectDirectSourceResources resolves each manifest-named blob under contentRoot // and produces sorted asset resources that stream from the blob cache with their // expected SHA-256. It never scans a materialized asset tree. func collectDirectSourceResources(manifest *SourceBuildManifest, contentRoot string) ([]assetResource, error) { if err := validateDirectSourceManifest(manifest); err != nil { return nil, err } if strings.TrimSpace(contentRoot) == "" { return nil, fmt.Errorf("content-addressed source manifest requires --content-addressed-root") } resources := make([]assetResource, 0, len(manifest.AssetSources)) for _, hak := range manifest.HAKs { for _, logicalPath := range hak.Assets { source := manifest.AssetSources[logicalPath] extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(logicalPath)), ".") resourceType, ok := erf.HAKResourceTypeForExtension(extension) if !ok { return nil, fmt.Errorf("unsupported HAK resource extension %q for %s", filepath.Ext(logicalPath), logicalPath) } resref := strings.ToLower(strings.TrimSuffix(filepath.Base(logicalPath), filepath.Ext(logicalPath))) blobPath, err := contentAddressedBlobPath(contentRoot, source.SHA256) if err != nil { return nil, err } info, err := os.Stat(blobPath) if err != nil { return nil, fmt.Errorf("missing source blob for %s: %w", logicalPath, err) } if info.Size() != source.SizeBytes { return nil, fmt.Errorf("source blob size mismatch for %s: expected %d, got %d", logicalPath, source.SizeBytes, info.Size()) } resource := erf.Resource{ Name: resref, Type: resourceType, SourcePath: blobPath, Size: source.SizeBytes, ExpectedSHA256: source.SHA256, } resources = append(resources, assetResource{ Rel: logicalPath, Resource: resource, Size: erf.ArchiveSize([]erf.Resource{resource}), ContentID: "sha256:" + source.SHA256, }) } } slices.SortFunc(resources, func(a, b assetResource) int { return compareResourceKeys(a.Resource, b.Resource) }) return resources, nil } // chunksFromSourceManifest converts direct manifest HAK entries into build chunks // reusing the legacy manifest chunker. func chunksFromSourceManifest(assets []assetResource, entries []SourceManifestHAK) ([]hakChunk, error) { converted := make([]BuildManifestHAK, len(entries)) for index, entry := range entries { converted[index] = BuildManifestHAK{ Name: entry.Name, Group: entry.Group, Priority: entry.Priority, MaxBytes: entry.MaxBytes, Optional: entry.Optional, Assets: entry.Assets, } } return chunksFromManifest(assets, converted) }