diff --git a/AGENTS.md b/AGENTS.md index 4d2f76c..85011e0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,10 +5,9 @@ alwaysApply: true # sow-tools / Crucible Agent Guide -This repo owns the **builder logic** for the migration: one Go module +This repo owns the **builder logic:** one Go module (`git.westgate.pw/ShadowsOverWestgate/sow-tools`) producing the `crucible` -dispatcher and the `crucible-` binaries (D11). Read -`../AGENTS.md` (migration hub) and `../../KICKOFF_PROMPT.md` first. +dispatcher and the `crucible-` binaries. ## What this repo owns / does not own @@ -20,19 +19,11 @@ is `sow-platform`). ## Rules -1. **Migrated logic, not a fresh rewrite.** The `internal/` packages (`app`, - `pipeline`, `project`, `erf`, `gff`, `topdata`, `music`, `changelog`, - `validator`) were folded in from `gitea/sow-tools` at cutover; wired builders - delegate to `internal/app`'s command surface. Keep them in step with upstream - fixes rather than diverging silently. -2. **Fail closed, never fake.** A builder with no migrated logic yet (`depot`) +1. **Fail closed, never fake.** A builder with no migrated logic yet (`depot`) exits `70`. Do not stub a builder to emit a placeholder artifact. -3. **Binaries are not committed.** They are CI artifacts / image layers (D19). +2. **Binaries are not committed.** They are CI artifacts / image layers. `/bin/`, `*.exe`, `nwn-tool`, `sow-toolkit` are gitignored. -4. **No home-dir / `NWN_ROOT` guessing.** Builders take roots explicitly via - flag or env (project resolution is CWD-based, never `$HOME`). See - [`docs/consumer-contract.md`](docs/consumer-contract.md). -5. **The registry is the command surface.** `internal/dispatch.Registry` is the +3. **The registry is the command surface.** `internal/dispatch.Registry` is the single source of truth; keep it in sync with `cmd/` and [`docs/command-surface.md`](docs/command-surface.md). Adding a builder = a `cmd/crucible-/main.go` shim + a `Registry` entry + a doc row. @@ -40,11 +31,8 @@ is `sow-platform`). ## Wiring a builder 1. Ensure the relevant `internal/` package(s) cover the work. -2. Add the legacy command(s) to the builder's `Legacy`/`Extra` set and set - `Wired: true` in `internal/dispatch`; the dispatcher delegates to - `app.Run`. `depot` is the remaining unwired builder. -3. Add tests; keep outputs deterministic (same input → same bytes). -4. `make check` must stay green; update `make smoke` to expect the wired exit. +2. Add tests; keep outputs deterministic (same input → same bytes). +3. `make check` must stay green; update `make smoke` to expect the wired exit. ## Commands @@ -54,7 +42,3 @@ make build # cmd/* -> ./bin make smoke # assert fail-closed contract make image # crucible: ``` - -## Git - -Never commit, branch, or push. Suggest a commit message; let the operator do it. diff --git a/internal/erf/erf.go b/internal/erf/erf.go index dd71df0..14c05f5 100644 --- a/internal/erf/erf.go +++ b/internal/erf/erf.go @@ -2,14 +2,19 @@ package erf import ( "bytes" + "crypto/sha256" "encoding/binary" + "encoding/hex" "fmt" "io" "os" + "regexp" "sort" "strings" ) +var expectedSHA256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + const ( headerSize = 160 versionV10 = "V1.0" @@ -30,6 +35,9 @@ type Resource struct { Data []byte SourcePath string Size int64 + // ExpectedSHA256, when set, is the lowercase hex SHA-256 the streamed + // SourcePath payload must hash to. A mismatch fails the write. + ExpectedSHA256 string } type header struct { @@ -407,18 +415,29 @@ func writeResourceData(w io.Writer, resource Resource) error { return err } + if resource.ExpectedSHA256 != "" && !expectedSHA256Pattern.MatchString(resource.ExpectedSHA256) { + return fmt.Errorf("resource %q has invalid expected sha256", resource.SourcePath) + } + file, err := os.Open(resource.SourcePath) if err != nil { return fmt.Errorf("open resource %q: %w", resource.SourcePath, err) } defer file.Close() - written, err := io.Copy(w, file) + hash := sha256.New() + written, err := io.Copy(io.MultiWriter(w, hash), file) if err != nil { return fmt.Errorf("copy resource %q: %w", resource.SourcePath, err) } if resource.Size > 0 && written != resource.Size { return fmt.Errorf("copy resource %q: expected %d bytes, wrote %d", resource.SourcePath, resource.Size, written) } + if resource.ExpectedSHA256 != "" { + got := hex.EncodeToString(hash.Sum(nil)) + if got != resource.ExpectedSHA256 { + return fmt.Errorf("resource %q sha256 mismatch: expected %s, got %s", resource.SourcePath, resource.ExpectedSHA256, got) + } + } return nil } diff --git a/internal/erf/erf_test.go b/internal/erf/erf_test.go index 2dbe828..dd441e3 100644 --- a/internal/erf/erf_test.go +++ b/internal/erf/erf_test.go @@ -2,9 +2,72 @@ package erf import ( "bytes" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" "testing" ) +func writeTempPayload(t *testing.T, payload []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "payload.bin") + if err := os.WriteFile(path, payload, 0o644); err != nil { + t.Fatalf("write payload: %v", err) + } + return path +} + +func TestWriteRejectsSourcePathSHA256Mismatch(t *testing.T) { + payload := []byte("hello world") + path := writeTempPayload(t, payload) + + archive := New("HAK ", []Resource{{ + Name: "asset", + Type: 0x0003, + SourcePath: path, + Size: int64(len(payload)), + ExpectedSHA256: strings.Repeat("a", 64), + }}) + + var buf bytes.Buffer + err := Write(&buf, archive) + if err == nil { + t.Fatal("expected sha256 mismatch error") + } + if !strings.Contains(err.Error(), "sha256 mismatch") { + t.Fatalf("error = %v, want sha256 mismatch", err) + } +} + +func TestWriteAcceptsMatchingSourcePathSHA256(t *testing.T) { + payload := []byte("hello world") + path := writeTempPayload(t, payload) + sum := sha256.Sum256(payload) + + archive := New("HAK ", []Resource{{ + Name: "asset", + Type: 0x0003, + SourcePath: path, + Size: int64(len(payload)), + ExpectedSHA256: hex.EncodeToString(sum[:]), + }}) + + var buf bytes.Buffer + if err := Write(&buf, archive); err != nil { + t.Fatalf("write: %v", err) + } + + decoded, err := Read(bytes.NewReader(buf.Bytes())) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(decoded.Resources) != 1 || string(decoded.Resources[0].Data) != string(payload) { + t.Fatalf("unexpected payload: %#v", decoded.Resources) + } +} + func TestArchiveRoundTrip(t *testing.T) { archive := New("MOD ", []Resource{ {Name: "module", Type: 0x07DE, Data: []byte("ifo")}, diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index 99abd54..ab145f4 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -106,11 +106,12 @@ type hakChunk struct { type ProgressFunc func(string) type BuildHAKOptions struct { - Progress ProgressFunc - ArchiveNames []string - SourceManifestPath string - SkipMusic bool - MusicDatasetIDs []string + Progress ProgressFunc + ArchiveNames []string + SourceManifestPath string + ContentAddressedRoot string + SkipMusic bool + MusicDatasetIDs []string } func Build(p *project.Project) (BuildResult, error) { @@ -196,7 +197,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu } func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) { - return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs) + return planOrBuildHAKs(p, opts.Progress, true, opts) } func PlanHAKs(p *project.Project) (BuildResult, error) { @@ -208,18 +209,18 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul } func PlanHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) { - return planOrBuildHAKs(p, opts.Progress, false, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs) + return planOrBuildHAKs(p, opts.Progress, false, opts) } func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { - return planOrBuildHAKs(p, progress, true, nil, "", false, nil) + return planOrBuildHAKs(p, progress, true, BuildHAKOptions{}) } func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { - return planOrBuildHAKs(p, progress, false, nil, "", false, nil) + return planOrBuildHAKs(p, progress, false, BuildHAKOptions{}) } -func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string, skipMusic bool, musicDatasetIDs []string) (result BuildResult, err error) { +func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, opts BuildHAKOptions) (result BuildResult, err error) { preserveExistingHAKs := p.EffectiveConfig().Build.KeepExistingHAKs || envBool("SOW_BUILD_HAKS_KEEP_EXISTING") progressf(progress, "Validating project...") @@ -227,15 +228,23 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo return BuildResult{}, err } + directManifest, err := loadDirectSourceManifest(opts.SourceManifestPath) + if err != nil { + return BuildResult{}, err + } + if directManifest != nil { + return buildDirectHAKs(p, progress, writeArchives, directManifest, opts) + } + progressf(progress, "Collecting asset resources...") - allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(sourceManifestPath) + allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(opts.SourceManifestPath) if err != nil { return BuildResult{}, err } musicAssets := &preparedMusicAssets{SkipSourceRel: map[string]struct{}{}} - if !skipMusic { + if !opts.SkipMusic { musicAssets, err = prepareMusicAssetsWithOptions(p, musicPrepareOptions{ - datasetIDs: musicDatasetIDs, + datasetIDs: opts.MusicDatasetIDs, write: writeArchives, }) if err != nil { @@ -294,7 +303,7 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo if err != nil { return BuildResult{}, err } - chunks, err = filterHAKChunksByName(chunks, archiveNames) + chunks, err = filterHAKChunksByName(chunks, opts.ArchiveNames) if err != nil { return BuildResult{}, err } @@ -408,6 +417,97 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo return result, nil } +// buildDirectHAKs packages HAKs directly from the content-addressed source blob +// cache. It skips music preparation, generated 2da discovery, git creation-time +// lookups, LFS discovery, and any materialized asset tree scan. +func buildDirectHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, manifest *SourceBuildManifest, opts BuildHAKOptions) (BuildResult, error) { + progressf(progress, "Collecting asset resources...") + assetResources, err := collectDirectSourceResources(manifest, opts.ContentAddressedRoot) + if err != nil { + return BuildResult{}, err + } + + result := BuildResult{HAKAssets: len(assetResources)} + + progressf(progress, "Planning HAK chunks...") + chunks, err := chunksFromSourceManifest(assetResources, manifest.HAKs) + if err != nil { + return BuildResult{}, err + } + chunks, err = filterHAKChunksByName(chunks, opts.ArchiveNames) + if err != nil { + return BuildResult{}, err + } + + buildManifest := BuildManifest{ + ModuleHAKs: append([]string(nil), manifest.ModuleHAKs...), + HAKs: make([]BuildManifestHAK, 0, len(chunks)), + } + for _, chunk := range chunks { + buildManifest.HAKs = append(buildManifest.HAKs, buildManifestEntry(chunk)) + } + + manifestPath := p.HAKManifestPath() + manifestBytes, err := json.MarshalIndent(buildManifest, "", " ") + if err != nil { + return BuildResult{}, fmt.Errorf("marshal hak manifest: %w", err) + } + manifestBytes = append(manifestBytes, '\n') + + if writeArchives { + progressf(progress, "Writing HAK archives...") + result.HAKSummary.Total = len(chunks) + result.HAKPaths = make([]string, 0, len(chunks)) + for index, chunk := range chunks { + hakPath := p.HAKArchivePath(chunk.Name) + progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets))) + if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err) + } + if err := writeHAKArchive(hakPath, resourceSlice(chunk.Assets)); err != nil { + return BuildResult{}, err + } + result.HAKSummary.Written++ + result.HAKSummary.Actions = append(result.HAKSummary.Actions, HAKArchiveAction{ + Name: chunk.Name, + AssetCount: len(chunk.Assets), + OutputPath: hakPath, + }) + result.HAKPaths = append(result.HAKPaths, hakPath) + } + } + + progressf(progress, "Writing HAK manifest...") + if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create hak manifest dir: %w", err) + } + if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil { + return BuildResult{}, fmt.Errorf("write hak manifest: %w", err) + } + result.Manifest = manifestPath + return result, nil +} + +// writeHAKArchive writes a HAK archive, removing any partial output if the ERF +// writer or a streamed source SHA verification fails, so a corrupt input never +// leaves a completed .hak behind. +func writeHAKArchive(hakPath string, resources []erf.Resource) error { + output, err := os.Create(hakPath) + if err != nil { + return fmt.Errorf("create hak archive: %w", err) + } + if err := erf.Write(output, erf.New("HAK ", resources)); err != nil { + output.Close() + os.Remove(hakPath) + return fmt.Errorf("write hak archive: %w", err) + } + if err := output.Close(); err != nil { + os.Remove(hakPath) + return fmt.Errorf("close hak archive: %w", err) + } + return nil +} + func writeAutogenManifestOutputs(progress ProgressFunc, manifests []topdata.ProducedAutogenManifest) error { if len(manifests) == 0 { return nil diff --git a/internal/pipeline/source_build_test.go b/internal/pipeline/source_build_test.go new file mode 100644 index 0000000..2a2c8f9 --- /dev/null +++ b/internal/pipeline/source_build_test.go @@ -0,0 +1,249 @@ +package pipeline + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +const directProjectConfig = `{ + "module": { + "name": "Test Module", + "resref": "testmod", + "hak_order": ["core_01"] + }, + "paths": { + "source": "src", + "assets": "assets", + "build": "build" + } +} +` + +func writeBlobAt(t *testing.T, root, sha, content string) { + t.Helper() + dir := filepath.Join(root, "sha256", sha[0:2], sha[2:4]) + mustMkdir(t, dir) + mustWriteFile(t, filepath.Join(dir, sha), content) +} + +func writeBlob(t *testing.T, root string, payload []byte) string { + t.Helper() + sum := sha256.Sum256(payload) + sha := hex.EncodeToString(sum[:]) + writeBlobAt(t, root, sha, string(payload)) + return sha +} + +func writeDirectManifest(t *testing.T, root string, sources map[string]SourceAsset, assets []string) string { + t.Helper() + manifest := SourceBuildManifest{ + Schema: 1, + BuilderID: buildinfo.String(), + ModuleHAKs: []string{"core_01"}, + HAKs: []SourceManifestHAK{{ + Name: "core_01", + Group: "core", + Priority: 1, + MaxBytes: 1 << 20, + Assets: assets, + }}, + AssetSources: sources, + } + raw, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal manifest: %v", err) + } + path := filepath.Join(root, "build-source-manifest.json") + mustWriteFile(t, path, string(raw)) + return path +} + +func loadDirectProject(t *testing.T, root string) *project.Project { + t.Helper() + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), directProjectConfig) + mustMkdir(t, filepath.Join(root, "build")) + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + return p +} + +func TestBuildHAKsFromContentAddressedManifestWithoutAssetsTree(t *testing.T) { + root := t.TempDir() + p := loadDirectProject(t, root) + + blobs := filepath.Join(root, "blobs") + payload := []byte("tga-payload-bytes") + sha := writeBlob(t, blobs, payload) + manifestPath := writeDirectManifest(t, root, + map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: int64(len(payload))}}, + []string{"core/a.tga"}) + + result, err := BuildHAKsWithOptions(p, BuildHAKOptions{ + SourceManifestPath: manifestPath, + ContentAddressedRoot: blobs, + }) + if err != nil { + t.Fatalf("build direct haks: %v", err) + } + if result.HAKAssets != 1 { + t.Fatalf("hak assets = %d, want 1", result.HAKAssets) + } + + if _, err := os.Stat(filepath.Join(root, "assets")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("direct build must not materialize an assets tree (stat err = %v)", err) + } + + hakPath := filepath.Join(root, "build", "core_01.hak") + raw, err := os.ReadFile(hakPath) + if err != nil { + t.Fatalf("read hak: %v", err) + } + archive, err := erf.Read(bytes.NewReader(raw)) + if err != nil { + t.Fatalf("decode hak: %v", err) + } + if len(archive.Resources) != 1 { + t.Fatalf("hak resources = %d, want 1", len(archive.Resources)) + } + if got := string(archive.Resources[0].Data); got != string(payload) { + t.Fatalf("resource payload = %q, want %q", got, payload) + } + if archive.Resources[0].Name != "a" { + t.Fatalf("resref = %q, want a", archive.Resources[0].Name) + } +} + +func TestBuildHAKsFromContentAddressedManifestRejectsMissingRoot(t *testing.T) { + root := t.TempDir() + p := loadDirectProject(t, root) + + blobs := filepath.Join(root, "blobs") + payload := []byte("tga-payload-bytes") + sha := writeBlob(t, blobs, payload) + manifestPath := writeDirectManifest(t, root, + map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: int64(len(payload))}}, + []string{"core/a.tga"}) + + _, err := BuildHAKsWithOptions(p, BuildHAKOptions{SourceManifestPath: manifestPath}) + if err == nil { + t.Fatal("expected failure when content-addressed root is missing") + } +} + +func TestBuildHAKsFromContentAddressedManifestRejectsMissingBlob(t *testing.T) { + root := t.TempDir() + p := loadDirectProject(t, root) + + blobs := filepath.Join(root, "blobs") + sha := strings.Repeat("b", 64) + manifestPath := writeDirectManifest(t, root, + map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: 4}}, + []string{"core/a.tga"}) + + _, err := BuildHAKsWithOptions(p, BuildHAKOptions{ + SourceManifestPath: manifestPath, + ContentAddressedRoot: blobs, + }) + if err == nil { + t.Fatal("expected failure for missing source blob") + } +} + +func TestBuildHAKsFromContentAddressedManifestRejectsCorruptBlob(t *testing.T) { + root := t.TempDir() + p := loadDirectProject(t, root) + + blobs := filepath.Join(root, "blobs") + declared := sha256.Sum256([]byte("abcd")) + sha := hex.EncodeToString(declared[:]) + // Same length, different bytes: size check passes, content hash mismatches. + writeBlobAt(t, blobs, sha, "wxyz") + manifestPath := writeDirectManifest(t, root, + map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: 4}}, + []string{"core/a.tga"}) + + _, err := BuildHAKsWithOptions(p, BuildHAKOptions{ + SourceManifestPath: manifestPath, + ContentAddressedRoot: blobs, + }) + if err == nil { + t.Fatal("expected sha256 mismatch failure for corrupt blob") + } + if !strings.Contains(err.Error(), "sha256 mismatch") { + t.Fatalf("error = %v, want sha256 mismatch", err) + } + if _, statErr := os.Stat(filepath.Join(root, "build", "core_01.hak")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("corrupt build must not leave a completed hak (stat err = %v)", statErr) + } +} + +func TestLegacySourceManifestStillBuildsFromAssetsTree(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "assets", "core")) + mustMkdir(t, filepath.Join(root, "build")) + mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": {"name": "Test Module", "resref": "testmod", "hak_order": ["group:core"]}, + "paths": {"source": "src", "assets": "assets", "build": "build"}, + "haks": [{"name": "core", "priority": 1, "include": ["core/**"]}] +} +`) + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": {"struct_type": 0, "fields": [ + {"label": "Mod_Name", "type": "CExoString", "value": "Test Module"}, + {"label": "Mod_HakList", "type": "List", "value": []} + ]} +} +`) + mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), strings.Repeat("a", 16)) + mustWriteFile(t, filepath.Join(root, "assets", "core", "b.tga"), strings.Repeat("b", 16)) + + manifestPath := filepath.Join(root, "legacy-source-manifest.json") + mustWriteFile(t, manifestPath, `{ + "module_haks": ["core"], + "haks": [{"name": "core", "group": "core", "priority": 1, "assets": ["core/a.tga", "core/b.tga"]}] +} +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + + result, err := BuildHAKsWithOptions(p, BuildHAKOptions{SourceManifestPath: manifestPath}) + if err != nil { + t.Fatalf("legacy build: %v", err) + } + if len(result.HAKPaths) != 1 { + t.Fatalf("hak paths = %d, want 1", len(result.HAKPaths)) + } + raw, err := os.ReadFile(filepath.Join(root, "build", "core.hak")) + if err != nil { + t.Fatalf("read legacy hak: %v", err) + } + archive, err := erf.Read(bytes.NewReader(raw)) + if err != nil { + t.Fatalf("decode legacy hak: %v", err) + } + if len(archive.Resources) != 2 { + t.Fatalf("legacy hak resources = %d, want 2", len(archive.Resources)) + } +} diff --git a/internal/pipeline/source_manifest.go b/internal/pipeline/source_manifest.go new file mode 100644 index 0000000..96c86a1 --- /dev/null +++ b/internal/pipeline/source_manifest.go @@ -0,0 +1,232 @@ +package pipeline + +import ( + "encoding/json" + "fmt" + "os" + pathpkg "path" + "path/filepath" + "regexp" + "slices" + "strings" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo" + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf" +) + +// SourceAsset is one content-addressed source blob referenced by a direct source +// manifest. The blob is resolved by sha256 under the content-addressed root. +type SourceAsset struct { + SHA256 string `json:"sha256"` + SizeBytes int64 `json:"size_bytes"` +} + +// SourceManifestHAK describes one output HAK and the logical asset paths it packs. +type SourceManifestHAK struct { + Name string `json:"name"` + Group string `json:"group"` + Priority int `json:"priority"` + MaxBytes int64 `json:"max_bytes"` + Optional bool `json:"optional,omitempty"` + Assets []string `json:"assets"` + BuildKey string `json:"build_key"` +} + +// SourceBuildManifest is the generated, content-addressed input to a HAK build. +// It carries asset_sources, so Crucible reads source bytes directly from the blob +// cache without scanning a materialized asset tree. +type SourceBuildManifest struct { + Schema int `json:"schema"` + BuilderID string `json:"builder_id"` + ModuleHAKs []string `json:"module_haks"` + HAKs []SourceManifestHAK `json:"haks"` + AssetSources map[string]SourceAsset `json:"asset_sources"` +} + +var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// hasAssetSources reports whether raw JSON carries the direct content-addressed +// schema. A legacy BuildManifest has no asset_sources object. +func hasAssetSources(raw []byte) bool { + var header struct { + AssetSources map[string]json.RawMessage `json:"asset_sources"` + } + if err := json.Unmarshal(raw, &header); err != nil { + return false + } + return header.AssetSources != nil +} + +func loadSourceBuildManifest(path string) (*SourceBuildManifest, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read source manifest %s: %w", path, err) + } + var manifest SourceBuildManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("parse source manifest %s: %w", path, err) + } + return &manifest, nil +} + +func validateDirectSourceManifest(manifest *SourceBuildManifest) error { + if manifest.Schema != 1 { + return fmt.Errorf("unsupported source manifest schema %d", manifest.Schema) + } + if manifest.BuilderID != buildinfo.String() { + return fmt.Errorf("source manifest builder_id %q does not match builder %q", manifest.BuilderID, buildinfo.String()) + } + if len(manifest.ModuleHAKs) == 0 { + return fmt.Errorf("source manifest module_haks is empty") + } + + referenced := make(map[string]struct{}, len(manifest.AssetSources)) + for _, hak := range manifest.HAKs { + if strings.TrimSpace(hak.Name) == "" { + return fmt.Errorf("source manifest hak has empty name") + } + for _, rel := range hak.Assets { + if !validLogicalAssetPath(rel) { + return fmt.Errorf("source manifest hak %s has invalid asset path %q", hak.Name, rel) + } + if _, dup := referenced[rel]; dup { + return fmt.Errorf("source manifest asset %q referenced by more than one hak", rel) + } + referenced[rel] = struct{}{} + source, ok := manifest.AssetSources[rel] + if !ok { + return fmt.Errorf("source manifest hak %s asset %q has no asset_sources entry", hak.Name, rel) + } + if !sha256Pattern.MatchString(source.SHA256) { + return fmt.Errorf("source manifest asset %q has invalid sha256 %q", rel, source.SHA256) + } + if source.SizeBytes < 0 { + return fmt.Errorf("source manifest asset %q has negative size %d", rel, source.SizeBytes) + } + } + } + + for rel := range manifest.AssetSources { + if !validLogicalAssetPath(rel) { + return fmt.Errorf("source manifest asset_sources has invalid path %q", rel) + } + if _, ok := referenced[rel]; !ok { + return fmt.Errorf("source manifest asset_sources entry %q is not referenced by any hak", rel) + } + } + return nil +} + +func validLogicalAssetPath(path string) bool { + if path == "" || filepath.IsAbs(path) || strings.Contains(path, `\`) { + return false + } + clean := pathpkg.Clean(path) + return clean == path && clean != "." && clean != ".." && + !strings.HasPrefix(clean, "../") +} + +func contentAddressedBlobPath(root, sha string) (string, error) { + if strings.TrimSpace(root) == "" { + return "", fmt.Errorf("content-addressed root is required") + } + if !sha256Pattern.MatchString(sha) { + return "", fmt.Errorf("invalid content-addressed sha256 %q", sha) + } + return filepath.Join(root, "sha256", sha[0:2], sha[2:4], sha), nil +} + +// loadDirectSourceManifest returns a parsed direct source manifest when path +// points at a content-addressed manifest. It returns (nil, nil) for an empty +// path or a legacy manifest without asset_sources, so the caller falls through +// to the legacy path-backed build. +func loadDirectSourceManifest(path string) (*SourceBuildManifest, error) { + if strings.TrimSpace(path) == "" { + return nil, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read source manifest %s: %w", path, err) + } + if !hasAssetSources(raw) { + return nil, nil + } + var manifest SourceBuildManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, fmt.Errorf("parse source manifest %s: %w", path, err) + } + return &manifest, nil +} + +// collectDirectSourceResources resolves each manifest-named blob under contentRoot +// and produces sorted asset resources that stream from the blob cache with their +// expected SHA-256. It never scans a materialized asset tree. +func collectDirectSourceResources(manifest *SourceBuildManifest, contentRoot string) ([]assetResource, error) { + if err := validateDirectSourceManifest(manifest); err != nil { + return nil, err + } + if strings.TrimSpace(contentRoot) == "" { + return nil, fmt.Errorf("content-addressed source manifest requires --content-addressed-root") + } + + resources := make([]assetResource, 0, len(manifest.AssetSources)) + for _, hak := range manifest.HAKs { + for _, logicalPath := range hak.Assets { + source := manifest.AssetSources[logicalPath] + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(logicalPath)), ".") + resourceType, ok := erf.HAKResourceTypeForExtension(extension) + if !ok { + return nil, fmt.Errorf("unsupported HAK resource extension %q for %s", filepath.Ext(logicalPath), logicalPath) + } + resref := strings.ToLower(strings.TrimSuffix(filepath.Base(logicalPath), filepath.Ext(logicalPath))) + + blobPath, err := contentAddressedBlobPath(contentRoot, source.SHA256) + if err != nil { + return nil, err + } + info, err := os.Stat(blobPath) + if err != nil { + return nil, fmt.Errorf("missing source blob for %s: %w", logicalPath, err) + } + if info.Size() != source.SizeBytes { + return nil, fmt.Errorf("source blob size mismatch for %s: expected %d, got %d", logicalPath, source.SizeBytes, info.Size()) + } + + resource := erf.Resource{ + Name: resref, + Type: resourceType, + SourcePath: blobPath, + Size: source.SizeBytes, + ExpectedSHA256: source.SHA256, + } + resources = append(resources, assetResource{ + Rel: logicalPath, + Resource: resource, + Size: erf.ArchiveSize([]erf.Resource{resource}), + ContentID: "sha256:" + source.SHA256, + }) + } + } + + slices.SortFunc(resources, func(a, b assetResource) int { + return compareResourceKeys(a.Resource, b.Resource) + }) + return resources, nil +} + +// chunksFromSourceManifest converts direct manifest HAK entries into build chunks +// reusing the legacy manifest chunker. +func chunksFromSourceManifest(assets []assetResource, entries []SourceManifestHAK) ([]hakChunk, error) { + converted := make([]BuildManifestHAK, len(entries)) + for index, entry := range entries { + converted[index] = BuildManifestHAK{ + Name: entry.Name, + Group: entry.Group, + Priority: entry.Priority, + MaxBytes: entry.MaxBytes, + Optional: entry.Optional, + Assets: entry.Assets, + } + } + return chunksFromManifest(assets, converted) +} diff --git a/internal/pipeline/source_manifest_test.go b/internal/pipeline/source_manifest_test.go new file mode 100644 index 0000000..cfaf446 --- /dev/null +++ b/internal/pipeline/source_manifest_test.go @@ -0,0 +1,160 @@ +package pipeline + +import ( + "encoding/json" + "path/filepath" + "testing" + + "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo" +) + +const validAsset = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +// directManifestFixture returns a minimal valid direct source manifest using the +// running builder identity, so validation does not reject it on builder mismatch. +func directManifestFixture() *SourceBuildManifest { + return &SourceBuildManifest{ + Schema: 1, + BuilderID: buildinfo.String(), + ModuleHAKs: []string{"core_01"}, + HAKs: []SourceManifestHAK{ + { + Name: "core_01", + Group: "core", + Priority: 1, + MaxBytes: 1024, + Assets: []string{"core/a.tga"}, + }, + }, + AssetSources: map[string]SourceAsset{ + "core/a.tga": {SHA256: validAsset, SizeBytes: 4}, + }, + } +} + +func writeManifestFixture(t *testing.T, manifest *SourceBuildManifest) string { + t.Helper() + raw, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + path := filepath.Join(t.TempDir(), "build-source-manifest.json") + mustWriteFile(t, path, string(raw)) + return path +} + +func TestLoadSourceBuildManifestDirect(t *testing.T) { + path := writeManifestFixture(t, directManifestFixture()) + + manifest, err := loadSourceBuildManifest(path) + if err != nil { + t.Fatalf("load: %v", err) + } + if manifest.Schema != 1 { + t.Fatalf("schema = %d, want 1", manifest.Schema) + } + if manifest.BuilderID != buildinfo.String() { + t.Fatalf("builder_id = %q, want %q", manifest.BuilderID, buildinfo.String()) + } + if len(manifest.HAKs) != 1 || manifest.HAKs[0].Name != "core_01" { + t.Fatalf("unexpected haks: %#v", manifest.HAKs) + } + source, ok := manifest.AssetSources["core/a.tga"] + if !ok || source.SHA256 != validAsset || source.SizeBytes != 4 { + t.Fatalf("unexpected asset source: %#v", manifest.AssetSources) + } + if err := validateDirectSourceManifest(manifest); err != nil { + t.Fatalf("validate valid manifest: %v", err) + } +} + +func TestValidateDirectSourceManifestRejectsUnsupportedSchema(t *testing.T) { + manifest := directManifestFixture() + manifest.Schema = 2 + if err := validateDirectSourceManifest(manifest); err == nil { + t.Fatal("expected schema rejection") + } +} + +func TestValidateDirectSourceManifestRejectsBadSHA(t *testing.T) { + manifest := directManifestFixture() + manifest.AssetSources["core/a.tga"] = SourceAsset{SHA256: "NOTHEX", SizeBytes: 4} + if err := validateDirectSourceManifest(manifest); err == nil { + t.Fatal("expected bad sha rejection") + } +} + +func TestValidateDirectSourceManifestRejectsNegativeSize(t *testing.T) { + manifest := directManifestFixture() + manifest.AssetSources["core/a.tga"] = SourceAsset{SHA256: validAsset, SizeBytes: -1} + if err := validateDirectSourceManifest(manifest); err == nil { + t.Fatal("expected negative size rejection") + } +} + +func TestValidateDirectSourceManifestRejectsMissingSource(t *testing.T) { + manifest := directManifestFixture() + delete(manifest.AssetSources, "core/a.tga") + if err := validateDirectSourceManifest(manifest); err == nil { + t.Fatal("expected missing source rejection") + } +} + +func TestValidateDirectSourceManifestRejectsUnusedSource(t *testing.T) { + manifest := directManifestFixture() + manifest.AssetSources["core/orphan.tga"] = SourceAsset{SHA256: validAsset, SizeBytes: 4} + if err := validateDirectSourceManifest(manifest); err == nil { + t.Fatal("expected unused source rejection") + } +} + +func TestValidateDirectSourceManifestRejectsDuplicateAssetAcrossHAKs(t *testing.T) { + manifest := directManifestFixture() + manifest.HAKs = append(manifest.HAKs, SourceManifestHAK{ + Name: "core_02", + Group: "core", + Priority: 1, + MaxBytes: 1024, + Assets: []string{"core/a.tga"}, + }) + if err := validateDirectSourceManifest(manifest); err == nil { + t.Fatal("expected duplicate-asset rejection") + } +} + +func TestValidateDirectSourceManifestRejectsTraversalPath(t *testing.T) { + manifest := directManifestFixture() + manifest.HAKs[0].Assets = []string{"../escape.tga"} + manifest.AssetSources = map[string]SourceAsset{ + "../escape.tga": {SHA256: validAsset, SizeBytes: 4}, + } + if err := validateDirectSourceManifest(manifest); err == nil { + t.Fatal("expected traversal rejection") + } +} + +func TestValidateDirectSourceManifestRejectsBuilderMismatch(t *testing.T) { + manifest := directManifestFixture() + manifest.BuilderID = "crucible deadbeef" + if err := validateDirectSourceManifest(manifest); err == nil { + t.Fatal("expected builder mismatch rejection") + } +} + +func TestContentAddressedBlobPath(t *testing.T) { + root := "/var/cache/blobs" + got, err := contentAddressedBlobPath(root, validAsset) + if err != nil { + t.Fatalf("blob path: %v", err) + } + want := filepath.Join(root, "sha256", "aa", "aa", validAsset) + if got != want { + t.Fatalf("blob path = %q, want %q", got, want) + } + if _, err := contentAddressedBlobPath(root, "NOTHEX"); err == nil { + t.Fatal("expected invalid sha rejection") + } + if _, err := contentAddressedBlobPath("", validAsset); err == nil { + t.Fatal("expected empty root rejection") + } +}