Compare commits

...
2 Commits
Author SHA1 Message Date
archvillainette cdbbba3181 add content-addressed hak source mode (#7)
test-image / build-image (push) Successful in 46s
test / test (push) Successful in 1m26s
build-binaries / build-binaries (push) Successful in 2m20s
build-image / publish (push) Successful in 31s
Reviewed-on: #7
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
2026-06-19 20:56:36 +00:00
archvillainette 223b9831c5 Always Gitkeep Cache (#6)
build-binaries / build-binaries (push) Successful in 2m2s
test-image / build-image (push) Successful in 43s
test / test (push) Successful in 1m20s
Reviewed-on: #6
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
2026-06-18 07:29:09 +00:00
13 changed files with 977 additions and 63 deletions
View File
+2 -2
View File
@@ -6,8 +6,8 @@ nwn-tool
sow-toolkit
# Go / build cache
.cache/
.cache/**
.cache/*
!.cache/.gitkeep
# nix build symlink
result
+7 -23
View File
@@ -5,10 +5,9 @@ alwaysApply: true
# sow-tools / Crucible Agent Guide
This repo owns the **builder logic** for the migration: one Go module
This repo owns the **builder logic:** one Go module
(`git.westgate.pw/ShadowsOverWestgate/sow-tools`) producing the `crucible`
dispatcher and the `crucible-<name>` binaries (D11). Read
`../AGENTS.md` (migration hub) and `../../KICKOFF_PROMPT.md` first.
dispatcher and the `crucible-<name>` binaries.
## What this repo owns / does not own
@@ -20,19 +19,11 @@ is `sow-platform`).
## Rules
1. **Migrated logic, not a fresh rewrite.** The `internal/` packages (`app`,
`pipeline`, `project`, `erf`, `gff`, `topdata`, `music`, `changelog`,
`validator`) were folded in from `gitea/sow-tools` at cutover; wired builders
delegate to `internal/app`'s command surface. Keep them in step with upstream
fixes rather than diverging silently.
2. **Fail closed, never fake.** A builder with no migrated logic yet (`depot`)
1. **Fail closed, never fake.** A builder with no migrated logic yet (`depot`)
exits `70`. Do not stub a builder to emit a placeholder artifact.
3. **Binaries are not committed.** They are CI artifacts / image layers (D19).
2. **Binaries are not committed.** They are CI artifacts / image layers.
`/bin/`, `*.exe`, `nwn-tool`, `sow-toolkit` are gitignored.
4. **No home-dir / `NWN_ROOT` guessing.** Builders take roots explicitly via
flag or env (project resolution is CWD-based, never `$HOME`). See
[`docs/consumer-contract.md`](docs/consumer-contract.md).
5. **The registry is the command surface.** `internal/dispatch.Registry` is the
3. **The registry is the command surface.** `internal/dispatch.Registry` is the
single source of truth; keep it in sync with `cmd/` and
[`docs/command-surface.md`](docs/command-surface.md). Adding a builder = a
`cmd/crucible-<name>/main.go` shim + a `Registry` entry + a doc row.
@@ -40,11 +31,8 @@ is `sow-platform`).
## Wiring a builder
1. Ensure the relevant `internal/` package(s) cover the work.
2. Add the legacy command(s) to the builder's `Legacy`/`Extra` set and set
`Wired: true` in `internal/dispatch`; the dispatcher delegates to
`app.Run`. `depot` is the remaining unwired builder.
3. Add tests; keep outputs deterministic (same input → same bytes).
4. `make check` must stay green; update `make smoke` to expect the wired exit.
2. Add tests; keep outputs deterministic (same input → same bytes).
3. `make check` must stay green; update `make smoke` to expect the wired exit.
## Commands
@@ -54,7 +42,3 @@ make build # cmd/* -> ./bin
make smoke # assert fail-closed contract
make image # crucible:<sha>
```
## Git
Never commit, branch, or push. Suggest a commit message; let the operator do it.
+16
View File
@@ -45,6 +45,22 @@ crucible changelog [args] -> legacy `build-changelog`
`crucible list` is machine-readable so CI can enumerate builders without parsing
help text.
## HAK builder source modes
```text
build-haks [--hak <hak-name> ...] [--archive <archive-name> ...]
[--source-manifest <path>] [--content-addressed-root <path>]
[--plan-only] [--skip-music] [--music-dataset <id> ...]
[--quiet|--verbose|--debug]
```
When a source manifest contains `asset_sources`, pass
`--content-addressed-root <directory>` (or
`--content-addressed-root=<directory>`). Crucible resolves each declared SHA-256
under `sha256/<first-two>/<next-two>/<full-sha256>` and verifies streamed bytes
while writing the selected HAK archives. Source manifests without
`asset_sources` continue to use the configured assets tree.
## Global flags (planned, at wiring time)
`--quiet`, `--verbose`, `--debug` (the legacy verbosity model), passed after the
+16 -1
View File
@@ -469,6 +469,7 @@ type buildHAKOptions struct {
filteredHAKs []string
filteredArchives []string
sourceManifest string
contentAddressedRoot string
musicDatasets []string
skipMusic bool
planOnly bool
@@ -1048,6 +1049,7 @@ func runBuildHAKs(ctx context) error {
Progress: console.progress,
ArchiveNames: opts.filteredArchives,
SourceManifestPath: opts.sourceManifest,
ContentAddressedRoot: opts.contentAddressedRoot,
SkipMusic: opts.skipMusic,
MusicDatasetIDs: opts.musicDatasets,
}
@@ -1074,7 +1076,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
arg := args[index]
switch arg {
case "-h", "--help":
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]")
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--content-addressed-root <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]")
case "--hak":
index++
if index >= len(args) {
@@ -1109,6 +1111,12 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
return opts, errors.New("--source-manifest requires a value")
}
opts.sourceManifest = args[index]
case "--content-addressed-root":
index++
if index >= len(args) {
return opts, errors.New("--content-addressed-root requires a value")
}
opts.contentAddressedRoot = args[index]
default:
if value, ok, err := requireInlineFlagValue(arg, "--hak"); ok || err != nil {
if err != nil {
@@ -1131,6 +1139,13 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
opts.sourceManifest = value
continue
}
if value, ok, err := requireInlineFlagValue(arg, "--content-addressed-root"); ok || err != nil {
if err != nil {
return opts, err
}
opts.contentAddressedRoot = value
continue
}
if value, ok, err := requireInlineFlagValue(arg, "--music-dataset"); ok || err != nil {
if err != nil {
return opts, err
+42
View File
@@ -11,6 +11,48 @@ import (
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline"
)
func TestParseBuildHAKArgsContentAddressedRoot(t *testing.T) {
tests := []struct {
name string
args []string
want string
}{
{
name: "separated",
args: []string{"--content-addressed-root", "/var/cache/blobs"},
want: "/var/cache/blobs",
},
{
name: "inline",
args: []string{"--content-addressed-root=/var/cache/blobs"},
want: "/var/cache/blobs",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
opts, err := parseBuildHAKArgs(tt.args)
if err != nil {
t.Fatalf("parse build-haks args: %v", err)
}
if opts.contentAddressedRoot != tt.want {
t.Fatalf("content-addressed root = %q, want %q", opts.contentAddressedRoot, tt.want)
}
})
}
}
func TestParseBuildHAKArgsHelpListsContentAddressedRoot(t *testing.T) {
_, err := parseBuildHAKArgs([]string{"--help"})
if err == nil {
t.Fatal("expected help usage error")
}
const want = "usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--content-addressed-root <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]"
if err.Error() != want {
t.Fatalf("help usage = %q, want %q", err.Error(), want)
}
}
func TestRunBuildTopPackageUsesCachedCompiledOutputs(t *testing.T) {
root := t.TempDir()
mkdirAll(t, filepath.Join(root, "build"))
+20 -1
View File
@@ -2,14 +2,19 @@ package erf
import (
"bytes"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"fmt"
"io"
"os"
"regexp"
"sort"
"strings"
)
var expectedSHA256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
const (
headerSize = 160
versionV10 = "V1.0"
@@ -30,6 +35,9 @@ type Resource struct {
Data []byte
SourcePath string
Size int64
// ExpectedSHA256, when set, is the lowercase hex SHA-256 the streamed
// SourcePath payload must hash to. A mismatch fails the write.
ExpectedSHA256 string
}
type header struct {
@@ -407,18 +415,29 @@ func writeResourceData(w io.Writer, resource Resource) error {
return err
}
if resource.ExpectedSHA256 != "" && !expectedSHA256Pattern.MatchString(resource.ExpectedSHA256) {
return fmt.Errorf("resource %q has invalid expected sha256", resource.SourcePath)
}
file, err := os.Open(resource.SourcePath)
if err != nil {
return fmt.Errorf("open resource %q: %w", resource.SourcePath, err)
}
defer file.Close()
written, err := io.Copy(w, file)
hash := sha256.New()
written, err := io.Copy(io.MultiWriter(w, hash), file)
if err != nil {
return fmt.Errorf("copy resource %q: %w", resource.SourcePath, err)
}
if resource.Size > 0 && written != resource.Size {
return fmt.Errorf("copy resource %q: expected %d bytes, wrote %d", resource.SourcePath, resource.Size, written)
}
if resource.ExpectedSHA256 != "" {
got := hex.EncodeToString(hash.Sum(nil))
if got != resource.ExpectedSHA256 {
return fmt.Errorf("resource %q sha256 mismatch: expected %s, got %s", resource.SourcePath, resource.ExpectedSHA256, got)
}
}
return nil
}
+63
View File
@@ -2,9 +2,72 @@ package erf
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"strings"
"testing"
)
func writeTempPayload(t *testing.T, payload []byte) string {
t.Helper()
path := filepath.Join(t.TempDir(), "payload.bin")
if err := os.WriteFile(path, payload, 0o644); err != nil {
t.Fatalf("write payload: %v", err)
}
return path
}
func TestWriteRejectsSourcePathSHA256Mismatch(t *testing.T) {
payload := []byte("hello world")
path := writeTempPayload(t, payload)
archive := New("HAK ", []Resource{{
Name: "asset",
Type: 0x0003,
SourcePath: path,
Size: int64(len(payload)),
ExpectedSHA256: strings.Repeat("a", 64),
}})
var buf bytes.Buffer
err := Write(&buf, archive)
if err == nil {
t.Fatal("expected sha256 mismatch error")
}
if !strings.Contains(err.Error(), "sha256 mismatch") {
t.Fatalf("error = %v, want sha256 mismatch", err)
}
}
func TestWriteAcceptsMatchingSourcePathSHA256(t *testing.T) {
payload := []byte("hello world")
path := writeTempPayload(t, payload)
sum := sha256.Sum256(payload)
archive := New("HAK ", []Resource{{
Name: "asset",
Type: 0x0003,
SourcePath: path,
Size: int64(len(payload)),
ExpectedSHA256: hex.EncodeToString(sum[:]),
}})
var buf bytes.Buffer
if err := Write(&buf, archive); err != nil {
t.Fatalf("write: %v", err)
}
decoded, err := Read(bytes.NewReader(buf.Bytes()))
if err != nil {
t.Fatalf("read: %v", err)
}
if len(decoded.Resources) != 1 || string(decoded.Resources[0].Data) != string(payload) {
t.Fatalf("unexpected payload: %#v", decoded.Resources)
}
}
func TestArchiveRoundTrip(t *testing.T) {
archive := New("MOD ", []Resource{
{Name: "module", Type: 0x07DE, Data: []byte("ifo")},
+124 -19
View File
@@ -109,6 +109,7 @@ type BuildHAKOptions struct {
Progress ProgressFunc
ArchiveNames []string
SourceManifestPath string
ContentAddressedRoot string
SkipMusic bool
MusicDatasetIDs []string
}
@@ -196,7 +197,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu
}
func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs)
return planOrBuildHAKs(p, opts.Progress, true, opts)
}
func PlanHAKs(p *project.Project) (BuildResult, error) {
@@ -208,18 +209,18 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul
}
func PlanHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
return planOrBuildHAKs(p, opts.Progress, false, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs)
return planOrBuildHAKs(p, opts.Progress, false, opts)
}
func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
return planOrBuildHAKs(p, progress, true, nil, "", false, nil)
return planOrBuildHAKs(p, progress, true, BuildHAKOptions{})
}
func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
return planOrBuildHAKs(p, progress, false, nil, "", false, nil)
return planOrBuildHAKs(p, progress, false, BuildHAKOptions{})
}
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string, skipMusic bool, musicDatasetIDs []string) (result BuildResult, err error) {
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, opts BuildHAKOptions) (result BuildResult, err error) {
preserveExistingHAKs := p.EffectiveConfig().Build.KeepExistingHAKs || envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
progressf(progress, "Validating project...")
@@ -227,15 +228,23 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
return BuildResult{}, err
}
directManifest, err := loadDirectSourceManifest(opts.SourceManifestPath)
if err != nil {
return BuildResult{}, err
}
if directManifest != nil {
return buildDirectHAKs(p, progress, writeArchives, directManifest, opts)
}
progressf(progress, "Collecting asset resources...")
allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(sourceManifestPath)
allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(opts.SourceManifestPath)
if err != nil {
return BuildResult{}, err
}
musicAssets := &preparedMusicAssets{SkipSourceRel: map[string]struct{}{}}
if !skipMusic {
if !opts.SkipMusic {
musicAssets, err = prepareMusicAssetsWithOptions(p, musicPrepareOptions{
datasetIDs: musicDatasetIDs,
datasetIDs: opts.MusicDatasetIDs,
write: writeArchives,
})
if err != nil {
@@ -294,7 +303,7 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
if err != nil {
return BuildResult{}, err
}
chunks, err = filterHAKChunksByName(chunks, archiveNames)
chunks, err = filterHAKChunksByName(chunks, opts.ArchiveNames)
if err != nil {
return BuildResult{}, err
}
@@ -369,16 +378,8 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil {
return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err)
}
hakOutput, err := os.Create(hakPath)
if err != nil {
return BuildResult{}, fmt.Errorf("create hak archive: %w", err)
}
if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil {
hakOutput.Close()
return BuildResult{}, fmt.Errorf("write hak archive: %w", err)
}
if err := hakOutput.Close(); err != nil {
return BuildResult{}, fmt.Errorf("close hak archive: %w", err)
if err := writeHAKArchive(hakPath, resourceSlice(chunk.Assets)); err != nil {
return BuildResult{}, err
}
}
@@ -408,6 +409,110 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
return result, nil
}
// buildDirectHAKs packages HAKs directly from the content-addressed source blob
// cache. It skips music preparation, generated 2da discovery, git creation-time
// lookups, LFS discovery, and any materialized asset tree scan.
func buildDirectHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, manifest *SourceBuildManifest, opts BuildHAKOptions) (BuildResult, error) {
progressf(progress, "Collecting asset resources...")
assetResources, err := collectDirectSourceResources(manifest, opts.ContentAddressedRoot)
if err != nil {
return BuildResult{}, err
}
result := BuildResult{HAKAssets: len(assetResources)}
progressf(progress, "Planning HAK chunks...")
chunks, err := chunksFromSourceManifest(assetResources, manifest.HAKs)
if err != nil {
return BuildResult{}, err
}
chunks, err = filterHAKChunksByName(chunks, opts.ArchiveNames)
if err != nil {
return BuildResult{}, err
}
buildManifest := BuildManifest{
ModuleHAKs: append([]string(nil), manifest.ModuleHAKs...),
HAKs: make([]BuildManifestHAK, 0, len(chunks)),
}
for _, chunk := range chunks {
buildManifest.HAKs = append(buildManifest.HAKs, buildManifestEntry(chunk))
}
manifestPath := p.HAKManifestPath()
manifestBytes, err := json.MarshalIndent(buildManifest, "", " ")
if err != nil {
return BuildResult{}, fmt.Errorf("marshal hak manifest: %w", err)
}
manifestBytes = append(manifestBytes, '\n')
if writeArchives {
progressf(progress, "Writing HAK archives...")
result.HAKSummary.Total = len(chunks)
result.HAKPaths = make([]string, 0, len(chunks))
for index, chunk := range chunks {
hakPath := p.HAKArchivePath(chunk.Name)
progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets)))
if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil {
return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err)
}
if err := writeHAKArchive(hakPath, resourceSlice(chunk.Assets)); err != nil {
return BuildResult{}, err
}
result.HAKSummary.Written++
result.HAKSummary.Actions = append(result.HAKSummary.Actions, HAKArchiveAction{
Name: chunk.Name,
AssetCount: len(chunk.Assets),
OutputPath: hakPath,
})
result.HAKPaths = append(result.HAKPaths, hakPath)
}
}
progressf(progress, "Writing HAK manifest...")
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
return BuildResult{}, fmt.Errorf("create hak manifest dir: %w", err)
}
if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil {
return BuildResult{}, fmt.Errorf("write hak manifest: %w", err)
}
result.Manifest = manifestPath
return result, nil
}
// writeHAKArchive publishes a completed HAK atomically, so a failed write never
// leaves a partial archive at the final path.
func writeHAKArchive(hakPath string, resources []erf.Resource) error {
tmpPath := hakPath + ".tmp"
if err := os.Remove(tmpPath); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("remove stale temporary hak %s: %w", tmpPath, err)
}
output, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
if err != nil {
return fmt.Errorf("create temporary hak archive %s: %w", tmpPath, err)
}
if err := erf.Write(output, erf.New("HAK ", resources)); err != nil {
output.Close()
os.Remove(tmpPath)
return fmt.Errorf("write hak archive: %w", err)
}
if err := output.Sync(); err != nil {
output.Close()
os.Remove(tmpPath)
return fmt.Errorf("sync hak archive: %w", err)
}
if err := output.Close(); err != nil {
os.Remove(tmpPath)
return fmt.Errorf("close hak archive: %w", err)
}
if err := os.Rename(tmpPath, hakPath); err != nil {
os.Remove(tmpPath)
return fmt.Errorf("publish hak archive %s: %w", hakPath, err)
}
return nil
}
func writeAutogenManifestOutputs(progress ProgressFunc, manifests []topdata.ProducedAutogenManifest) error {
if len(manifests) == 0 {
return nil
+29
View File
@@ -17,6 +17,35 @@ import (
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
func TestBuildHAKsCleansFailedTemporaryArchive(t *testing.T) {
root := t.TempDir()
p := loadDirectProject(t, root)
blobs := filepath.Join(root, "blobs")
sha := strings.Repeat("a", 64)
writeBlobAt(t, blobs, sha, "wxyz")
manifestPath := writeDirectManifest(t, root,
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: 4}},
[]string{"core/a.tga"})
tmpPath := filepath.Join(root, "build", "core_01.hak.tmp")
mustWriteFile(t, tmpPath, "stale partial archive")
_, err := BuildHAKsWithOptions(p, BuildHAKOptions{
SourceManifestPath: manifestPath,
ContentAddressedRoot: blobs,
})
if err == nil {
t.Fatal("expected corrupt input to fail")
}
if _, err := os.Stat(filepath.Join(root, "build", "core_01.hak")); !errors.Is(err, os.ErrNotExist) {
t.Fatal("corrupt input must not leave a completed hak")
}
if _, err := os.Stat(tmpPath); !errors.Is(err, os.ErrNotExist) {
t.Fatal("failed build must clean temporary hak")
}
}
func TestBuildThenExtract(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
+249
View File
@@ -0,0 +1,249 @@
package pipeline
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
const directProjectConfig = `{
"module": {
"name": "Test Module",
"resref": "testmod",
"hak_order": ["core_01"]
},
"paths": {
"source": "src",
"assets": "assets",
"build": "build"
}
}
`
func writeBlobAt(t *testing.T, root, sha, content string) {
t.Helper()
dir := filepath.Join(root, "sha256", sha[0:2], sha[2:4])
mustMkdir(t, dir)
mustWriteFile(t, filepath.Join(dir, sha), content)
}
func writeBlob(t *testing.T, root string, payload []byte) string {
t.Helper()
sum := sha256.Sum256(payload)
sha := hex.EncodeToString(sum[:])
writeBlobAt(t, root, sha, string(payload))
return sha
}
func writeDirectManifest(t *testing.T, root string, sources map[string]SourceAsset, assets []string) string {
t.Helper()
manifest := SourceBuildManifest{
Schema: 1,
BuilderID: buildinfo.String(),
ModuleHAKs: []string{"core_01"},
HAKs: []SourceManifestHAK{{
Name: "core_01",
Group: "core",
Priority: 1,
MaxBytes: 1 << 20,
Assets: assets,
}},
AssetSources: sources,
}
raw, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("marshal manifest: %v", err)
}
path := filepath.Join(root, "build-source-manifest.json")
mustWriteFile(t, path, string(raw))
return path
}
func loadDirectProject(t *testing.T, root string) *project.Project {
t.Helper()
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), directProjectConfig)
mustMkdir(t, filepath.Join(root, "build"))
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
return p
}
func TestBuildHAKsFromContentAddressedManifestWithoutAssetsTree(t *testing.T) {
root := t.TempDir()
p := loadDirectProject(t, root)
blobs := filepath.Join(root, "blobs")
payload := []byte("tga-payload-bytes")
sha := writeBlob(t, blobs, payload)
manifestPath := writeDirectManifest(t, root,
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: int64(len(payload))}},
[]string{"core/a.tga"})
result, err := BuildHAKsWithOptions(p, BuildHAKOptions{
SourceManifestPath: manifestPath,
ContentAddressedRoot: blobs,
})
if err != nil {
t.Fatalf("build direct haks: %v", err)
}
if result.HAKAssets != 1 {
t.Fatalf("hak assets = %d, want 1", result.HAKAssets)
}
if _, err := os.Stat(filepath.Join(root, "assets")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("direct build must not materialize an assets tree (stat err = %v)", err)
}
hakPath := filepath.Join(root, "build", "core_01.hak")
raw, err := os.ReadFile(hakPath)
if err != nil {
t.Fatalf("read hak: %v", err)
}
archive, err := erf.Read(bytes.NewReader(raw))
if err != nil {
t.Fatalf("decode hak: %v", err)
}
if len(archive.Resources) != 1 {
t.Fatalf("hak resources = %d, want 1", len(archive.Resources))
}
if got := string(archive.Resources[0].Data); got != string(payload) {
t.Fatalf("resource payload = %q, want %q", got, payload)
}
if archive.Resources[0].Name != "a" {
t.Fatalf("resref = %q, want a", archive.Resources[0].Name)
}
}
func TestBuildHAKsFromContentAddressedManifestRejectsMissingRoot(t *testing.T) {
root := t.TempDir()
p := loadDirectProject(t, root)
blobs := filepath.Join(root, "blobs")
payload := []byte("tga-payload-bytes")
sha := writeBlob(t, blobs, payload)
manifestPath := writeDirectManifest(t, root,
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: int64(len(payload))}},
[]string{"core/a.tga"})
_, err := BuildHAKsWithOptions(p, BuildHAKOptions{SourceManifestPath: manifestPath})
if err == nil {
t.Fatal("expected failure when content-addressed root is missing")
}
}
func TestBuildHAKsFromContentAddressedManifestRejectsMissingBlob(t *testing.T) {
root := t.TempDir()
p := loadDirectProject(t, root)
blobs := filepath.Join(root, "blobs")
sha := strings.Repeat("b", 64)
manifestPath := writeDirectManifest(t, root,
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: 4}},
[]string{"core/a.tga"})
_, err := BuildHAKsWithOptions(p, BuildHAKOptions{
SourceManifestPath: manifestPath,
ContentAddressedRoot: blobs,
})
if err == nil {
t.Fatal("expected failure for missing source blob")
}
}
func TestBuildHAKsFromContentAddressedManifestRejectsCorruptBlob(t *testing.T) {
root := t.TempDir()
p := loadDirectProject(t, root)
blobs := filepath.Join(root, "blobs")
declared := sha256.Sum256([]byte("abcd"))
sha := hex.EncodeToString(declared[:])
// Same length, different bytes: size check passes, content hash mismatches.
writeBlobAt(t, blobs, sha, "wxyz")
manifestPath := writeDirectManifest(t, root,
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: 4}},
[]string{"core/a.tga"})
_, err := BuildHAKsWithOptions(p, BuildHAKOptions{
SourceManifestPath: manifestPath,
ContentAddressedRoot: blobs,
})
if err == nil {
t.Fatal("expected sha256 mismatch failure for corrupt blob")
}
if !strings.Contains(err.Error(), "sha256 mismatch") {
t.Fatalf("error = %v, want sha256 mismatch", err)
}
if _, statErr := os.Stat(filepath.Join(root, "build", "core_01.hak")); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("corrupt build must not leave a completed hak (stat err = %v)", statErr)
}
}
func TestLegacySourceManifestStillBuildsFromAssetsTree(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "assets", "core"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
"module": {"name": "Test Module", "resref": "testmod", "hak_order": ["group:core"]},
"paths": {"source": "src", "assets": "assets", "build": "build"},
"haks": [{"name": "core", "priority": 1, "include": ["core/**"]}]
}
`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {"struct_type": 0, "fields": [
{"label": "Mod_Name", "type": "CExoString", "value": "Test Module"},
{"label": "Mod_HakList", "type": "List", "value": []}
]}
}
`)
mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), strings.Repeat("a", 16))
mustWriteFile(t, filepath.Join(root, "assets", "core", "b.tga"), strings.Repeat("b", 16))
manifestPath := filepath.Join(root, "legacy-source-manifest.json")
mustWriteFile(t, manifestPath, `{
"module_haks": ["core"],
"haks": [{"name": "core", "group": "core", "priority": 1, "assets": ["core/a.tga", "core/b.tga"]}]
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
result, err := BuildHAKsWithOptions(p, BuildHAKOptions{SourceManifestPath: manifestPath})
if err != nil {
t.Fatalf("legacy build: %v", err)
}
if len(result.HAKPaths) != 1 {
t.Fatalf("hak paths = %d, want 1", len(result.HAKPaths))
}
raw, err := os.ReadFile(filepath.Join(root, "build", "core.hak"))
if err != nil {
t.Fatalf("read legacy hak: %v", err)
}
archive, err := erf.Read(bytes.NewReader(raw))
if err != nil {
t.Fatalf("decode legacy hak: %v", err)
}
if len(archive.Resources) != 2 {
t.Fatalf("legacy hak resources = %d, want 2", len(archive.Resources))
}
}
+232
View File
@@ -0,0 +1,232 @@
package pipeline
import (
"encoding/json"
"fmt"
"os"
pathpkg "path"
"path/filepath"
"regexp"
"slices"
"strings"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
)
// SourceAsset is one content-addressed source blob referenced by a direct source
// manifest. The blob is resolved by sha256 under the content-addressed root.
type SourceAsset struct {
SHA256 string `json:"sha256"`
SizeBytes int64 `json:"size_bytes"`
}
// SourceManifestHAK describes one output HAK and the logical asset paths it packs.
type SourceManifestHAK struct {
Name string `json:"name"`
Group string `json:"group"`
Priority int `json:"priority"`
MaxBytes int64 `json:"max_bytes"`
Optional bool `json:"optional,omitempty"`
Assets []string `json:"assets"`
BuildKey string `json:"build_key"`
}
// SourceBuildManifest is the generated, content-addressed input to a HAK build.
// It carries asset_sources, so Crucible reads source bytes directly from the blob
// cache without scanning a materialized asset tree.
type SourceBuildManifest struct {
Schema int `json:"schema"`
BuilderID string `json:"builder_id"`
ModuleHAKs []string `json:"module_haks"`
HAKs []SourceManifestHAK `json:"haks"`
AssetSources map[string]SourceAsset `json:"asset_sources"`
}
var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
// hasAssetSources reports whether raw JSON carries the direct content-addressed
// schema. A legacy BuildManifest has no asset_sources object.
func hasAssetSources(raw []byte) bool {
var header struct {
AssetSources map[string]json.RawMessage `json:"asset_sources"`
}
if err := json.Unmarshal(raw, &header); err != nil {
return false
}
return header.AssetSources != nil
}
func loadSourceBuildManifest(path string) (*SourceBuildManifest, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read source manifest %s: %w", path, err)
}
var manifest SourceBuildManifest
if err := json.Unmarshal(raw, &manifest); err != nil {
return nil, fmt.Errorf("parse source manifest %s: %w", path, err)
}
return &manifest, nil
}
func validateDirectSourceManifest(manifest *SourceBuildManifest) error {
if manifest.Schema != 1 {
return fmt.Errorf("unsupported source manifest schema %d", manifest.Schema)
}
if manifest.BuilderID != buildinfo.String() {
return fmt.Errorf("source manifest builder_id %q does not match builder %q", manifest.BuilderID, buildinfo.String())
}
if len(manifest.ModuleHAKs) == 0 {
return fmt.Errorf("source manifest module_haks is empty")
}
referenced := make(map[string]struct{}, len(manifest.AssetSources))
for _, hak := range manifest.HAKs {
if strings.TrimSpace(hak.Name) == "" {
return fmt.Errorf("source manifest hak has empty name")
}
for _, rel := range hak.Assets {
if !validLogicalAssetPath(rel) {
return fmt.Errorf("source manifest hak %s has invalid asset path %q", hak.Name, rel)
}
if _, dup := referenced[rel]; dup {
return fmt.Errorf("source manifest asset %q referenced by more than one hak", rel)
}
referenced[rel] = struct{}{}
source, ok := manifest.AssetSources[rel]
if !ok {
return fmt.Errorf("source manifest hak %s asset %q has no asset_sources entry", hak.Name, rel)
}
if !sha256Pattern.MatchString(source.SHA256) {
return fmt.Errorf("source manifest asset %q has invalid sha256 %q", rel, source.SHA256)
}
if source.SizeBytes < 0 {
return fmt.Errorf("source manifest asset %q has negative size %d", rel, source.SizeBytes)
}
}
}
for rel := range manifest.AssetSources {
if !validLogicalAssetPath(rel) {
return fmt.Errorf("source manifest asset_sources has invalid path %q", rel)
}
if _, ok := referenced[rel]; !ok {
return fmt.Errorf("source manifest asset_sources entry %q is not referenced by any hak", rel)
}
}
return nil
}
func validLogicalAssetPath(path string) bool {
if path == "" || filepath.IsAbs(path) || strings.Contains(path, `\`) {
return false
}
clean := pathpkg.Clean(path)
return clean == path && clean != "." && clean != ".." &&
!strings.HasPrefix(clean, "../")
}
func contentAddressedBlobPath(root, sha string) (string, error) {
if strings.TrimSpace(root) == "" {
return "", fmt.Errorf("content-addressed root is required")
}
if !sha256Pattern.MatchString(sha) {
return "", fmt.Errorf("invalid content-addressed sha256 %q", sha)
}
return filepath.Join(root, "sha256", sha[0:2], sha[2:4], sha), nil
}
// loadDirectSourceManifest returns a parsed direct source manifest when path
// points at a content-addressed manifest. It returns (nil, nil) for an empty
// path or a legacy manifest without asset_sources, so the caller falls through
// to the legacy path-backed build.
func loadDirectSourceManifest(path string) (*SourceBuildManifest, error) {
if strings.TrimSpace(path) == "" {
return nil, nil
}
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read source manifest %s: %w", path, err)
}
if !hasAssetSources(raw) {
return nil, nil
}
var manifest SourceBuildManifest
if err := json.Unmarshal(raw, &manifest); err != nil {
return nil, fmt.Errorf("parse source manifest %s: %w", path, err)
}
return &manifest, nil
}
// collectDirectSourceResources resolves each manifest-named blob under contentRoot
// and produces sorted asset resources that stream from the blob cache with their
// expected SHA-256. It never scans a materialized asset tree.
func collectDirectSourceResources(manifest *SourceBuildManifest, contentRoot string) ([]assetResource, error) {
if err := validateDirectSourceManifest(manifest); err != nil {
return nil, err
}
if strings.TrimSpace(contentRoot) == "" {
return nil, fmt.Errorf("content-addressed source manifest requires --content-addressed-root")
}
resources := make([]assetResource, 0, len(manifest.AssetSources))
for _, hak := range manifest.HAKs {
for _, logicalPath := range hak.Assets {
source := manifest.AssetSources[logicalPath]
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(logicalPath)), ".")
resourceType, ok := erf.HAKResourceTypeForExtension(extension)
if !ok {
return nil, fmt.Errorf("unsupported HAK resource extension %q for %s", filepath.Ext(logicalPath), logicalPath)
}
resref := strings.ToLower(strings.TrimSuffix(filepath.Base(logicalPath), filepath.Ext(logicalPath)))
blobPath, err := contentAddressedBlobPath(contentRoot, source.SHA256)
if err != nil {
return nil, err
}
info, err := os.Stat(blobPath)
if err != nil {
return nil, fmt.Errorf("missing source blob for %s: %w", logicalPath, err)
}
if info.Size() != source.SizeBytes {
return nil, fmt.Errorf("source blob size mismatch for %s: expected %d, got %d", logicalPath, source.SizeBytes, info.Size())
}
resource := erf.Resource{
Name: resref,
Type: resourceType,
SourcePath: blobPath,
Size: source.SizeBytes,
ExpectedSHA256: source.SHA256,
}
resources = append(resources, assetResource{
Rel: logicalPath,
Resource: resource,
Size: erf.ArchiveSize([]erf.Resource{resource}),
ContentID: "sha256:" + source.SHA256,
})
}
}
slices.SortFunc(resources, func(a, b assetResource) int {
return compareResourceKeys(a.Resource, b.Resource)
})
return resources, nil
}
// chunksFromSourceManifest converts direct manifest HAK entries into build chunks
// reusing the legacy manifest chunker.
func chunksFromSourceManifest(assets []assetResource, entries []SourceManifestHAK) ([]hakChunk, error) {
converted := make([]BuildManifestHAK, len(entries))
for index, entry := range entries {
converted[index] = BuildManifestHAK{
Name: entry.Name,
Group: entry.Group,
Priority: entry.Priority,
MaxBytes: entry.MaxBytes,
Optional: entry.Optional,
Assets: entry.Assets,
}
}
return chunksFromManifest(assets, converted)
}
+160
View File
@@ -0,0 +1,160 @@
package pipeline
import (
"encoding/json"
"path/filepath"
"testing"
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
)
const validAsset = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
// directManifestFixture returns a minimal valid direct source manifest using the
// running builder identity, so validation does not reject it on builder mismatch.
func directManifestFixture() *SourceBuildManifest {
return &SourceBuildManifest{
Schema: 1,
BuilderID: buildinfo.String(),
ModuleHAKs: []string{"core_01"},
HAKs: []SourceManifestHAK{
{
Name: "core_01",
Group: "core",
Priority: 1,
MaxBytes: 1024,
Assets: []string{"core/a.tga"},
},
},
AssetSources: map[string]SourceAsset{
"core/a.tga": {SHA256: validAsset, SizeBytes: 4},
},
}
}
func writeManifestFixture(t *testing.T, manifest *SourceBuildManifest) string {
t.Helper()
raw, err := json.Marshal(manifest)
if err != nil {
t.Fatalf("marshal fixture: %v", err)
}
path := filepath.Join(t.TempDir(), "build-source-manifest.json")
mustWriteFile(t, path, string(raw))
return path
}
func TestLoadSourceBuildManifestDirect(t *testing.T) {
path := writeManifestFixture(t, directManifestFixture())
manifest, err := loadSourceBuildManifest(path)
if err != nil {
t.Fatalf("load: %v", err)
}
if manifest.Schema != 1 {
t.Fatalf("schema = %d, want 1", manifest.Schema)
}
if manifest.BuilderID != buildinfo.String() {
t.Fatalf("builder_id = %q, want %q", manifest.BuilderID, buildinfo.String())
}
if len(manifest.HAKs) != 1 || manifest.HAKs[0].Name != "core_01" {
t.Fatalf("unexpected haks: %#v", manifest.HAKs)
}
source, ok := manifest.AssetSources["core/a.tga"]
if !ok || source.SHA256 != validAsset || source.SizeBytes != 4 {
t.Fatalf("unexpected asset source: %#v", manifest.AssetSources)
}
if err := validateDirectSourceManifest(manifest); err != nil {
t.Fatalf("validate valid manifest: %v", err)
}
}
func TestValidateDirectSourceManifestRejectsUnsupportedSchema(t *testing.T) {
manifest := directManifestFixture()
manifest.Schema = 2
if err := validateDirectSourceManifest(manifest); err == nil {
t.Fatal("expected schema rejection")
}
}
func TestValidateDirectSourceManifestRejectsBadSHA(t *testing.T) {
manifest := directManifestFixture()
manifest.AssetSources["core/a.tga"] = SourceAsset{SHA256: "NOTHEX", SizeBytes: 4}
if err := validateDirectSourceManifest(manifest); err == nil {
t.Fatal("expected bad sha rejection")
}
}
func TestValidateDirectSourceManifestRejectsNegativeSize(t *testing.T) {
manifest := directManifestFixture()
manifest.AssetSources["core/a.tga"] = SourceAsset{SHA256: validAsset, SizeBytes: -1}
if err := validateDirectSourceManifest(manifest); err == nil {
t.Fatal("expected negative size rejection")
}
}
func TestValidateDirectSourceManifestRejectsMissingSource(t *testing.T) {
manifest := directManifestFixture()
delete(manifest.AssetSources, "core/a.tga")
if err := validateDirectSourceManifest(manifest); err == nil {
t.Fatal("expected missing source rejection")
}
}
func TestValidateDirectSourceManifestRejectsUnusedSource(t *testing.T) {
manifest := directManifestFixture()
manifest.AssetSources["core/orphan.tga"] = SourceAsset{SHA256: validAsset, SizeBytes: 4}
if err := validateDirectSourceManifest(manifest); err == nil {
t.Fatal("expected unused source rejection")
}
}
func TestValidateDirectSourceManifestRejectsDuplicateAssetAcrossHAKs(t *testing.T) {
manifest := directManifestFixture()
manifest.HAKs = append(manifest.HAKs, SourceManifestHAK{
Name: "core_02",
Group: "core",
Priority: 1,
MaxBytes: 1024,
Assets: []string{"core/a.tga"},
})
if err := validateDirectSourceManifest(manifest); err == nil {
t.Fatal("expected duplicate-asset rejection")
}
}
func TestValidateDirectSourceManifestRejectsTraversalPath(t *testing.T) {
manifest := directManifestFixture()
manifest.HAKs[0].Assets = []string{"../escape.tga"}
manifest.AssetSources = map[string]SourceAsset{
"../escape.tga": {SHA256: validAsset, SizeBytes: 4},
}
if err := validateDirectSourceManifest(manifest); err == nil {
t.Fatal("expected traversal rejection")
}
}
func TestValidateDirectSourceManifestRejectsBuilderMismatch(t *testing.T) {
manifest := directManifestFixture()
manifest.BuilderID = "crucible deadbeef"
if err := validateDirectSourceManifest(manifest); err == nil {
t.Fatal("expected builder mismatch rejection")
}
}
func TestContentAddressedBlobPath(t *testing.T) {
root := "/var/cache/blobs"
got, err := contentAddressedBlobPath(root, validAsset)
if err != nil {
t.Fatalf("blob path: %v", err)
}
want := filepath.Join(root, "sha256", "aa", "aa", validAsset)
if got != want {
t.Fatalf("blob path = %q, want %q", got, want)
}
if _, err := contentAddressedBlobPath(root, "NOTHEX"); err == nil {
t.Fatal("expected invalid sha rejection")
}
if _, err := contentAddressedBlobPath("", validAsset); err == nil {
t.Fatal("expected empty root rejection")
}
}