add content-addressed hak source mode (#7)
Reviewed-on: #7 Co-authored-by: vickydotbat <vickydotbat@tutamail.com> Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
This commit is contained in:
@@ -5,10 +5,9 @@ alwaysApply: true
|
||||
|
||||
# sow-tools / Crucible Agent Guide
|
||||
|
||||
This repo owns the **builder logic** for the migration: one Go module
|
||||
This repo owns the **builder logic:** one Go module
|
||||
(`git.westgate.pw/ShadowsOverWestgate/sow-tools`) producing the `crucible`
|
||||
dispatcher and the `crucible-<name>` binaries (D11). Read
|
||||
`../AGENTS.md` (migration hub) and `../../KICKOFF_PROMPT.md` first.
|
||||
dispatcher and the `crucible-<name>` binaries.
|
||||
|
||||
## What this repo owns / does not own
|
||||
|
||||
@@ -20,19 +19,11 @@ is `sow-platform`).
|
||||
|
||||
## Rules
|
||||
|
||||
1. **Migrated logic, not a fresh rewrite.** The `internal/` packages (`app`,
|
||||
`pipeline`, `project`, `erf`, `gff`, `topdata`, `music`, `changelog`,
|
||||
`validator`) were folded in from `gitea/sow-tools` at cutover; wired builders
|
||||
delegate to `internal/app`'s command surface. Keep them in step with upstream
|
||||
fixes rather than diverging silently.
|
||||
2. **Fail closed, never fake.** A builder with no migrated logic yet (`depot`)
|
||||
1. **Fail closed, never fake.** A builder with no migrated logic yet (`depot`)
|
||||
exits `70`. Do not stub a builder to emit a placeholder artifact.
|
||||
3. **Binaries are not committed.** They are CI artifacts / image layers (D19).
|
||||
2. **Binaries are not committed.** They are CI artifacts / image layers.
|
||||
`/bin/`, `*.exe`, `nwn-tool`, `sow-toolkit` are gitignored.
|
||||
4. **No home-dir / `NWN_ROOT` guessing.** Builders take roots explicitly via
|
||||
flag or env (project resolution is CWD-based, never `$HOME`). See
|
||||
[`docs/consumer-contract.md`](docs/consumer-contract.md).
|
||||
5. **The registry is the command surface.** `internal/dispatch.Registry` is the
|
||||
3. **The registry is the command surface.** `internal/dispatch.Registry` is the
|
||||
single source of truth; keep it in sync with `cmd/` and
|
||||
[`docs/command-surface.md`](docs/command-surface.md). Adding a builder = a
|
||||
`cmd/crucible-<name>/main.go` shim + a `Registry` entry + a doc row.
|
||||
@@ -40,11 +31,8 @@ is `sow-platform`).
|
||||
## Wiring a builder
|
||||
|
||||
1. Ensure the relevant `internal/` package(s) cover the work.
|
||||
2. Add the legacy command(s) to the builder's `Legacy`/`Extra` set and set
|
||||
`Wired: true` in `internal/dispatch`; the dispatcher delegates to
|
||||
`app.Run`. `depot` is the remaining unwired builder.
|
||||
3. Add tests; keep outputs deterministic (same input → same bytes).
|
||||
4. `make check` must stay green; update `make smoke` to expect the wired exit.
|
||||
2. Add tests; keep outputs deterministic (same input → same bytes).
|
||||
3. `make check` must stay green; update `make smoke` to expect the wired exit.
|
||||
|
||||
## Commands
|
||||
|
||||
@@ -54,7 +42,3 @@ make build # cmd/* -> ./bin
|
||||
make smoke # assert fail-closed contract
|
||||
make image # crucible:<sha>
|
||||
```
|
||||
|
||||
## Git
|
||||
|
||||
Never commit, branch, or push. Suggest a commit message; let the operator do it.
|
||||
|
||||
@@ -45,6 +45,22 @@ crucible changelog [args] -> legacy `build-changelog`
|
||||
`crucible list` is machine-readable so CI can enumerate builders without parsing
|
||||
help text.
|
||||
|
||||
## HAK builder source modes
|
||||
|
||||
```text
|
||||
build-haks [--hak <hak-name> ...] [--archive <archive-name> ...]
|
||||
[--source-manifest <path>] [--content-addressed-root <path>]
|
||||
[--plan-only] [--skip-music] [--music-dataset <id> ...]
|
||||
[--quiet|--verbose|--debug]
|
||||
```
|
||||
|
||||
When a source manifest contains `asset_sources`, pass
|
||||
`--content-addressed-root <directory>` (or
|
||||
`--content-addressed-root=<directory>`). Crucible resolves each declared SHA-256
|
||||
under `sha256/<first-two>/<next-two>/<full-sha256>` and verifies streamed bytes
|
||||
while writing the selected HAK archives. Source manifests without
|
||||
`asset_sources` continue to use the configured assets tree.
|
||||
|
||||
## Global flags (planned, at wiring time)
|
||||
|
||||
`--quiet`, `--verbose`, `--debug` (the legacy verbosity model), passed after the
|
||||
|
||||
+16
-1
@@ -469,6 +469,7 @@ type buildHAKOptions struct {
|
||||
filteredHAKs []string
|
||||
filteredArchives []string
|
||||
sourceManifest string
|
||||
contentAddressedRoot string
|
||||
musicDatasets []string
|
||||
skipMusic bool
|
||||
planOnly bool
|
||||
@@ -1048,6 +1049,7 @@ func runBuildHAKs(ctx context) error {
|
||||
Progress: console.progress,
|
||||
ArchiveNames: opts.filteredArchives,
|
||||
SourceManifestPath: opts.sourceManifest,
|
||||
ContentAddressedRoot: opts.contentAddressedRoot,
|
||||
SkipMusic: opts.skipMusic,
|
||||
MusicDatasetIDs: opts.musicDatasets,
|
||||
}
|
||||
@@ -1074,7 +1076,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
||||
arg := args[index]
|
||||
switch arg {
|
||||
case "-h", "--help":
|
||||
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]")
|
||||
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--content-addressed-root <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]")
|
||||
case "--hak":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
@@ -1109,6 +1111,12 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
||||
return opts, errors.New("--source-manifest requires a value")
|
||||
}
|
||||
opts.sourceManifest = args[index]
|
||||
case "--content-addressed-root":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, errors.New("--content-addressed-root requires a value")
|
||||
}
|
||||
opts.contentAddressedRoot = args[index]
|
||||
default:
|
||||
if value, ok, err := requireInlineFlagValue(arg, "--hak"); ok || err != nil {
|
||||
if err != nil {
|
||||
@@ -1131,6 +1139,13 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
||||
opts.sourceManifest = value
|
||||
continue
|
||||
}
|
||||
if value, ok, err := requireInlineFlagValue(arg, "--content-addressed-root"); ok || err != nil {
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
opts.contentAddressedRoot = value
|
||||
continue
|
||||
}
|
||||
if value, ok, err := requireInlineFlagValue(arg, "--music-dataset"); ok || err != nil {
|
||||
if err != nil {
|
||||
return opts, err
|
||||
|
||||
@@ -11,6 +11,48 @@ import (
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline"
|
||||
)
|
||||
|
||||
func TestParseBuildHAKArgsContentAddressedRoot(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "separated",
|
||||
args: []string{"--content-addressed-root", "/var/cache/blobs"},
|
||||
want: "/var/cache/blobs",
|
||||
},
|
||||
{
|
||||
name: "inline",
|
||||
args: []string{"--content-addressed-root=/var/cache/blobs"},
|
||||
want: "/var/cache/blobs",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
opts, err := parseBuildHAKArgs(tt.args)
|
||||
if err != nil {
|
||||
t.Fatalf("parse build-haks args: %v", err)
|
||||
}
|
||||
if opts.contentAddressedRoot != tt.want {
|
||||
t.Fatalf("content-addressed root = %q, want %q", opts.contentAddressedRoot, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBuildHAKArgsHelpListsContentAddressedRoot(t *testing.T) {
|
||||
_, err := parseBuildHAKArgs([]string{"--help"})
|
||||
if err == nil {
|
||||
t.Fatal("expected help usage error")
|
||||
}
|
||||
const want = "usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--content-addressed-root <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]"
|
||||
if err.Error() != want {
|
||||
t.Fatalf("help usage = %q, want %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBuildTopPackageUsesCachedCompiledOutputs(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mkdirAll(t, filepath.Join(root, "build"))
|
||||
|
||||
+20
-1
@@ -2,14 +2,19 @@ package erf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var expectedSHA256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
|
||||
const (
|
||||
headerSize = 160
|
||||
versionV10 = "V1.0"
|
||||
@@ -30,6 +35,9 @@ type Resource struct {
|
||||
Data []byte
|
||||
SourcePath string
|
||||
Size int64
|
||||
// ExpectedSHA256, when set, is the lowercase hex SHA-256 the streamed
|
||||
// SourcePath payload must hash to. A mismatch fails the write.
|
||||
ExpectedSHA256 string
|
||||
}
|
||||
|
||||
type header struct {
|
||||
@@ -407,18 +415,29 @@ func writeResourceData(w io.Writer, resource Resource) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if resource.ExpectedSHA256 != "" && !expectedSHA256Pattern.MatchString(resource.ExpectedSHA256) {
|
||||
return fmt.Errorf("resource %q has invalid expected sha256", resource.SourcePath)
|
||||
}
|
||||
|
||||
file, err := os.Open(resource.SourcePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open resource %q: %w", resource.SourcePath, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
written, err := io.Copy(w, file)
|
||||
hash := sha256.New()
|
||||
written, err := io.Copy(io.MultiWriter(w, hash), file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copy resource %q: %w", resource.SourcePath, err)
|
||||
}
|
||||
if resource.Size > 0 && written != resource.Size {
|
||||
return fmt.Errorf("copy resource %q: expected %d bytes, wrote %d", resource.SourcePath, resource.Size, written)
|
||||
}
|
||||
if resource.ExpectedSHA256 != "" {
|
||||
got := hex.EncodeToString(hash.Sum(nil))
|
||||
if got != resource.ExpectedSHA256 {
|
||||
return fmt.Errorf("resource %q sha256 mismatch: expected %s, got %s", resource.SourcePath, resource.ExpectedSHA256, got)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2,9 +2,72 @@ package erf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeTempPayload(t *testing.T, payload []byte) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "payload.bin")
|
||||
if err := os.WriteFile(path, payload, 0o644); err != nil {
|
||||
t.Fatalf("write payload: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestWriteRejectsSourcePathSHA256Mismatch(t *testing.T) {
|
||||
payload := []byte("hello world")
|
||||
path := writeTempPayload(t, payload)
|
||||
|
||||
archive := New("HAK ", []Resource{{
|
||||
Name: "asset",
|
||||
Type: 0x0003,
|
||||
SourcePath: path,
|
||||
Size: int64(len(payload)),
|
||||
ExpectedSHA256: strings.Repeat("a", 64),
|
||||
}})
|
||||
|
||||
var buf bytes.Buffer
|
||||
err := Write(&buf, archive)
|
||||
if err == nil {
|
||||
t.Fatal("expected sha256 mismatch error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "sha256 mismatch") {
|
||||
t.Fatalf("error = %v, want sha256 mismatch", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteAcceptsMatchingSourcePathSHA256(t *testing.T) {
|
||||
payload := []byte("hello world")
|
||||
path := writeTempPayload(t, payload)
|
||||
sum := sha256.Sum256(payload)
|
||||
|
||||
archive := New("HAK ", []Resource{{
|
||||
Name: "asset",
|
||||
Type: 0x0003,
|
||||
SourcePath: path,
|
||||
Size: int64(len(payload)),
|
||||
ExpectedSHA256: hex.EncodeToString(sum[:]),
|
||||
}})
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := Write(&buf, archive); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
decoded, err := Read(bytes.NewReader(buf.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if len(decoded.Resources) != 1 || string(decoded.Resources[0].Data) != string(payload) {
|
||||
t.Fatalf("unexpected payload: %#v", decoded.Resources)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveRoundTrip(t *testing.T) {
|
||||
archive := New("MOD ", []Resource{
|
||||
{Name: "module", Type: 0x07DE, Data: []byte("ifo")},
|
||||
|
||||
+124
-19
@@ -109,6 +109,7 @@ type BuildHAKOptions struct {
|
||||
Progress ProgressFunc
|
||||
ArchiveNames []string
|
||||
SourceManifestPath string
|
||||
ContentAddressedRoot string
|
||||
SkipMusic bool
|
||||
MusicDatasetIDs []string
|
||||
}
|
||||
@@ -196,7 +197,7 @@ func BuildHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResu
|
||||
}
|
||||
|
||||
func BuildHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
|
||||
return planOrBuildHAKs(p, opts.Progress, true, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs)
|
||||
return planOrBuildHAKs(p, opts.Progress, true, opts)
|
||||
}
|
||||
|
||||
func PlanHAKs(p *project.Project) (BuildResult, error) {
|
||||
@@ -208,18 +209,18 @@ func PlanHAKsWithProgress(p *project.Project, progress ProgressFunc) (BuildResul
|
||||
}
|
||||
|
||||
func PlanHAKsWithOptions(p *project.Project, opts BuildHAKOptions) (BuildResult, error) {
|
||||
return planOrBuildHAKs(p, opts.Progress, false, opts.ArchiveNames, opts.SourceManifestPath, opts.SkipMusic, opts.MusicDatasetIDs)
|
||||
return planOrBuildHAKs(p, opts.Progress, false, opts)
|
||||
}
|
||||
|
||||
func buildHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
|
||||
return planOrBuildHAKs(p, progress, true, nil, "", false, nil)
|
||||
return planOrBuildHAKs(p, progress, true, BuildHAKOptions{})
|
||||
}
|
||||
|
||||
func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
|
||||
return planOrBuildHAKs(p, progress, false, nil, "", false, nil)
|
||||
return planOrBuildHAKs(p, progress, false, BuildHAKOptions{})
|
||||
}
|
||||
|
||||
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string, skipMusic bool, musicDatasetIDs []string) (result BuildResult, err error) {
|
||||
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, opts BuildHAKOptions) (result BuildResult, err error) {
|
||||
preserveExistingHAKs := p.EffectiveConfig().Build.KeepExistingHAKs || envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
|
||||
|
||||
progressf(progress, "Validating project...")
|
||||
@@ -227,15 +228,23 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
||||
return BuildResult{}, err
|
||||
}
|
||||
|
||||
directManifest, err := loadDirectSourceManifest(opts.SourceManifestPath)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
if directManifest != nil {
|
||||
return buildDirectHAKs(p, progress, writeArchives, directManifest, opts)
|
||||
}
|
||||
|
||||
progressf(progress, "Collecting asset resources...")
|
||||
allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(sourceManifestPath)
|
||||
allowedAssets, sourceManifest, err := loadSourceManifestAssetSet(opts.SourceManifestPath)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
musicAssets := &preparedMusicAssets{SkipSourceRel: map[string]struct{}{}}
|
||||
if !skipMusic {
|
||||
if !opts.SkipMusic {
|
||||
musicAssets, err = prepareMusicAssetsWithOptions(p, musicPrepareOptions{
|
||||
datasetIDs: musicDatasetIDs,
|
||||
datasetIDs: opts.MusicDatasetIDs,
|
||||
write: writeArchives,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -294,7 +303,7 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
chunks, err = filterHAKChunksByName(chunks, archiveNames)
|
||||
chunks, err = filterHAKChunksByName(chunks, opts.ArchiveNames)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
@@ -369,16 +378,8 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
||||
if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err)
|
||||
}
|
||||
hakOutput, err := os.Create(hakPath)
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("create hak archive: %w", err)
|
||||
}
|
||||
if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil {
|
||||
hakOutput.Close()
|
||||
return BuildResult{}, fmt.Errorf("write hak archive: %w", err)
|
||||
}
|
||||
if err := hakOutput.Close(); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("close hak archive: %w", err)
|
||||
if err := writeHAKArchive(hakPath, resourceSlice(chunk.Assets)); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -408,6 +409,110 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// buildDirectHAKs packages HAKs directly from the content-addressed source blob
|
||||
// cache. It skips music preparation, generated 2da discovery, git creation-time
|
||||
// lookups, LFS discovery, and any materialized asset tree scan.
|
||||
func buildDirectHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, manifest *SourceBuildManifest, opts BuildHAKOptions) (BuildResult, error) {
|
||||
progressf(progress, "Collecting asset resources...")
|
||||
assetResources, err := collectDirectSourceResources(manifest, opts.ContentAddressedRoot)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
|
||||
result := BuildResult{HAKAssets: len(assetResources)}
|
||||
|
||||
progressf(progress, "Planning HAK chunks...")
|
||||
chunks, err := chunksFromSourceManifest(assetResources, manifest.HAKs)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
chunks, err = filterHAKChunksByName(chunks, opts.ArchiveNames)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
|
||||
buildManifest := BuildManifest{
|
||||
ModuleHAKs: append([]string(nil), manifest.ModuleHAKs...),
|
||||
HAKs: make([]BuildManifestHAK, 0, len(chunks)),
|
||||
}
|
||||
for _, chunk := range chunks {
|
||||
buildManifest.HAKs = append(buildManifest.HAKs, buildManifestEntry(chunk))
|
||||
}
|
||||
|
||||
manifestPath := p.HAKManifestPath()
|
||||
manifestBytes, err := json.MarshalIndent(buildManifest, "", " ")
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("marshal hak manifest: %w", err)
|
||||
}
|
||||
manifestBytes = append(manifestBytes, '\n')
|
||||
|
||||
if writeArchives {
|
||||
progressf(progress, "Writing HAK archives...")
|
||||
result.HAKSummary.Total = len(chunks)
|
||||
result.HAKPaths = make([]string, 0, len(chunks))
|
||||
for index, chunk := range chunks {
|
||||
hakPath := p.HAKArchivePath(chunk.Name)
|
||||
progressf(progress, fmt.Sprintf("Writing HAK %d/%d: %s (%d assets)", index+1, len(chunks), chunk.Name, len(chunk.Assets)))
|
||||
if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err)
|
||||
}
|
||||
if err := writeHAKArchive(hakPath, resourceSlice(chunk.Assets)); err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
result.HAKSummary.Written++
|
||||
result.HAKSummary.Actions = append(result.HAKSummary.Actions, HAKArchiveAction{
|
||||
Name: chunk.Name,
|
||||
AssetCount: len(chunk.Assets),
|
||||
OutputPath: hakPath,
|
||||
})
|
||||
result.HAKPaths = append(result.HAKPaths, hakPath)
|
||||
}
|
||||
}
|
||||
|
||||
progressf(progress, "Writing HAK manifest...")
|
||||
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("create hak manifest dir: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("write hak manifest: %w", err)
|
||||
}
|
||||
result.Manifest = manifestPath
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// writeHAKArchive publishes a completed HAK atomically, so a failed write never
|
||||
// leaves a partial archive at the final path.
|
||||
func writeHAKArchive(hakPath string, resources []erf.Resource) error {
|
||||
tmpPath := hakPath + ".tmp"
|
||||
if err := os.Remove(tmpPath); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale temporary hak %s: %w", tmpPath, err)
|
||||
}
|
||||
|
||||
output, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create temporary hak archive %s: %w", tmpPath, err)
|
||||
}
|
||||
if err := erf.Write(output, erf.New("HAK ", resources)); err != nil {
|
||||
output.Close()
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("write hak archive: %w", err)
|
||||
}
|
||||
if err := output.Sync(); err != nil {
|
||||
output.Close()
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("sync hak archive: %w", err)
|
||||
}
|
||||
if err := output.Close(); err != nil {
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("close hak archive: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmpPath, hakPath); err != nil {
|
||||
os.Remove(tmpPath)
|
||||
return fmt.Errorf("publish hak archive %s: %w", hakPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeAutogenManifestOutputs(progress ProgressFunc, manifests []topdata.ProducedAutogenManifest) error {
|
||||
if len(manifests) == 0 {
|
||||
return nil
|
||||
|
||||
@@ -17,6 +17,35 @@ import (
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
func TestBuildHAKsCleansFailedTemporaryArchive(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
p := loadDirectProject(t, root)
|
||||
|
||||
blobs := filepath.Join(root, "blobs")
|
||||
sha := strings.Repeat("a", 64)
|
||||
writeBlobAt(t, blobs, sha, "wxyz")
|
||||
manifestPath := writeDirectManifest(t, root,
|
||||
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: 4}},
|
||||
[]string{"core/a.tga"})
|
||||
|
||||
tmpPath := filepath.Join(root, "build", "core_01.hak.tmp")
|
||||
mustWriteFile(t, tmpPath, "stale partial archive")
|
||||
|
||||
_, err := BuildHAKsWithOptions(p, BuildHAKOptions{
|
||||
SourceManifestPath: manifestPath,
|
||||
ContentAddressedRoot: blobs,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected corrupt input to fail")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "build", "core_01.hak")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("corrupt input must not leave a completed hak")
|
||||
}
|
||||
if _, err := os.Stat(tmpPath); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("failed build must clean temporary hak")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildThenExtract(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "src"))
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
)
|
||||
|
||||
const directProjectConfig = `{
|
||||
"module": {
|
||||
"name": "Test Module",
|
||||
"resref": "testmod",
|
||||
"hak_order": ["core_01"]
|
||||
},
|
||||
"paths": {
|
||||
"source": "src",
|
||||
"assets": "assets",
|
||||
"build": "build"
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
func writeBlobAt(t *testing.T, root, sha, content string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "sha256", sha[0:2], sha[2:4])
|
||||
mustMkdir(t, dir)
|
||||
mustWriteFile(t, filepath.Join(dir, sha), content)
|
||||
}
|
||||
|
||||
func writeBlob(t *testing.T, root string, payload []byte) string {
|
||||
t.Helper()
|
||||
sum := sha256.Sum256(payload)
|
||||
sha := hex.EncodeToString(sum[:])
|
||||
writeBlobAt(t, root, sha, string(payload))
|
||||
return sha
|
||||
}
|
||||
|
||||
func writeDirectManifest(t *testing.T, root string, sources map[string]SourceAsset, assets []string) string {
|
||||
t.Helper()
|
||||
manifest := SourceBuildManifest{
|
||||
Schema: 1,
|
||||
BuilderID: buildinfo.String(),
|
||||
ModuleHAKs: []string{"core_01"},
|
||||
HAKs: []SourceManifestHAK{{
|
||||
Name: "core_01",
|
||||
Group: "core",
|
||||
Priority: 1,
|
||||
MaxBytes: 1 << 20,
|
||||
Assets: assets,
|
||||
}},
|
||||
AssetSources: sources,
|
||||
}
|
||||
raw, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal manifest: %v", err)
|
||||
}
|
||||
path := filepath.Join(root, "build-source-manifest.json")
|
||||
mustWriteFile(t, path, string(raw))
|
||||
return path
|
||||
}
|
||||
|
||||
func loadDirectProject(t *testing.T, root string) *project.Project {
|
||||
t.Helper()
|
||||
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), directProjectConfig)
|
||||
mustMkdir(t, filepath.Join(root, "build"))
|
||||
p, err := project.Load(root)
|
||||
if err != nil {
|
||||
t.Fatalf("load project: %v", err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestBuildHAKsFromContentAddressedManifestWithoutAssetsTree(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
p := loadDirectProject(t, root)
|
||||
|
||||
blobs := filepath.Join(root, "blobs")
|
||||
payload := []byte("tga-payload-bytes")
|
||||
sha := writeBlob(t, blobs, payload)
|
||||
manifestPath := writeDirectManifest(t, root,
|
||||
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: int64(len(payload))}},
|
||||
[]string{"core/a.tga"})
|
||||
|
||||
result, err := BuildHAKsWithOptions(p, BuildHAKOptions{
|
||||
SourceManifestPath: manifestPath,
|
||||
ContentAddressedRoot: blobs,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("build direct haks: %v", err)
|
||||
}
|
||||
if result.HAKAssets != 1 {
|
||||
t.Fatalf("hak assets = %d, want 1", result.HAKAssets)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(root, "assets")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("direct build must not materialize an assets tree (stat err = %v)", err)
|
||||
}
|
||||
|
||||
hakPath := filepath.Join(root, "build", "core_01.hak")
|
||||
raw, err := os.ReadFile(hakPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read hak: %v", err)
|
||||
}
|
||||
archive, err := erf.Read(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("decode hak: %v", err)
|
||||
}
|
||||
if len(archive.Resources) != 1 {
|
||||
t.Fatalf("hak resources = %d, want 1", len(archive.Resources))
|
||||
}
|
||||
if got := string(archive.Resources[0].Data); got != string(payload) {
|
||||
t.Fatalf("resource payload = %q, want %q", got, payload)
|
||||
}
|
||||
if archive.Resources[0].Name != "a" {
|
||||
t.Fatalf("resref = %q, want a", archive.Resources[0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHAKsFromContentAddressedManifestRejectsMissingRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
p := loadDirectProject(t, root)
|
||||
|
||||
blobs := filepath.Join(root, "blobs")
|
||||
payload := []byte("tga-payload-bytes")
|
||||
sha := writeBlob(t, blobs, payload)
|
||||
manifestPath := writeDirectManifest(t, root,
|
||||
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: int64(len(payload))}},
|
||||
[]string{"core/a.tga"})
|
||||
|
||||
_, err := BuildHAKsWithOptions(p, BuildHAKOptions{SourceManifestPath: manifestPath})
|
||||
if err == nil {
|
||||
t.Fatal("expected failure when content-addressed root is missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHAKsFromContentAddressedManifestRejectsMissingBlob(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
p := loadDirectProject(t, root)
|
||||
|
||||
blobs := filepath.Join(root, "blobs")
|
||||
sha := strings.Repeat("b", 64)
|
||||
manifestPath := writeDirectManifest(t, root,
|
||||
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: 4}},
|
||||
[]string{"core/a.tga"})
|
||||
|
||||
_, err := BuildHAKsWithOptions(p, BuildHAKOptions{
|
||||
SourceManifestPath: manifestPath,
|
||||
ContentAddressedRoot: blobs,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected failure for missing source blob")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHAKsFromContentAddressedManifestRejectsCorruptBlob(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
p := loadDirectProject(t, root)
|
||||
|
||||
blobs := filepath.Join(root, "blobs")
|
||||
declared := sha256.Sum256([]byte("abcd"))
|
||||
sha := hex.EncodeToString(declared[:])
|
||||
// Same length, different bytes: size check passes, content hash mismatches.
|
||||
writeBlobAt(t, blobs, sha, "wxyz")
|
||||
manifestPath := writeDirectManifest(t, root,
|
||||
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: 4}},
|
||||
[]string{"core/a.tga"})
|
||||
|
||||
_, err := BuildHAKsWithOptions(p, BuildHAKOptions{
|
||||
SourceManifestPath: manifestPath,
|
||||
ContentAddressedRoot: blobs,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected sha256 mismatch failure for corrupt blob")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "sha256 mismatch") {
|
||||
t.Fatalf("error = %v, want sha256 mismatch", err)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(root, "build", "core_01.hak")); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("corrupt build must not leave a completed hak (stat err = %v)", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacySourceManifestStillBuildsFromAssetsTree(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mustMkdir(t, filepath.Join(root, "src", "module"))
|
||||
mustMkdir(t, filepath.Join(root, "assets", "core"))
|
||||
mustMkdir(t, filepath.Join(root, "build"))
|
||||
mustWriteFile(t, filepath.Join(root, "nwn-tool.json"), `{
|
||||
"module": {"name": "Test Module", "resref": "testmod", "hak_order": ["group:core"]},
|
||||
"paths": {"source": "src", "assets": "assets", "build": "build"},
|
||||
"haks": [{"name": "core", "priority": 1, "include": ["core/**"]}]
|
||||
}
|
||||
`)
|
||||
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
|
||||
"file_type": "IFO ",
|
||||
"file_version": "V3.2",
|
||||
"root": {"struct_type": 0, "fields": [
|
||||
{"label": "Mod_Name", "type": "CExoString", "value": "Test Module"},
|
||||
{"label": "Mod_HakList", "type": "List", "value": []}
|
||||
]}
|
||||
}
|
||||
`)
|
||||
mustWriteFile(t, filepath.Join(root, "assets", "core", "a.tga"), strings.Repeat("a", 16))
|
||||
mustWriteFile(t, filepath.Join(root, "assets", "core", "b.tga"), strings.Repeat("b", 16))
|
||||
|
||||
manifestPath := filepath.Join(root, "legacy-source-manifest.json")
|
||||
mustWriteFile(t, manifestPath, `{
|
||||
"module_haks": ["core"],
|
||||
"haks": [{"name": "core", "group": "core", "priority": 1, "assets": ["core/a.tga", "core/b.tga"]}]
|
||||
}
|
||||
`)
|
||||
|
||||
p, err := project.Load(root)
|
||||
if err != nil {
|
||||
t.Fatalf("load project: %v", err)
|
||||
}
|
||||
if err := p.Scan(); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
result, err := BuildHAKsWithOptions(p, BuildHAKOptions{SourceManifestPath: manifestPath})
|
||||
if err != nil {
|
||||
t.Fatalf("legacy build: %v", err)
|
||||
}
|
||||
if len(result.HAKPaths) != 1 {
|
||||
t.Fatalf("hak paths = %d, want 1", len(result.HAKPaths))
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(root, "build", "core.hak"))
|
||||
if err != nil {
|
||||
t.Fatalf("read legacy hak: %v", err)
|
||||
}
|
||||
archive, err := erf.Read(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("decode legacy hak: %v", err)
|
||||
}
|
||||
if len(archive.Resources) != 2 {
|
||||
t.Fatalf("legacy hak resources = %d, want 2", len(archive.Resources))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
pathpkg "path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
)
|
||||
|
||||
// SourceAsset is one content-addressed source blob referenced by a direct source
|
||||
// manifest. The blob is resolved by sha256 under the content-addressed root.
|
||||
type SourceAsset struct {
|
||||
SHA256 string `json:"sha256"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
}
|
||||
|
||||
// SourceManifestHAK describes one output HAK and the logical asset paths it packs.
|
||||
type SourceManifestHAK struct {
|
||||
Name string `json:"name"`
|
||||
Group string `json:"group"`
|
||||
Priority int `json:"priority"`
|
||||
MaxBytes int64 `json:"max_bytes"`
|
||||
Optional bool `json:"optional,omitempty"`
|
||||
Assets []string `json:"assets"`
|
||||
BuildKey string `json:"build_key"`
|
||||
}
|
||||
|
||||
// SourceBuildManifest is the generated, content-addressed input to a HAK build.
|
||||
// It carries asset_sources, so Crucible reads source bytes directly from the blob
|
||||
// cache without scanning a materialized asset tree.
|
||||
type SourceBuildManifest struct {
|
||||
Schema int `json:"schema"`
|
||||
BuilderID string `json:"builder_id"`
|
||||
ModuleHAKs []string `json:"module_haks"`
|
||||
HAKs []SourceManifestHAK `json:"haks"`
|
||||
AssetSources map[string]SourceAsset `json:"asset_sources"`
|
||||
}
|
||||
|
||||
var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
|
||||
// hasAssetSources reports whether raw JSON carries the direct content-addressed
|
||||
// schema. A legacy BuildManifest has no asset_sources object.
|
||||
func hasAssetSources(raw []byte) bool {
|
||||
var header struct {
|
||||
AssetSources map[string]json.RawMessage `json:"asset_sources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &header); err != nil {
|
||||
return false
|
||||
}
|
||||
return header.AssetSources != nil
|
||||
}
|
||||
|
||||
func loadSourceBuildManifest(path string) (*SourceBuildManifest, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read source manifest %s: %w", path, err)
|
||||
}
|
||||
var manifest SourceBuildManifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("parse source manifest %s: %w", path, err)
|
||||
}
|
||||
return &manifest, nil
|
||||
}
|
||||
|
||||
func validateDirectSourceManifest(manifest *SourceBuildManifest) error {
|
||||
if manifest.Schema != 1 {
|
||||
return fmt.Errorf("unsupported source manifest schema %d", manifest.Schema)
|
||||
}
|
||||
if manifest.BuilderID != buildinfo.String() {
|
||||
return fmt.Errorf("source manifest builder_id %q does not match builder %q", manifest.BuilderID, buildinfo.String())
|
||||
}
|
||||
if len(manifest.ModuleHAKs) == 0 {
|
||||
return fmt.Errorf("source manifest module_haks is empty")
|
||||
}
|
||||
|
||||
referenced := make(map[string]struct{}, len(manifest.AssetSources))
|
||||
for _, hak := range manifest.HAKs {
|
||||
if strings.TrimSpace(hak.Name) == "" {
|
||||
return fmt.Errorf("source manifest hak has empty name")
|
||||
}
|
||||
for _, rel := range hak.Assets {
|
||||
if !validLogicalAssetPath(rel) {
|
||||
return fmt.Errorf("source manifest hak %s has invalid asset path %q", hak.Name, rel)
|
||||
}
|
||||
if _, dup := referenced[rel]; dup {
|
||||
return fmt.Errorf("source manifest asset %q referenced by more than one hak", rel)
|
||||
}
|
||||
referenced[rel] = struct{}{}
|
||||
source, ok := manifest.AssetSources[rel]
|
||||
if !ok {
|
||||
return fmt.Errorf("source manifest hak %s asset %q has no asset_sources entry", hak.Name, rel)
|
||||
}
|
||||
if !sha256Pattern.MatchString(source.SHA256) {
|
||||
return fmt.Errorf("source manifest asset %q has invalid sha256 %q", rel, source.SHA256)
|
||||
}
|
||||
if source.SizeBytes < 0 {
|
||||
return fmt.Errorf("source manifest asset %q has negative size %d", rel, source.SizeBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for rel := range manifest.AssetSources {
|
||||
if !validLogicalAssetPath(rel) {
|
||||
return fmt.Errorf("source manifest asset_sources has invalid path %q", rel)
|
||||
}
|
||||
if _, ok := referenced[rel]; !ok {
|
||||
return fmt.Errorf("source manifest asset_sources entry %q is not referenced by any hak", rel)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validLogicalAssetPath(path string) bool {
|
||||
if path == "" || filepath.IsAbs(path) || strings.Contains(path, `\`) {
|
||||
return false
|
||||
}
|
||||
clean := pathpkg.Clean(path)
|
||||
return clean == path && clean != "." && clean != ".." &&
|
||||
!strings.HasPrefix(clean, "../")
|
||||
}
|
||||
|
||||
func contentAddressedBlobPath(root, sha string) (string, error) {
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return "", fmt.Errorf("content-addressed root is required")
|
||||
}
|
||||
if !sha256Pattern.MatchString(sha) {
|
||||
return "", fmt.Errorf("invalid content-addressed sha256 %q", sha)
|
||||
}
|
||||
return filepath.Join(root, "sha256", sha[0:2], sha[2:4], sha), nil
|
||||
}
|
||||
|
||||
// loadDirectSourceManifest returns a parsed direct source manifest when path
|
||||
// points at a content-addressed manifest. It returns (nil, nil) for an empty
|
||||
// path or a legacy manifest without asset_sources, so the caller falls through
|
||||
// to the legacy path-backed build.
|
||||
func loadDirectSourceManifest(path string) (*SourceBuildManifest, error) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read source manifest %s: %w", path, err)
|
||||
}
|
||||
if !hasAssetSources(raw) {
|
||||
return nil, nil
|
||||
}
|
||||
var manifest SourceBuildManifest
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("parse source manifest %s: %w", path, err)
|
||||
}
|
||||
return &manifest, nil
|
||||
}
|
||||
|
||||
// collectDirectSourceResources resolves each manifest-named blob under contentRoot
|
||||
// and produces sorted asset resources that stream from the blob cache with their
|
||||
// expected SHA-256. It never scans a materialized asset tree.
|
||||
func collectDirectSourceResources(manifest *SourceBuildManifest, contentRoot string) ([]assetResource, error) {
|
||||
if err := validateDirectSourceManifest(manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(contentRoot) == "" {
|
||||
return nil, fmt.Errorf("content-addressed source manifest requires --content-addressed-root")
|
||||
}
|
||||
|
||||
resources := make([]assetResource, 0, len(manifest.AssetSources))
|
||||
for _, hak := range manifest.HAKs {
|
||||
for _, logicalPath := range hak.Assets {
|
||||
source := manifest.AssetSources[logicalPath]
|
||||
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(logicalPath)), ".")
|
||||
resourceType, ok := erf.HAKResourceTypeForExtension(extension)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unsupported HAK resource extension %q for %s", filepath.Ext(logicalPath), logicalPath)
|
||||
}
|
||||
resref := strings.ToLower(strings.TrimSuffix(filepath.Base(logicalPath), filepath.Ext(logicalPath)))
|
||||
|
||||
blobPath, err := contentAddressedBlobPath(contentRoot, source.SHA256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(blobPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("missing source blob for %s: %w", logicalPath, err)
|
||||
}
|
||||
if info.Size() != source.SizeBytes {
|
||||
return nil, fmt.Errorf("source blob size mismatch for %s: expected %d, got %d", logicalPath, source.SizeBytes, info.Size())
|
||||
}
|
||||
|
||||
resource := erf.Resource{
|
||||
Name: resref,
|
||||
Type: resourceType,
|
||||
SourcePath: blobPath,
|
||||
Size: source.SizeBytes,
|
||||
ExpectedSHA256: source.SHA256,
|
||||
}
|
||||
resources = append(resources, assetResource{
|
||||
Rel: logicalPath,
|
||||
Resource: resource,
|
||||
Size: erf.ArchiveSize([]erf.Resource{resource}),
|
||||
ContentID: "sha256:" + source.SHA256,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
slices.SortFunc(resources, func(a, b assetResource) int {
|
||||
return compareResourceKeys(a.Resource, b.Resource)
|
||||
})
|
||||
return resources, nil
|
||||
}
|
||||
|
||||
// chunksFromSourceManifest converts direct manifest HAK entries into build chunks
|
||||
// reusing the legacy manifest chunker.
|
||||
func chunksFromSourceManifest(assets []assetResource, entries []SourceManifestHAK) ([]hakChunk, error) {
|
||||
converted := make([]BuildManifestHAK, len(entries))
|
||||
for index, entry := range entries {
|
||||
converted[index] = BuildManifestHAK{
|
||||
Name: entry.Name,
|
||||
Group: entry.Group,
|
||||
Priority: entry.Priority,
|
||||
MaxBytes: entry.MaxBytes,
|
||||
Optional: entry.Optional,
|
||||
Assets: entry.Assets,
|
||||
}
|
||||
}
|
||||
return chunksFromManifest(assets, converted)
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||
)
|
||||
|
||||
const validAsset = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
// directManifestFixture returns a minimal valid direct source manifest using the
|
||||
// running builder identity, so validation does not reject it on builder mismatch.
|
||||
func directManifestFixture() *SourceBuildManifest {
|
||||
return &SourceBuildManifest{
|
||||
Schema: 1,
|
||||
BuilderID: buildinfo.String(),
|
||||
ModuleHAKs: []string{"core_01"},
|
||||
HAKs: []SourceManifestHAK{
|
||||
{
|
||||
Name: "core_01",
|
||||
Group: "core",
|
||||
Priority: 1,
|
||||
MaxBytes: 1024,
|
||||
Assets: []string{"core/a.tga"},
|
||||
},
|
||||
},
|
||||
AssetSources: map[string]SourceAsset{
|
||||
"core/a.tga": {SHA256: validAsset, SizeBytes: 4},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func writeManifestFixture(t *testing.T, manifest *SourceBuildManifest) string {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal fixture: %v", err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "build-source-manifest.json")
|
||||
mustWriteFile(t, path, string(raw))
|
||||
return path
|
||||
}
|
||||
|
||||
func TestLoadSourceBuildManifestDirect(t *testing.T) {
|
||||
path := writeManifestFixture(t, directManifestFixture())
|
||||
|
||||
manifest, err := loadSourceBuildManifest(path)
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
if manifest.Schema != 1 {
|
||||
t.Fatalf("schema = %d, want 1", manifest.Schema)
|
||||
}
|
||||
if manifest.BuilderID != buildinfo.String() {
|
||||
t.Fatalf("builder_id = %q, want %q", manifest.BuilderID, buildinfo.String())
|
||||
}
|
||||
if len(manifest.HAKs) != 1 || manifest.HAKs[0].Name != "core_01" {
|
||||
t.Fatalf("unexpected haks: %#v", manifest.HAKs)
|
||||
}
|
||||
source, ok := manifest.AssetSources["core/a.tga"]
|
||||
if !ok || source.SHA256 != validAsset || source.SizeBytes != 4 {
|
||||
t.Fatalf("unexpected asset source: %#v", manifest.AssetSources)
|
||||
}
|
||||
if err := validateDirectSourceManifest(manifest); err != nil {
|
||||
t.Fatalf("validate valid manifest: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDirectSourceManifestRejectsUnsupportedSchema(t *testing.T) {
|
||||
manifest := directManifestFixture()
|
||||
manifest.Schema = 2
|
||||
if err := validateDirectSourceManifest(manifest); err == nil {
|
||||
t.Fatal("expected schema rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDirectSourceManifestRejectsBadSHA(t *testing.T) {
|
||||
manifest := directManifestFixture()
|
||||
manifest.AssetSources["core/a.tga"] = SourceAsset{SHA256: "NOTHEX", SizeBytes: 4}
|
||||
if err := validateDirectSourceManifest(manifest); err == nil {
|
||||
t.Fatal("expected bad sha rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDirectSourceManifestRejectsNegativeSize(t *testing.T) {
|
||||
manifest := directManifestFixture()
|
||||
manifest.AssetSources["core/a.tga"] = SourceAsset{SHA256: validAsset, SizeBytes: -1}
|
||||
if err := validateDirectSourceManifest(manifest); err == nil {
|
||||
t.Fatal("expected negative size rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDirectSourceManifestRejectsMissingSource(t *testing.T) {
|
||||
manifest := directManifestFixture()
|
||||
delete(manifest.AssetSources, "core/a.tga")
|
||||
if err := validateDirectSourceManifest(manifest); err == nil {
|
||||
t.Fatal("expected missing source rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDirectSourceManifestRejectsUnusedSource(t *testing.T) {
|
||||
manifest := directManifestFixture()
|
||||
manifest.AssetSources["core/orphan.tga"] = SourceAsset{SHA256: validAsset, SizeBytes: 4}
|
||||
if err := validateDirectSourceManifest(manifest); err == nil {
|
||||
t.Fatal("expected unused source rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDirectSourceManifestRejectsDuplicateAssetAcrossHAKs(t *testing.T) {
|
||||
manifest := directManifestFixture()
|
||||
manifest.HAKs = append(manifest.HAKs, SourceManifestHAK{
|
||||
Name: "core_02",
|
||||
Group: "core",
|
||||
Priority: 1,
|
||||
MaxBytes: 1024,
|
||||
Assets: []string{"core/a.tga"},
|
||||
})
|
||||
if err := validateDirectSourceManifest(manifest); err == nil {
|
||||
t.Fatal("expected duplicate-asset rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDirectSourceManifestRejectsTraversalPath(t *testing.T) {
|
||||
manifest := directManifestFixture()
|
||||
manifest.HAKs[0].Assets = []string{"../escape.tga"}
|
||||
manifest.AssetSources = map[string]SourceAsset{
|
||||
"../escape.tga": {SHA256: validAsset, SizeBytes: 4},
|
||||
}
|
||||
if err := validateDirectSourceManifest(manifest); err == nil {
|
||||
t.Fatal("expected traversal rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDirectSourceManifestRejectsBuilderMismatch(t *testing.T) {
|
||||
manifest := directManifestFixture()
|
||||
manifest.BuilderID = "crucible deadbeef"
|
||||
if err := validateDirectSourceManifest(manifest); err == nil {
|
||||
t.Fatal("expected builder mismatch rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentAddressedBlobPath(t *testing.T) {
|
||||
root := "/var/cache/blobs"
|
||||
got, err := contentAddressedBlobPath(root, validAsset)
|
||||
if err != nil {
|
||||
t.Fatalf("blob path: %v", err)
|
||||
}
|
||||
want := filepath.Join(root, "sha256", "aa", "aa", validAsset)
|
||||
if got != want {
|
||||
t.Fatalf("blob path = %q, want %q", got, want)
|
||||
}
|
||||
if _, err := contentAddressedBlobPath(root, "NOTHEX"); err == nil {
|
||||
t.Fatal("expected invalid sha rejection")
|
||||
}
|
||||
if _, err := contentAddressedBlobPath("", validAsset); err == nil {
|
||||
t.Fatal("expected empty root rejection")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user