diff --git a/docs/command-surface.md b/docs/command-surface.md index 3b5ef68..24a6ceb 100644 --- a/docs/command-surface.md +++ b/docs/command-surface.md @@ -45,6 +45,22 @@ crucible changelog [args] -> legacy `build-changelog` `crucible list` is machine-readable so CI can enumerate builders without parsing help text. +## HAK builder source modes + +```text +build-haks [--hak ...] [--archive ...] + [--source-manifest ] [--content-addressed-root ] + [--plan-only] [--skip-music] [--music-dataset ...] + [--quiet|--verbose|--debug] +``` + +When a source manifest contains `asset_sources`, pass +`--content-addressed-root ` (or +`--content-addressed-root=`). Crucible resolves each declared SHA-256 +under `sha256///` and verifies streamed bytes +while writing the selected HAK archives. Source manifests without +`asset_sources` continue to use the configured assets tree. + ## Global flags (planned, at wiring time) `--quiet`, `--verbose`, `--debug` (the legacy verbosity model), passed after the diff --git a/internal/app/app.go b/internal/app/app.go index d4f0c3d..0a606bb 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -466,13 +466,14 @@ func relPathFromRoot(root, path string) string { } type buildHAKOptions struct { - filteredHAKs []string - filteredArchives []string - sourceManifest string - musicDatasets []string - skipMusic bool - planOnly bool - logLevel logLevel + filteredHAKs []string + filteredArchives []string + sourceManifest string + contentAddressedRoot string + musicDatasets []string + skipMusic bool + planOnly bool + logLevel logLevel } type logLevel int @@ -1045,11 +1046,12 @@ func runBuildHAKs(ctx context) error { defer spin.stop() var result pipeline.BuildResult pipelineOpts := pipeline.BuildHAKOptions{ - Progress: console.progress, - ArchiveNames: opts.filteredArchives, - SourceManifestPath: opts.sourceManifest, - SkipMusic: opts.skipMusic, - MusicDatasetIDs: opts.musicDatasets, + Progress: console.progress, + ArchiveNames: opts.filteredArchives, + SourceManifestPath: opts.sourceManifest, + ContentAddressedRoot: opts.contentAddressedRoot, + SkipMusic: opts.skipMusic, + MusicDatasetIDs: opts.musicDatasets, } if opts.planOnly { result, err = pipeline.PlanHAKsWithOptions(p, pipelineOpts) @@ -1074,7 +1076,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) { arg := args[index] switch arg { case "-h", "--help": - return opts, errors.New("usage: build-haks [--hak ...] [--archive ...] [--source-manifest ] [--plan-only] [--skip-music] [--music-dataset ...] [--quiet|--verbose|--debug]") + return opts, errors.New("usage: build-haks [--hak ...] [--archive ...] [--source-manifest ] [--content-addressed-root ] [--plan-only] [--skip-music] [--music-dataset ...] [--quiet|--verbose|--debug]") case "--hak": index++ if index >= len(args) { @@ -1109,6 +1111,12 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) { return opts, errors.New("--source-manifest requires a value") } opts.sourceManifest = args[index] + case "--content-addressed-root": + index++ + if index >= len(args) { + return opts, errors.New("--content-addressed-root requires a value") + } + opts.contentAddressedRoot = args[index] default: if value, ok, err := requireInlineFlagValue(arg, "--hak"); ok || err != nil { if err != nil { @@ -1131,6 +1139,13 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) { opts.sourceManifest = value continue } + if value, ok, err := requireInlineFlagValue(arg, "--content-addressed-root"); ok || err != nil { + if err != nil { + return opts, err + } + opts.contentAddressedRoot = value + continue + } if value, ok, err := requireInlineFlagValue(arg, "--music-dataset"); ok || err != nil { if err != nil { return opts, err diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 89e3ffb..6f5e0ee 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -11,6 +11,48 @@ import ( "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline" ) +func TestParseBuildHAKArgsContentAddressedRoot(t *testing.T) { + tests := []struct { + name string + args []string + want string + }{ + { + name: "separated", + args: []string{"--content-addressed-root", "/var/cache/blobs"}, + want: "/var/cache/blobs", + }, + { + name: "inline", + args: []string{"--content-addressed-root=/var/cache/blobs"}, + want: "/var/cache/blobs", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opts, err := parseBuildHAKArgs(tt.args) + if err != nil { + t.Fatalf("parse build-haks args: %v", err) + } + if opts.contentAddressedRoot != tt.want { + t.Fatalf("content-addressed root = %q, want %q", opts.contentAddressedRoot, tt.want) + } + }) + } +} + +func TestParseBuildHAKArgsHelpListsContentAddressedRoot(t *testing.T) { + _, err := parseBuildHAKArgs([]string{"--help"}) + if err == nil { + t.Fatal("expected help usage error") + } + const want = "usage: build-haks [--hak ...] [--archive ...] [--source-manifest ] [--content-addressed-root ] [--plan-only] [--skip-music] [--music-dataset ...] [--quiet|--verbose|--debug]" + if err.Error() != want { + t.Fatalf("help usage = %q, want %q", err.Error(), want) + } +} + func TestRunBuildTopPackageUsesCachedCompiledOutputs(t *testing.T) { root := t.TempDir() mkdirAll(t, filepath.Join(root, "build")) diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index ab145f4..a6fb128 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -378,16 +378,8 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil { return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err) } - hakOutput, err := os.Create(hakPath) - if err != nil { - return BuildResult{}, fmt.Errorf("create hak archive: %w", err) - } - if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil { - hakOutput.Close() - return BuildResult{}, fmt.Errorf("write hak archive: %w", err) - } - if err := hakOutput.Close(); err != nil { - return BuildResult{}, fmt.Errorf("close hak archive: %w", err) + if err := writeHAKArchive(hakPath, resourceSlice(chunk.Assets)); err != nil { + return BuildResult{}, err } } @@ -488,23 +480,36 @@ func buildDirectHAKs(p *project.Project, progress ProgressFunc, writeArchives bo return result, nil } -// writeHAKArchive writes a HAK archive, removing any partial output if the ERF -// writer or a streamed source SHA verification fails, so a corrupt input never -// leaves a completed .hak behind. +// writeHAKArchive publishes a completed HAK atomically, so a failed write never +// leaves a partial archive at the final path. func writeHAKArchive(hakPath string, resources []erf.Resource) error { - output, err := os.Create(hakPath) + tmpPath := hakPath + ".tmp" + if err := os.Remove(tmpPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove stale temporary hak %s: %w", tmpPath, err) + } + + output, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) if err != nil { - return fmt.Errorf("create hak archive: %w", err) + return fmt.Errorf("create temporary hak archive %s: %w", tmpPath, err) } if err := erf.Write(output, erf.New("HAK ", resources)); err != nil { output.Close() - os.Remove(hakPath) + os.Remove(tmpPath) return fmt.Errorf("write hak archive: %w", err) } + if err := output.Sync(); err != nil { + output.Close() + os.Remove(tmpPath) + return fmt.Errorf("sync hak archive: %w", err) + } if err := output.Close(); err != nil { - os.Remove(hakPath) + os.Remove(tmpPath) return fmt.Errorf("close hak archive: %w", err) } + if err := os.Rename(tmpPath, hakPath); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("publish hak archive %s: %w", hakPath, err) + } return nil } diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 23a3694..33f546e 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -17,6 +17,35 @@ import ( "git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) +func TestBuildHAKsCleansFailedTemporaryArchive(t *testing.T) { + root := t.TempDir() + p := loadDirectProject(t, root) + + blobs := filepath.Join(root, "blobs") + sha := strings.Repeat("a", 64) + writeBlobAt(t, blobs, sha, "wxyz") + manifestPath := writeDirectManifest(t, root, + map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: 4}}, + []string{"core/a.tga"}) + + tmpPath := filepath.Join(root, "build", "core_01.hak.tmp") + mustWriteFile(t, tmpPath, "stale partial archive") + + _, err := BuildHAKsWithOptions(p, BuildHAKOptions{ + SourceManifestPath: manifestPath, + ContentAddressedRoot: blobs, + }) + if err == nil { + t.Fatal("expected corrupt input to fail") + } + if _, err := os.Stat(filepath.Join(root, "build", "core_01.hak")); !errors.Is(err, os.ErrNotExist) { + t.Fatal("corrupt input must not leave a completed hak") + } + if _, err := os.Stat(tmpPath); !errors.Is(err, os.ErrNotExist) { + t.Fatal("failed build must clean temporary hak") + } +} + func TestBuildThenExtract(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src"))