add content-addressed hak source mode #7
@@ -45,6 +45,22 @@ crucible changelog [args] -> legacy `build-changelog`
|
|||||||
`crucible list` is machine-readable so CI can enumerate builders without parsing
|
`crucible list` is machine-readable so CI can enumerate builders without parsing
|
||||||
help text.
|
help text.
|
||||||
|
|
||||||
|
## HAK builder source modes
|
||||||
|
|
||||||
|
```text
|
||||||
|
build-haks [--hak <hak-name> ...] [--archive <archive-name> ...]
|
||||||
|
[--source-manifest <path>] [--content-addressed-root <path>]
|
||||||
|
[--plan-only] [--skip-music] [--music-dataset <id> ...]
|
||||||
|
[--quiet|--verbose|--debug]
|
||||||
|
```
|
||||||
|
|
||||||
|
When a source manifest contains `asset_sources`, pass
|
||||||
|
`--content-addressed-root <directory>` (or
|
||||||
|
`--content-addressed-root=<directory>`). Crucible resolves each declared SHA-256
|
||||||
|
under `sha256/<first-two>/<next-two>/<full-sha256>` and verifies streamed bytes
|
||||||
|
while writing the selected HAK archives. Source manifests without
|
||||||
|
`asset_sources` continue to use the configured assets tree.
|
||||||
|
|
||||||
## Global flags (planned, at wiring time)
|
## Global flags (planned, at wiring time)
|
||||||
|
|
||||||
`--quiet`, `--verbose`, `--debug` (the legacy verbosity model), passed after the
|
`--quiet`, `--verbose`, `--debug` (the legacy verbosity model), passed after the
|
||||||
|
|||||||
+16
-1
@@ -469,6 +469,7 @@ type buildHAKOptions struct {
|
|||||||
filteredHAKs []string
|
filteredHAKs []string
|
||||||
filteredArchives []string
|
filteredArchives []string
|
||||||
sourceManifest string
|
sourceManifest string
|
||||||
|
contentAddressedRoot string
|
||||||
musicDatasets []string
|
musicDatasets []string
|
||||||
skipMusic bool
|
skipMusic bool
|
||||||
planOnly bool
|
planOnly bool
|
||||||
@@ -1048,6 +1049,7 @@ func runBuildHAKs(ctx context) error {
|
|||||||
Progress: console.progress,
|
Progress: console.progress,
|
||||||
ArchiveNames: opts.filteredArchives,
|
ArchiveNames: opts.filteredArchives,
|
||||||
SourceManifestPath: opts.sourceManifest,
|
SourceManifestPath: opts.sourceManifest,
|
||||||
|
ContentAddressedRoot: opts.contentAddressedRoot,
|
||||||
SkipMusic: opts.skipMusic,
|
SkipMusic: opts.skipMusic,
|
||||||
MusicDatasetIDs: opts.musicDatasets,
|
MusicDatasetIDs: opts.musicDatasets,
|
||||||
}
|
}
|
||||||
@@ -1074,7 +1076,7 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
|||||||
arg := args[index]
|
arg := args[index]
|
||||||
switch arg {
|
switch arg {
|
||||||
case "-h", "--help":
|
case "-h", "--help":
|
||||||
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]")
|
return opts, errors.New("usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--content-addressed-root <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]")
|
||||||
case "--hak":
|
case "--hak":
|
||||||
index++
|
index++
|
||||||
if index >= len(args) {
|
if index >= len(args) {
|
||||||
@@ -1109,6 +1111,12 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
|||||||
return opts, errors.New("--source-manifest requires a value")
|
return opts, errors.New("--source-manifest requires a value")
|
||||||
}
|
}
|
||||||
opts.sourceManifest = args[index]
|
opts.sourceManifest = args[index]
|
||||||
|
case "--content-addressed-root":
|
||||||
|
index++
|
||||||
|
if index >= len(args) {
|
||||||
|
return opts, errors.New("--content-addressed-root requires a value")
|
||||||
|
}
|
||||||
|
opts.contentAddressedRoot = args[index]
|
||||||
default:
|
default:
|
||||||
if value, ok, err := requireInlineFlagValue(arg, "--hak"); ok || err != nil {
|
if value, ok, err := requireInlineFlagValue(arg, "--hak"); ok || err != nil {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1131,6 +1139,13 @@ func parseBuildHAKArgs(args []string) (buildHAKOptions, error) {
|
|||||||
opts.sourceManifest = value
|
opts.sourceManifest = value
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if value, ok, err := requireInlineFlagValue(arg, "--content-addressed-root"); ok || err != nil {
|
||||||
|
if err != nil {
|
||||||
|
return opts, err
|
||||||
|
}
|
||||||
|
opts.contentAddressedRoot = value
|
||||||
|
continue
|
||||||
|
}
|
||||||
if value, ok, err := requireInlineFlagValue(arg, "--music-dataset"); ok || err != nil {
|
if value, ok, err := requireInlineFlagValue(arg, "--music-dataset"); ok || err != nil {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return opts, err
|
return opts, err
|
||||||
|
|||||||
@@ -11,6 +11,48 @@ import (
|
|||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline"
|
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestParseBuildHAKArgsContentAddressedRoot(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "separated",
|
||||||
|
args: []string{"--content-addressed-root", "/var/cache/blobs"},
|
||||||
|
want: "/var/cache/blobs",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "inline",
|
||||||
|
args: []string{"--content-addressed-root=/var/cache/blobs"},
|
||||||
|
want: "/var/cache/blobs",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
opts, err := parseBuildHAKArgs(tt.args)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse build-haks args: %v", err)
|
||||||
|
}
|
||||||
|
if opts.contentAddressedRoot != tt.want {
|
||||||
|
t.Fatalf("content-addressed root = %q, want %q", opts.contentAddressedRoot, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseBuildHAKArgsHelpListsContentAddressedRoot(t *testing.T) {
|
||||||
|
_, err := parseBuildHAKArgs([]string{"--help"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected help usage error")
|
||||||
|
}
|
||||||
|
const want = "usage: build-haks [--hak <hak-name> ...] [--archive <archive-name> ...] [--source-manifest <path>] [--content-addressed-root <path>] [--plan-only] [--skip-music] [--music-dataset <id> ...] [--quiet|--verbose|--debug]"
|
||||||
|
if err.Error() != want {
|
||||||
|
t.Fatalf("help usage = %q, want %q", err.Error(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunBuildTopPackageUsesCachedCompiledOutputs(t *testing.T) {
|
func TestRunBuildTopPackageUsesCachedCompiledOutputs(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
mkdirAll(t, filepath.Join(root, "build"))
|
mkdirAll(t, filepath.Join(root, "build"))
|
||||||
|
|||||||
+22
-17
@@ -378,16 +378,8 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
|||||||
if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(hakPath), 0o755); err != nil {
|
||||||
return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err)
|
return BuildResult{}, fmt.Errorf("create hak archive dir: %w", err)
|
||||||
}
|
}
|
||||||
hakOutput, err := os.Create(hakPath)
|
if err := writeHAKArchive(hakPath, resourceSlice(chunk.Assets)); err != nil {
|
||||||
if err != nil {
|
return BuildResult{}, err
|
||||||
return BuildResult{}, fmt.Errorf("create hak archive: %w", err)
|
|
||||||
}
|
|
||||||
if err := erf.Write(hakOutput, erf.New("HAK ", resourceSlice(chunk.Assets))); err != nil {
|
|
||||||
hakOutput.Close()
|
|
||||||
return BuildResult{}, fmt.Errorf("write hak archive: %w", err)
|
|
||||||
}
|
|
||||||
if err := hakOutput.Close(); err != nil {
|
|
||||||
return BuildResult{}, fmt.Errorf("close hak archive: %w", err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -488,23 +480,36 @@ func buildDirectHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeHAKArchive writes a HAK archive, removing any partial output if the ERF
|
// writeHAKArchive publishes a completed HAK atomically, so a failed write never
|
||||||
// writer or a streamed source SHA verification fails, so a corrupt input never
|
// leaves a partial archive at the final path.
|
||||||
// leaves a completed .hak behind.
|
|
||||||
func writeHAKArchive(hakPath string, resources []erf.Resource) error {
|
func writeHAKArchive(hakPath string, resources []erf.Resource) error {
|
||||||
output, err := os.Create(hakPath)
|
tmpPath := hakPath + ".tmp"
|
||||||
|
if err := os.Remove(tmpPath); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return fmt.Errorf("remove stale temporary hak %s: %w", tmpPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("create hak archive: %w", err)
|
return fmt.Errorf("create temporary hak archive %s: %w", tmpPath, err)
|
||||||
}
|
}
|
||||||
if err := erf.Write(output, erf.New("HAK ", resources)); err != nil {
|
if err := erf.Write(output, erf.New("HAK ", resources)); err != nil {
|
||||||
output.Close()
|
output.Close()
|
||||||
os.Remove(hakPath)
|
os.Remove(tmpPath)
|
||||||
return fmt.Errorf("write hak archive: %w", err)
|
return fmt.Errorf("write hak archive: %w", err)
|
||||||
}
|
}
|
||||||
|
if err := output.Sync(); err != nil {
|
||||||
|
output.Close()
|
||||||
|
os.Remove(tmpPath)
|
||||||
|
return fmt.Errorf("sync hak archive: %w", err)
|
||||||
|
}
|
||||||
if err := output.Close(); err != nil {
|
if err := output.Close(); err != nil {
|
||||||
os.Remove(hakPath)
|
os.Remove(tmpPath)
|
||||||
return fmt.Errorf("close hak archive: %w", err)
|
return fmt.Errorf("close hak archive: %w", err)
|
||||||
}
|
}
|
||||||
|
if err := os.Rename(tmpPath, hakPath); err != nil {
|
||||||
|
os.Remove(tmpPath)
|
||||||
|
return fmt.Errorf("publish hak archive %s: %w", hakPath, err)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,35 @@ import (
|
|||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestBuildHAKsCleansFailedTemporaryArchive(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
p := loadDirectProject(t, root)
|
||||||
|
|
||||||
|
blobs := filepath.Join(root, "blobs")
|
||||||
|
sha := strings.Repeat("a", 64)
|
||||||
|
writeBlobAt(t, blobs, sha, "wxyz")
|
||||||
|
manifestPath := writeDirectManifest(t, root,
|
||||||
|
map[string]SourceAsset{"core/a.tga": {SHA256: sha, SizeBytes: 4}},
|
||||||
|
[]string{"core/a.tga"})
|
||||||
|
|
||||||
|
tmpPath := filepath.Join(root, "build", "core_01.hak.tmp")
|
||||||
|
mustWriteFile(t, tmpPath, "stale partial archive")
|
||||||
|
|
||||||
|
_, err := BuildHAKsWithOptions(p, BuildHAKOptions{
|
||||||
|
SourceManifestPath: manifestPath,
|
||||||
|
ContentAddressedRoot: blobs,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected corrupt input to fail")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(root, "build", "core_01.hak")); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatal("corrupt input must not leave a completed hak")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(tmpPath); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatal("failed build must clean temporary hak")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildThenExtract(t *testing.T) {
|
func TestBuildThenExtract(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
mustMkdir(t, filepath.Join(root, "src"))
|
mustMkdir(t, filepath.Join(root, "src"))
|
||||||
|
|||||||
Reference in New Issue
Block a user