feat(nwsync): emit blobs and per-artifact NSYM, assemble merged manifests (#71)
Builds sow-tools#53. Format spec followed is the resolution comment of sow-platform#94, checked line by line against niv/neverwinter.nim at HEAD (`nwsync.nim`, `compressedbuf.nim`, `nwsync/private/libupdate.nim`). ## What lands `crucible nwsync emit <artifact> --out DIR` — explodes one `.hak`/`.erf`, or one loose file such as the TLK, into NWSync blobs plus a NSYM v3 manifest covering only that artifact, with the same `.json` sidecar upstream writes. Blob path `data/sha1/<h0h1>/<h2h3>/<sha1>`, body in NWCompressedBuffer framing (magic `NSYC`, version 3, algorithm 2, uncompressed size, zstd header version 1, dictionary 0, raw zstd frame). The sha1 that names a blob is over the uncompressed bytes. `crucible nwsync assemble --order NAMES --entries DIR --out DIR [--group-id N]` — merges the per-artifact manifests into one, reading no bulk data at all. Merge rule is resref shadowing, not concatenation: a resref in more than one artifact resolves to the earliest artifact in `--order`, which is how the game resolves it. `--group-id` stays caller-supplied (1 current, 2 testing; 0 is absent, matching upstream omitting a zero integer meta field). Rules taken from upstream and not re-invented: `nss`/`ndb`/`gic` always skipped; an unresolvable restype is a hard error, not a skip; a resource over 15 MB fails closed; no `latest` file and no `.origin` file, ever. A `.mod` is refused outright — a persistent world publishes no module contents, so the module contributes no bytes. ## Two deliberate departures - **Emitter version is its own field, not the build revision.** `emitter_version` is a constant bumped only when emitted bytes change. Keying the refuse-to-merge check on `created_with` would invalidate every published index on every unrelated crucible commit and force a re-emit of the whole 15 GB corpus — the opposite of "nothing downstream ever needs the hak again". - **`SOURCE_DATE_EPOCH` pins the sidecar timestamp.** The manifest itself was already deterministic; the sidecar's `created` was not, against the determinism rule in `docs/consumer-contract.md`. ## Not in this PR, and why - **Direct upload.** Only the local `--out` sink exists, which is the conformance path. The upload sink and the mid-hak-failure question are sow-tools#60, and the consumer wiring is #65. - **The conformance run against upstream.** sow-tools#59 owns getting `nwn_nwsync_write` running and capturing reference output. The format here was read from upstream source rather than from its output, so the byte-for-byte manifest comparison and the after-decompression blob comparison still have to happen — that is what #59 is for. The tests in this PR check the layout against the spec, so a shared misreading would pass them. - **The `artifacts/haks/sha256/<a>/<b>/<sha256>.nsym` location.** emit writes `<out>/<name>.nsym`; where a publisher puts it is the publisher's business (#65). - **The acceptance gate** — a real client syncing from an assembled manifest — is unchanged and still open. ## Checks `make check` green (vet, unit tests, shellcheck, yamllint, workflow contract), `make smoke` green with the new builder, `nix build .#crucible` produces `crucible-nwsync`. 🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #71 Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
This commit was merged in pull request #71.
This commit is contained in:
@@ -21,6 +21,7 @@ import (
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/depot"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/menu"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/nwsync"
|
||||
)
|
||||
|
||||
// Exit codes follow the sysexits(3) convention so CI can distinguish
|
||||
@@ -104,6 +105,16 @@ var Registry = []Builder{
|
||||
},
|
||||
Wired: true,
|
||||
},
|
||||
{
|
||||
Name: "nwsync",
|
||||
Bin: "crucible-nwsync",
|
||||
Summary: "publish NWSync blobs and manifests (emit/assemble)",
|
||||
Commands: []Command{
|
||||
{Name: "emit", Summary: "explode one artifact into blobs plus its own NSYM manifest", Usage: "crucible nwsync emit <artifact> --out DIR"},
|
||||
{Name: "assemble", Summary: "merge per-artifact NSYM manifests into one", Usage: "crucible nwsync assemble --order NAMES --entries DIR --out DIR [--group-id N]"},
|
||||
},
|
||||
Wired: true,
|
||||
},
|
||||
{
|
||||
Name: "assets",
|
||||
Bin: "crucible-assets",
|
||||
@@ -241,7 +252,7 @@ var Registry = []Builder{
|
||||
"2da-to-module [flags] <input.2da> [output.json]",
|
||||
"json-to-2da <input.json> <output.2da>",
|
||||
},
|
||||
Aliases: []CommandAlias{{Name: "convert-topdata"}},
|
||||
Aliases: []CommandAlias{{Name: "convert-topdata"}},
|
||||
},
|
||||
},
|
||||
Wired: true,
|
||||
@@ -392,7 +403,7 @@ func runBuilder(name string, args []string, out, errw io.Writer) int {
|
||||
}
|
||||
}
|
||||
if b.Wired {
|
||||
// depot and assets are self-contained builders: they parse their own
|
||||
// depot, assets and nwsync are self-contained builders: they parse their own
|
||||
// subcommands and own their exit contract, bypassing the
|
||||
// b.Commands/delegateLegacy legacy routing entirely.
|
||||
switch b.Name {
|
||||
@@ -400,6 +411,8 @@ func runBuilder(name string, args []string, out, errw io.Writer) int {
|
||||
return depot.Run(args, out, errw, os.Getenv)
|
||||
case "assets":
|
||||
return assets.Run(args, out, errw, os.Getenv)
|
||||
case "nwsync":
|
||||
return nwsync.Run(args, out, errw)
|
||||
}
|
||||
}
|
||||
if !b.Wired {
|
||||
|
||||
@@ -148,6 +148,10 @@ func TestBuilderHelpIsOK(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// selfContained builders parse their own subcommands instead of delegating to
|
||||
// the legacy internal/app surface.
|
||||
var selfContained = map[string]bool{"depot": true, "assets": true, "nwsync": true}
|
||||
|
||||
func TestCanonicalCommandSurface(t *testing.T) {
|
||||
want := map[string][]string{
|
||||
"depot": {"status", "push", "verify", "get", "pull"},
|
||||
@@ -156,6 +160,7 @@ func TestCanonicalCommandSurface(t *testing.T) {
|
||||
"module": {"build", "extract", "validate", "compare", "manifest"},
|
||||
"topdata": {"validate", "build", "package", "compare", "convert"},
|
||||
"wiki": {"build", "deploy"},
|
||||
"nwsync": {"emit", "assemble"},
|
||||
}
|
||||
for _, builder := range Registry {
|
||||
got := builder.subcommands()
|
||||
@@ -173,10 +178,10 @@ func TestRegistryCommandNamesAndAliasesAreUnambiguous(t *testing.T) {
|
||||
for _, builder := range Registry {
|
||||
seen := map[string]bool{}
|
||||
for _, command := range builder.Commands {
|
||||
// depot and assets parse their own subcommands and bypass AppCommand
|
||||
// routing entirely (see the self-contained-builder branch in
|
||||
// runBuilder), so their Commands carry no AppCommand.
|
||||
requireAppCommand := builder.Name != "depot" && builder.Name != "assets"
|
||||
// depot, assets and nwsync parse their own subcommands and bypass
|
||||
// AppCommand routing entirely (see the self-contained-builder branch
|
||||
// in runBuilder), so their Commands carry no AppCommand.
|
||||
requireAppCommand := !selfContained[builder.Name]
|
||||
if command.Name == "" || command.Summary == "" || command.Usage == "" || (requireAppCommand && command.AppCommand == "") {
|
||||
t.Errorf("%s has incomplete command metadata: %#v", builder.Name, command)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user