feat(nwsync): emit blobs and per-artifact NSYM, assemble merged manifests (#71)

Builds sow-tools#53. Format spec followed is the resolution comment of sow-platform#94, checked line by line against niv/neverwinter.nim at HEAD (`nwsync.nim`, `compressedbuf.nim`, `nwsync/private/libupdate.nim`). ## What lands `crucible nwsync emit <artifact> --out DIR` — explodes one `.hak`/`.erf`, or one loose file such as the TLK, into NWSync blobs plus a NSYM v3 manifest covering only that artifact, with the same `.json` sidecar upstream writes. Blob path `data/sha1/<h0h1>/<h2h3>/<sha1>`, body in NWCompressedBuffer framing (magic `NSYC`, version 3, algorithm 2, uncompressed size, zstd header version 1, dictionary 0, raw zstd frame). The sha1 that names a blob is over the uncompressed bytes. `crucible nwsync assemble --order NAMES --entries DIR --out DIR [--group-id N]` — merges the per-artifact manifests into one, reading no bulk data at all. Merge rule is resref shadowing, not concatenation: a resref in more than one artifact resolves to the earliest artifact in `--order`, which is how the game resolves it. `--group-id` stays caller-supplied (1 current, 2 testing; 0 is absent, matching upstream omitting a zero integer meta field). Rules taken from upstream and not re-invented: `nss`/`ndb`/`gic` always skipped; an unresolvable restype is a hard error, not a skip; a resource over 15 MB fails closed; no `latest` file and no `.origin` file, ever. A `.mod` is refused outright — a persistent world publishes no module contents, so the module contributes no bytes. ## Two deliberate departures - **Emitter version is its own field, not the build revision.** `emitter_version` is a constant bumped only when emitted bytes change. Keying the refuse-to-merge check on `created_with` would invalidate every published index on every unrelated crucible commit and force a re-emit of the whole 15 GB corpus — the opposite of "nothing downstream ever needs the hak again". - **`SOURCE_DATE_EPOCH` pins the sidecar timestamp.** The manifest itself was already deterministic; the sidecar's `created` was not, against the determinism rule in `docs/consumer-contract.md`. ## Not in this PR, and why - **Direct upload.** Only the local `--out` sink exists, which is the conformance path. The upload sink and the mid-hak-failure question are sow-tools#60, and the consumer wiring is #65. - **The conformance run against upstream.** sow-tools#59 owns getting `nwn_nwsync_write` running and capturing reference output. The format here was read from upstream source rather than from its output, so the byte-for-byte manifest comparison and the after-decompression blob comparison still have to happen — that is what #59 is for. The tests in this PR check the layout against the spec, so a shared misreading would pass them. - **The `artifacts/haks/sha256/<a>/<b>/<sha256>.nsym` location.** emit writes `<out>/<name>.nsym`; where a publisher puts it is the publisher's business (#65). - **The acceptance gate** — a real client syncing from an assembled manifest — is unchanged and still open. ## Checks `make check` green (vet, unit tests, shellcheck, yamllint, workflow contract), `make smoke` green with the new builder, `nix build .#crucible` produces `crucible-nwsync`. 🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #71 Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
2026-07-29 10:50:05 +00:00
parent 4d03085996
commit a131b25e5b
15 changed files with 1265 additions and 8 deletions
@@ -0,0 +1,450 @@
+package nwsync
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
+)
+
+func restype(t *testing.T, extension string) uint16 {
+	t.Helper()
+	value, ok := erf.ResourceTypeForExtension(extension)
+	if !ok {
+		t.Fatalf("unknown restype %q", extension)
+	}
+	return value
+}
+
+// writeHak builds a HAK fixture from resref.ext => body pairs.
+func writeHak(t *testing.T, path string, contents map[string][]byte) {
+	t.Helper()
+	resources := make([]erf.Resource, 0, len(contents))
+	for name, body := range contents {
+		stem, extension, _ := strings.Cut(name, ".")
+		resources = append(resources, erf.Resource{
+			Name: stem,
+			Type: restype(t, extension),
+			Data: body,
+			Size: int64(len(body)),
+		})
+	}
+	var out bytes.Buffer
+	if err := erf.Write(&out, erf.New("HAK", resources)); err != nil {
+		t.Fatalf("write hak: %v", err)
+	}
+	if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil {
+		t.Fatalf("write hak file: %v", err)
+	}
+}
+
+func TestBlobFramingRoundTrips(t *testing.T) {
+	data := []byte("the quick brown fox jumps over the lazy dog, repeatedly and at length")
+	blob := compressBlob(data)
+
+	header := make([]uint32, 6)
+	if err := binary.Read(bytes.NewReader(blob[:blobHeaderBytes]), binary.LittleEndian, header); err != nil {
+		t.Fatalf("read header: %v", err)
+	}
+	want := []uint32{blobMagic, 3, 2, uint32(len(data)), 1, 0}
+	for i := range want {
+		if header[i] != want[i] {
+			t.Errorf("header field %d = %d, want %d", i, header[i], want[i])
+		}
+	}
+	if string(blob[:4]) != "NSYC" {
+		t.Errorf("magic bytes = %q, want NSYC", blob[:4])
+	}
+
+	got, err := decompressBlob(blob)
+	if err != nil {
+		t.Fatalf("decompress: %v", err)
+	}
+	if !bytes.Equal(got, data) {
+		t.Errorf("round trip mismatch: %q", got)
+	}
+}
+
+func TestManifestBytesMatchUpstreamLayout(t *testing.T) {
+	shared := sha1.Sum([]byte("shared"))
+	other := sha1.Sum([]byte("other"))
+	// Deliberately out of order, and with two resrefs sharing one sha1: the
+	// second one must become a mapping, not a second entry.
+	entries := []Entry{
+		{SHA1: other, Size: 5, ResRef: "zzz", ResType: 1},
+		{SHA1: shared, Size: 6, ResRef: "bbb", ResType: 2},
+		{SHA1: shared, Size: 6, ResRef: "aaa", ResType: 3},
+	}
+	data, err := writeManifest(entries)
+	if err != nil {
+		t.Fatalf("write manifest: %v", err)
+	}
+
+	if string(data[:4]) != "NSYM" {
+		t.Fatalf("magic = %q", data[:4])
+	}
+	var version, entryCount, mappingCount uint32
+	reader := bytes.NewReader(data[4:16])
+	for _, field := range []*uint32{&version, &entryCount, &mappingCount} {
+		_ = binary.Read(reader, binary.LittleEndian, field)
+	}
+	if version != 3 || entryCount != 2 || mappingCount != 1 {
+		t.Fatalf("header = version %d, %d entries, %d mappings; want 3/2/1", version, entryCount, mappingCount)
+	}
+	wantSize := 16 + int(entryCount)*(20+4+16+2) + int(mappingCount)*(4+16+2)
+	if len(data) != wantSize {
+		t.Fatalf("manifest is %d bytes, want %d", len(data), wantSize)
+	}
+
+	// Sorted by sha1 hex then resref, so the shared hash's "aaa" is the entry
+	// and "bbb" is demoted to a mapping.
+	round, err := readManifest(data)
+	if err != nil {
+		t.Fatalf("read manifest: %v", err)
+	}
+	if len(round) != 3 {
+		t.Fatalf("round trip returned %d entries, want 3", len(round))
+	}
+	byResRef := map[string]Entry{}
+	for _, entry := range round {
+		byResRef[entry.ResRef] = entry
+	}
+	for _, entry := range entries {
+		got, ok := byResRef[entry.ResRef]
+		if !ok {
+			t.Fatalf("resref %q lost in round trip", entry.ResRef)
+		}
+		if got != entry {
+			t.Errorf("resref %q = %+v, want %+v", entry.ResRef, got, entry)
+		}
+	}
+	if round[0].ResRef != "aaa" && round[1].ResRef != "aaa" {
+		t.Errorf("entries are not sorted by sha1 then resref: %+v", round)
+	}
+}
+
+func TestEmitWritesBlobsAndManifest(t *testing.T) {
+	dir := t.TempDir()
+	hak := filepath.Join(dir, "sow_test_01.hak")
+	body := []byte("texture bytes")
+	writeHak(t, hak, map[string][]byte{
+		"bloodstain1.tga": body,
+		"copy1.txi":       body, // same content, different resref: one blob
+		"script1.nss":     []byte("void main() {}"),
+		"debug1.ndb":      []byte("debug"),
+		"comment1.gic":    []byte("comment"),
+	})
+
+	out := filepath.Join(dir, "out")
+	result, err := Emit(hak, out)
+	if err != nil {
+		t.Fatalf("emit: %v", err)
+	}
+	if result.Entries != 2 {
+		t.Errorf("emitted %d entries, want 2 (nss/ndb/gic are always skipped)", result.Entries)
+	}
+	if result.BlobsWritten != 1 {
+		t.Errorf("wrote %d blobs, want 1 (identical content shares a blob)", result.BlobsWritten)
+	}
+
+	// The blob is named by the sha1 of the uncompressed bytes, under a depth-2
+	// hash tree, and decompresses back to exactly those bytes.
+	sum := sha1.Sum(body)
+	name := hex.EncodeToString(sum[:])
+	path := filepath.Join(out, "data", "sha1", name[0:2], name[2:4], name)
+	blob, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("blob missing at %s: %v", path, err)
+	}
+	got, err := decompressBlob(blob)
+	if err != nil {
+		t.Fatalf("decompress blob: %v", err)
+	}
+	if !bytes.Equal(got, body) {
+		t.Errorf("blob decompressed to %q, want %q", got, body)
+	}
+
+	entries := readEmitted(t, out, "sow_test_01")
+	for _, entry := range entries {
+		if entry.ResType == restype(t, "nss") || entry.ResType == restype(t, "ndb") || entry.ResType == restype(t, "gic") {
+			t.Errorf("skipped restype leaked into the manifest: %+v", entry)
+		}
+	}
+
+	var sidecar Sidecar
+	body2, err := os.ReadFile(result.ManifestPath + ".json")
+	if err != nil {
+		t.Fatalf("sidecar missing: %v", err)
+	}
+	if err := json.Unmarshal(body2, &sidecar); err != nil {
+		t.Fatalf("sidecar json: %v", err)
+	}
+	if sidecar.TotalFiles != 2 || sidecar.HashTreeDepth != 2 || sidecar.Version != 3 {
+		t.Errorf("sidecar = %+v", sidecar)
+	}
+	if sidecar.IncludesModuleContents {
+		t.Error("sidecar claims module contents; a published manifest never has them")
+	}
+	if strings.Contains(string(body2), "group_id") {
+		t.Error("per-artifact sidecar should omit group_id (0 means absent)")
+	}
+	if _, err := os.Stat(filepath.Join(out, "latest")); err == nil {
+		t.Error("a latest file was written; there must never be one")
+	}
+}
+
+func readEmitted(t *testing.T, dir, name string) []Entry {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join(dir, name+".nsym"))
+	if err != nil {
+		t.Fatalf("read emitted manifest: %v", err)
+	}
+	entries, err := readManifest(data)
+	if err != nil {
+		t.Fatalf("parse emitted manifest: %v", err)
+	}
+	return entries
+}
+
+func TestEmitFailsClosedOnOversizeResource(t *testing.T) {
+	dir := t.TempDir()
+	hak := filepath.Join(dir, "big.hak")
+	writeHak(t, hak, map[string][]byte{"huge1.tga": make([]byte, fileSizeLimit+1)})
+
+	if _, err := Emit(hak, filepath.Join(dir, "out")); err == nil {
+		t.Fatal("emit accepted a resource over the 15 MB limit")
+	}
+}
+
+func TestEmitLooseFile(t *testing.T) {
+	dir := t.TempDir()
+	tlk := filepath.Join(dir, "sow_tlk.tlk")
+	if err := os.WriteFile(tlk, []byte("TLK V3.0 payload"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	out := filepath.Join(dir, "out")
+	result, err := Emit(tlk, out)
+	if err != nil {
+		t.Fatalf("emit tlk: %v", err)
+	}
+	entries := readEmitted(t, out, "sow_tlk")
+	if len(entries) != 1 || entries[0].ResRef != "sow_tlk" || entries[0].ResType != restype(t, "tlk") {
+		t.Fatalf("tlk emitted as %+v", entries)
+	}
+	if result.BlobsWritten != 1 {
+		t.Errorf("wrote %d blobs, want 1", result.BlobsWritten)
+	}
+}
+
+// emitFixture emits two haks that share a resref, so the merge rule is
+// observable: "top" holds the winning body, "assets" the shadowed one.
+func emitFixture(t *testing.T) (string, []byte, []byte) {
+	t.Helper()
+	dir := t.TempDir()
+	topBody := []byte("2da from sow_top")
+	assetBody := []byte("2da from the asset hak")
+	writeHak(t, filepath.Join(dir, "sow_top.hak"), map[string][]byte{"appearance.2da": topBody})
+	writeHak(t, filepath.Join(dir, "sow_core_01.hak"), map[string][]byte{
+		"appearance.2da":  assetBody,
+		"bloodstain1.tga": []byte("blood"),
+	})
+	out := filepath.Join(dir, "out")
+	for _, name := range []string{"sow_top", "sow_core_01"} {
+		if _, err := Emit(filepath.Join(dir, name+".hak"), out); err != nil {
+			t.Fatalf("emit %s: %v", name, err)
+		}
+	}
+	return out, topBody, assetBody
+}
+
+func TestAssembleShadowsByOrder(t *testing.T) {
+	entriesDir, topBody, assetBody := emitFixture(t)
+	out := t.TempDir()
+
+	result, err := Assemble(AssembleOptions{
+		Order:      []string{"sow_top", "sow_core_01"},
+		EntriesDir: entriesDir,
+		OutDir:     out,
+		GroupID:    2,
+	})
+	if err != nil {
+		t.Fatalf("assemble: %v", err)
+	}
+	if result.Entries != 2 {
+		t.Fatalf("merged %d entries, want 2 (appearance.2da is shadowed, not duplicated)", result.Entries)
+	}
+
+	data, err := os.ReadFile(result.ManifestPath)
+	if err != nil {
+		t.Fatalf("read merged manifest: %v", err)
+	}
+	merged := sha1.Sum(data)
+	if hex.EncodeToString(merged[:]) != result.SHA1 || filepath.Base(result.ManifestPath) != result.SHA1 {
+		t.Errorf("manifest is not named by its own sha1: %s", result.ManifestPath)
+	}
+	entries, err := readManifest(data)
+	if err != nil {
+		t.Fatalf("parse merged manifest: %v", err)
+	}
+	for _, entry := range entries {
+		if entry.ResRef != "appearance" {
+			continue
+		}
+		if entry.SHA1 != sha1.Sum(topBody) {
+			t.Errorf("appearance.2da resolved to the wrong hak; want the earliest in --order")
+		}
+		if entry.SHA1 == sha1.Sum(assetBody) {
+			t.Error("appearance.2da resolved to the shadowed hak")
+		}
+	}
+
+	sidecar := Sidecar{}
+	body, err := os.ReadFile(result.ManifestPath + ".json")
+	if err != nil {
+		t.Fatalf("merged sidecar missing: %v", err)
+	}
+	if err := json.Unmarshal(body, &sidecar); err != nil {
+		t.Fatalf("merged sidecar json: %v", err)
+	}
+	if sidecar.GroupID != 2 {
+		t.Errorf("group_id = %d, want 2 (testing)", sidecar.GroupID)
+	}
+	if sidecar.SHA1 != result.SHA1 {
+		t.Errorf("sidecar sha1 = %s, want %s", sidecar.SHA1, result.SHA1)
+	}
+	if sidecar.TotalFiles != 2 {
+		t.Errorf("total_files = %d, want 2", sidecar.TotalFiles)
+	}
+}
+
+func TestAssembleReversedOrderPicksTheOtherHak(t *testing.T) {
+	entriesDir, topBody, assetBody := emitFixture(t)
+	out := t.TempDir()
+
+	result, err := Assemble(AssembleOptions{
+		Order:      []string{"sow_core_01", "sow_top"},
+		EntriesDir: entriesDir,
+		OutDir:     out,
+	})
+	if err != nil {
+		t.Fatalf("assemble: %v", err)
+	}
+	data, _ := os.ReadFile(result.ManifestPath)
+	entries, err := readManifest(data)
+	if err != nil {
+		t.Fatalf("parse merged manifest: %v", err)
+	}
+	for _, entry := range entries {
+		if entry.ResRef == "appearance" && entry.SHA1 != sha1.Sum(assetBody) {
+			t.Errorf("appearance.2da did not follow --order; still resolves to %x", entry.SHA1)
+		}
+	}
+	_ = topBody
+}
+
+func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) {
+	entriesDir, _, _ := emitFixture(t)
+
+	path := filepath.Join(entriesDir, "sow_core_01.nsym.json")
+	var sidecar Sidecar
+	body, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := json.Unmarshal(body, &sidecar); err != nil {
+		t.Fatal(err)
+	}
+	sidecar.EmitterVersion = "0"
+	patched, err := marshalSidecar(sidecar)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, patched, 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = Assemble(AssembleOptions{
+		Order:      []string{"sow_top", "sow_core_01"},
+		EntriesDir: entriesDir,
+		OutDir:     t.TempDir(),
+	})
+	if err == nil || !strings.Contains(err.Error(), "emitter version mismatch") {
+		t.Fatalf("assemble merged across emitter versions: %v", err)
+	}
+}
+
+func TestEmitHonoursSourceDateEpoch(t *testing.T) {
+	t.Setenv("SOURCE_DATE_EPOCH", "1700000000")
+	dir := t.TempDir()
+	hak := filepath.Join(dir, "pinned.hak")
+	writeHak(t, hak, map[string][]byte{"one1.tga": []byte("body")})
+	result, err := Emit(hak, filepath.Join(dir, "out"))
+	if err != nil {
+		t.Fatalf("emit: %v", err)
+	}
+	body, err := os.ReadFile(result.ManifestPath + ".json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var sidecar Sidecar
+	if err := json.Unmarshal(body, &sidecar); err != nil {
+		t.Fatal(err)
+	}
+	if sidecar.Created != 1700000000 {
+		t.Errorf("created = %d, want the pinned SOURCE_DATE_EPOCH", sidecar.Created)
+	}
+}
+
+func TestEmitRejectsAModule(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "sow.mod")
+	var out bytes.Buffer
+	if err := erf.Write(&out, erf.New("MOD", []erf.Resource{
+		{Name: "module", Type: restype(t, "ifo"), Data: []byte("ifo"), Size: 3},
+	})); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Emit(path, filepath.Join(dir, "out")); err == nil {
+		t.Fatal("emit accepted a .mod; a manifest never carries module contents")
+	}
+}
+
+func TestAssembleFailsClosedOnMissingIndex(t *testing.T) {
+	entriesDir, _, _ := emitFixture(t)
+	_, err := Assemble(AssembleOptions{
+		Order:      []string{"sow_top", "sow_never_published"},
+		EntriesDir: entriesDir,
+		OutDir:     t.TempDir(),
+	})
+	if err == nil || !strings.Contains(err.Error(), "sow_never_published") {
+		t.Fatalf("assemble did not fail closed and name the missing artifact: %v", err)
+	}
+}
+
+func TestRunUsageErrors(t *testing.T) {
+	cases := [][]string{
+		nil,
+		{"nope"},
+		{"emit"},
+		{"emit", "artifact.hak"},
+		{"assemble", "--entries", "x", "--out", "y"},
+		{"assemble", "--order", "a", "--out", "y"},
+		{"assemble", "--order", "a", "--entries", "x"},
+	}
+	for _, args := range cases {
+		var out, errw bytes.Buffer
+		if code := Run(args, &out, &errw); code != exitUsage {
+			t.Errorf("Run(%v) exit=%d, want %d", args, code, exitUsage)
+		}
+	}
+}