feat(nwsync): emit blobs and per-artifact NSYM, assemble merged manifests (#71)

Builds sow-tools#53. Format spec followed is the resolution comment of sow-platform#94, checked line by line against niv/neverwinter.nim at HEAD (`nwsync.nim`, `compressedbuf.nim`, `nwsync/private/libupdate.nim`). ## What lands `crucible nwsync emit <artifact> --out DIR` — explodes one `.hak`/`.erf`, or one loose file such as the TLK, into NWSync blobs plus a NSYM v3 manifest covering only that artifact, with the same `.json` sidecar upstream writes. Blob path `data/sha1/<h0h1>/<h2h3>/<sha1>`, body in NWCompressedBuffer framing (magic `NSYC`, version 3, algorithm 2, uncompressed size, zstd header version 1, dictionary 0, raw zstd frame). The sha1 that names a blob is over the uncompressed bytes. `crucible nwsync assemble --order NAMES --entries DIR --out DIR [--group-id N]` — merges the per-artifact manifests into one, reading no bulk data at all. Merge rule is resref shadowing, not concatenation: a resref in more than one artifact resolves to the earliest artifact in `--order`, which is how the game resolves it. `--group-id` stays caller-supplied (1 current, 2 testing; 0 is absent, matching upstream omitting a zero integer meta field). Rules taken from upstream and not re-invented: `nss`/`ndb`/`gic` always skipped; an unresolvable restype is a hard error, not a skip; a resource over 15 MB fails closed; no `latest` file and no `.origin` file, ever. A `.mod` is refused outright — a persistent world publishes no module contents, so the module contributes no bytes. ## Two deliberate departures - **Emitter version is its own field, not the build revision.** `emitter_version` is a constant bumped only when emitted bytes change. Keying the refuse-to-merge check on `created_with` would invalidate every published index on every unrelated crucible commit and force a re-emit of the whole 15 GB corpus — the opposite of "nothing downstream ever needs the hak again". - **`SOURCE_DATE_EPOCH` pins the sidecar timestamp.** The manifest itself was already deterministic; the sidecar's `created` was not, against the determinism rule in `docs/consumer-contract.md`. ## Not in this PR, and why - **Direct upload.** Only the local `--out` sink exists, which is the conformance path. The upload sink and the mid-hak-failure question are sow-tools#60, and the consumer wiring is #65. - **The conformance run against upstream.** sow-tools#59 owns getting `nwn_nwsync_write` running and capturing reference output. The format here was read from upstream source rather than from its output, so the byte-for-byte manifest comparison and the after-decompression blob comparison still have to happen — that is what #59 is for. The tests in this PR check the layout against the spec, so a shared misreading would pass them. - **The `artifacts/haks/sha256/<a>/<b>/<sha256>.nsym` location.** emit writes `<out>/<name>.nsym`; where a publisher puts it is the publisher's business (#65). - **The acceptance gate** — a real client syncing from an assembled manifest — is unchanged and still open. ## Checks `make check` green (vet, unit tests, shellcheck, yamllint, workflow contract), `make smoke` green with the new builder, `nix build .#crucible` produces `crucible-nwsync`. 🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #71 Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
2026-07-29 10:50:05 +00:00
parent 4d03085996
commit a131b25e5b
15 changed files with 1265 additions and 8 deletions
@@ -0,0 +1,247 @@
+package nwsync
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
+)
+
+// fileSizeLimit matches upstream's --limit-file-size default of 15 MB. A
+// resource over it is a hard failure, not a skip: upstream quit(1)s and so do
+// we. Our largest resource today is 13.66 MiB, so the headroom is thin.
+const fileSizeLimit = 15 * 1024 * 1024
+
+// skippedTypes are never published, matching upstream's GobalResTypeSkipList.
+var skippedTypes = resTypes("nss", "ndb", "gic")
+
+// emitterVersion identifies the blob/manifest byte format this package
+// produces. assemble refuses to merge indexes that disagree on it, because two
+// producers of blobs mean a skewed emitter can otherwise write blobs the merged
+// manifest quietly disagrees with. Bump it only when emitted bytes change — it
+// is deliberately not the build revision, which would invalidate every
+// published index on every unrelated commit.
+const emitterVersion = "1"
+
+// serverTypes are loaded only server-side; a manifest holding nothing else
+// has no client contents. Mirrors upstream's GlobalResTypeServerList, whose
+// trailing 0 is RESTYPE_INVALID.
+var serverTypes = append(resTypes(
+	"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl", "ncs", "ndb",
+	"nss", "ptm", "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
+), 0)
+
+func resTypes(extensions ...string) []uint16 {
+	types := make([]uint16, 0, len(extensions))
+	for _, extension := range extensions {
+		restype, ok := erf.ResourceTypeForExtension(extension)
+		if !ok {
+			panic("nwsync: unknown restype " + extension)
+		}
+		types = append(types, restype)
+	}
+	return types
+}
+
+// EmitResult reports what one emit run produced.
+type EmitResult struct {
+	Name         string // artifact name, without extension
+	ManifestPath string
+	Entries      int
+	BlobsWritten int
+}
+
+// Emit explodes one artifact — a .hak/.erf/.mod or a loose file such as the
+// TLK — into NWSync blobs plus a NSYM manifest describing only that artifact.
+func Emit(artifactPath, outDir string) (EmitResult, error) {
+	resources, err := readArtifact(artifactPath)
+	if err != nil {
+		return EmitResult{}, err
+	}
+	name := strings.TrimSuffix(filepath.Base(artifactPath), filepath.Ext(artifactPath))
+
+	entries, blobs, onDiskBytes, err := emitResources(resources, outDir)
+	if err != nil {
+		return EmitResult{}, err
+	}
+	if len(entries) == 0 {
+		return EmitResult{}, fmt.Errorf("%s: nothing to index (no publishable resources)", artifactPath)
+	}
+
+	data, err := writeManifest(entries)
+	if err != nil {
+		return EmitResult{}, err
+	}
+	manifestPath := filepath.Join(outDir, name+".nsym")
+	if err := writeManifestPair(manifestPath, data, entries, onDiskBytes, Sidecar{ModuleName: name}); err != nil {
+		return EmitResult{}, err
+	}
+	return EmitResult{Name: name, ManifestPath: manifestPath, Entries: len(entries), BlobsWritten: blobs}, nil
+}
+
+// readArtifact returns the resources of an ERF/HAK/MOD, or the single resource
+// a loose file represents. Upstream's resman does the same dispatch on the
+// file's first three bytes.
+func readArtifact(path string) ([]erf.Resource, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read artifact: %w", err)
+	}
+	if len(data) >= 3 {
+		switch string(data[:3]) {
+		case "ERF", "HAK":
+			archive, err := erf.Read(bytes.NewReader(data))
+			if err != nil {
+				return nil, fmt.Errorf("%s: %w", path, err)
+			}
+			return archive.Resources, nil
+		case "MOD":
+			// A persistent world never publishes module contents, so the .mod
+			// contributes no bytes to a manifest — it only says which haks and
+			// which TLK the manifest covers.
+			return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path)
+		}
+	}
+	base := filepath.Base(path)
+	extension := filepath.Ext(base)
+	restype, ok := erf.ResourceTypeForExtension(extension)
+	if !ok {
+		return nil, fmt.Errorf("%s: unknown resource type %q", path, extension)
+	}
+	return []erf.Resource{{
+		Name: strings.TrimSuffix(base, extension),
+		Type: restype,
+		Data: data,
+	}}, nil
+}
+
+func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64, error) {
+	// A resref appearing twice inside one artifact resolves to the last one,
+	// the way resman lets the last container added win.
+	order := make([]Identity, 0, len(resources))
+	latest := make(map[Identity]erf.Resource, len(resources))
+	var tooBig []string
+	for _, resource := range resources {
+		if _, ok := erf.ExtensionForResourceType(resource.Type); !ok {
+			return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", resource.Name, resource.Type)
+		}
+		if slices.Contains(skippedTypes, resource.Type) {
+			continue
+		}
+		if len(resource.Data) > fileSizeLimit {
+			tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", resource.Name, len(resource.Data), fileSizeLimit))
+			continue
+		}
+		identity := Identity{ResRef: strings.ToLower(resource.Name), ResType: resource.Type}
+		if _, seen := latest[identity]; !seen {
+			order = append(order, identity)
+		}
+		latest[identity] = resource
+	}
+	if len(tooBig) > 0 {
+		sort.Strings(tooBig)
+		return nil, 0, 0, fmt.Errorf("resources exceed the file size limit:\n  %s", strings.Join(tooBig, "\n  "))
+	}
+
+	entries := make([]Entry, 0, len(order))
+	var blobs int
+	var onDiskBytes int64
+	for _, identity := range order {
+		resource := latest[identity]
+		sum := sha1.Sum(resource.Data)
+		entries = append(entries, Entry{
+			SHA1:    sum,
+			Size:    uint32(len(resource.Data)),
+			ResRef:  identity.ResRef,
+			ResType: identity.ResType,
+		})
+		written, err := writeBlob(outDir, sum, resource.Data)
+		if err != nil {
+			return nil, 0, 0, err
+		}
+		if written > 0 {
+			blobs++
+			onDiskBytes += written
+		}
+	}
+	return entries, blobs, onDiskBytes, nil
+}
+
+// created is the sidecar timestamp. SOURCE_DATE_EPOCH pins it so a build can
+// be reproduced byte for byte; the manifest itself is deterministic already.
+func created() int64 {
+	if raw := os.Getenv("SOURCE_DATE_EPOCH"); raw != "" {
+		if seconds, err := strconv.ParseInt(raw, 10, 64); err == nil {
+			return seconds
+		}
+	}
+	return time.Now().Unix()
+}
+
+// writeBlob writes one NWCompressedBuffer blob and returns its size on disk,
+// or 0 if the blob already existed. Blob names are content hashes, so an
+// existing name is existing content.
+func writeBlob(outDir string, sum [20]byte, data []byte) (int64, error) {
+	path := blobPath(outDir, fmt.Sprintf("%x", sum))
+	if _, err := os.Stat(path); err == nil {
+		return 0, nil
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return 0, fmt.Errorf("create blob directory: %w", err)
+	}
+	blob := compressBlob(data)
+	if err := os.WriteFile(path, blob, 0o644); err != nil {
+		return 0, fmt.Errorf("write blob: %w", err)
+	}
+	return int64(len(blob)), nil
+}
+
+// writeManifestPair writes a NSYM manifest and its .json sidecar. The caller
+// supplies the sidecar fields it knows; the rest are derived from the entries.
+// data must be the serialised form of entries.
+func writeManifestPair(manifestPath string, data []byte, entries []Entry, onDiskBytes int64, sidecar Sidecar) error {
+	if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
+		return fmt.Errorf("create manifest directory: %w", err)
+	}
+	if err := os.WriteFile(manifestPath, data, 0o644); err != nil {
+		return fmt.Errorf("write manifest: %w", err)
+	}
+
+	var totalBytes int64
+	clientContents := false
+	for _, entry := range entries {
+		totalBytes += int64(entry.Size)
+		if !slices.Contains(serverTypes, entry.ResType) {
+			clientContents = true
+		}
+	}
+	sidecar.Version = manifestVersion
+	sidecar.SHA1 = fmt.Sprintf("%x", sha1.Sum(data))
+	sidecar.HashTreeDepth = hashTreeDepth
+	sidecar.IncludesModuleContents = false
+	sidecar.IncludesClientContents = clientContents
+	sidecar.TotalFiles = len(entries)
+	sidecar.TotalBytes = totalBytes
+	sidecar.OnDiskBytes = onDiskBytes
+	sidecar.Created = created()
+	sidecar.CreatedWith = buildinfo.String()
+	sidecar.EmitterVersion = emitterVersion
+
+	body, err := marshalSidecar(sidecar)
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(manifestPath+".json", body, 0o644); err != nil {
+		return fmt.Errorf("write sidecar: %w", err)
+	}
+	return nil
+}