fix(module): run scripts/fetch-upstream-manifests (#84)

Closes #83. Part of #54; implements #63 Decision 5 ("Coordinated rename flip").

## What changed

One string literal in `internal/app/app.go:388`:

```go
[]string{"scripts", "fetch-hak-manifest"} -> []string{"scripts", "fetch-upstream-manifests"}
```

sow-module renamed its half in ShadowsOverWestgate/sow-module#57
(`scripts/fetch-hak-manifest.sh` -> `scripts/fetch-upstream-manifests.sh`,
extended to resolve the topdata channel as well). The script name is a
hardcoded path convention shared by the two repos, not config, so both sides
only work when they carry the same name.

No doc in this repo named the old script (`grep` over the tree found the one
call site only), so nothing else needed touching.

## No fallback, by decision

Per #63 Decision 5: no transitional symlink, no Go-side fallback. Both sides
flip and the short broken window is accepted, because the failure is loud and
unmistakable (`required project script is missing`). A fallback would keep
both names alive forever.

## Merge order matters

`sow-module/.gitea/workflows/release.yml` runs `nix flake update sow-tools`, so
it always builds against the latest crucible, unpinned. Every crucible module
build in sow-module fails between sow-module#57 merging and a crucible release
carrying this flip. **Merge sow-module#57 and cut a crucible release back to
back** to keep that gap short.

## Checks

- `go build ./...`, `go vet ./internal/app/`, and the full `go test ./...` suite pass.
- No test covers this call path, and none was added: it is a single hardcoded
  constant that has to match the other repo, so a test here could only assert
  the literal against itself. The real check is the sow-module build.

## Still open on the issue's "done when"

- [x] app.go runs `scripts/fetch-upstream-manifests`
- [x] no doc in this repo names `fetch-hak-manifest`
- [ ] a crucible release is cut, and the crucible module build succeeds in a
      sow-module checkout at #57's head — needs a release after this merges

🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #84
Reviewed-by: xtul <mpiasecki720@protonmail.com>
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>

docs/command-surface.md

@@ -48,10 +48,19 @@ beside the artifact itself with the extension replaced, so `emit` and
 `assemble` agree on where it is without being told.
 
 ```
-nwsync emit     [--as NAME] [--out DIR] <artifact-key> <file>
+nwsync emit     [--as NAME] [--out DIR] [--jobs N] <artifact-key> <file>
 nwsync assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>...
 ```
 
+`emit` is latency-bound, not CPU-bound: every blob costs an existence probe
+plus an upload, and a measured backfill spent 26 seconds of CPU across 9.5
+minutes of wall clock. `--jobs N` (default 16) sets how many resources are in
+flight at once. The manifest is byte-identical at any value — the number of
+workers is never observable in the output. Peak memory is `N` times the
+per-resource limit of 15 MB plus its compressed copy, so raising `N` far past
+the default costs real memory for little gain: the transport keeps 16 idle
+connections per host, and past that a worker pays a fresh TLS handshake.
+
 Both verbs upload by default; nothing bulky is ever written to the runner's
 disk. `--out DIR` writes a local repository tree instead, which is the
 conformance path against upstream `nwn_nwsync_write`. The zone comes from
@@ -70,6 +79,35 @@ the zone and no index. Blob names are content hashes, so re-running skips
 whatever already landed, and `assemble` fails closed on an artifact with no
 index rather than publishing a manifest that is missing a hak.
 
+### What an emitted tree looks like
+
+`--out DIR` produces the same tree `emit` would upload, which makes it the way
+to check a zone by hand without touching one:
+
+```
+<artifact-sha>.nsym          binary index
+<artifact-sha>.nsym.json     the same index, readable
+data/sha1/a7/4a/a74aa84a...  one blob per resource, two-level fanout
+```
+
+A blob's name is the SHA-1 of the resource's **original** bytes, but the file on
+disk is not those bytes: each blob is wrapped in NWCompressedBuffer framing, a
+24-byte `NSYC` header followed by a zstd frame. Hashing the file directly will
+not match its name, which is the obvious
+first thing to try and the obvious first thing to be confused by. Strip the
+header first:
+
+```
+tail -c +25 <blob> | zstd -dc | sha1sum   # == the blob's filename
+```
+
+The header carries the uncompressed length as a little-endian `uint32` at offset
+12, so the decompressed size is checkable without decompressing. Compression is
+worth roughly a 4:1
+saving on hak content: a 250 MB hak emitted 2296 blobs totalling 59 MB on disk
+against 249 MB of resources, as recorded in the sidecar's `on_disk_bytes` and
+`total_bytes`.
+
 ## Hidden compatibility aliases
 
 Existing scripts may continue using these names indefinitely. They are accepted

internal/app/app.go

@@ -385,7 +385,7 @@ func refreshBuildModuleManifest(ctx context, p *project.Project, progress func(s
 	}
 
 	progress("Refreshing hak list from the latest published sow-assets manifest...")
-	if err := runProjectScript(ctx, p, []string{"scripts", "fetch-hak-manifest"}, manifestPath); err != nil {
+	if err := runProjectScript(ctx, p, []string{"scripts", "fetch-upstream-manifests"}, manifestPath); err != nil {
 		return "", "", err
 	}
 	if _, err := pipeline.ApplyHAKManifest(p, manifestPath); err != nil {

internal/erf/erf.go

@@ -311,63 +311,110 @@ func Write(w io.Writer, archive Archive) error {
 	return nil
 }
 
+// IndexEntry locates one resource inside an archive without holding its
+// payload. Streaming callers read one payload at a time from these, so peak
+// memory tracks the largest resource instead of the whole archive.
+type IndexEntry struct {
+	Name   string
+	Type   uint16
+	Offset int64
+	Size   int64
+}
+
+// Index is the header plus the resource table of an ERF: everything except the
+// payloads.
+type Index struct {
+	FileType string
+	Version  string
+	Entries  []IndexEntry
+}
+
+// ReadIndex parses the tables of an ERF of the given size, reading only the
+// header, the key list and the resource list.
+func ReadIndex(r io.ReaderAt, size int64) (Index, error) {
+	if size < headerSize {
+		return Index{}, fmt.Errorf("erf file too small: %d bytes", size)
+	}
+
+	var hdr header
+	if err := binary.Read(io.NewSectionReader(r, 0, headerSize), binary.LittleEndian, &hdr); err != nil {
+		return Index{}, fmt.Errorf("decode erf header: %w", err)
+	}
+
+	if int64(hdr.KeyListOffset)+int64(hdr.EntryCount)*24 > size {
+		return Index{}, fmt.Errorf("erf key list exceeds file bounds")
+	}
+	keys := make([]keyEntry, hdr.EntryCount)
+	keyReader := io.NewSectionReader(r, int64(hdr.KeyListOffset), int64(hdr.EntryCount)*24)
+	if err := binary.Read(keyReader, binary.LittleEndian, &keys); err != nil {
+		return Index{}, fmt.Errorf("decode key list: %w", err)
+	}
+
+	if int64(hdr.ResourceListOffset)+int64(hdr.EntryCount)*8 > size {
+		return Index{}, fmt.Errorf("erf resource list exceeds file bounds")
+	}
+	entries := make([]resourceEntry, hdr.EntryCount)
+	entryReader := io.NewSectionReader(r, int64(hdr.ResourceListOffset), int64(hdr.EntryCount)*8)
+	if err := binary.Read(entryReader, binary.LittleEndian, &entries); err != nil {
+		return Index{}, fmt.Errorf("decode resource list: %w", err)
+	}
+
+	index := Index{
+		FileType: string(hdr.FileType[:]),
+		Version:  string(hdr.Version[:]),
+		Entries:  make([]IndexEntry, 0, hdr.EntryCount),
+	}
+	for position, key := range keys {
+		entry := entries[position]
+		if int64(entry.Offset)+int64(entry.Size) > size {
+			return Index{}, fmt.Errorf("resource %d exceeds file bounds", position)
+		}
+		index.Entries = append(index.Entries, IndexEntry{
+			Name:   string(bytes.TrimRight(key.ResRef[:], "\x00")),
+			Type:   key.ResourceType,
+			Offset: int64(entry.Offset),
+			Size:   int64(entry.Size),
+		})
+	}
+	return index, nil
+}
+
+// ReadPayload returns one resource's bytes.
+func ReadPayload(r io.ReaderAt, entry IndexEntry) ([]byte, error) {
+	payload := make([]byte, entry.Size)
+	if _, err := r.ReadAt(payload, entry.Offset); err != nil {
+		return nil, fmt.Errorf("read resource %q: %w", entry.Name, err)
+	}
+	return payload, nil
+}
+
+// Read materialises a whole archive. Payloads are subslices of the buffer the
+// archive was read into, so nothing is copied twice: a caller must not mutate
+// Data. Callers that only need one resource at a time should use ReadIndex
+// instead, which never holds the archive at all.
 func Read(r io.Reader) (Archive, error) {
 	data, err := io.ReadAll(r)
 	if err != nil {
 		return Archive{}, fmt.Errorf("read erf: %w", err)
 	}
-	if len(data) < headerSize {
-		return Archive{}, fmt.Errorf("erf file too small: %d bytes", len(data))
+	index, err := ReadIndex(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		return Archive{}, err
 	}
 
-	var hdr header
-	if err := binary.Read(bytes.NewReader(data[:headerSize]), binary.LittleEndian, &hdr); err != nil {
-		return Archive{}, fmt.Errorf("decode erf header: %w", err)
-	}
-
-	keyStart := int(hdr.KeyListOffset)
-	keyEnd := keyStart + int(hdr.EntryCount)*24
-	if keyEnd > len(data) {
-		return Archive{}, fmt.Errorf("erf key list exceeds file bounds")
-	}
-	keys := make([]keyEntry, hdr.EntryCount)
-	if err := binary.Read(bytes.NewReader(data[keyStart:keyEnd]), binary.LittleEndian, &keys); err != nil {
-		return Archive{}, fmt.Errorf("decode key list: %w", err)
-	}
-
-	resourceStart := int(hdr.ResourceListOffset)
-	resourceEnd := resourceStart + int(hdr.EntryCount)*8
-	if resourceEnd > len(data) {
-		return Archive{}, fmt.Errorf("erf resource list exceeds file bounds")
-	}
-	entries := make([]resourceEntry, hdr.EntryCount)
-	if err := binary.Read(bytes.NewReader(data[resourceStart:resourceEnd]), binary.LittleEndian, &entries); err != nil {
-		return Archive{}, fmt.Errorf("decode resource list: %w", err)
-	}
-
-	resources := make([]Resource, 0, hdr.EntryCount)
-	for index, key := range keys {
-		entry := entries[index]
-		start := int(entry.Offset)
-		end := start + int(entry.Size)
-		if end > len(data) {
-			return Archive{}, fmt.Errorf("resource %d exceeds file bounds", index)
-		}
-
-		resref := string(bytes.TrimRight(key.ResRef[:], "\x00"))
-		payload := make([]byte, entry.Size)
-		copy(payload, data[start:end])
+	resources := make([]Resource, 0, len(index.Entries))
+	for _, entry := range index.Entries {
 		resources = append(resources, Resource{
-			Name: resref,
-			Type: key.ResourceType,
-			Data: payload,
-			Size: int64(entry.Size),
+			Name: entry.Name,
+			Type: entry.Type,
+			Data: data[entry.Offset : entry.Offset+entry.Size],
+			Size: entry.Size,
 		})
 	}
 
 	return Archive{
-		FileType:  string(hdr.FileType[:]),
-		Version:   string(hdr.Version[:]),
+		FileType:  index.FileType,
+		Version:   index.Version,
 		Resources: resources,
 	}, nil
 }

internal/nwsync/compressedbuf.go

@@ -22,9 +22,13 @@ const (
 	blobHeaderBytes = 24
 )
 
+// EncodeAll/DecodeAll are single-threaded per call, so the default pool of one
+// encoder per CPU only buys idle memory: each holds a window-sized history, so
+// on a 24-core runner that is ~200 MB of live heap doing nothing. Concurrency 1
+// produces byte-identical output.
 var (
-	blobEncoder, _ = zstd.NewWriter(nil)
-	blobDecoder, _ = zstd.NewReader(nil)
+	blobEncoder, _ = zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1))
+	blobDecoder, _ = zstd.NewReader(nil, zstd.WithDecoderConcurrency(1))
 )
 
 // compressBlob wraps data in NWCompressedBuffer framing.

internal/nwsync/emit.go

@@ -1,10 +1,10 @@
 package nwsync
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha1"
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -12,6 +12,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
@@ -62,12 +63,21 @@ type EmitResult struct {
 	BlobsWritten int
 }
 
+// defaultEmitJobs is how many resources are hashed, compressed and stored at
+// once. Emit is latency-bound, not CPU-bound: a blob costs a probe round-trip
+// plus an upload round-trip, and a measured backfill spent 26 s of CPU across
+// 9.5 minutes of wall clock. The figure matches depot's DEPOT_JOBS default and
+// the transport's MaxIdleConnsPerHost, so a worker per connection needs no new
+// TLS handshake.
+const defaultEmitJobs = 16
+
 // EmitOptions describes one emit run.
 type EmitOptions struct {
 	ArtifactKey  string // depot key of the artifact; the NSYM key is derived from it
 	ArtifactPath string // the file on disk
 	As           string // name override, for a TLK whose filename is not its published name
 	OutDir       string // write locally instead of uploading — the conformance path
+	Jobs         int    // resources in flight at once; 0 means defaultEmitJobs
 	Sink         sink   // test seam; nil means OutDir or the zone
 }
 
@@ -79,11 +89,18 @@ type EmitOptions struct {
 // real blobs in the zone and no index, which is unambiguous. Blob names are
 // content hashes, so re-running skips whatever already landed.
 func Emit(options EmitOptions) (EmitResult, error) {
-	artifact, err := os.ReadFile(options.ArtifactPath)
+	artifact, err := os.Open(options.ArtifactPath)
 	if err != nil {
 		return EmitResult{}, fmt.Errorf("read artifact: %w", err)
 	}
-	if err := checkArtifactKey(options.ArtifactKey, artifact); err != nil {
+	defer artifact.Close()
+	info, err := artifact.Stat()
+	if err != nil {
+		return EmitResult{}, fmt.Errorf("read artifact: %w", err)
+	}
+	// A section reader, not the file itself: hashing must not move the file
+	// offset out from under everything that reads the artifact afterwards.
+	if err := checkArtifactKey(options.ArtifactKey, io.NewSectionReader(artifact, 0, info.Size())); err != nil {
 		return EmitResult{}, err
 	}
 	name := options.As
@@ -98,7 +115,7 @@ func Emit(options EmitOptions) (EmitResult, error) {
 		return EmitResult{}, err
 	}
 
-	resources, err := readArtifact(options.ArtifactPath, artifact, name)
+	index, err := readArtifactIndex(options.ArtifactPath, artifact, info.Size(), name)
 	if err != nil {
 		return EmitResult{}, err
 	}
@@ -108,7 +125,11 @@ func Emit(options EmitOptions) (EmitResult, error) {
 		return EmitResult{}, err
 	}
 
-	entries, blobs, onDiskBytes, err := emitResources(resources, target)
+	jobs := options.Jobs
+	if jobs < 1 {
+		jobs = defaultEmitJobs
+	}
+	entries, blobs, onDiskBytes, err := emitResources(artifact, index, target, jobs)
 	if err != nil {
 		return EmitResult{}, err
 	}
@@ -137,87 +158,164 @@ func openSink(outDir string, injected sink) (sink, error) {
 	return newZoneSink(context.Background(), os.Getenv)
 }
 
-// readArtifact returns the resources of an ERF/HAK/MOD, or the single resource
-// a loose file represents. Upstream's resman does the same dispatch on the
-// file's first three bytes. name is the artifact's published name, which for a
-// loose file is also its resref.
-func readArtifact(path string, data []byte, name string) ([]erf.Resource, error) {
-	if len(data) >= 3 {
-		switch string(data[:3]) {
+// readArtifactIndex locates the resources of an ERF/HAK/MOD, or the single
+// resource a loose file represents, without reading any payload. Upstream's
+// resman does the same dispatch on the file's first three bytes. name is the
+// artifact's published name, which for a loose file is also its resref.
+func readArtifactIndex(path string, artifact io.ReaderAt, size int64, name string) ([]erf.IndexEntry, error) {
+	magic := make([]byte, 3)
+	if size >= 3 {
+		if _, err := artifact.ReadAt(magic, 0); err != nil {
+			return nil, fmt.Errorf("%s: %w", path, err)
+		}
+	}
+	switch string(magic) {
 	case "ERF", "HAK":
-			archive, err := erf.Read(bytes.NewReader(data))
+		index, err := erf.ReadIndex(artifact, size)
 		if err != nil {
 			return nil, fmt.Errorf("%s: %w", path, err)
 		}
-			return archive.Resources, nil
+		return index.Entries, nil
 	case "MOD":
 		// A persistent world never publishes module contents, so the .mod
 		// contributes no bytes to a manifest — it only says which haks and
 		// which TLK the manifest covers.
 		return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path)
 	}
-	}
 	extension := filepath.Ext(filepath.Base(path))
 	restype, ok := erf.ResourceTypeForExtension(extension)
 	if !ok {
 		return nil, fmt.Errorf("%s: unknown resource type %q", path, extension)
 	}
-	return []erf.Resource{{
-		Name: name,
-		Type: restype,
-		Data: data,
-	}}, nil
+	return []erf.IndexEntry{{Name: name, Type: restype, Offset: 0, Size: size}}, nil
 }
 
-func emitResources(resources []erf.Resource, target sink) ([]Entry, int, int64, error) {
+// emitResources hashes, compresses and stores resources, reading each payload
+// from the artifact only when its turn comes. Peak memory tracks the resources
+// in flight, not the archive: a 2 GB hak must emit inside a runner's few spare
+// GB. jobs of them are in flight at once, so the ceiling is jobs multiplied by
+// fileSizeLimit and its compressed copy — bounded, and bounded by a constant
+// this package enforces itself.
+//
+// The returned entries are in artifact order whatever order the workers finish
+// in, because a manifest's bytes are promised deterministic by emitterVersion.
+func emitResources(artifact io.ReaderAt, index []erf.IndexEntry, target sink, jobs int) ([]Entry, int, int64, error) {
 	// A resref appearing twice inside one artifact resolves to the last one,
 	// the way resman lets the last container added win.
-	order := make([]Identity, 0, len(resources))
-	latest := make(map[Identity]erf.Resource, len(resources))
+	order := make([]Identity, 0, len(index))
+	latest := make(map[Identity]erf.IndexEntry, len(index))
 	var tooBig []string
-	for _, resource := range resources {
-		if _, ok := erf.ExtensionForResourceType(resource.Type); !ok {
-			return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", resource.Name, resource.Type)
+	for _, entry := range index {
+		if _, ok := erf.ExtensionForResourceType(entry.Type); !ok {
+			return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", entry.Name, entry.Type)
 		}
-		if slices.Contains(skippedTypes, resource.Type) {
+		if slices.Contains(skippedTypes, entry.Type) {
 			continue
 		}
-		if len(resource.Data) > fileSizeLimit {
-			tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", resource.Name, len(resource.Data), fileSizeLimit))
+		if entry.Size > fileSizeLimit {
+			tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", entry.Name, entry.Size, fileSizeLimit))
 			continue
 		}
-		identity := Identity{ResRef: strings.ToLower(resource.Name), ResType: resource.Type}
+		identity := Identity{ResRef: strings.ToLower(entry.Name), ResType: entry.Type}
 		if _, seen := latest[identity]; !seen {
 			order = append(order, identity)
 		}
-		latest[identity] = resource
+		latest[identity] = entry
 	}
 	if len(tooBig) > 0 {
 		sort.Strings(tooBig)
 		return nil, 0, 0, fmt.Errorf("resources exceed the file size limit:\n  %s", strings.Join(tooBig, "\n  "))
 	}
 
-	entries := make([]Entry, 0, len(order))
+	// Index-addressed, never appended to: a worker owns entries[i] alone, so
+	// the slice comes back in artifact order and needs no lock.
+	entries := make([]Entry, len(order))
 	var blobs int
 	var onDiskBytes int64
-	for _, identity := range order {
-		resource := latest[identity]
-		sum := sha1.Sum(resource.Data)
-		entries = append(entries, Entry{
+	var mu sync.Mutex
+	var firstErr error
+	// Two resrefs in one artifact can hold identical bytes, and therefore one
+	// blob. Serially the sink's existence check absorbed that; in parallel both
+	// workers would probe, both miss, and both upload. Claiming the sha1 here
+	// restores the dedupe and skips the probe round-trip as well.
+	claimed := make(map[[20]byte]bool, len(order))
+
+	failed := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return firstErr != nil
+	}
+
+	store := func(i int) {
+		identity := order[i]
+		payload, err := erf.ReadPayload(artifact, latest[identity])
+		if err != nil {
+			mu.Lock()
+			if firstErr == nil {
+				firstErr = err
+			}
+			mu.Unlock()
+			return
+		}
+		sum := sha1.Sum(payload)
+		entries[i] = Entry{
 			SHA1:    sum,
-			Size:    uint32(len(resource.Data)),
+			Size:    uint32(len(payload)),
 			ResRef:  identity.ResRef,
 			ResType: identity.ResType,
-		})
-		written, err := target.putBlob(fmt.Sprintf("%x", sum), func() []byte { return compressBlob(resource.Data) })
+		}
+		mu.Lock()
+		duplicate := claimed[sum]
+		claimed[sum] = true
+		mu.Unlock()
+		if duplicate {
+			return
+		}
+		written, err := target.putBlob(fmt.Sprintf("%x", sum), func() []byte { return compressBlob(payload) })
+		mu.Lock()
+		defer mu.Unlock()
 		if err != nil {
-			return nil, 0, 0, err
+			if firstErr == nil {
+				firstErr = err
+			}
+			return
 		}
 		if written > 0 {
 			blobs++
 			onDiskBytes += written
 		}
 	}
+
+	if jobs < 1 {
+		jobs = 1
+	}
+	work := make(chan int)
+	var wg sync.WaitGroup
+	for range jobs {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := range work {
+				// After a failure the run is over — the caller discards
+				// everything and no index is written. Draining the rest of the
+				// channel cheaply, rather than returning, keeps the feeder from
+				// blocking on workers that have gone away.
+				if failed() {
+					continue
+				}
+				store(i)
+			}
+		}()
+	}
+	for i := range order {
+		work <- i
+	}
+	close(work)
+	wg.Wait()
+
+	if firstErr != nil {
+		return nil, 0, 0, firstErr
+	}
 	return entries, blobs, onDiskBytes, nil
 }
 

internal/nwsync/jobs_test.go

@@ -0,0 +1,134 @@
+package nwsync
+
+import (
+	"bytes"
+	"fmt"
+	"path/filepath"
+	"runtime/debug"
+	"strings"
+	"testing"
+)
+
+// manyResources is a hak body with enough distinct resources that a worker pool
+// actually interleaves. Payloads differ so nothing is deduplicated away.
+func manyResources(count int) map[string][]byte {
+	contents := make(map[string][]byte, count)
+	for i := range count {
+		contents[fmt.Sprintf("res%05d.tga", i)] = []byte(fmt.Sprintf("payload %d", i))
+	}
+	return contents
+}
+
+// TestEmitProducesTheSameIndexAtEveryJobCount is the contract that lets emit be
+// parallel at all: emitterVersion promises a manifest's bytes are a function of
+// its artifact, so the number of workers must not be observable in the output.
+func TestEmitProducesTheSameIndexAtEveryJobCount(t *testing.T) {
+	// The sidecar stamps a wall-clock time unless this is set, which would make
+	// two runs differ for a reason that has nothing to do with job count.
+	t.Setenv("SOURCE_DATE_EPOCH", "1700000000")
+
+	dir := t.TempDir()
+	hak := filepath.Join(dir, "sow_test_01.hak")
+	writeHak(t, hak, manyResources(64))
+	key := artifactKey(t, hak)
+
+	emit := func(jobs int) (manifest, sidecar []byte, result EmitResult) {
+		out := filepath.Join(t.TempDir(), "out")
+		result, err := Emit(EmitOptions{
+			ArtifactKey:  key,
+			ArtifactPath: hak,
+			OutDir:       out,
+			Jobs:         jobs,
+		})
+		if err != nil {
+			t.Fatalf("emit at -jobs %d: %v", jobs, err)
+		}
+		manifest, sidecar, err = dirSink{root: out}.getIndex(filepath.Base(result.ManifestPath))
+		if err != nil {
+			t.Fatalf("read index at -jobs %d: %v", jobs, err)
+		}
+		return manifest, sidecar, result
+	}
+
+	serialManifest, serialSidecar, serial := emit(1)
+	parallelManifest, parallelSidecar, parallel := emit(16)
+
+	if !bytes.Equal(serialManifest, parallelManifest) {
+		t.Errorf("manifest bytes differ between -jobs 1 and -jobs 16")
+	}
+	if !bytes.Equal(serialSidecar, parallelSidecar) {
+		t.Errorf("sidecar bytes differ between -jobs 1 and -jobs 16:\n %s\n %s", serialSidecar, parallelSidecar)
+	}
+	if serial.Entries != parallel.Entries || serial.BlobsWritten != parallel.BlobsWritten {
+		t.Errorf("-jobs 1 wrote %d entries/%d blobs, -jobs 16 wrote %d/%d",
+			serial.Entries, serial.BlobsWritten, parallel.Entries, parallel.BlobsWritten)
+	}
+}
+
+// TestEmitLeavesNoIndexWhenAParallelUploadFails is the fail-closed check with
+// workers in flight: several uploads are in the air when the first one fails,
+// and the index must still never appear. Run under -race this also covers the
+// shared counters.
+func TestEmitLeavesNoIndexWhenAParallelUploadFails(t *testing.T) {
+	fixture := newZoneFixture(t)
+	fixture.zone.failOn = func(key string) bool { return strings.HasPrefix(key, "data/sha1/") }
+	dir := t.TempDir()
+	hak := filepath.Join(dir, "sow_test_01.hak")
+	writeHak(t, hak, manyResources(64))
+
+	if _, err := Emit(EmitOptions{
+		ArtifactKey:  artifactKey(t, hak),
+		ArtifactPath: hak,
+		Sink:         fixture.sink,
+		Jobs:         16,
+	}); err == nil {
+		t.Fatal("emit reported success after an upload failed")
+	}
+	fixture.zone.mu.Lock()
+	defer fixture.zone.mu.Unlock()
+	for key := range fixture.zone.objects {
+		if strings.HasSuffix(key, ".nsym") {
+			t.Errorf("a half-emitted artifact published an index: %s", key)
+		}
+	}
+}
+
+// TestEmitPeakMemoryIsBoundedByJobCount pins the ceiling the parallel emit
+// rests on. Peak still must not track the archive — it tracks the resources in
+// flight, so a bigger hak at the same job count costs the same.
+func TestEmitPeakMemoryIsBoundedByJobCount(t *testing.T) {
+	if testing.Short() {
+		t.Skip("writes a 64 MB fixture")
+	}
+	defer debug.SetGCPercent(debug.SetGCPercent(10))
+
+	measure := func(count, jobs int) uint64 {
+		dir := t.TempDir()
+		hak := filepath.Join(dir, "big.hak")
+		writeStreamedHak(t, hak, count)
+		options := EmitOptions{
+			ArtifactKey:  artifactKey(t, hak),
+			ArtifactPath: hak,
+			As:           filepath.Base(hak),
+			OutDir:       filepath.Join(dir, "out"),
+			Jobs:         jobs,
+		}
+		return peakHeapDuring(func() {
+			if _, err := Emit(options); err != nil {
+				t.Fatalf("emit %d resources at -jobs %d: %v", count, jobs, err)
+			}
+		})
+	}
+
+	const jobs = 8
+	small := measure(8, jobs)  // 8 MB
+	large := measure(64, jobs) // 64 MB
+	// Each worker may hold one resourceSize payload plus its compressed copy,
+	// so the pool itself is the slack — not the archive.
+	const slack = 24 << 20
+
+	t.Logf("peak heap at -jobs %d: 8 MB hak %d bytes, 64 MB hak %d bytes", jobs, small, large)
+	if large > small+slack {
+		t.Fatalf("peak heap scaled with artifact size at -jobs %d: 8 MB hak peaked at %d bytes, 64 MB hak at %d", jobs, small, large)
+	}
+}

internal/nwsync/memory_test.go

@@ -0,0 +1,150 @@
+package nwsync
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/debug"
+	"testing"
+	"time"
+
+	"github.com/klauspost/compress/zstd"
+
+	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
+)
+
+// TestSingleThreadedEncoderMatchesDefault pins the claim the blob encoder's
+// concurrency setting rests on: it saves memory only, and a published blob is
+// the same bytes either way.
+func TestSingleThreadedEncoderMatchesDefault(t *testing.T) {
+	standard, err := zstd.NewWriter(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer standard.Close()
+
+	body := make([]byte, 4<<20)
+	random := rand.New(rand.NewSource(1))
+	random.Read(body[:len(body)/2])
+	for _, size := range []int{0, 1, 4 << 10, len(body)} {
+		if !bytes.Equal(blobEncoder.EncodeAll(body[:size], nil), standard.EncodeAll(body[:size], nil)) {
+			t.Fatalf("%d bytes compress differently at concurrency 1", size)
+		}
+	}
+}
+
+// resourceSize is one payload in the memory fixtures. Real haks hold a few MB
+// per resource, and peak memory is meant to track that, not the archive.
+const resourceSize = 1 << 20
+
+// writeStreamedHak builds a hak of count resources without ever holding the
+// archive in memory, so the fixture itself does not decide the measurement.
+// Payloads are distinct, so no blob is deduplicated away.
+func writeStreamedHak(t *testing.T, path string, count int) {
+	t.Helper()
+	payload := filepath.Join(t.TempDir(), "payload.bin")
+	body := make([]byte, resourceSize)
+	for index := range body {
+		body[index] = byte(index)
+	}
+
+	resources := make([]erf.Resource, 0, count)
+	for index := range count {
+		// A distinct first byte per resource is enough to give every payload
+		// its own sha1 while still streaming from one file per resource.
+		unique := filepath.Join(filepath.Dir(payload), fmt.Sprintf("p%d.bin", index))
+		body[0] = byte(index)
+		body[1] = byte(index >> 8)
+		if err := os.WriteFile(unique, body, 0o644); err != nil {
+			t.Fatal(err)
+		}
+		resources = append(resources, erf.Resource{
+			Name:       fmt.Sprintf("res%05d", index),
+			Type:       restype(t, "tga"),
+			SourcePath: unique,
+			Size:       resourceSize,
+		})
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	if err := erf.Write(file, erf.New("HAK", resources)); err != nil {
+		t.Fatalf("write hak: %v", err)
+	}
+}
+
+// peakHeapDuring runs work while sampling the heap, and returns the largest
+// live heap it saw.
+func peakHeapDuring(work func()) uint64 {
+	runtime.GC()
+	done := make(chan struct{})
+	peak := make(chan uint64, 1)
+	go func() {
+		var highest uint64
+		var stats runtime.MemStats
+		for {
+			select {
+			case <-done:
+				peak <- highest
+				return
+			default:
+			}
+			runtime.ReadMemStats(&stats)
+			if stats.HeapAlloc > highest {
+				highest = stats.HeapAlloc
+			}
+			time.Sleep(time.Millisecond)
+		}
+	}()
+	work()
+	close(done)
+	return <-peak
+}
+
+// TestEmitPeakMemoryDoesNotScaleWithArtifactSize is the regression check for
+// the OOM kills on large haks: emit used to hold the whole archive (twice), so
+// a 2 GB hak needed about 10 GB. Emitting an archive 8× bigger must not cost
+// meaningfully more memory.
+func TestEmitPeakMemoryDoesNotScaleWithArtifactSize(t *testing.T) {
+	if testing.Short() {
+		t.Skip("writes a 64 MB fixture")
+	}
+	// A lazy GC lets garbage pile up in proportion to the live heap, which
+	// hides the thing under test. Collecting eagerly makes the sampled heap
+	// track what emit actually holds.
+	defer debug.SetGCPercent(debug.SetGCPercent(10))
+
+	measure := func(count int) uint64 {
+		dir := t.TempDir()
+		hak := filepath.Join(dir, "big.hak")
+		writeStreamedHak(t, hak, count)
+		// The key is computed outside the measurement: the test helper reads
+		// the whole file to hash it, which emit itself no longer does.
+		options := EmitOptions{
+			ArtifactKey:  artifactKey(t, hak),
+			ArtifactPath: hak,
+			As:           filepath.Base(hak),
+			OutDir:       filepath.Join(dir, "out"),
+		}
+		return peakHeapDuring(func() {
+			if _, err := Emit(options); err != nil {
+				t.Fatalf("emit %d resources: %v", count, err)
+			}
+		})
+	}
+
+	small := measure(8)  // 8 MB
+	large := measure(64) // 64 MB
+	const slack = 24 << 20
+
+	t.Logf("peak heap: 8 MB hak %d bytes, 64 MB hak %d bytes", small, large)
+	if large > small+slack {
+		t.Fatalf("peak heap scaled with artifact size: 8 MB hak peaked at %d bytes, 64 MB hak at %d", small, large)
+	}
+}

internal/nwsync/nwsync_test.go

@@ -458,6 +458,33 @@ func TestAssembleFailsClosedOnMissingIndex(t *testing.T) {
 	}
 }
 
+// Callers capture a script's stdout as a value: `dir="$(pack-haks.sh)"`. A
+// summary line on stdout gets glued onto that value, so both summaries belong
+// on stderr.
+func TestRunKeepsSummariesOffStdout(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "sow_top.hak")
+	writeHak(t, path, map[string][]byte{"appearance.2da": []byte("2da from sow_top")})
+	key := artifactKey(t, path)
+	out := filepath.Join(dir, "out")
+
+	for _, args := range [][]string{
+		{"emit", "--out", out, "--as", "sow_top.hak", key, path},
+		{"assemble", "--out", out, key},
+	} {
+		var stdout, stderr bytes.Buffer
+		if code := Run(args, &stdout, &stderr); code != exitOK {
+			t.Fatalf("Run(%v) exit=%d: %s", args, code, stderr.String())
+		}
+		if stdout.Len() != 0 {
+			t.Errorf("Run(%v) wrote to stdout: %q", args, stdout.String())
+		}
+		if stderr.Len() == 0 {
+			t.Errorf("Run(%v) reported no summary on stderr", args)
+		}
+	}
+}
+
 func TestRunUsageErrors(t *testing.T) {
 	cases := [][]string{
 		nil,

internal/nwsync/run.go

@@ -29,9 +29,9 @@ func Run(args []string, stdout, stderr io.Writer) int {
 	}
 	switch args[0] {
 	case "emit":
-		return runEmit(args[1:], stdout, stderr)
+		return runEmit(args[1:], stderr)
 	case "assemble":
-		return runAssemble(args[1:], stdout, stderr)
+		return runAssemble(args[1:], stderr)
 	case "-h", "--help", "help":
 		printRunUsage(stdout)
 		return exitOK
@@ -76,11 +76,12 @@ func parseArgs(fs *flag.FlagSet, args []string) ([]string, error) {
 	}
 }
 
-func runEmit(args []string, stdout, stderr io.Writer) int {
+func runEmit(args []string, stderr io.Writer) int {
 	fs := flag.NewFlagSet("emit", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	as := fs.String("as", "", "published name of the artifact, when it differs from the key")
 	out := fs.String("out", "", "write to a local repository tree instead of uploading")
+	jobs := fs.Int("jobs", defaultEmitJobs, "resources to hash, compress and store at once")
 	positional, err := parseArgs(fs, args)
 	if err != nil {
 		return exitUsage
@@ -89,23 +90,28 @@ func runEmit(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "nwsync emit: <artifact-key> and <file> are both required\n")
 		return exitUsage
 	}
+	if *jobs < 1 {
+		fmt.Fprintf(stderr, "nwsync emit: -jobs must be at least 1, got %d\n", *jobs)
+		return exitUsage
+	}
 
 	result, err := Emit(EmitOptions{
 		ArtifactKey:  positional[0],
 		ArtifactPath: positional[1],
 		As:           *as,
 		OutDir:       *out,
+		Jobs:         *jobs,
 	})
 	if err != nil {
 		fmt.Fprintf(stderr, "nwsync emit: %v\n", err)
 		return exitInternal
 	}
-	fmt.Fprintf(stdout, "emitted %s: %d resources, %d new blobs, index %s\n",
+	fmt.Fprintf(stderr, "emitted %s: %d resources, %d new blobs, index %s\n",
 		result.Name, result.Entries, result.BlobsWritten, result.ManifestPath)
 	return exitOK
 }
 
-func runAssemble(args []string, stdout, stderr io.Writer) int {
+func runAssemble(args []string, stderr io.Writer) int {
 	fs := flag.NewFlagSet("assemble", flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	tlkKey := fs.String("tlk-key", "", "depot key of the TLK, which shadows nothing and merges last")
@@ -134,7 +140,7 @@ func runAssemble(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "nwsync assemble: %v\n", err)
 		return exitInternal
 	}
-	fmt.Fprintf(stdout, "assembled manifest %s: %d resources, %s\n",
+	fmt.Fprintf(stderr, "assembled manifest %s: %d resources, %s\n",
 		result.SHA1, result.Entries, result.ManifestPath)
 	return exitOK
 }

internal/nwsync/sink.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -193,14 +194,18 @@ func resolveIndexKey(artifactKey, outDir string) (string, error) {
 // checkArtifactKey fails closed when the key's embedded digest is not the
 // digest of the bytes being emitted. Publishing an index under the wrong key
 // silently pairs a manifest with the wrong artifact.
-func checkArtifactKey(artifactKey string, artifact []byte) error {
+// artifact is hashed by streaming, so a multi-gigabyte hak is never resident.
+func checkArtifactKey(artifactKey string, artifact io.Reader) error {
 	base := path.Base(artifactKey)
 	digest := strings.TrimSuffix(base, path.Ext(base))
 	if len(digest) != 64 {
 		return fmt.Errorf("artifact key %q does not name a sha256", artifactKey)
 	}
-	sum := sha256.Sum256(artifact)
-	if got := hex.EncodeToString(sum[:]); got != digest {
+	hash := sha256.New()
+	if _, err := io.Copy(hash, artifact); err != nil {
+		return fmt.Errorf("hash artifact: %w", err)
+	}
+	if got := hex.EncodeToString(hash.Sum(nil)); got != digest {
 		return fmt.Errorf("artifact key %q names digest %s but the file hashes to %s", artifactKey, digest, got)
 	}
 	return nil