fix(nwsync): stream emit so peak memory tracks the largest resource (#76)

Emit read the whole artifact into memory and erf.Read then allocated a
second full copy of every payload, so a hak cost about 5x its size in
RAM. The 7 GB runner was OOM-killed on any hak over ~1.4 GB, which
blocks the NWSync backfill and every release that rebuilds a large hak.

Emit now opens the artifact, hashes it by streaming for the key check,
parses only the header and resource table via erf.ReadIndex, and reads,
hashes, compresses and stores one payload at a time. erf.Read keeps its
old shape but returns payloads as subslices instead of fresh copies,
which removes the second copy for the other callers too.

The zstd encoder pool also held one window-sized history per CPU — about
200 MB of live heap on a 24-core runner. EncodeAll is single-threaded
per call, so concurrency 1 gives byte-identical blobs for far less
memory.

Peak heap is now flat at ~22 MB for both an 8 MB and a 64 MB hak, and a
regression test asserts it does not scale with artifact size.

Closes #76

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

internal/nwsync/emit.go

@@ -1,10 +1,10 @@
 package nwsync
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha1"
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -79,7 +79,12 @@ type EmitOptions struct {
 // real blobs in the zone and no index, which is unambiguous. Blob names are
 // content hashes, so re-running skips whatever already landed.
 func Emit(options EmitOptions) (EmitResult, error) {
-	artifact, err := os.ReadFile(options.ArtifactPath)
+	artifact, err := os.Open(options.ArtifactPath)
+	if err != nil {
+		return EmitResult{}, fmt.Errorf("read artifact: %w", err)
+	}
+	defer artifact.Close()
+	info, err := artifact.Stat()
 	if err != nil {
 		return EmitResult{}, fmt.Errorf("read artifact: %w", err)
 	}
@@ -98,7 +103,7 @@ func Emit(options EmitOptions) (EmitResult, error) {
 		return EmitResult{}, err
 	}
 
-	resources, err := readArtifact(options.ArtifactPath, artifact, name)
+	index, err := readArtifactIndex(options.ArtifactPath, artifact, info.Size(), name)
 	if err != nil {
 		return EmitResult{}, err
 	}
@@ -108,7 +113,7 @@ func Emit(options EmitOptions) (EmitResult, error) {
 		return EmitResult{}, err
 	}
 
-	entries, blobs, onDiskBytes, err := emitResources(resources, target)
+	entries, blobs, onDiskBytes, err := emitResources(artifact, index, target)
 	if err != nil {
 		return EmitResult{}, err
 	}
@@ -137,60 +142,64 @@ func openSink(outDir string, injected sink) (sink, error) {
 	return newZoneSink(context.Background(), os.Getenv)
 }
 
-// readArtifact returns the resources of an ERF/HAK/MOD, or the single resource
-// a loose file represents. Upstream's resman does the same dispatch on the
-// file's first three bytes. name is the artifact's published name, which for a
-// loose file is also its resref.
-func readArtifact(path string, data []byte, name string) ([]erf.Resource, error) {
-	if len(data) >= 3 {
-		switch string(data[:3]) {
-		case "ERF", "HAK":
-			archive, err := erf.Read(bytes.NewReader(data))
-			if err != nil {
-				return nil, fmt.Errorf("%s: %w", path, err)
-			}
-			return archive.Resources, nil
-		case "MOD":
-			// A persistent world never publishes module contents, so the .mod
-			// contributes no bytes to a manifest — it only says which haks and
-			// which TLK the manifest covers.
-			return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path)
+// readArtifactIndex locates the resources of an ERF/HAK/MOD, or the single
+// resource a loose file represents, without reading any payload. Upstream's
+// resman does the same dispatch on the file's first three bytes. name is the
+// artifact's published name, which for a loose file is also its resref.
+func readArtifactIndex(path string, artifact io.ReaderAt, size int64, name string) ([]erf.IndexEntry, error) {
+	magic := make([]byte, 3)
+	if size >= 3 {
+		if _, err := artifact.ReadAt(magic, 0); err != nil {
+			return nil, fmt.Errorf("%s: %w", path, err)
 		}
 	}
+	switch string(magic) {
+	case "ERF", "HAK":
+		index, err := erf.ReadIndex(artifact, size)
+		if err != nil {
+			return nil, fmt.Errorf("%s: %w", path, err)
+		}
+		return index.Entries, nil
+	case "MOD":
+		// A persistent world never publishes module contents, so the .mod
+		// contributes no bytes to a manifest — it only says which haks and
+		// which TLK the manifest covers.
+		return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path)
+	}
 	extension := filepath.Ext(filepath.Base(path))
 	restype, ok := erf.ResourceTypeForExtension(extension)
 	if !ok {
 		return nil, fmt.Errorf("%s: unknown resource type %q", path, extension)
 	}
-	return []erf.Resource{{
-		Name: name,
-		Type: restype,
-		Data: data,
-	}}, nil
+	return []erf.IndexEntry{{Name: name, Type: restype, Offset: 0, Size: size}}, nil
 }
 
-func emitResources(resources []erf.Resource, target sink) ([]Entry, int, int64, error) {
+// emitResources hashes, compresses and stores one resource at a time, reading
+// each payload from the artifact only when its turn comes. Peak memory
+// therefore tracks the largest single resource, not the archive: a 2 GB hak
+// must emit inside a runner's few spare GB.
+func emitResources(artifact io.ReaderAt, index []erf.IndexEntry, target sink) ([]Entry, int, int64, error) {
 	// A resref appearing twice inside one artifact resolves to the last one,
 	// the way resman lets the last container added win.
-	order := make([]Identity, 0, len(resources))
-	latest := make(map[Identity]erf.Resource, len(resources))
+	order := make([]Identity, 0, len(index))
+	latest := make(map[Identity]erf.IndexEntry, len(index))
 	var tooBig []string
-	for _, resource := range resources {
-		if _, ok := erf.ExtensionForResourceType(resource.Type); !ok {
-			return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", resource.Name, resource.Type)
+	for _, entry := range index {
+		if _, ok := erf.ExtensionForResourceType(entry.Type); !ok {
+			return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", entry.Name, entry.Type)
 		}
-		if slices.Contains(skippedTypes, resource.Type) {
+		if slices.Contains(skippedTypes, entry.Type) {
 			continue
 		}
-		if len(resource.Data) > fileSizeLimit {
-			tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", resource.Name, len(resource.Data), fileSizeLimit))
+		if entry.Size > fileSizeLimit {
+			tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", entry.Name, entry.Size, fileSizeLimit))
 			continue
 		}
-		identity := Identity{ResRef: strings.ToLower(resource.Name), ResType: resource.Type}
+		identity := Identity{ResRef: strings.ToLower(entry.Name), ResType: entry.Type}
 		if _, seen := latest[identity]; !seen {
 			order = append(order, identity)
 		}
-		latest[identity] = resource
+		latest[identity] = entry
 	}
 	if len(tooBig) > 0 {
 		sort.Strings(tooBig)
@@ -201,15 +210,18 @@ func emitResources(resources []erf.Resource, target sink) ([]Entry, int, int64,
 	var blobs int
 	var onDiskBytes int64
 	for _, identity := range order {
-		resource := latest[identity]
-		sum := sha1.Sum(resource.Data)
+		payload, err := erf.ReadPayload(artifact, latest[identity])
+		if err != nil {
+			return nil, 0, 0, err
+		}
+		sum := sha1.Sum(payload)
 		entries = append(entries, Entry{
 			SHA1:    sum,
-			Size:    uint32(len(resource.Data)),
+			Size:    uint32(len(payload)),
 			ResRef:  identity.ResRef,
 			ResType: identity.ResType,
 		})
-		written, err := target.putBlob(fmt.Sprintf("%x", sum), func() []byte { return compressBlob(resource.Data) })
+		written, err := target.putBlob(fmt.Sprintf("%x", sum), func() []byte { return compressBlob(payload) })
 		if err != nil {
 			return nil, 0, 0, err
 		}