feat(nwsync): emit blobs in parallel with a bounded worker pool (#79)

Emit is latency-bound, not CPU-bound. Every blob costs two serial HTTP
round-trips to the zone — an existence probe, then an upload — so a hak
with a few thousand resources pays a few thousand serialised latencies.
A measured backfill spent 26 seconds of CPU across 9.5 minutes of wall
clock, on a host with three of four cores idle and 5 GB free.

Emit now hashes, compresses and stores `--jobs N` resources at once
(default 16, matching DEPOT_JOBS and the transport's idle connections
per host).

Three properties had to survive, and each has a test:

- The manifest's bytes are promised deterministic by emitterVersion, so
  entries is index-addressed rather than appended to: a worker owns
  entries[i] alone and the slice comes back in artifact order whatever
  order the uploads finish in.
- The index is still the publication marker, so any worker's failure
  aborts the run before a manifest is written. Workers drain the rest of
  the channel instead of returning, which keeps the feeder from blocking
  on workers that have gone away.
- Two resrefs holding identical bytes still share one blob. Serially the
  sink's existence check absorbed that; in parallel both workers would
  probe, both miss, and both upload. Claiming the sha1 in-process
  restores the dedupe and skips a probe round-trip as well.

Peak memory is now the resources in flight rather than one resource, so
the ceiling is N times the 15 MB per-resource limit and its compressed
copy — bounded by a constant this package enforces itself, and still
nowhere near tracking the archive.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

internal/nwsync/emit.go

@@ -12,6 +12,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
@@ -62,12 +63,21 @@ type EmitResult struct {
 	BlobsWritten int
 }
 
+// defaultEmitJobs is how many resources are hashed, compressed and stored at
+// once. Emit is latency-bound, not CPU-bound: a blob costs a probe round-trip
+// plus an upload round-trip, and a measured backfill spent 26 s of CPU across
+// 9.5 minutes of wall clock. The figure matches depot's DEPOT_JOBS default and
+// the transport's MaxIdleConnsPerHost, so a worker per connection needs no new
+// TLS handshake.
+const defaultEmitJobs = 16
+
 // EmitOptions describes one emit run.
 type EmitOptions struct {
 	ArtifactKey  string // depot key of the artifact; the NSYM key is derived from it
 	ArtifactPath string // the file on disk
 	As           string // name override, for a TLK whose filename is not its published name
 	OutDir       string // write locally instead of uploading — the conformance path
+	Jobs         int    // resources in flight at once; 0 means defaultEmitJobs
 	Sink         sink   // test seam; nil means OutDir or the zone
 }
 
@@ -115,7 +125,11 @@ func Emit(options EmitOptions) (EmitResult, error) {
 		return EmitResult{}, err
 	}
 
-	entries, blobs, onDiskBytes, err := emitResources(artifact, index, target)
+	jobs := options.Jobs
+	if jobs < 1 {
+		jobs = defaultEmitJobs
+	}
+	entries, blobs, onDiskBytes, err := emitResources(artifact, index, target, jobs)
 	if err != nil {
 		return EmitResult{}, err
 	}
@@ -176,11 +190,16 @@ func readArtifactIndex(path string, artifact io.ReaderAt, size int64, name strin
 	return []erf.IndexEntry{{Name: name, Type: restype, Offset: 0, Size: size}}, nil
 }
 
-// emitResources hashes, compresses and stores one resource at a time, reading
-// each payload from the artifact only when its turn comes. Peak memory
-// therefore tracks the largest single resource, not the archive: a 2 GB hak
-// must emit inside a runner's few spare GB.
-func emitResources(artifact io.ReaderAt, index []erf.IndexEntry, target sink) ([]Entry, int, int64, error) {
+// emitResources hashes, compresses and stores resources, reading each payload
+// from the artifact only when its turn comes. Peak memory tracks the resources
+// in flight, not the archive: a 2 GB hak must emit inside a runner's few spare
+// GB. jobs of them are in flight at once, so the ceiling is jobs multiplied by
+// fileSizeLimit and its compressed copy — bounded, and bounded by a constant
+// this package enforces itself.
+//
+// The returned entries are in artifact order whatever order the workers finish
+// in, because a manifest's bytes are promised deterministic by emitterVersion.
+func emitResources(artifact io.ReaderAt, index []erf.IndexEntry, target sink, jobs int) ([]Entry, int, int64, error) {
 	// A resref appearing twice inside one artifact resolves to the last one,
 	// the way resman lets the last container added win.
 	order := make([]Identity, 0, len(index))
@@ -208,30 +227,95 @@ func emitResources(artifact io.ReaderAt, index []erf.IndexEntry, target sink) ([
 		return nil, 0, 0, fmt.Errorf("resources exceed the file size limit:\n  %s", strings.Join(tooBig, "\n  "))
 	}
 
-	entries := make([]Entry, 0, len(order))
+	// Index-addressed, never appended to: a worker owns entries[i] alone, so
+	// the slice comes back in artifact order and needs no lock.
+	entries := make([]Entry, len(order))
 	var blobs int
 	var onDiskBytes int64
-	for _, identity := range order {
+	var mu sync.Mutex
+	var firstErr error
+	// Two resrefs in one artifact can hold identical bytes, and therefore one
+	// blob. Serially the sink's existence check absorbed that; in parallel both
+	// workers would probe, both miss, and both upload. Claiming the sha1 here
+	// restores the dedupe and skips the probe round-trip as well.
+	claimed := make(map[[20]byte]bool, len(order))
+
+	failed := func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return firstErr != nil
+	}
+
+	store := func(i int) {
+		identity := order[i]
 		payload, err := erf.ReadPayload(artifact, latest[identity])
 		if err != nil {
-			return nil, 0, 0, err
+			mu.Lock()
+			if firstErr == nil {
+				firstErr = err
+			}
+			mu.Unlock()
+			return
 		}
 		sum := sha1.Sum(payload)
-		entries = append(entries, Entry{
+		entries[i] = Entry{
 			SHA1:    sum,
 			Size:    uint32(len(payload)),
 			ResRef:  identity.ResRef,
 			ResType: identity.ResType,
-		})
+		}
+		mu.Lock()
+		duplicate := claimed[sum]
+		claimed[sum] = true
+		mu.Unlock()
+		if duplicate {
+			return
+		}
 		written, err := target.putBlob(fmt.Sprintf("%x", sum), func() []byte { return compressBlob(payload) })
+		mu.Lock()
+		defer mu.Unlock()
 		if err != nil {
-			return nil, 0, 0, err
+			if firstErr == nil {
+				firstErr = err
+			}
+			return
 		}
 		if written > 0 {
 			blobs++
 			onDiskBytes += written
 		}
 	}
+
+	if jobs < 1 {
+		jobs = 1
+	}
+	work := make(chan int)
+	var wg sync.WaitGroup
+	for range jobs {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := range work {
+				// After a failure the run is over — the caller discards
+				// everything and no index is written. Draining the rest of the
+				// channel cheaply, rather than returning, keeps the feeder from
+				// blocking on workers that have gone away.
+				if failed() {
+					continue
+				}
+				store(i)
+			}
+		}()
+	}
+	for i := range order {
+		work <- i
+	}
+	close(work)
+	wg.Wait()
+
+	if firstErr != nil {
+		return nil, 0, 0, firstErr
+	}
 	return entries, blobs, onDiskBytes, nil
 }