fix(nwsync): stream emit so peak memory tracks the largest resource (#76)
Emit read the whole artifact into memory and erf.Read then allocated a second full copy of every payload, so a hak cost about 5x its size in RAM. The 7 GB runner was OOM-killed on any hak over ~1.4 GB, which blocks the NWSync backfill and every release that rebuilds a large hak. Emit now opens the artifact, hashes it by streaming for the key check, parses only the header and resource table via erf.ReadIndex, and reads, hashes, compresses and stores one payload at a time. erf.Read keeps its old shape but returns payloads as subslices instead of fresh copies, which removes the second copy for the other callers too. The zstd encoder pool also held one window-sized history per CPU — about 200 MB of live heap on a 24-core runner. EncodeAll is single-threaded per call, so concurrency 1 gives byte-identical blobs for far less memory. Peak heap is now flat at ~22 MB for both an 8 MB and a 64 MB hak, and a regression test asserts it does not scale with artifact size. Closes #76 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -193,14 +194,18 @@ func resolveIndexKey(artifactKey, outDir string) (string, error) {
|
||||
// checkArtifactKey fails closed when the key's embedded digest is not the
|
||||
// digest of the bytes being emitted. Publishing an index under the wrong key
|
||||
// silently pairs a manifest with the wrong artifact.
|
||||
func checkArtifactKey(artifactKey string, artifact []byte) error {
|
||||
// artifact is hashed by streaming, so a multi-gigabyte hak is never resident.
|
||||
func checkArtifactKey(artifactKey string, artifact io.Reader) error {
|
||||
base := path.Base(artifactKey)
|
||||
digest := strings.TrimSuffix(base, path.Ext(base))
|
||||
if len(digest) != 64 {
|
||||
return fmt.Errorf("artifact key %q does not name a sha256", artifactKey)
|
||||
}
|
||||
sum := sha256.Sum256(artifact)
|
||||
if got := hex.EncodeToString(sum[:]); got != digest {
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, artifact); err != nil {
|
||||
return fmt.Errorf("hash artifact: %w", err)
|
||||
}
|
||||
if got := hex.EncodeToString(hash.Sum(nil)); got != digest {
|
||||
return fmt.Errorf("artifact key %q names digest %s but the file hashes to %s", artifactKey, digest, got)
|
||||
}
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user