archvillainette
fa32dd411f
build-binaries / build-binaries (push) Successful in 2m13s
Closes #76. Part of #54. `nwsync emit` held roughly 5× the artifact size in RAM, so ovh-main (7 GB, no swap) OOM-killed it on any hak over ~1.4 GB. That blocked the backfill in sow-assets-manifest and would have killed the next release rebuilding a large hak. ## What it does - `erf.ReadIndex` / `erf.ReadPayload` — parse the header and resource table only, read one payload on demand. `erf.Read` keeps its shape but returns payloads as subslices of the buffer instead of fresh copies, which removes one full copy for the `pipeline` callers too. Callers must not mutate `Resource.Data`; the doc comment says so and no caller does. - `nwsync.Emit` opens the artifact, hashes it by streaming for the key check (through a section reader, so the file offset stays put), then hashes, compresses and stores one resource at a time. The archive is never resident. Shadowed duplicate resrefs are now never read at all. - Bounds checks in `ReadIndex` moved to `int64`, so key/resource-list offsets can no longer overflow. ## Not in the issue, but memory-motivated The zstd blob encoder ran at the default concurrency, which is one encoder per CPU, each holding a window-sized history — about 200 MB of live heap doing nothing on a 24-core runner. `EncodeAll` is single-threaded per call, so concurrency 1 costs nothing. `TestSingleThreadedEncoderMatchesDefault` pins the claim that blobs come out byte-identical. ## Measured Peak heap during emit, sampled 1 ms: | hak | before | after | |-----|--------|-------| | 8 MB | 155 MB | 21 MB | | 64 MB | 289 MB | 22 MB | Flat, as the acceptance asks. `TestEmitPeakMemoryDoesNotScaleWithArtifactSize` fails if the 64 MB fixture costs more than the 8 MB one plus 24 MB of slack. ## Gaps - The regression check measures Go heap, not RSS, and its largest fixture is 64 MB — a multi-GB run was not done here. A 2.15 GB hak now needs about the same ~22 MB the 64 MB one does, so the 5 GB budget is not close, but that is inference from the flat curve, not a measurement. - mmap was suggested in the issue and skipped. Payload buffers are still anonymous, but they are one resource each (≤15 MiB), so making them file-backed buys nothing now. 🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #78 Reviewed-by: xtul <mpiasecki720@protonmail.com> Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
81 lines
2.8 KiB
Go
81 lines
2.8 KiB
Go
package nwsync
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/binary"
|
|
"fmt"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
// NWCompressedBuffer framing, as upstream's neverwinter/compressedbuf.nim
|
|
// writes it for NWSync blobs. All fields are little-endian uint32:
|
|
//
|
|
// magic "NSYC", version 3, algorithm 2 (zstd), uncompressed size,
|
|
// zstd header version 1, dictionary 0, then the raw zstd frame.
|
|
const (
|
|
blobMagic = 0x4359534E // "NSYC" little-endian
|
|
blobVersion = 3
|
|
algorithmZstd = 2
|
|
zstdHeaderVer = 1
|
|
zstdDictionary = 0
|
|
blobHeaderBytes = 24
|
|
)
|
|
|
|
// EncodeAll/DecodeAll are single-threaded per call, so the default pool of one
|
|
// encoder per CPU only buys idle memory: each holds a window-sized history, so
|
|
// on a 24-core runner that is ~200 MB of live heap doing nothing. Concurrency 1
|
|
// produces byte-identical output.
|
|
var (
|
|
blobEncoder, _ = zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1))
|
|
blobDecoder, _ = zstd.NewReader(nil, zstd.WithDecoderConcurrency(1))
|
|
)
|
|
|
|
// compressBlob wraps data in NWCompressedBuffer framing.
|
|
func compressBlob(data []byte) []byte {
|
|
var out bytes.Buffer
|
|
header := []uint32{blobMagic, blobVersion, algorithmZstd, uint32(len(data)), zstdHeaderVer, zstdDictionary}
|
|
for _, field := range header {
|
|
_ = binary.Write(&out, binary.LittleEndian, field)
|
|
}
|
|
out.Write(blobEncoder.EncodeAll(data, nil))
|
|
return out.Bytes()
|
|
}
|
|
|
|
// decompressBlob unwraps NWCompressedBuffer framing. It exists so a blob this
|
|
// package wrote — or one upstream wrote — can be compared by its uncompressed
|
|
// bytes, which is the only comparison that is meaningful across zstd
|
|
// implementations.
|
|
func decompressBlob(blob []byte) ([]byte, error) {
|
|
if len(blob) < blobHeaderBytes {
|
|
return nil, fmt.Errorf("blob too small: %d bytes", len(blob))
|
|
}
|
|
header := make([]uint32, 6)
|
|
if err := binary.Read(bytes.NewReader(blob[:blobHeaderBytes]), binary.LittleEndian, header); err != nil {
|
|
return nil, fmt.Errorf("decode blob header: %w", err)
|
|
}
|
|
switch {
|
|
case header[0] != blobMagic:
|
|
return nil, fmt.Errorf("invalid blob magic: %#x", header[0])
|
|
case header[1] != blobVersion:
|
|
return nil, fmt.Errorf("unsupported blob version: %d", header[1])
|
|
case header[2] != algorithmZstd:
|
|
return nil, fmt.Errorf("unsupported compression algorithm: %d", header[2])
|
|
case header[4] != zstdHeaderVer:
|
|
return nil, fmt.Errorf("unsupported zstd header version: %d", header[4])
|
|
case header[5] != zstdDictionary:
|
|
return nil, fmt.Errorf("zstd dictionaries are not supported")
|
|
}
|
|
if header[3] == 0 {
|
|
return nil, nil
|
|
}
|
|
data, err := blobDecoder.DecodeAll(blob[blobHeaderBytes:], nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decompress blob: %w", err)
|
|
}
|
|
if uint32(len(data)) != header[3] {
|
|
return nil, fmt.Errorf("blob size mismatch: header says %d, got %d", header[3], len(data))
|
|
}
|
|
return data, nil
|
|
}
|