sow-tools/internal/nwsync/compressedbuf.go

package nwsync

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"

	"github.com/klauspost/compress/zstd"
)

// NWCompressedBuffer framing, as upstream's neverwinter/compressedbuf.nim
// writes it for NWSync blobs. All fields are little-endian uint32:
//
//	magic "NSYC", version 3, algorithm 2 (zstd), uncompressed size,
//	zstd header version 1, dictionary 0, then the raw zstd frame.
const (
	blobMagic       = 0x4359534E // "NSYC" little-endian
	blobVersion     = 3
	algorithmZstd   = 2
	zstdHeaderVer   = 1
	zstdDictionary  = 0
	blobHeaderBytes = 24
)

// EncodeAll/DecodeAll are single-threaded per call, so the default pool of one
// encoder per CPU only buys idle memory: each holds a window-sized history, so
// on a 24-core runner that is ~200 MB of live heap doing nothing. Concurrency 1
// produces byte-identical output.
var (
	blobEncoder, _ = zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1))
	blobDecoder, _ = zstd.NewReader(nil, zstd.WithDecoderConcurrency(1))
)

// zstd frame header bits we care about. A frame starts with the magic, then a
// one-byte Frame_Header_Descriptor: bits 7-6 size the Frame_Content_Size field,
// bit 5 is Single_Segment_flag, bits 1-0 size the Dictionary_ID field.
const (
	zstdFrameMagic      = "\x28\xb5\x2f\xfd"
	frameSingleSegment  = 1 << 5
	frameDictionaryMask = 0x03
	// oneByteContentSizeCeiling is the size above which a Frame_Content_Size no
	// longer fits in one byte. Below it the field's size flag is 0, which is
	// what lets klauspost/compress leave the field out entirely.
	oneByteContentSizeCeiling = 256
)

// compressBlob wraps data in NWCompressedBuffer framing.
func compressBlob(data []byte) []byte {
	var out bytes.Buffer
	header := []uint32{blobMagic, blobVersion, algorithmZstd, uint32(len(data)), zstdHeaderVer, zstdDictionary}
	for _, field := range header {
		_ = binary.Write(&out, binary.LittleEndian, field)
	}
	frame := declareFrameContentSize(blobEncoder.EncodeAll(data, nil), len(data))
	// Fail closed rather than publish a blob no client can decode. An encoder
	// upgrade that finds a new way to omit the field would otherwise reproduce
	// #86 in silence, and a blob is skipped by every later emit once written.
	if !frameDeclaresContentSize(frame) {
		panic(fmt.Sprintf("nwsync: refusing to emit a %d-byte blob whose zstd frame declares no content size (descriptor %#x)",
			len(data), frame[4]))
	}
	out.Write(frame)
	return out.Bytes()
}

// inspectBlob unwraps a stored blob the way the game client reads it, and is the
// only reader that should be trusted to judge a published blob.
//
// It asserts the frame property on top of the round trip. Go's decoder — like
// the zstd CLI — streams a frame that declares no content size, so a check that
// only decompresses and hashes is a *more* capable decoder than the client's: it
// certifies exactly the blobs the client rejects, which is how #86 reached
// production and survived an audit.
func inspectBlob(blob []byte) ([]byte, error) {
	data, err := decompressBlob(blob)
	if err != nil {
		return nil, fmt.Errorf("malformed framing: %w", err)
	}
	if len(blob) > blobHeaderBytes && !frameDeclaresContentSize(blob[blobHeaderBytes:]) {
		return nil, fmt.Errorf("malformed framing: the zstd frame declares no content size, which the game client cannot decode")
	}
	return data, nil
}

// blobMatchesName holds a stored blob to its own file name: a blob is named
// after the sha1 of its uncompressed bytes, so the name is a complete statement
// about the contents and nothing else is needed to check it.
func blobMatchesName(blob []byte, sha1Hex string) error {
	data, err := inspectBlob(blob)
	if err != nil {
		return err
	}
	if got := hex.EncodeToString(sha1Sum(data)); got != sha1Hex {
		return fmt.Errorf("blob %s holds the contents of %s", sha1Hex, got)
	}
	return nil
}

// frameDeclaresContentSize reports whether a zstd frame states how many bytes it
// decompresses to. A frame with a zero-sized Frame_Content_Size field declares
// one only when Single_Segment_flag is set; otherwise the size is unknown.
func frameDeclaresContentSize(frame []byte) bool {
	if len(frame) < 5 || string(frame[:4]) != zstdFrameMagic {
		return false
	}
	descriptor := frame[4]
	return descriptor>>6 != 0 || descriptor&frameSingleSegment != 0
}

// declareFrameContentSize rewrites a frame that does not declare its
// Frame_Content_Size so that it does, and returns any other frame unchanged.
//
// klauspost/compress omits the field for inputs under 256 bytes, which the spec
// permits. Reference libzstd never does, so the NWN client — which sizes its
// output buffer from ZSTD_getFrameContentSize and has therefore never met a
// frame without one — rejects the blob outright with an empty "potential
// compression error" (#86). No encoder option changes this, so the frame is
// re-headered here.
//
// The result is the shape libzstd itself emits for the same input: setting
// Single_Segment_flag drops the Window_Descriptor byte, and the freed byte pays
// for a one-byte Frame_Content_Size. Window_Size then equals the content size,
// which is sound because the content is under 256 bytes and every match in it
// therefore falls inside that window. Same length in, same length out.
func declareFrameContentSize(frame []byte, size int) []byte {
	if size <= 0 || size >= oneByteContentSizeCeiling || len(frame) < 6 || string(frame[:4]) != zstdFrameMagic {
		return frame
	}
	descriptor := frame[4]
	// Rewrite only the exact shape a small input produces: no declared size, no
	// single segment, no dictionary. Anything else either declares a size
	// already or is not a frame this reinterpretation is safe on.
	if descriptor>>6 != 0 || descriptor&frameSingleSegment != 0 || descriptor&frameDictionaryMask != 0 {
		return frame
	}
	reframed := make([]byte, len(frame))
	copy(reframed, frame)
	reframed[4] = descriptor | frameSingleSegment
	reframed[5] = byte(size) // replaces Window_Descriptor
	return reframed
}

// decompressBlob unwraps NWCompressedBuffer framing. It exists so a blob this
// package wrote — or one upstream wrote — can be compared by its uncompressed
// bytes, which is the only comparison that is meaningful across zstd
// implementations.
func decompressBlob(blob []byte) ([]byte, error) {
	if len(blob) < blobHeaderBytes {
		return nil, fmt.Errorf("blob too small: %d bytes", len(blob))
	}
	header := make([]uint32, 6)
	if err := binary.Read(bytes.NewReader(blob[:blobHeaderBytes]), binary.LittleEndian, header); err != nil {
		return nil, fmt.Errorf("decode blob header: %w", err)
	}
	switch {
	case header[0] != blobMagic:
		return nil, fmt.Errorf("invalid blob magic: %#x", header[0])
	case header[1] != blobVersion:
		return nil, fmt.Errorf("unsupported blob version: %d", header[1])
	case header[2] != algorithmZstd:
		return nil, fmt.Errorf("unsupported compression algorithm: %d", header[2])
	case header[4] != zstdHeaderVer:
		return nil, fmt.Errorf("unsupported zstd header version: %d", header[4])
	case header[5] != zstdDictionary:
		return nil, fmt.Errorf("zstd dictionaries are not supported")
	}
	if header[3] == 0 {
		return nil, nil
	}
	data, err := blobDecoder.DecodeAll(blob[blobHeaderBytes:], nil)
	if err != nil {
		return nil, fmt.Errorf("decompress blob: %w", err)
	}
	if uint32(len(data)) != header[3] {
		return nil, fmt.Errorf("blob size mismatch: header says %d, got %d", header[3], len(data))
	}
	return data, nil
}