sow-tools/internal/nwsync/manifest.go

package nwsync

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"path/filepath"
	"sort"
	"strings"
)

// NSYM manifest, version 3, exactly as upstream neverwinter/nwsync.nim writes
// it. Everything is little-endian:
//
//	"NSYM", uint32 version, uint32 entry count, uint32 mapping count,
//	entries:  byte[20] raw sha1, uint32 size, char[16] resref, uint16 restype
//	mappings: uint32 entry index,             char[16] resref, uint16 restype
//
// Entries are sorted by lowercase sha1 hex then resref, and a resource whose
// sha1 was already written becomes a mapping instead of a second entry.
const (
	manifestVersion = 3
	hashTreeDepth   = 2
	resRefBytes     = 16
)

// Entry is one resource in a manifest.
type Entry struct {
	SHA1    [20]byte
	Size    uint32
	ResRef  string // lowercase, no extension, at most 16 characters
	ResType uint16
}

func (e Entry) sha1Hex() string { return hex.EncodeToString(e.SHA1[:]) }

// Identity is what a resref resolves by: name plus type. It is the merge key
// inside one artifact and across artifacts alike.
type Identity struct {
	ResRef  string
	ResType uint16
}

func (e Entry) identity() Identity { return Identity{ResRef: e.ResRef, ResType: e.ResType} }

// writeManifest serialises entries into NSYM v3 bytes.
func writeManifest(entries []Entry) ([]byte, error) {
	sorted := make([]Entry, len(entries))
	copy(sorted, entries)
	sort.SliceStable(sorted, func(i, j int) bool {
		left, right := sorted[i].sha1Hex(), sorted[j].sha1Hex()
		if left != right {
			return left < right
		}
		return sorted[i].ResRef < sorted[j].ResRef
	})

	var body, mappings bytes.Buffer
	seen := make(map[string]uint32, len(sorted))
	var entryCount, mappingCount uint32
	for _, entry := range sorted {
		padded, err := padResRef(entry.ResRef)
		if err != nil {
			return nil, err
		}
		if index, ok := seen[entry.sha1Hex()]; ok {
			_ = binary.Write(&mappings, binary.LittleEndian, index)
			mappings.Write(padded)
			_ = binary.Write(&mappings, binary.LittleEndian, entry.ResType)
			mappingCount++
			continue
		}
		seen[entry.sha1Hex()] = entryCount
		entryCount++
		body.Write(entry.SHA1[:])
		_ = binary.Write(&body, binary.LittleEndian, entry.Size)
		body.Write(padded)
		_ = binary.Write(&body, binary.LittleEndian, entry.ResType)
	}

	var out bytes.Buffer
	out.WriteString("NSYM")
	for _, field := range []uint32{manifestVersion, entryCount, mappingCount} {
		_ = binary.Write(&out, binary.LittleEndian, field)
	}
	out.Write(body.Bytes())
	out.Write(mappings.Bytes())
	return out.Bytes(), nil
}

// readManifest parses NSYM v3 bytes. Mappings are expanded back into entries,
// the way upstream's reader does, so a caller sees one entry per resref.
func readManifest(data []byte) ([]Entry, error) {
	reader := bytes.NewReader(data)
	magic := make([]byte, 4)
	if _, err := io.ReadFull(reader, magic); err != nil || string(magic) != "NSYM" {
		return nil, fmt.Errorf("not a manifest (invalid magic bytes)")
	}
	var version, entryCount, mappingCount uint32
	for _, field := range []*uint32{&version, &entryCount, &mappingCount} {
		if err := binary.Read(reader, binary.LittleEndian, field); err != nil {
			return nil, fmt.Errorf("truncated manifest header: %w", err)
		}
	}
	if version != manifestVersion {
		return nil, fmt.Errorf("unsupported manifest version %d", version)
	}

	entries := make([]Entry, 0, entryCount+mappingCount)
	for i := uint32(0); i < entryCount; i++ {
		var entry Entry
		if _, err := io.ReadFull(reader, entry.SHA1[:]); err != nil {
			return nil, fmt.Errorf("truncated entry %d: %w", i, err)
		}
		if err := binary.Read(reader, binary.LittleEndian, &entry.Size); err != nil {
			return nil, fmt.Errorf("truncated entry %d: %w", i, err)
		}
		resref, restype, err := readResRef(reader)
		if err != nil {
			return nil, fmt.Errorf("truncated entry %d: %w", i, err)
		}
		entry.ResRef, entry.ResType = resref, restype
		entries = append(entries, entry)
	}
	for i := uint32(0); i < mappingCount; i++ {
		var index uint32
		if err := binary.Read(reader, binary.LittleEndian, &index); err != nil {
			return nil, fmt.Errorf("truncated mapping %d: %w", i, err)
		}
		if index >= entryCount {
			return nil, fmt.Errorf("mapping %d references non-existent entry %d", i, index)
		}
		resref, restype, err := readResRef(reader)
		if err != nil {
			return nil, fmt.Errorf("truncated mapping %d: %w", i, err)
		}
		target := entries[index]
		entries = append(entries, Entry{SHA1: target.SHA1, Size: target.Size, ResRef: resref, ResType: restype})
	}
	return entries, nil
}

func readResRef(reader *bytes.Reader) (string, uint16, error) {
	raw := make([]byte, resRefBytes)
	if _, err := io.ReadFull(reader, raw); err != nil {
		return "", 0, err
	}
	var restype uint16
	if err := binary.Read(reader, binary.LittleEndian, &restype); err != nil {
		return "", 0, err
	}
	return strings.ToLower(string(bytes.TrimRight(raw, "\x00"))), restype, nil
}

func padResRef(resref string) ([]byte, error) {
	if len(resref) > resRefBytes {
		return nil, fmt.Errorf("resref %q exceeds %d characters", resref, resRefBytes)
	}
	padded := make([]byte, resRefBytes)
	copy(padded, strings.ToLower(resref))
	return padded, nil
}

// Sidecar is the .json file written next to every manifest. Clients fetch it
// (nwn_nwsync_fetch.nim), so the field names and order match upstream.
// Upstream omits an integer meta field whose value is 0, hence group_id's
// omitempty: 0 means absent, not "group zero".
type Sidecar struct {
	Version                int    `json:"version"`
	SHA1                   string `json:"sha1"`
	HashTreeDepth          int    `json:"hash_tree_depth"`
	ModuleName             string `json:"module_name"`
	Description            string `json:"description"`
	IncludesModuleContents bool   `json:"includes_module_contents"`
	IncludesClientContents bool   `json:"includes_client_contents"`
	TotalFiles             int    `json:"total_files"`
	TotalBytes             int64  `json:"total_bytes"`
	OnDiskBytes            int64  `json:"on_disk_bytes"`
	Created                int64  `json:"created"`
	CreatedWith            string `json:"created_with"`
	EmitterVersion         string `json:"emitter_version,omitempty"`
	GroupID                int    `json:"group_id,omitempty"`
}

func marshalSidecar(sidecar Sidecar) ([]byte, error) {
	body, err := json.MarshalIndent(sidecar, "", "  ")
	if err != nil {
		return nil, err
	}
	// Upstream terminates the file with CRLF; match it.
	return append(body, '\r', '\n'), nil
}

// blobPath is the data store path for a blob, hash tree depth 2.
func blobPath(root, sha1Hex string) string {
	return filepath.Join(root, "data", "sha1", sha1Hex[0:2], sha1Hex[2:4], sha1Hex)
}