feat(nwsync): emit blobs and per-artifact NSYM, assemble merged manifests
Adds `crucible nwsync emit` and `crucible nwsync assemble`, the split replacement for upstream nwn_nwsync_write, which cannot be used because it wants every hak, the TLK and the module present in one run on one disk. emit explodes one artifact — a .hak/.erf or a loose file such as the TLK — into NWSync blobs (NWCompressedBuffer framing, magic NSYC, zstd) plus a NSYM v3 manifest describing only that artifact, with the same .json sidecar upstream writes. Blob names are the sha1 of the uncompressed bytes, restypes nss/ndb/gic are always skipped, an unresolvable restype is a hard error, a resource over 15 MB fails closed, and no latest or .origin file is ever written. assemble merges the per-artifact manifests into one, reading no bulk data. The merge rule is resref shadowing, not concatenation: a resref present in more than one artifact resolves to the earliest artifact in --order, the way the game resolves it. It refuses to merge across mismatched emitter versions, and takes --group-id from the caller (1 current, 2 testing; 0 is absent). Refs #53.
This commit is contained in:
@@ -0,0 +1,411 @@
|
||||
package nwsync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha1"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
)
|
||||
|
||||
func restype(t *testing.T, extension string) uint16 {
|
||||
t.Helper()
|
||||
value, ok := erf.ResourceTypeForExtension(extension)
|
||||
if !ok {
|
||||
t.Fatalf("unknown restype %q", extension)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// writeHak builds a HAK fixture from resref.ext => body pairs.
|
||||
func writeHak(t *testing.T, path string, contents map[string][]byte) {
|
||||
t.Helper()
|
||||
resources := make([]erf.Resource, 0, len(contents))
|
||||
for name, body := range contents {
|
||||
stem, extension, _ := strings.Cut(name, ".")
|
||||
resources = append(resources, erf.Resource{
|
||||
Name: stem,
|
||||
Type: restype(t, extension),
|
||||
Data: body,
|
||||
Size: int64(len(body)),
|
||||
})
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := erf.Write(&out, erf.New("HAK", resources)); err != nil {
|
||||
t.Fatalf("write hak: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil {
|
||||
t.Fatalf("write hak file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobFramingRoundTrips(t *testing.T) {
|
||||
data := []byte("the quick brown fox jumps over the lazy dog, repeatedly and at length")
|
||||
blob := compressBlob(data)
|
||||
|
||||
header := make([]uint32, 6)
|
||||
if err := binary.Read(bytes.NewReader(blob[:blobHeaderBytes]), binary.LittleEndian, header); err != nil {
|
||||
t.Fatalf("read header: %v", err)
|
||||
}
|
||||
want := []uint32{blobMagic, 3, 2, uint32(len(data)), 1, 0}
|
||||
for i := range want {
|
||||
if header[i] != want[i] {
|
||||
t.Errorf("header field %d = %d, want %d", i, header[i], want[i])
|
||||
}
|
||||
}
|
||||
if string(blob[:4]) != "NSYC" {
|
||||
t.Errorf("magic bytes = %q, want NSYC", blob[:4])
|
||||
}
|
||||
|
||||
got, err := decompressBlob(blob)
|
||||
if err != nil {
|
||||
t.Fatalf("decompress: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Errorf("round trip mismatch: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestBytesMatchUpstreamLayout(t *testing.T) {
|
||||
shared := sha1.Sum([]byte("shared"))
|
||||
other := sha1.Sum([]byte("other"))
|
||||
// Deliberately out of order, and with two resrefs sharing one sha1: the
|
||||
// second one must become a mapping, not a second entry.
|
||||
entries := []Entry{
|
||||
{SHA1: other, Size: 5, ResRef: "zzz", ResType: 1},
|
||||
{SHA1: shared, Size: 6, ResRef: "bbb", ResType: 2},
|
||||
{SHA1: shared, Size: 6, ResRef: "aaa", ResType: 3},
|
||||
}
|
||||
data, err := writeManifest(entries)
|
||||
if err != nil {
|
||||
t.Fatalf("write manifest: %v", err)
|
||||
}
|
||||
|
||||
if string(data[:4]) != "NSYM" {
|
||||
t.Fatalf("magic = %q", data[:4])
|
||||
}
|
||||
var version, entryCount, mappingCount uint32
|
||||
reader := bytes.NewReader(data[4:16])
|
||||
for _, field := range []*uint32{&version, &entryCount, &mappingCount} {
|
||||
_ = binary.Read(reader, binary.LittleEndian, field)
|
||||
}
|
||||
if version != 3 || entryCount != 2 || mappingCount != 1 {
|
||||
t.Fatalf("header = version %d, %d entries, %d mappings; want 3/2/1", version, entryCount, mappingCount)
|
||||
}
|
||||
wantSize := 16 + int(entryCount)*(20+4+16+2) + int(mappingCount)*(4+16+2)
|
||||
if len(data) != wantSize {
|
||||
t.Fatalf("manifest is %d bytes, want %d", len(data), wantSize)
|
||||
}
|
||||
|
||||
// Sorted by sha1 hex then resref, so the shared hash's "aaa" is the entry
|
||||
// and "bbb" is demoted to a mapping.
|
||||
round, err := readManifest(data)
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest: %v", err)
|
||||
}
|
||||
if len(round) != 3 {
|
||||
t.Fatalf("round trip returned %d entries, want 3", len(round))
|
||||
}
|
||||
byResRef := map[string]Entry{}
|
||||
for _, entry := range round {
|
||||
byResRef[entry.ResRef] = entry
|
||||
}
|
||||
for _, entry := range entries {
|
||||
got, ok := byResRef[entry.ResRef]
|
||||
if !ok {
|
||||
t.Fatalf("resref %q lost in round trip", entry.ResRef)
|
||||
}
|
||||
if got != entry {
|
||||
t.Errorf("resref %q = %+v, want %+v", entry.ResRef, got, entry)
|
||||
}
|
||||
}
|
||||
if round[0].ResRef != "aaa" && round[1].ResRef != "aaa" {
|
||||
t.Errorf("entries are not sorted by sha1 then resref: %+v", round)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitWritesBlobsAndManifest(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
hak := filepath.Join(dir, "sow_test_01.hak")
|
||||
body := []byte("texture bytes")
|
||||
writeHak(t, hak, map[string][]byte{
|
||||
"bloodstain1.tga": body,
|
||||
"copy1.txi": body, // same content, different resref: one blob
|
||||
"script1.nss": []byte("void main() {}"),
|
||||
"debug1.ndb": []byte("debug"),
|
||||
"comment1.gic": []byte("comment"),
|
||||
})
|
||||
|
||||
out := filepath.Join(dir, "out")
|
||||
result, err := Emit(hak, out)
|
||||
if err != nil {
|
||||
t.Fatalf("emit: %v", err)
|
||||
}
|
||||
if result.Entries != 2 {
|
||||
t.Errorf("emitted %d entries, want 2 (nss/ndb/gic are always skipped)", result.Entries)
|
||||
}
|
||||
if result.BlobsWritten != 1 {
|
||||
t.Errorf("wrote %d blobs, want 1 (identical content shares a blob)", result.BlobsWritten)
|
||||
}
|
||||
|
||||
// The blob is named by the sha1 of the uncompressed bytes, under a depth-2
|
||||
// hash tree, and decompresses back to exactly those bytes.
|
||||
sum := sha1.Sum(body)
|
||||
name := hex.EncodeToString(sum[:])
|
||||
path := filepath.Join(out, "data", "sha1", name[0:2], name[2:4], name)
|
||||
blob, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("blob missing at %s: %v", path, err)
|
||||
}
|
||||
got, err := decompressBlob(blob)
|
||||
if err != nil {
|
||||
t.Fatalf("decompress blob: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, body) {
|
||||
t.Errorf("blob decompressed to %q, want %q", got, body)
|
||||
}
|
||||
|
||||
entries := readEmitted(t, out, "sow_test_01")
|
||||
for _, entry := range entries {
|
||||
if entry.ResType == restype(t, "nss") || entry.ResType == restype(t, "ndb") || entry.ResType == restype(t, "gic") {
|
||||
t.Errorf("skipped restype leaked into the manifest: %+v", entry)
|
||||
}
|
||||
}
|
||||
|
||||
var sidecar Sidecar
|
||||
body2, err := os.ReadFile(result.ManifestPath + ".json")
|
||||
if err != nil {
|
||||
t.Fatalf("sidecar missing: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(body2, &sidecar); err != nil {
|
||||
t.Fatalf("sidecar json: %v", err)
|
||||
}
|
||||
if sidecar.TotalFiles != 2 || sidecar.HashTreeDepth != 2 || sidecar.Version != 3 {
|
||||
t.Errorf("sidecar = %+v", sidecar)
|
||||
}
|
||||
if sidecar.IncludesModuleContents {
|
||||
t.Error("sidecar claims module contents; a published manifest never has them")
|
||||
}
|
||||
if strings.Contains(string(body2), "group_id") {
|
||||
t.Error("per-artifact sidecar should omit group_id (0 means absent)")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(out, "latest")); err == nil {
|
||||
t.Error("a latest file was written; there must never be one")
|
||||
}
|
||||
}
|
||||
|
||||
func readEmitted(t *testing.T, dir, name string) []Entry {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(dir, name+".nsym"))
|
||||
if err != nil {
|
||||
t.Fatalf("read emitted manifest: %v", err)
|
||||
}
|
||||
entries, err := readManifest(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse emitted manifest: %v", err)
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func TestEmitFailsClosedOnOversizeResource(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
hak := filepath.Join(dir, "big.hak")
|
||||
writeHak(t, hak, map[string][]byte{"huge1.tga": make([]byte, fileSizeLimit+1)})
|
||||
|
||||
if _, err := Emit(hak, filepath.Join(dir, "out")); err == nil {
|
||||
t.Fatal("emit accepted a resource over the 15 MB limit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitLooseFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tlk := filepath.Join(dir, "sow_tlk.tlk")
|
||||
if err := os.WriteFile(tlk, []byte("TLK V3.0 payload"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := filepath.Join(dir, "out")
|
||||
result, err := Emit(tlk, out)
|
||||
if err != nil {
|
||||
t.Fatalf("emit tlk: %v", err)
|
||||
}
|
||||
entries := readEmitted(t, out, "sow_tlk")
|
||||
if len(entries) != 1 || entries[0].ResRef != "sow_tlk" || entries[0].ResType != restype(t, "tlk") {
|
||||
t.Fatalf("tlk emitted as %+v", entries)
|
||||
}
|
||||
if result.BlobsWritten != 1 {
|
||||
t.Errorf("wrote %d blobs, want 1", result.BlobsWritten)
|
||||
}
|
||||
}
|
||||
|
||||
// emitFixture emits two haks that share a resref, so the merge rule is
|
||||
// observable: "top" holds the winning body, "assets" the shadowed one.
|
||||
func emitFixture(t *testing.T) (string, []byte, []byte) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
topBody := []byte("2da from sow_top")
|
||||
assetBody := []byte("2da from the asset hak")
|
||||
writeHak(t, filepath.Join(dir, "sow_top.hak"), map[string][]byte{"appearance.2da": topBody})
|
||||
writeHak(t, filepath.Join(dir, "sow_core_01.hak"), map[string][]byte{
|
||||
"appearance.2da": assetBody,
|
||||
"bloodstain1.tga": []byte("blood"),
|
||||
})
|
||||
out := filepath.Join(dir, "out")
|
||||
for _, name := range []string{"sow_top", "sow_core_01"} {
|
||||
if _, err := Emit(filepath.Join(dir, name+".hak"), out); err != nil {
|
||||
t.Fatalf("emit %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
return out, topBody, assetBody
|
||||
}
|
||||
|
||||
func TestAssembleShadowsByOrder(t *testing.T) {
|
||||
entriesDir, topBody, assetBody := emitFixture(t)
|
||||
out := t.TempDir()
|
||||
|
||||
result, err := Assemble(AssembleOptions{
|
||||
Order: []string{"sow_top", "sow_core_01"},
|
||||
EntriesDir: entriesDir,
|
||||
OutDir: out,
|
||||
GroupID: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("assemble: %v", err)
|
||||
}
|
||||
if result.Entries != 2 {
|
||||
t.Fatalf("merged %d entries, want 2 (appearance.2da is shadowed, not duplicated)", result.Entries)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(result.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read merged manifest: %v", err)
|
||||
}
|
||||
merged := sha1.Sum(data)
|
||||
if hex.EncodeToString(merged[:]) != result.SHA1 || filepath.Base(result.ManifestPath) != result.SHA1 {
|
||||
t.Errorf("manifest is not named by its own sha1: %s", result.ManifestPath)
|
||||
}
|
||||
entries, err := readManifest(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse merged manifest: %v", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.ResRef != "appearance" {
|
||||
continue
|
||||
}
|
||||
if entry.SHA1 != sha1.Sum(topBody) {
|
||||
t.Errorf("appearance.2da resolved to the wrong hak; want the earliest in --order")
|
||||
}
|
||||
if entry.SHA1 == sha1.Sum(assetBody) {
|
||||
t.Error("appearance.2da resolved to the shadowed hak")
|
||||
}
|
||||
}
|
||||
|
||||
sidecar := Sidecar{}
|
||||
body, err := os.ReadFile(result.ManifestPath + ".json")
|
||||
if err != nil {
|
||||
t.Fatalf("merged sidecar missing: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(body, &sidecar); err != nil {
|
||||
t.Fatalf("merged sidecar json: %v", err)
|
||||
}
|
||||
if sidecar.GroupID != 2 {
|
||||
t.Errorf("group_id = %d, want 2 (testing)", sidecar.GroupID)
|
||||
}
|
||||
if sidecar.SHA1 != result.SHA1 {
|
||||
t.Errorf("sidecar sha1 = %s, want %s", sidecar.SHA1, result.SHA1)
|
||||
}
|
||||
if sidecar.TotalFiles != 2 {
|
||||
t.Errorf("total_files = %d, want 2", sidecar.TotalFiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleReversedOrderPicksTheOtherHak(t *testing.T) {
|
||||
entriesDir, topBody, assetBody := emitFixture(t)
|
||||
out := t.TempDir()
|
||||
|
||||
result, err := Assemble(AssembleOptions{
|
||||
Order: []string{"sow_core_01", "sow_top"},
|
||||
EntriesDir: entriesDir,
|
||||
OutDir: out,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("assemble: %v", err)
|
||||
}
|
||||
data, _ := os.ReadFile(result.ManifestPath)
|
||||
entries, err := readManifest(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse merged manifest: %v", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.ResRef == "appearance" && entry.SHA1 != sha1.Sum(assetBody) {
|
||||
t.Errorf("appearance.2da did not follow --order; still resolves to %x", entry.SHA1)
|
||||
}
|
||||
}
|
||||
_ = topBody
|
||||
}
|
||||
|
||||
func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) {
|
||||
entriesDir, _, _ := emitFixture(t)
|
||||
|
||||
path := filepath.Join(entriesDir, "sow_core_01.nsym.json")
|
||||
var sidecar Sidecar
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(body, &sidecar); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sidecar.CreatedWith = "crucible some-other-build"
|
||||
patched, err := marshalSidecar(sidecar)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, patched, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = Assemble(AssembleOptions{
|
||||
Order: []string{"sow_top", "sow_core_01"},
|
||||
EntriesDir: entriesDir,
|
||||
OutDir: t.TempDir(),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "emitter version mismatch") {
|
||||
t.Fatalf("assemble merged across emitter versions: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleFailsClosedOnMissingIndex(t *testing.T) {
|
||||
entriesDir, _, _ := emitFixture(t)
|
||||
_, err := Assemble(AssembleOptions{
|
||||
Order: []string{"sow_top", "sow_never_published"},
|
||||
EntriesDir: entriesDir,
|
||||
OutDir: t.TempDir(),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "sow_never_published") {
|
||||
t.Fatalf("assemble did not fail closed and name the missing artifact: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsageErrors(t *testing.T) {
|
||||
cases := [][]string{
|
||||
nil,
|
||||
{"nope"},
|
||||
{"emit"},
|
||||
{"emit", "artifact.hak"},
|
||||
{"assemble", "--entries", "x", "--out", "y"},
|
||||
{"assemble", "--order", "a", "--out", "y"},
|
||||
{"assemble", "--order", "a", "--entries", "x"},
|
||||
}
|
||||
for _, args := range cases {
|
||||
var out, errw bytes.Buffer
|
||||
if code := Run(args, &out, &errw); code != exitUsage {
|
||||
t.Errorf("Run(%v) exit=%d, want %d", args, code, exitUsage)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user