flake.nix

@@ -28,6 +28,7 @@
               "cmd/crucible-depot"
               "cmd/crucible-hak"
               "cmd/crucible-module"
+              "cmd/crucible-nwsync"
               "cmd/crucible-topdata"
               "cmd/crucible-wiki"
             ];

go.mod

@@ -7,4 +7,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-require github.com/klauspost/compress v1.19.1 // indirect
+require github.com/klauspost/compress v1.19.1

internal/nwsync/assemble.go

@@ -37,15 +37,9 @@ func Assemble(options AssembleOptions) (AssembleResult, error) {
 		return AssembleResult{}, fmt.Errorf("assemble: --order names no artifacts")
 	}
 
-	type key struct {
-		resref  string
-		restype uint16
-	}
 	merged := make([]Entry, 0, 1024)
-	winner := make(map[key]bool, 1024)
+	winner := make(map[Identity]bool, 1024)
 	var onDiskBytes int64
-	emitterVersion := ""
-	emitterSource := ""
 
 	for _, name := range options.Order {
 		manifestPath := filepath.Join(options.EntriesDir, name+".nsym")
@@ -64,19 +58,17 @@ func Assemble(options AssembleOptions) (AssembleResult, error) {
 		// Two producers of blobs means a skewed emitter can write blobs the
 		// merged manifest quietly disagrees with. Refuse to merge across
 		// mismatched emitter versions.
-		if emitterVersion == "" {
+		if sidecar.EmitterVersion != emitterVersion {
-			emitterVersion, emitterSource = sidecar.CreatedWith, name
-		} else if sidecar.CreatedWith != emitterVersion {
 			return AssembleResult{}, fmt.Errorf(
-				"assemble: emitter version mismatch: %s was emitted with %q, %s with %q",
+				"assemble: emitter version mismatch: %s was emitted by emitter %q, this is emitter %q",
-				emitterSource, emitterVersion, name, sidecar.CreatedWith)
+				name, sidecar.EmitterVersion, emitterVersion)
 		}
 		// on_disk_bytes overcounts by the handful of cross-artifact
 		// duplicates. It is a display statistic; no dedupe pass for it.
 		onDiskBytes += sidecar.OnDiskBytes
 
 		for _, entry := range entries {
-			identity := key{resref: entry.ResRef, restype: entry.ResType}
+			identity := entry.identity()
 			if winner[identity] {
 				continue
 			}

internal/nwsync/emit.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"slices"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -23,8 +24,17 @@ const fileSizeLimit = 15 * 1024 * 1024
 // skippedTypes are never published, matching upstream's GobalResTypeSkipList.
 var skippedTypes = resTypes("nss", "ndb", "gic")
 
+// emitterVersion identifies the blob/manifest byte format this package
+// produces. assemble refuses to merge indexes that disagree on it, because two
+// producers of blobs mean a skewed emitter can otherwise write blobs the merged
+// manifest quietly disagrees with. Bump it only when emitted bytes change — it
+// is deliberately not the build revision, which would invalidate every
+// published index on every unrelated commit.
+const emitterVersion = "1"
+
 // serverTypes are loaded only server-side; a manifest holding nothing else
-// has no client contents. Mirrors upstream's GlobalResTypeServerList.
+// has no client contents. Mirrors upstream's GlobalResTypeServerList, whose
+// trailing 0 is RESTYPE_INVALID.
 var serverTypes = append(resTypes(
 	"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl", "ncs", "ndb",
 	"nss", "ptm", "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
@@ -88,12 +98,17 @@ func readArtifact(path string) ([]erf.Resource, error) {
 	}
 	if len(data) >= 3 {
 		switch string(data[:3]) {
-		case "ERF", "HAK", "MOD":
+		case "ERF", "HAK":
 			archive, err := erf.Read(bytes.NewReader(data))
 			if err != nil {
 				return nil, fmt.Errorf("%s: %w", path, err)
 			}
 			return archive.Resources, nil
+		case "MOD":
+			// A persistent world never publishes module contents, so the .mod
+			// contributes no bytes to a manifest — it only says which haks and
+			// which TLK the manifest covers.
+			return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path)
 		}
 	}
 	base := filepath.Base(path)
@@ -112,12 +127,8 @@ func readArtifact(path string) ([]erf.Resource, error) {
 func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64, error) {
 	// A resref appearing twice inside one artifact resolves to the last one,
 	// the way resman lets the last container added win.
-	type key struct {
+	order := make([]Identity, 0, len(resources))
-		resref  string
+	latest := make(map[Identity]erf.Resource, len(resources))
-		restype uint16
-	}
-	order := make([]key, 0, len(resources))
-	latest := make(map[key]erf.Resource, len(resources))
 	var tooBig []string
 	for _, resource := range resources {
 		if _, ok := erf.ExtensionForResourceType(resource.Type); !ok {
@@ -130,7 +141,7 @@ func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64
 			tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", resource.Name, len(resource.Data), fileSizeLimit))
 			continue
 		}
-		identity := key{resref: strings.ToLower(resource.Name), restype: resource.Type}
+		identity := Identity{ResRef: strings.ToLower(resource.Name), ResType: resource.Type}
 		if _, seen := latest[identity]; !seen {
 			order = append(order, identity)
 		}
@@ -150,8 +161,8 @@ func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64
 		entries = append(entries, Entry{
 			SHA1:    sum,
 			Size:    uint32(len(resource.Data)),
-			ResRef:  identity.resref,
+			ResRef:  identity.ResRef,
-			ResType: identity.restype,
+			ResType: identity.ResType,
 		})
 		written, err := writeBlob(outDir, sum, resource.Data)
 		if err != nil {
@@ -165,6 +176,17 @@ func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64
 	return entries, blobs, onDiskBytes, nil
 }
 
+// created is the sidecar timestamp. SOURCE_DATE_EPOCH pins it so a build can
+// be reproduced byte for byte; the manifest itself is deterministic already.
+func created() int64 {
+	if raw := os.Getenv("SOURCE_DATE_EPOCH"); raw != "" {
+		if seconds, err := strconv.ParseInt(raw, 10, 64); err == nil {
+			return seconds
+		}
+	}
+	return time.Now().Unix()
+}
+
 // writeBlob writes one NWCompressedBuffer blob and returns its size on disk,
 // or 0 if the blob already existed. Blob names are content hashes, so an
 // existing name is existing content.
@@ -210,8 +232,9 @@ func writeManifestPair(manifestPath string, data []byte, entries []Entry, onDisk
 	sidecar.TotalFiles = len(entries)
 	sidecar.TotalBytes = totalBytes
 	sidecar.OnDiskBytes = onDiskBytes
-	sidecar.Created = time.Now().Unix()
+	sidecar.Created = created()
 	sidecar.CreatedWith = buildinfo.String()
+	sidecar.EmitterVersion = emitterVersion
 
 	body, err := marshalSidecar(sidecar)
 	if err != nil {

internal/nwsync/manifest.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -36,6 +37,15 @@ type Entry struct {
 
 func (e Entry) sha1Hex() string { return hex.EncodeToString(e.SHA1[:]) }
 
+// Identity is what a resref resolves by: name plus type. It is the merge key
+// inside one artifact and across artifacts alike.
+type Identity struct {
+	ResRef  string
+	ResType uint16
+}
+
+func (e Entry) identity() Identity { return Identity{ResRef: e.ResRef, ResType: e.ResType} }
+
 // writeManifest serialises entries into NSYM v3 bytes.
 func writeManifest(entries []Entry) ([]byte, error) {
 	sorted := make([]Entry, len(entries))
@@ -86,7 +96,7 @@ func writeManifest(entries []Entry) ([]byte, error) {
 func readManifest(data []byte) ([]Entry, error) {
 	reader := bytes.NewReader(data)
 	magic := make([]byte, 4)
-	if _, err := reader.Read(magic); err != nil || string(magic) != "NSYM" {
+	if _, err := io.ReadFull(reader, magic); err != nil || string(magic) != "NSYM" {
 		return nil, fmt.Errorf("not a manifest (invalid magic bytes)")
 	}
 	var version, entryCount, mappingCount uint32
@@ -102,7 +112,7 @@ func readManifest(data []byte) ([]Entry, error) {
 	entries := make([]Entry, 0, entryCount+mappingCount)
 	for i := uint32(0); i < entryCount; i++ {
 		var entry Entry
-		if _, err := reader.Read(entry.SHA1[:]); err != nil {
+		if _, err := io.ReadFull(reader, entry.SHA1[:]); err != nil {
 			return nil, fmt.Errorf("truncated entry %d: %w", i, err)
 		}
 		if err := binary.Read(reader, binary.LittleEndian, &entry.Size); err != nil {
@@ -135,7 +145,7 @@ func readManifest(data []byte) ([]Entry, error) {
 
 func readResRef(reader *bytes.Reader) (string, uint16, error) {
 	raw := make([]byte, resRefBytes)
-	if _, err := reader.Read(raw); err != nil {
+	if _, err := io.ReadFull(reader, raw); err != nil {
 		return "", 0, err
 	}
 	var restype uint16
@@ -171,6 +181,7 @@ type Sidecar struct {
 	OnDiskBytes            int64  `json:"on_disk_bytes"`
 	Created                int64  `json:"created"`
 	CreatedWith            string `json:"created_with"`
+	EmitterVersion         string `json:"emitter_version,omitempty"`
 	GroupID                int    `json:"group_id,omitempty"`
 }
 

internal/nwsync/nwsync_test.go

@@ -361,7 +361,7 @@ func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) {
 	if err := json.Unmarshal(body, &sidecar); err != nil {
 		t.Fatal(err)
 	}
-	sidecar.CreatedWith = "crucible some-other-build"
+	sidecar.EmitterVersion = "0"
 	patched, err := marshalSidecar(sidecar)
 	if err != nil {
 		t.Fatal(err)
@@ -380,6 +380,45 @@ func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) {
 	}
 }
 
+func TestEmitHonoursSourceDateEpoch(t *testing.T) {
+	t.Setenv("SOURCE_DATE_EPOCH", "1700000000")
+	dir := t.TempDir()
+	hak := filepath.Join(dir, "pinned.hak")
+	writeHak(t, hak, map[string][]byte{"one1.tga": []byte("body")})
+	result, err := Emit(hak, filepath.Join(dir, "out"))
+	if err != nil {
+		t.Fatalf("emit: %v", err)
+	}
+	body, err := os.ReadFile(result.ManifestPath + ".json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var sidecar Sidecar
+	if err := json.Unmarshal(body, &sidecar); err != nil {
+		t.Fatal(err)
+	}
+	if sidecar.Created != 1700000000 {
+		t.Errorf("created = %d, want the pinned SOURCE_DATE_EPOCH", sidecar.Created)
+	}
+}
+
+func TestEmitRejectsAModule(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "sow.mod")
+	var out bytes.Buffer
+	if err := erf.Write(&out, erf.New("MOD", []erf.Resource{
+		{Name: "module", Type: restype(t, "ifo"), Data: []byte("ifo"), Size: 3},
+	})); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Emit(path, filepath.Join(dir, "out")); err == nil {
+		t.Fatal("emit accepted a .mod; a manifest never carries module contents")
+	}
+}
+
 func TestAssembleFailsClosedOnMissingIndex(t *testing.T) {
 	entriesDir, _, _ := emitFixture(t)
 	_, err := Assemble(AssembleOptions{