diff --git a/flake.nix b/flake.nix index c38bb33..0ed890e 100644 --- a/flake.nix +++ b/flake.nix @@ -28,6 +28,7 @@ "cmd/crucible-depot" "cmd/crucible-hak" "cmd/crucible-module" + "cmd/crucible-nwsync" "cmd/crucible-topdata" "cmd/crucible-wiki" ]; diff --git a/go.mod b/go.mod index c19595f..87a1e81 100644 --- a/go.mod +++ b/go.mod @@ -7,4 +7,4 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) -require github.com/klauspost/compress v1.19.1 // indirect +require github.com/klauspost/compress v1.19.1 diff --git a/internal/nwsync/assemble.go b/internal/nwsync/assemble.go index 1a79a37..666cb41 100644 --- a/internal/nwsync/assemble.go +++ b/internal/nwsync/assemble.go @@ -37,15 +37,9 @@ func Assemble(options AssembleOptions) (AssembleResult, error) { return AssembleResult{}, fmt.Errorf("assemble: --order names no artifacts") } - type key struct { - resref string - restype uint16 - } merged := make([]Entry, 0, 1024) - winner := make(map[key]bool, 1024) + winner := make(map[Identity]bool, 1024) var onDiskBytes int64 - emitterVersion := "" - emitterSource := "" for _, name := range options.Order { manifestPath := filepath.Join(options.EntriesDir, name+".nsym") @@ -64,19 +58,17 @@ func Assemble(options AssembleOptions) (AssembleResult, error) { // Two producers of blobs means a skewed emitter can write blobs the // merged manifest quietly disagrees with. Refuse to merge across // mismatched emitter versions. - if emitterVersion == "" { - emitterVersion, emitterSource = sidecar.CreatedWith, name - } else if sidecar.CreatedWith != emitterVersion { + if sidecar.EmitterVersion != emitterVersion { return AssembleResult{}, fmt.Errorf( - "assemble: emitter version mismatch: %s was emitted with %q, %s with %q", - emitterSource, emitterVersion, name, sidecar.CreatedWith) + "assemble: emitter version mismatch: %s was emitted by emitter %q, this is emitter %q", + name, sidecar.EmitterVersion, emitterVersion) } // on_disk_bytes overcounts by the handful of cross-artifact // duplicates. It is a display statistic; no dedupe pass for it. onDiskBytes += sidecar.OnDiskBytes for _, entry := range entries { - identity := key{resref: entry.ResRef, restype: entry.ResType} + identity := entry.identity() if winner[identity] { continue } diff --git a/internal/nwsync/emit.go b/internal/nwsync/emit.go index bde78a0..8317ae7 100644 --- a/internal/nwsync/emit.go +++ b/internal/nwsync/emit.go @@ -8,6 +8,7 @@ import ( "path/filepath" "slices" "sort" + "strconv" "strings" "time" @@ -23,8 +24,17 @@ const fileSizeLimit = 15 * 1024 * 1024 // skippedTypes are never published, matching upstream's GobalResTypeSkipList. var skippedTypes = resTypes("nss", "ndb", "gic") +// emitterVersion identifies the blob/manifest byte format this package +// produces. assemble refuses to merge indexes that disagree on it, because two +// producers of blobs mean a skewed emitter can otherwise write blobs the merged +// manifest quietly disagrees with. Bump it only when emitted bytes change — it +// is deliberately not the build revision, which would invalidate every +// published index on every unrelated commit. +const emitterVersion = "1" + // serverTypes are loaded only server-side; a manifest holding nothing else -// has no client contents. Mirrors upstream's GlobalResTypeServerList. +// has no client contents. Mirrors upstream's GlobalResTypeServerList, whose +// trailing 0 is RESTYPE_INVALID. var serverTypes = append(resTypes( "are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl", "ncs", "ndb", "nss", "ptm", "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw", @@ -88,12 +98,17 @@ func readArtifact(path string) ([]erf.Resource, error) { } if len(data) >= 3 { switch string(data[:3]) { - case "ERF", "HAK", "MOD": + case "ERF", "HAK": archive, err := erf.Read(bytes.NewReader(data)) if err != nil { return nil, fmt.Errorf("%s: %w", path, err) } return archive.Resources, nil + case "MOD": + // A persistent world never publishes module contents, so the .mod + // contributes no bytes to a manifest — it only says which haks and + // which TLK the manifest covers. + return nil, fmt.Errorf("%s: a module is never emitted; a manifest is haks plus the TLK", path) } } base := filepath.Base(path) @@ -112,12 +127,8 @@ func readArtifact(path string) ([]erf.Resource, error) { func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64, error) { // A resref appearing twice inside one artifact resolves to the last one, // the way resman lets the last container added win. - type key struct { - resref string - restype uint16 - } - order := make([]key, 0, len(resources)) - latest := make(map[key]erf.Resource, len(resources)) + order := make([]Identity, 0, len(resources)) + latest := make(map[Identity]erf.Resource, len(resources)) var tooBig []string for _, resource := range resources { if _, ok := erf.ExtensionForResourceType(resource.Type); !ok { @@ -130,7 +141,7 @@ func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64 tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", resource.Name, len(resource.Data), fileSizeLimit)) continue } - identity := key{resref: strings.ToLower(resource.Name), restype: resource.Type} + identity := Identity{ResRef: strings.ToLower(resource.Name), ResType: resource.Type} if _, seen := latest[identity]; !seen { order = append(order, identity) } @@ -150,8 +161,8 @@ func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64 entries = append(entries, Entry{ SHA1: sum, Size: uint32(len(resource.Data)), - ResRef: identity.resref, - ResType: identity.restype, + ResRef: identity.ResRef, + ResType: identity.ResType, }) written, err := writeBlob(outDir, sum, resource.Data) if err != nil { @@ -165,6 +176,17 @@ func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64 return entries, blobs, onDiskBytes, nil } +// created is the sidecar timestamp. SOURCE_DATE_EPOCH pins it so a build can +// be reproduced byte for byte; the manifest itself is deterministic already. +func created() int64 { + if raw := os.Getenv("SOURCE_DATE_EPOCH"); raw != "" { + if seconds, err := strconv.ParseInt(raw, 10, 64); err == nil { + return seconds + } + } + return time.Now().Unix() +} + // writeBlob writes one NWCompressedBuffer blob and returns its size on disk, // or 0 if the blob already existed. Blob names are content hashes, so an // existing name is existing content. @@ -210,8 +232,9 @@ func writeManifestPair(manifestPath string, data []byte, entries []Entry, onDisk sidecar.TotalFiles = len(entries) sidecar.TotalBytes = totalBytes sidecar.OnDiskBytes = onDiskBytes - sidecar.Created = time.Now().Unix() + sidecar.Created = created() sidecar.CreatedWith = buildinfo.String() + sidecar.EmitterVersion = emitterVersion body, err := marshalSidecar(sidecar) if err != nil { diff --git a/internal/nwsync/manifest.go b/internal/nwsync/manifest.go index 53192fa..17ddd4b 100644 --- a/internal/nwsync/manifest.go +++ b/internal/nwsync/manifest.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" "path/filepath" "sort" "strings" @@ -36,6 +37,15 @@ type Entry struct { func (e Entry) sha1Hex() string { return hex.EncodeToString(e.SHA1[:]) } +// Identity is what a resref resolves by: name plus type. It is the merge key +// inside one artifact and across artifacts alike. +type Identity struct { + ResRef string + ResType uint16 +} + +func (e Entry) identity() Identity { return Identity{ResRef: e.ResRef, ResType: e.ResType} } + // writeManifest serialises entries into NSYM v3 bytes. func writeManifest(entries []Entry) ([]byte, error) { sorted := make([]Entry, len(entries)) @@ -86,7 +96,7 @@ func writeManifest(entries []Entry) ([]byte, error) { func readManifest(data []byte) ([]Entry, error) { reader := bytes.NewReader(data) magic := make([]byte, 4) - if _, err := reader.Read(magic); err != nil || string(magic) != "NSYM" { + if _, err := io.ReadFull(reader, magic); err != nil || string(magic) != "NSYM" { return nil, fmt.Errorf("not a manifest (invalid magic bytes)") } var version, entryCount, mappingCount uint32 @@ -102,7 +112,7 @@ func readManifest(data []byte) ([]Entry, error) { entries := make([]Entry, 0, entryCount+mappingCount) for i := uint32(0); i < entryCount; i++ { var entry Entry - if _, err := reader.Read(entry.SHA1[:]); err != nil { + if _, err := io.ReadFull(reader, entry.SHA1[:]); err != nil { return nil, fmt.Errorf("truncated entry %d: %w", i, err) } if err := binary.Read(reader, binary.LittleEndian, &entry.Size); err != nil { @@ -135,7 +145,7 @@ func readManifest(data []byte) ([]Entry, error) { func readResRef(reader *bytes.Reader) (string, uint16, error) { raw := make([]byte, resRefBytes) - if _, err := reader.Read(raw); err != nil { + if _, err := io.ReadFull(reader, raw); err != nil { return "", 0, err } var restype uint16 @@ -171,6 +181,7 @@ type Sidecar struct { OnDiskBytes int64 `json:"on_disk_bytes"` Created int64 `json:"created"` CreatedWith string `json:"created_with"` + EmitterVersion string `json:"emitter_version,omitempty"` GroupID int `json:"group_id,omitempty"` } diff --git a/internal/nwsync/nwsync_test.go b/internal/nwsync/nwsync_test.go index fa3eacf..5af837f 100644 --- a/internal/nwsync/nwsync_test.go +++ b/internal/nwsync/nwsync_test.go @@ -361,7 +361,7 @@ func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) { if err := json.Unmarshal(body, &sidecar); err != nil { t.Fatal(err) } - sidecar.CreatedWith = "crucible some-other-build" + sidecar.EmitterVersion = "0" patched, err := marshalSidecar(sidecar) if err != nil { t.Fatal(err) @@ -380,6 +380,45 @@ func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) { } } +func TestEmitHonoursSourceDateEpoch(t *testing.T) { + t.Setenv("SOURCE_DATE_EPOCH", "1700000000") + dir := t.TempDir() + hak := filepath.Join(dir, "pinned.hak") + writeHak(t, hak, map[string][]byte{"one1.tga": []byte("body")}) + result, err := Emit(hak, filepath.Join(dir, "out")) + if err != nil { + t.Fatalf("emit: %v", err) + } + body, err := os.ReadFile(result.ManifestPath + ".json") + if err != nil { + t.Fatal(err) + } + var sidecar Sidecar + if err := json.Unmarshal(body, &sidecar); err != nil { + t.Fatal(err) + } + if sidecar.Created != 1700000000 { + t.Errorf("created = %d, want the pinned SOURCE_DATE_EPOCH", sidecar.Created) + } +} + +func TestEmitRejectsAModule(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sow.mod") + var out bytes.Buffer + if err := erf.Write(&out, erf.New("MOD", []erf.Resource{ + {Name: "module", Type: restype(t, "ifo"), Data: []byte("ifo"), Size: 3}, + })); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Emit(path, filepath.Join(dir, "out")); err == nil { + t.Fatal("emit accepted a .mod; a manifest never carries module contents") + } +} + func TestAssembleFailsClosedOnMissingIndex(t *testing.T) { entriesDir, _, _ := emitFixture(t) _, err := Assemble(AssembleOptions{