feat(nwsync): emit blobs and per-artifact NSYM, assemble merged manifests
Adds `crucible nwsync emit` and `crucible nwsync assemble`, the split replacement for upstream nwn_nwsync_write, which cannot be used because it wants every hak, the TLK and the module present in one run on one disk. emit explodes one artifact — a .hak/.erf or a loose file such as the TLK — into NWSync blobs (NWCompressedBuffer framing, magic NSYC, zstd) plus a NSYM v3 manifest describing only that artifact, with the same .json sidecar upstream writes. Blob names are the sha1 of the uncompressed bytes, restypes nss/ndb/gic are always skipped, an unresolvable restype is a hard error, a resource over 15 MB fails closed, and no latest or .origin file is ever written. assemble merges the per-artifact manifests into one, reading no bulk data. The merge rule is resref shadowing, not concatenation: a resref present in more than one artifact resolves to the earliest artifact in --order, the way the game resolves it. It refuses to merge across mismatched emitter versions, and takes --group-id from the caller (1 current, 2 testing; 0 is absent). Refs #53.
This commit is contained in:
@@ -0,0 +1,119 @@
|
||||
package nwsync
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// AssembleOptions describes one merged manifest.
|
||||
type AssembleOptions struct {
|
||||
Order []string // artifact names, highest priority first
|
||||
EntriesDir string // directory holding <name>.nsym and <name>.nsym.json
|
||||
OutDir string // repository root; the manifest lands in <out>/manifests
|
||||
GroupID int // 1 = current, 2 = testing; 0 means absent
|
||||
ModuleName string
|
||||
Description string
|
||||
}
|
||||
|
||||
// AssembleResult reports what one assemble run produced.
|
||||
type AssembleResult struct {
|
||||
SHA1 string
|
||||
ManifestPath string
|
||||
Entries int
|
||||
}
|
||||
|
||||
// Assemble merges the per-artifact NSYM manifests named by Order into one
|
||||
// manifest. It reads no bulk data at all — only the small index files.
|
||||
//
|
||||
// Merge rule is resref shadowing, not concatenation: a resref present in more
|
||||
// than one artifact resolves to the earliest artifact in Order, which is how
|
||||
// the game resolves it (upstream's resman adds haks in reverse and lets the
|
||||
// last one win). Get this backwards and the wrong texture ships silently.
|
||||
func Assemble(options AssembleOptions) (AssembleResult, error) {
|
||||
if len(options.Order) == 0 {
|
||||
return AssembleResult{}, fmt.Errorf("assemble: --order names no artifacts")
|
||||
}
|
||||
|
||||
type key struct {
|
||||
resref string
|
||||
restype uint16
|
||||
}
|
||||
merged := make([]Entry, 0, 1024)
|
||||
winner := make(map[key]bool, 1024)
|
||||
var onDiskBytes int64
|
||||
emitterVersion := ""
|
||||
emitterSource := ""
|
||||
|
||||
for _, name := range options.Order {
|
||||
manifestPath := filepath.Join(options.EntriesDir, name+".nsym")
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return AssembleResult{}, fmt.Errorf("assemble: no index for %q: %w", name, err)
|
||||
}
|
||||
entries, err := readManifest(data)
|
||||
if err != nil {
|
||||
return AssembleResult{}, fmt.Errorf("%s: %w", manifestPath, err)
|
||||
}
|
||||
sidecar, err := readSidecar(manifestPath + ".json")
|
||||
if err != nil {
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
// Two producers of blobs means a skewed emitter can write blobs the
|
||||
// merged manifest quietly disagrees with. Refuse to merge across
|
||||
// mismatched emitter versions.
|
||||
if emitterVersion == "" {
|
||||
emitterVersion, emitterSource = sidecar.CreatedWith, name
|
||||
} else if sidecar.CreatedWith != emitterVersion {
|
||||
return AssembleResult{}, fmt.Errorf(
|
||||
"assemble: emitter version mismatch: %s was emitted with %q, %s with %q",
|
||||
emitterSource, emitterVersion, name, sidecar.CreatedWith)
|
||||
}
|
||||
// on_disk_bytes overcounts by the handful of cross-artifact
|
||||
// duplicates. It is a display statistic; no dedupe pass for it.
|
||||
onDiskBytes += sidecar.OnDiskBytes
|
||||
|
||||
for _, entry := range entries {
|
||||
identity := key{resref: entry.ResRef, restype: entry.ResType}
|
||||
if winner[identity] {
|
||||
continue
|
||||
}
|
||||
winner[identity] = true
|
||||
merged = append(merged, entry)
|
||||
}
|
||||
}
|
||||
|
||||
if len(merged) == 0 {
|
||||
return AssembleResult{}, fmt.Errorf("assemble: merged manifest is empty")
|
||||
}
|
||||
|
||||
data, err := writeManifest(merged)
|
||||
if err != nil {
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
sha1Hex := fmt.Sprintf("%x", sha1.Sum(data))
|
||||
manifestPath := filepath.Join(options.OutDir, "manifests", sha1Hex)
|
||||
sidecar := Sidecar{
|
||||
ModuleName: options.ModuleName,
|
||||
Description: options.Description,
|
||||
GroupID: options.GroupID,
|
||||
}
|
||||
if err := writeManifestPair(manifestPath, data, merged, onDiskBytes, sidecar); err != nil {
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
return AssembleResult{SHA1: sha1Hex, ManifestPath: manifestPath, Entries: len(merged)}, nil
|
||||
}
|
||||
|
||||
func readSidecar(path string) (Sidecar, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Sidecar{}, fmt.Errorf("assemble: missing sidecar: %w", err)
|
||||
}
|
||||
var sidecar Sidecar
|
||||
if err := json.Unmarshal(data, &sidecar); err != nil {
|
||||
return Sidecar{}, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
return sidecar, nil
|
||||
}
|
||||
Reference in New Issue
Block a user