feat(nwsync): emit blobs and per-artifact NSYM, assemble merged manifests
Adds `crucible nwsync emit` and `crucible nwsync assemble`, the split replacement for upstream nwn_nwsync_write, which cannot be used because it wants every hak, the TLK and the module present in one run on one disk. emit explodes one artifact — a .hak/.erf or a loose file such as the TLK — into NWSync blobs (NWCompressedBuffer framing, magic NSYC, zstd) plus a NSYM v3 manifest describing only that artifact, with the same .json sidecar upstream writes. Blob names are the sha1 of the uncompressed bytes, restypes nss/ndb/gic are always skipped, an unresolvable restype is a hard error, a resource over 15 MB fails closed, and no latest or .origin file is ever written. assemble merges the per-artifact manifests into one, reading no bulk data. The merge rule is resref shadowing, not concatenation: a resref present in more than one artifact resolves to the earliest artifact in --order, the way the game resolves it. It refuses to merge across mismatched emitter versions, and takes --group-id from the caller (1 current, 2 testing; 0 is absent). Refs #53.
This commit is contained in:
@@ -0,0 +1,224 @@
|
||||
package nwsync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
)
|
||||
|
||||
// fileSizeLimit matches upstream's --limit-file-size default of 15 MB. A
|
||||
// resource over it is a hard failure, not a skip: upstream quit(1)s and so do
|
||||
// we. Our largest resource today is 13.66 MiB, so the headroom is thin.
|
||||
const fileSizeLimit = 15 * 1024 * 1024
|
||||
|
||||
// skippedTypes are never published, matching upstream's GobalResTypeSkipList.
|
||||
var skippedTypes = resTypes("nss", "ndb", "gic")
|
||||
|
||||
// serverTypes are loaded only server-side; a manifest holding nothing else
|
||||
// has no client contents. Mirrors upstream's GlobalResTypeServerList.
|
||||
var serverTypes = append(resTypes(
|
||||
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl", "ncs", "ndb",
|
||||
"nss", "ptm", "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
|
||||
), 0)
|
||||
|
||||
func resTypes(extensions ...string) []uint16 {
|
||||
types := make([]uint16, 0, len(extensions))
|
||||
for _, extension := range extensions {
|
||||
restype, ok := erf.ResourceTypeForExtension(extension)
|
||||
if !ok {
|
||||
panic("nwsync: unknown restype " + extension)
|
||||
}
|
||||
types = append(types, restype)
|
||||
}
|
||||
return types
|
||||
}
|
||||
|
||||
// EmitResult reports what one emit run produced.
|
||||
type EmitResult struct {
|
||||
Name string // artifact name, without extension
|
||||
ManifestPath string
|
||||
Entries int
|
||||
BlobsWritten int
|
||||
}
|
||||
|
||||
// Emit explodes one artifact — a .hak/.erf/.mod or a loose file such as the
|
||||
// TLK — into NWSync blobs plus a NSYM manifest describing only that artifact.
|
||||
func Emit(artifactPath, outDir string) (EmitResult, error) {
|
||||
resources, err := readArtifact(artifactPath)
|
||||
if err != nil {
|
||||
return EmitResult{}, err
|
||||
}
|
||||
name := strings.TrimSuffix(filepath.Base(artifactPath), filepath.Ext(artifactPath))
|
||||
|
||||
entries, blobs, onDiskBytes, err := emitResources(resources, outDir)
|
||||
if err != nil {
|
||||
return EmitResult{}, err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return EmitResult{}, fmt.Errorf("%s: nothing to index (no publishable resources)", artifactPath)
|
||||
}
|
||||
|
||||
data, err := writeManifest(entries)
|
||||
if err != nil {
|
||||
return EmitResult{}, err
|
||||
}
|
||||
manifestPath := filepath.Join(outDir, name+".nsym")
|
||||
if err := writeManifestPair(manifestPath, data, entries, onDiskBytes, Sidecar{ModuleName: name}); err != nil {
|
||||
return EmitResult{}, err
|
||||
}
|
||||
return EmitResult{Name: name, ManifestPath: manifestPath, Entries: len(entries), BlobsWritten: blobs}, nil
|
||||
}
|
||||
|
||||
// readArtifact returns the resources of an ERF/HAK/MOD, or the single resource
|
||||
// a loose file represents. Upstream's resman does the same dispatch on the
|
||||
// file's first three bytes.
|
||||
func readArtifact(path string) ([]erf.Resource, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read artifact: %w", err)
|
||||
}
|
||||
if len(data) >= 3 {
|
||||
switch string(data[:3]) {
|
||||
case "ERF", "HAK", "MOD":
|
||||
archive, err := erf.Read(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
return archive.Resources, nil
|
||||
}
|
||||
}
|
||||
base := filepath.Base(path)
|
||||
extension := filepath.Ext(base)
|
||||
restype, ok := erf.ResourceTypeForExtension(extension)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s: unknown resource type %q", path, extension)
|
||||
}
|
||||
return []erf.Resource{{
|
||||
Name: strings.TrimSuffix(base, extension),
|
||||
Type: restype,
|
||||
Data: data,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64, error) {
|
||||
// A resref appearing twice inside one artifact resolves to the last one,
|
||||
// the way resman lets the last container added win.
|
||||
type key struct {
|
||||
resref string
|
||||
restype uint16
|
||||
}
|
||||
order := make([]key, 0, len(resources))
|
||||
latest := make(map[key]erf.Resource, len(resources))
|
||||
var tooBig []string
|
||||
for _, resource := range resources {
|
||||
if _, ok := erf.ExtensionForResourceType(resource.Type); !ok {
|
||||
return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", resource.Name, resource.Type)
|
||||
}
|
||||
if slices.Contains(skippedTypes, resource.Type) {
|
||||
continue
|
||||
}
|
||||
if len(resource.Data) > fileSizeLimit {
|
||||
tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", resource.Name, len(resource.Data), fileSizeLimit))
|
||||
continue
|
||||
}
|
||||
identity := key{resref: strings.ToLower(resource.Name), restype: resource.Type}
|
||||
if _, seen := latest[identity]; !seen {
|
||||
order = append(order, identity)
|
||||
}
|
||||
latest[identity] = resource
|
||||
}
|
||||
if len(tooBig) > 0 {
|
||||
sort.Strings(tooBig)
|
||||
return nil, 0, 0, fmt.Errorf("resources exceed the file size limit:\n %s", strings.Join(tooBig, "\n "))
|
||||
}
|
||||
|
||||
entries := make([]Entry, 0, len(order))
|
||||
var blobs int
|
||||
var onDiskBytes int64
|
||||
for _, identity := range order {
|
||||
resource := latest[identity]
|
||||
sum := sha1.Sum(resource.Data)
|
||||
entries = append(entries, Entry{
|
||||
SHA1: sum,
|
||||
Size: uint32(len(resource.Data)),
|
||||
ResRef: identity.resref,
|
||||
ResType: identity.restype,
|
||||
})
|
||||
written, err := writeBlob(outDir, sum, resource.Data)
|
||||
if err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
if written > 0 {
|
||||
blobs++
|
||||
onDiskBytes += written
|
||||
}
|
||||
}
|
||||
return entries, blobs, onDiskBytes, nil
|
||||
}
|
||||
|
||||
// writeBlob writes one NWCompressedBuffer blob and returns its size on disk,
|
||||
// or 0 if the blob already existed. Blob names are content hashes, so an
|
||||
// existing name is existing content.
|
||||
func writeBlob(outDir string, sum [20]byte, data []byte) (int64, error) {
|
||||
path := blobPath(outDir, fmt.Sprintf("%x", sum))
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return 0, nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return 0, fmt.Errorf("create blob directory: %w", err)
|
||||
}
|
||||
blob := compressBlob(data)
|
||||
if err := os.WriteFile(path, blob, 0o644); err != nil {
|
||||
return 0, fmt.Errorf("write blob: %w", err)
|
||||
}
|
||||
return int64(len(blob)), nil
|
||||
}
|
||||
|
||||
// writeManifestPair writes a NSYM manifest and its .json sidecar. The caller
|
||||
// supplies the sidecar fields it knows; the rest are derived from the entries.
|
||||
// data must be the serialised form of entries.
|
||||
func writeManifestPair(manifestPath string, data []byte, entries []Entry, onDiskBytes int64, sidecar Sidecar) error {
|
||||
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create manifest directory: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, data, 0o644); err != nil {
|
||||
return fmt.Errorf("write manifest: %w", err)
|
||||
}
|
||||
|
||||
var totalBytes int64
|
||||
clientContents := false
|
||||
for _, entry := range entries {
|
||||
totalBytes += int64(entry.Size)
|
||||
if !slices.Contains(serverTypes, entry.ResType) {
|
||||
clientContents = true
|
||||
}
|
||||
}
|
||||
sidecar.Version = manifestVersion
|
||||
sidecar.SHA1 = fmt.Sprintf("%x", sha1.Sum(data))
|
||||
sidecar.HashTreeDepth = hashTreeDepth
|
||||
sidecar.IncludesModuleContents = false
|
||||
sidecar.IncludesClientContents = clientContents
|
||||
sidecar.TotalFiles = len(entries)
|
||||
sidecar.TotalBytes = totalBytes
|
||||
sidecar.OnDiskBytes = onDiskBytes
|
||||
sidecar.Created = time.Now().Unix()
|
||||
sidecar.CreatedWith = buildinfo.String()
|
||||
|
||||
body, err := marshalSidecar(sidecar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(manifestPath+".json", body, 0o644); err != nil {
|
||||
return fmt.Errorf("write sidecar: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user