feat(nwsync): emit blobs and per-artifact NSYM, assemble merged manifests
Adds `crucible nwsync emit` and `crucible nwsync assemble`, the split replacement for upstream nwn_nwsync_write, which cannot be used because it wants every hak, the TLK and the module present in one run on one disk. emit explodes one artifact — a .hak/.erf or a loose file such as the TLK — into NWSync blobs (NWCompressedBuffer framing, magic NSYC, zstd) plus a NSYM v3 manifest describing only that artifact, with the same .json sidecar upstream writes. Blob names are the sha1 of the uncompressed bytes, restypes nss/ndb/gic are always skipped, an unresolvable restype is a hard error, a resource over 15 MB fails closed, and no latest or .origin file is ever written. assemble merges the per-artifact manifests into one, reading no bulk data. The merge rule is resref shadowing, not concatenation: a resref present in more than one artifact resolves to the earliest artifact in --order, the way the game resolves it. It refuses to merge across mismatched emitter versions, and takes --group-id from the caller (1 current, 2 testing; 0 is absent). Refs #53.
This commit is contained in:
@@ -0,0 +1,119 @@
|
||||
package nwsync
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// AssembleOptions describes one merged manifest.
|
||||
type AssembleOptions struct {
|
||||
Order []string // artifact names, highest priority first
|
||||
EntriesDir string // directory holding <name>.nsym and <name>.nsym.json
|
||||
OutDir string // repository root; the manifest lands in <out>/manifests
|
||||
GroupID int // 1 = current, 2 = testing; 0 means absent
|
||||
ModuleName string
|
||||
Description string
|
||||
}
|
||||
|
||||
// AssembleResult reports what one assemble run produced.
|
||||
type AssembleResult struct {
|
||||
SHA1 string
|
||||
ManifestPath string
|
||||
Entries int
|
||||
}
|
||||
|
||||
// Assemble merges the per-artifact NSYM manifests named by Order into one
|
||||
// manifest. It reads no bulk data at all — only the small index files.
|
||||
//
|
||||
// Merge rule is resref shadowing, not concatenation: a resref present in more
|
||||
// than one artifact resolves to the earliest artifact in Order, which is how
|
||||
// the game resolves it (upstream's resman adds haks in reverse and lets the
|
||||
// last one win). Get this backwards and the wrong texture ships silently.
|
||||
func Assemble(options AssembleOptions) (AssembleResult, error) {
|
||||
if len(options.Order) == 0 {
|
||||
return AssembleResult{}, fmt.Errorf("assemble: --order names no artifacts")
|
||||
}
|
||||
|
||||
type key struct {
|
||||
resref string
|
||||
restype uint16
|
||||
}
|
||||
merged := make([]Entry, 0, 1024)
|
||||
winner := make(map[key]bool, 1024)
|
||||
var onDiskBytes int64
|
||||
emitterVersion := ""
|
||||
emitterSource := ""
|
||||
|
||||
for _, name := range options.Order {
|
||||
manifestPath := filepath.Join(options.EntriesDir, name+".nsym")
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return AssembleResult{}, fmt.Errorf("assemble: no index for %q: %w", name, err)
|
||||
}
|
||||
entries, err := readManifest(data)
|
||||
if err != nil {
|
||||
return AssembleResult{}, fmt.Errorf("%s: %w", manifestPath, err)
|
||||
}
|
||||
sidecar, err := readSidecar(manifestPath + ".json")
|
||||
if err != nil {
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
// Two producers of blobs means a skewed emitter can write blobs the
|
||||
// merged manifest quietly disagrees with. Refuse to merge across
|
||||
// mismatched emitter versions.
|
||||
if emitterVersion == "" {
|
||||
emitterVersion, emitterSource = sidecar.CreatedWith, name
|
||||
} else if sidecar.CreatedWith != emitterVersion {
|
||||
return AssembleResult{}, fmt.Errorf(
|
||||
"assemble: emitter version mismatch: %s was emitted with %q, %s with %q",
|
||||
emitterSource, emitterVersion, name, sidecar.CreatedWith)
|
||||
}
|
||||
// on_disk_bytes overcounts by the handful of cross-artifact
|
||||
// duplicates. It is a display statistic; no dedupe pass for it.
|
||||
onDiskBytes += sidecar.OnDiskBytes
|
||||
|
||||
for _, entry := range entries {
|
||||
identity := key{resref: entry.ResRef, restype: entry.ResType}
|
||||
if winner[identity] {
|
||||
continue
|
||||
}
|
||||
winner[identity] = true
|
||||
merged = append(merged, entry)
|
||||
}
|
||||
}
|
||||
|
||||
if len(merged) == 0 {
|
||||
return AssembleResult{}, fmt.Errorf("assemble: merged manifest is empty")
|
||||
}
|
||||
|
||||
data, err := writeManifest(merged)
|
||||
if err != nil {
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
sha1Hex := fmt.Sprintf("%x", sha1.Sum(data))
|
||||
manifestPath := filepath.Join(options.OutDir, "manifests", sha1Hex)
|
||||
sidecar := Sidecar{
|
||||
ModuleName: options.ModuleName,
|
||||
Description: options.Description,
|
||||
GroupID: options.GroupID,
|
||||
}
|
||||
if err := writeManifestPair(manifestPath, data, merged, onDiskBytes, sidecar); err != nil {
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
return AssembleResult{SHA1: sha1Hex, ManifestPath: manifestPath, Entries: len(merged)}, nil
|
||||
}
|
||||
|
||||
func readSidecar(path string) (Sidecar, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Sidecar{}, fmt.Errorf("assemble: missing sidecar: %w", err)
|
||||
}
|
||||
var sidecar Sidecar
|
||||
if err := json.Unmarshal(data, &sidecar); err != nil {
|
||||
return Sidecar{}, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
return sidecar, nil
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package nwsync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// NWCompressedBuffer framing, as upstream's neverwinter/compressedbuf.nim
|
||||
// writes it for NWSync blobs. All fields are little-endian uint32:
|
||||
//
|
||||
// magic "NSYC", version 3, algorithm 2 (zstd), uncompressed size,
|
||||
// zstd header version 1, dictionary 0, then the raw zstd frame.
|
||||
const (
|
||||
blobMagic = 0x4359534E // "NSYC" little-endian
|
||||
blobVersion = 3
|
||||
algorithmZstd = 2
|
||||
zstdHeaderVer = 1
|
||||
zstdDictionary = 0
|
||||
blobHeaderBytes = 24
|
||||
)
|
||||
|
||||
var (
|
||||
blobEncoder, _ = zstd.NewWriter(nil)
|
||||
blobDecoder, _ = zstd.NewReader(nil)
|
||||
)
|
||||
|
||||
// compressBlob wraps data in NWCompressedBuffer framing.
|
||||
func compressBlob(data []byte) []byte {
|
||||
var out bytes.Buffer
|
||||
header := []uint32{blobMagic, blobVersion, algorithmZstd, uint32(len(data)), zstdHeaderVer, zstdDictionary}
|
||||
for _, field := range header {
|
||||
_ = binary.Write(&out, binary.LittleEndian, field)
|
||||
}
|
||||
out.Write(blobEncoder.EncodeAll(data, nil))
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// decompressBlob unwraps NWCompressedBuffer framing. It exists so a blob this
|
||||
// package wrote — or one upstream wrote — can be compared by its uncompressed
|
||||
// bytes, which is the only comparison that is meaningful across zstd
|
||||
// implementations.
|
||||
func decompressBlob(blob []byte) ([]byte, error) {
|
||||
if len(blob) < blobHeaderBytes {
|
||||
return nil, fmt.Errorf("blob too small: %d bytes", len(blob))
|
||||
}
|
||||
header := make([]uint32, 6)
|
||||
if err := binary.Read(bytes.NewReader(blob[:blobHeaderBytes]), binary.LittleEndian, header); err != nil {
|
||||
return nil, fmt.Errorf("decode blob header: %w", err)
|
||||
}
|
||||
switch {
|
||||
case header[0] != blobMagic:
|
||||
return nil, fmt.Errorf("invalid blob magic: %#x", header[0])
|
||||
case header[1] != blobVersion:
|
||||
return nil, fmt.Errorf("unsupported blob version: %d", header[1])
|
||||
case header[2] != algorithmZstd:
|
||||
return nil, fmt.Errorf("unsupported compression algorithm: %d", header[2])
|
||||
case header[4] != zstdHeaderVer:
|
||||
return nil, fmt.Errorf("unsupported zstd header version: %d", header[4])
|
||||
case header[5] != zstdDictionary:
|
||||
return nil, fmt.Errorf("zstd dictionaries are not supported")
|
||||
}
|
||||
if header[3] == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
data, err := blobDecoder.DecodeAll(blob[blobHeaderBytes:], nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decompress blob: %w", err)
|
||||
}
|
||||
if uint32(len(data)) != header[3] {
|
||||
return nil, fmt.Errorf("blob size mismatch: header says %d, got %d", header[3], len(data))
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package nwsync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
)
|
||||
|
||||
// fileSizeLimit matches upstream's --limit-file-size default of 15 MB. A
|
||||
// resource over it is a hard failure, not a skip: upstream quit(1)s and so do
|
||||
// we. Our largest resource today is 13.66 MiB, so the headroom is thin.
|
||||
const fileSizeLimit = 15 * 1024 * 1024
|
||||
|
||||
// skippedTypes are never published, matching upstream's GobalResTypeSkipList.
|
||||
var skippedTypes = resTypes("nss", "ndb", "gic")
|
||||
|
||||
// serverTypes are loaded only server-side; a manifest holding nothing else
|
||||
// has no client contents. Mirrors upstream's GlobalResTypeServerList.
|
||||
var serverTypes = append(resTypes(
|
||||
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl", "ncs", "ndb",
|
||||
"nss", "ptm", "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
|
||||
), 0)
|
||||
|
||||
func resTypes(extensions ...string) []uint16 {
|
||||
types := make([]uint16, 0, len(extensions))
|
||||
for _, extension := range extensions {
|
||||
restype, ok := erf.ResourceTypeForExtension(extension)
|
||||
if !ok {
|
||||
panic("nwsync: unknown restype " + extension)
|
||||
}
|
||||
types = append(types, restype)
|
||||
}
|
||||
return types
|
||||
}
|
||||
|
||||
// EmitResult reports what one emit run produced.
|
||||
type EmitResult struct {
|
||||
Name string // artifact name, without extension
|
||||
ManifestPath string
|
||||
Entries int
|
||||
BlobsWritten int
|
||||
}
|
||||
|
||||
// Emit explodes one artifact — a .hak/.erf/.mod or a loose file such as the
|
||||
// TLK — into NWSync blobs plus a NSYM manifest describing only that artifact.
|
||||
func Emit(artifactPath, outDir string) (EmitResult, error) {
|
||||
resources, err := readArtifact(artifactPath)
|
||||
if err != nil {
|
||||
return EmitResult{}, err
|
||||
}
|
||||
name := strings.TrimSuffix(filepath.Base(artifactPath), filepath.Ext(artifactPath))
|
||||
|
||||
entries, blobs, onDiskBytes, err := emitResources(resources, outDir)
|
||||
if err != nil {
|
||||
return EmitResult{}, err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return EmitResult{}, fmt.Errorf("%s: nothing to index (no publishable resources)", artifactPath)
|
||||
}
|
||||
|
||||
data, err := writeManifest(entries)
|
||||
if err != nil {
|
||||
return EmitResult{}, err
|
||||
}
|
||||
manifestPath := filepath.Join(outDir, name+".nsym")
|
||||
if err := writeManifestPair(manifestPath, data, entries, onDiskBytes, Sidecar{ModuleName: name}); err != nil {
|
||||
return EmitResult{}, err
|
||||
}
|
||||
return EmitResult{Name: name, ManifestPath: manifestPath, Entries: len(entries), BlobsWritten: blobs}, nil
|
||||
}
|
||||
|
||||
// readArtifact returns the resources of an ERF/HAK/MOD, or the single resource
|
||||
// a loose file represents. Upstream's resman does the same dispatch on the
|
||||
// file's first three bytes.
|
||||
func readArtifact(path string) ([]erf.Resource, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read artifact: %w", err)
|
||||
}
|
||||
if len(data) >= 3 {
|
||||
switch string(data[:3]) {
|
||||
case "ERF", "HAK", "MOD":
|
||||
archive, err := erf.Read(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
return archive.Resources, nil
|
||||
}
|
||||
}
|
||||
base := filepath.Base(path)
|
||||
extension := filepath.Ext(base)
|
||||
restype, ok := erf.ResourceTypeForExtension(extension)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s: unknown resource type %q", path, extension)
|
||||
}
|
||||
return []erf.Resource{{
|
||||
Name: strings.TrimSuffix(base, extension),
|
||||
Type: restype,
|
||||
Data: data,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func emitResources(resources []erf.Resource, outDir string) ([]Entry, int, int64, error) {
|
||||
// A resref appearing twice inside one artifact resolves to the last one,
|
||||
// the way resman lets the last container added win.
|
||||
type key struct {
|
||||
resref string
|
||||
restype uint16
|
||||
}
|
||||
order := make([]key, 0, len(resources))
|
||||
latest := make(map[key]erf.Resource, len(resources))
|
||||
var tooBig []string
|
||||
for _, resource := range resources {
|
||||
if _, ok := erf.ExtensionForResourceType(resource.Type); !ok {
|
||||
return nil, 0, 0, fmt.Errorf("resref %s is not resolvable (unknown restype %d)", resource.Name, resource.Type)
|
||||
}
|
||||
if slices.Contains(skippedTypes, resource.Type) {
|
||||
continue
|
||||
}
|
||||
if len(resource.Data) > fileSizeLimit {
|
||||
tooBig = append(tooBig, fmt.Sprintf("%s: %d bytes > %d", resource.Name, len(resource.Data), fileSizeLimit))
|
||||
continue
|
||||
}
|
||||
identity := key{resref: strings.ToLower(resource.Name), restype: resource.Type}
|
||||
if _, seen := latest[identity]; !seen {
|
||||
order = append(order, identity)
|
||||
}
|
||||
latest[identity] = resource
|
||||
}
|
||||
if len(tooBig) > 0 {
|
||||
sort.Strings(tooBig)
|
||||
return nil, 0, 0, fmt.Errorf("resources exceed the file size limit:\n %s", strings.Join(tooBig, "\n "))
|
||||
}
|
||||
|
||||
entries := make([]Entry, 0, len(order))
|
||||
var blobs int
|
||||
var onDiskBytes int64
|
||||
for _, identity := range order {
|
||||
resource := latest[identity]
|
||||
sum := sha1.Sum(resource.Data)
|
||||
entries = append(entries, Entry{
|
||||
SHA1: sum,
|
||||
Size: uint32(len(resource.Data)),
|
||||
ResRef: identity.resref,
|
||||
ResType: identity.restype,
|
||||
})
|
||||
written, err := writeBlob(outDir, sum, resource.Data)
|
||||
if err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
if written > 0 {
|
||||
blobs++
|
||||
onDiskBytes += written
|
||||
}
|
||||
}
|
||||
return entries, blobs, onDiskBytes, nil
|
||||
}
|
||||
|
||||
// writeBlob writes one NWCompressedBuffer blob and returns its size on disk,
|
||||
// or 0 if the blob already existed. Blob names are content hashes, so an
|
||||
// existing name is existing content.
|
||||
func writeBlob(outDir string, sum [20]byte, data []byte) (int64, error) {
|
||||
path := blobPath(outDir, fmt.Sprintf("%x", sum))
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return 0, nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return 0, fmt.Errorf("create blob directory: %w", err)
|
||||
}
|
||||
blob := compressBlob(data)
|
||||
if err := os.WriteFile(path, blob, 0o644); err != nil {
|
||||
return 0, fmt.Errorf("write blob: %w", err)
|
||||
}
|
||||
return int64(len(blob)), nil
|
||||
}
|
||||
|
||||
// writeManifestPair writes a NSYM manifest and its .json sidecar. The caller
|
||||
// supplies the sidecar fields it knows; the rest are derived from the entries.
|
||||
// data must be the serialised form of entries.
|
||||
func writeManifestPair(manifestPath string, data []byte, entries []Entry, onDiskBytes int64, sidecar Sidecar) error {
|
||||
if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create manifest directory: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(manifestPath, data, 0o644); err != nil {
|
||||
return fmt.Errorf("write manifest: %w", err)
|
||||
}
|
||||
|
||||
var totalBytes int64
|
||||
clientContents := false
|
||||
for _, entry := range entries {
|
||||
totalBytes += int64(entry.Size)
|
||||
if !slices.Contains(serverTypes, entry.ResType) {
|
||||
clientContents = true
|
||||
}
|
||||
}
|
||||
sidecar.Version = manifestVersion
|
||||
sidecar.SHA1 = fmt.Sprintf("%x", sha1.Sum(data))
|
||||
sidecar.HashTreeDepth = hashTreeDepth
|
||||
sidecar.IncludesModuleContents = false
|
||||
sidecar.IncludesClientContents = clientContents
|
||||
sidecar.TotalFiles = len(entries)
|
||||
sidecar.TotalBytes = totalBytes
|
||||
sidecar.OnDiskBytes = onDiskBytes
|
||||
sidecar.Created = time.Now().Unix()
|
||||
sidecar.CreatedWith = buildinfo.String()
|
||||
|
||||
body, err := marshalSidecar(sidecar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(manifestPath+".json", body, 0o644); err != nil {
|
||||
return fmt.Errorf("write sidecar: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package nwsync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// NSYM manifest, version 3, exactly as upstream neverwinter/nwsync.nim writes
|
||||
// it. Everything is little-endian:
|
||||
//
|
||||
// "NSYM", uint32 version, uint32 entry count, uint32 mapping count,
|
||||
// entries: byte[20] raw sha1, uint32 size, char[16] resref, uint16 restype
|
||||
// mappings: uint32 entry index, char[16] resref, uint16 restype
|
||||
//
|
||||
// Entries are sorted by lowercase sha1 hex then resref, and a resource whose
|
||||
// sha1 was already written becomes a mapping instead of a second entry.
|
||||
const (
|
||||
manifestVersion = 3
|
||||
hashTreeDepth = 2
|
||||
resRefBytes = 16
|
||||
)
|
||||
|
||||
// Entry is one resource in a manifest.
|
||||
type Entry struct {
|
||||
SHA1 [20]byte
|
||||
Size uint32
|
||||
ResRef string // lowercase, no extension, at most 16 characters
|
||||
ResType uint16
|
||||
}
|
||||
|
||||
func (e Entry) sha1Hex() string { return hex.EncodeToString(e.SHA1[:]) }
|
||||
|
||||
// writeManifest serialises entries into NSYM v3 bytes.
|
||||
func writeManifest(entries []Entry) ([]byte, error) {
|
||||
sorted := make([]Entry, len(entries))
|
||||
copy(sorted, entries)
|
||||
sort.SliceStable(sorted, func(i, j int) bool {
|
||||
left, right := sorted[i].sha1Hex(), sorted[j].sha1Hex()
|
||||
if left != right {
|
||||
return left < right
|
||||
}
|
||||
return sorted[i].ResRef < sorted[j].ResRef
|
||||
})
|
||||
|
||||
var body, mappings bytes.Buffer
|
||||
seen := make(map[string]uint32, len(sorted))
|
||||
var entryCount, mappingCount uint32
|
||||
for _, entry := range sorted {
|
||||
padded, err := padResRef(entry.ResRef)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if index, ok := seen[entry.sha1Hex()]; ok {
|
||||
_ = binary.Write(&mappings, binary.LittleEndian, index)
|
||||
mappings.Write(padded)
|
||||
_ = binary.Write(&mappings, binary.LittleEndian, entry.ResType)
|
||||
mappingCount++
|
||||
continue
|
||||
}
|
||||
seen[entry.sha1Hex()] = entryCount
|
||||
entryCount++
|
||||
body.Write(entry.SHA1[:])
|
||||
_ = binary.Write(&body, binary.LittleEndian, entry.Size)
|
||||
body.Write(padded)
|
||||
_ = binary.Write(&body, binary.LittleEndian, entry.ResType)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
out.WriteString("NSYM")
|
||||
for _, field := range []uint32{manifestVersion, entryCount, mappingCount} {
|
||||
_ = binary.Write(&out, binary.LittleEndian, field)
|
||||
}
|
||||
out.Write(body.Bytes())
|
||||
out.Write(mappings.Bytes())
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
|
||||
// readManifest parses NSYM v3 bytes. Mappings are expanded back into entries,
|
||||
// the way upstream's reader does, so a caller sees one entry per resref.
|
||||
func readManifest(data []byte) ([]Entry, error) {
|
||||
reader := bytes.NewReader(data)
|
||||
magic := make([]byte, 4)
|
||||
if _, err := reader.Read(magic); err != nil || string(magic) != "NSYM" {
|
||||
return nil, fmt.Errorf("not a manifest (invalid magic bytes)")
|
||||
}
|
||||
var version, entryCount, mappingCount uint32
|
||||
for _, field := range []*uint32{&version, &entryCount, &mappingCount} {
|
||||
if err := binary.Read(reader, binary.LittleEndian, field); err != nil {
|
||||
return nil, fmt.Errorf("truncated manifest header: %w", err)
|
||||
}
|
||||
}
|
||||
if version != manifestVersion {
|
||||
return nil, fmt.Errorf("unsupported manifest version %d", version)
|
||||
}
|
||||
|
||||
entries := make([]Entry, 0, entryCount+mappingCount)
|
||||
for i := uint32(0); i < entryCount; i++ {
|
||||
var entry Entry
|
||||
if _, err := reader.Read(entry.SHA1[:]); err != nil {
|
||||
return nil, fmt.Errorf("truncated entry %d: %w", i, err)
|
||||
}
|
||||
if err := binary.Read(reader, binary.LittleEndian, &entry.Size); err != nil {
|
||||
return nil, fmt.Errorf("truncated entry %d: %w", i, err)
|
||||
}
|
||||
resref, restype, err := readResRef(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("truncated entry %d: %w", i, err)
|
||||
}
|
||||
entry.ResRef, entry.ResType = resref, restype
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
for i := uint32(0); i < mappingCount; i++ {
|
||||
var index uint32
|
||||
if err := binary.Read(reader, binary.LittleEndian, &index); err != nil {
|
||||
return nil, fmt.Errorf("truncated mapping %d: %w", i, err)
|
||||
}
|
||||
if index >= entryCount {
|
||||
return nil, fmt.Errorf("mapping %d references non-existent entry %d", i, index)
|
||||
}
|
||||
resref, restype, err := readResRef(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("truncated mapping %d: %w", i, err)
|
||||
}
|
||||
target := entries[index]
|
||||
entries = append(entries, Entry{SHA1: target.SHA1, Size: target.Size, ResRef: resref, ResType: restype})
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func readResRef(reader *bytes.Reader) (string, uint16, error) {
|
||||
raw := make([]byte, resRefBytes)
|
||||
if _, err := reader.Read(raw); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
var restype uint16
|
||||
if err := binary.Read(reader, binary.LittleEndian, &restype); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return strings.ToLower(string(bytes.TrimRight(raw, "\x00"))), restype, nil
|
||||
}
|
||||
|
||||
func padResRef(resref string) ([]byte, error) {
|
||||
if len(resref) > resRefBytes {
|
||||
return nil, fmt.Errorf("resref %q exceeds %d characters", resref, resRefBytes)
|
||||
}
|
||||
padded := make([]byte, resRefBytes)
|
||||
copy(padded, strings.ToLower(resref))
|
||||
return padded, nil
|
||||
}
|
||||
|
||||
// Sidecar is the .json file written next to every manifest. Clients fetch it
|
||||
// (nwn_nwsync_fetch.nim), so the field names and order match upstream.
|
||||
// Upstream omits an integer meta field whose value is 0, hence group_id's
|
||||
// omitempty: 0 means absent, not "group zero".
|
||||
type Sidecar struct {
|
||||
Version int `json:"version"`
|
||||
SHA1 string `json:"sha1"`
|
||||
HashTreeDepth int `json:"hash_tree_depth"`
|
||||
ModuleName string `json:"module_name"`
|
||||
Description string `json:"description"`
|
||||
IncludesModuleContents bool `json:"includes_module_contents"`
|
||||
IncludesClientContents bool `json:"includes_client_contents"`
|
||||
TotalFiles int `json:"total_files"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
OnDiskBytes int64 `json:"on_disk_bytes"`
|
||||
Created int64 `json:"created"`
|
||||
CreatedWith string `json:"created_with"`
|
||||
GroupID int `json:"group_id,omitempty"`
|
||||
}
|
||||
|
||||
func marshalSidecar(sidecar Sidecar) ([]byte, error) {
|
||||
body, err := json.MarshalIndent(sidecar, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Upstream terminates the file with CRLF; match it.
|
||||
return append(body, '\r', '\n'), nil
|
||||
}
|
||||
|
||||
// blobPath is the data store path for a blob, hash tree depth 2.
|
||||
func blobPath(root, sha1Hex string) string {
|
||||
return filepath.Join(root, "data", "sha1", sha1Hex[0:2], sha1Hex[2:4], sha1Hex)
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
package nwsync
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha1"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
|
||||
)
|
||||
|
||||
func restype(t *testing.T, extension string) uint16 {
|
||||
t.Helper()
|
||||
value, ok := erf.ResourceTypeForExtension(extension)
|
||||
if !ok {
|
||||
t.Fatalf("unknown restype %q", extension)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// writeHak builds a HAK fixture from resref.ext => body pairs.
|
||||
func writeHak(t *testing.T, path string, contents map[string][]byte) {
|
||||
t.Helper()
|
||||
resources := make([]erf.Resource, 0, len(contents))
|
||||
for name, body := range contents {
|
||||
stem, extension, _ := strings.Cut(name, ".")
|
||||
resources = append(resources, erf.Resource{
|
||||
Name: stem,
|
||||
Type: restype(t, extension),
|
||||
Data: body,
|
||||
Size: int64(len(body)),
|
||||
})
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := erf.Write(&out, erf.New("HAK", resources)); err != nil {
|
||||
t.Fatalf("write hak: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil {
|
||||
t.Fatalf("write hak file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobFramingRoundTrips(t *testing.T) {
|
||||
data := []byte("the quick brown fox jumps over the lazy dog, repeatedly and at length")
|
||||
blob := compressBlob(data)
|
||||
|
||||
header := make([]uint32, 6)
|
||||
if err := binary.Read(bytes.NewReader(blob[:blobHeaderBytes]), binary.LittleEndian, header); err != nil {
|
||||
t.Fatalf("read header: %v", err)
|
||||
}
|
||||
want := []uint32{blobMagic, 3, 2, uint32(len(data)), 1, 0}
|
||||
for i := range want {
|
||||
if header[i] != want[i] {
|
||||
t.Errorf("header field %d = %d, want %d", i, header[i], want[i])
|
||||
}
|
||||
}
|
||||
if string(blob[:4]) != "NSYC" {
|
||||
t.Errorf("magic bytes = %q, want NSYC", blob[:4])
|
||||
}
|
||||
|
||||
got, err := decompressBlob(blob)
|
||||
if err != nil {
|
||||
t.Fatalf("decompress: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Errorf("round trip mismatch: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestBytesMatchUpstreamLayout(t *testing.T) {
|
||||
shared := sha1.Sum([]byte("shared"))
|
||||
other := sha1.Sum([]byte("other"))
|
||||
// Deliberately out of order, and with two resrefs sharing one sha1: the
|
||||
// second one must become a mapping, not a second entry.
|
||||
entries := []Entry{
|
||||
{SHA1: other, Size: 5, ResRef: "zzz", ResType: 1},
|
||||
{SHA1: shared, Size: 6, ResRef: "bbb", ResType: 2},
|
||||
{SHA1: shared, Size: 6, ResRef: "aaa", ResType: 3},
|
||||
}
|
||||
data, err := writeManifest(entries)
|
||||
if err != nil {
|
||||
t.Fatalf("write manifest: %v", err)
|
||||
}
|
||||
|
||||
if string(data[:4]) != "NSYM" {
|
||||
t.Fatalf("magic = %q", data[:4])
|
||||
}
|
||||
var version, entryCount, mappingCount uint32
|
||||
reader := bytes.NewReader(data[4:16])
|
||||
for _, field := range []*uint32{&version, &entryCount, &mappingCount} {
|
||||
_ = binary.Read(reader, binary.LittleEndian, field)
|
||||
}
|
||||
if version != 3 || entryCount != 2 || mappingCount != 1 {
|
||||
t.Fatalf("header = version %d, %d entries, %d mappings; want 3/2/1", version, entryCount, mappingCount)
|
||||
}
|
||||
wantSize := 16 + int(entryCount)*(20+4+16+2) + int(mappingCount)*(4+16+2)
|
||||
if len(data) != wantSize {
|
||||
t.Fatalf("manifest is %d bytes, want %d", len(data), wantSize)
|
||||
}
|
||||
|
||||
// Sorted by sha1 hex then resref, so the shared hash's "aaa" is the entry
|
||||
// and "bbb" is demoted to a mapping.
|
||||
round, err := readManifest(data)
|
||||
if err != nil {
|
||||
t.Fatalf("read manifest: %v", err)
|
||||
}
|
||||
if len(round) != 3 {
|
||||
t.Fatalf("round trip returned %d entries, want 3", len(round))
|
||||
}
|
||||
byResRef := map[string]Entry{}
|
||||
for _, entry := range round {
|
||||
byResRef[entry.ResRef] = entry
|
||||
}
|
||||
for _, entry := range entries {
|
||||
got, ok := byResRef[entry.ResRef]
|
||||
if !ok {
|
||||
t.Fatalf("resref %q lost in round trip", entry.ResRef)
|
||||
}
|
||||
if got != entry {
|
||||
t.Errorf("resref %q = %+v, want %+v", entry.ResRef, got, entry)
|
||||
}
|
||||
}
|
||||
if round[0].ResRef != "aaa" && round[1].ResRef != "aaa" {
|
||||
t.Errorf("entries are not sorted by sha1 then resref: %+v", round)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitWritesBlobsAndManifest(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
hak := filepath.Join(dir, "sow_test_01.hak")
|
||||
body := []byte("texture bytes")
|
||||
writeHak(t, hak, map[string][]byte{
|
||||
"bloodstain1.tga": body,
|
||||
"copy1.txi": body, // same content, different resref: one blob
|
||||
"script1.nss": []byte("void main() {}"),
|
||||
"debug1.ndb": []byte("debug"),
|
||||
"comment1.gic": []byte("comment"),
|
||||
})
|
||||
|
||||
out := filepath.Join(dir, "out")
|
||||
result, err := Emit(hak, out)
|
||||
if err != nil {
|
||||
t.Fatalf("emit: %v", err)
|
||||
}
|
||||
if result.Entries != 2 {
|
||||
t.Errorf("emitted %d entries, want 2 (nss/ndb/gic are always skipped)", result.Entries)
|
||||
}
|
||||
if result.BlobsWritten != 1 {
|
||||
t.Errorf("wrote %d blobs, want 1 (identical content shares a blob)", result.BlobsWritten)
|
||||
}
|
||||
|
||||
// The blob is named by the sha1 of the uncompressed bytes, under a depth-2
|
||||
// hash tree, and decompresses back to exactly those bytes.
|
||||
sum := sha1.Sum(body)
|
||||
name := hex.EncodeToString(sum[:])
|
||||
path := filepath.Join(out, "data", "sha1", name[0:2], name[2:4], name)
|
||||
blob, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("blob missing at %s: %v", path, err)
|
||||
}
|
||||
got, err := decompressBlob(blob)
|
||||
if err != nil {
|
||||
t.Fatalf("decompress blob: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, body) {
|
||||
t.Errorf("blob decompressed to %q, want %q", got, body)
|
||||
}
|
||||
|
||||
entries := readEmitted(t, out, "sow_test_01")
|
||||
for _, entry := range entries {
|
||||
if entry.ResType == restype(t, "nss") || entry.ResType == restype(t, "ndb") || entry.ResType == restype(t, "gic") {
|
||||
t.Errorf("skipped restype leaked into the manifest: %+v", entry)
|
||||
}
|
||||
}
|
||||
|
||||
var sidecar Sidecar
|
||||
body2, err := os.ReadFile(result.ManifestPath + ".json")
|
||||
if err != nil {
|
||||
t.Fatalf("sidecar missing: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(body2, &sidecar); err != nil {
|
||||
t.Fatalf("sidecar json: %v", err)
|
||||
}
|
||||
if sidecar.TotalFiles != 2 || sidecar.HashTreeDepth != 2 || sidecar.Version != 3 {
|
||||
t.Errorf("sidecar = %+v", sidecar)
|
||||
}
|
||||
if sidecar.IncludesModuleContents {
|
||||
t.Error("sidecar claims module contents; a published manifest never has them")
|
||||
}
|
||||
if strings.Contains(string(body2), "group_id") {
|
||||
t.Error("per-artifact sidecar should omit group_id (0 means absent)")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(out, "latest")); err == nil {
|
||||
t.Error("a latest file was written; there must never be one")
|
||||
}
|
||||
}
|
||||
|
||||
func readEmitted(t *testing.T, dir, name string) []Entry {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(dir, name+".nsym"))
|
||||
if err != nil {
|
||||
t.Fatalf("read emitted manifest: %v", err)
|
||||
}
|
||||
entries, err := readManifest(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse emitted manifest: %v", err)
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func TestEmitFailsClosedOnOversizeResource(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
hak := filepath.Join(dir, "big.hak")
|
||||
writeHak(t, hak, map[string][]byte{"huge1.tga": make([]byte, fileSizeLimit+1)})
|
||||
|
||||
if _, err := Emit(hak, filepath.Join(dir, "out")); err == nil {
|
||||
t.Fatal("emit accepted a resource over the 15 MB limit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitLooseFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tlk := filepath.Join(dir, "sow_tlk.tlk")
|
||||
if err := os.WriteFile(tlk, []byte("TLK V3.0 payload"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := filepath.Join(dir, "out")
|
||||
result, err := Emit(tlk, out)
|
||||
if err != nil {
|
||||
t.Fatalf("emit tlk: %v", err)
|
||||
}
|
||||
entries := readEmitted(t, out, "sow_tlk")
|
||||
if len(entries) != 1 || entries[0].ResRef != "sow_tlk" || entries[0].ResType != restype(t, "tlk") {
|
||||
t.Fatalf("tlk emitted as %+v", entries)
|
||||
}
|
||||
if result.BlobsWritten != 1 {
|
||||
t.Errorf("wrote %d blobs, want 1", result.BlobsWritten)
|
||||
}
|
||||
}
|
||||
|
||||
// emitFixture emits two haks that share a resref, so the merge rule is
|
||||
// observable: "top" holds the winning body, "assets" the shadowed one.
|
||||
func emitFixture(t *testing.T) (string, []byte, []byte) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
topBody := []byte("2da from sow_top")
|
||||
assetBody := []byte("2da from the asset hak")
|
||||
writeHak(t, filepath.Join(dir, "sow_top.hak"), map[string][]byte{"appearance.2da": topBody})
|
||||
writeHak(t, filepath.Join(dir, "sow_core_01.hak"), map[string][]byte{
|
||||
"appearance.2da": assetBody,
|
||||
"bloodstain1.tga": []byte("blood"),
|
||||
})
|
||||
out := filepath.Join(dir, "out")
|
||||
for _, name := range []string{"sow_top", "sow_core_01"} {
|
||||
if _, err := Emit(filepath.Join(dir, name+".hak"), out); err != nil {
|
||||
t.Fatalf("emit %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
return out, topBody, assetBody
|
||||
}
|
||||
|
||||
func TestAssembleShadowsByOrder(t *testing.T) {
|
||||
entriesDir, topBody, assetBody := emitFixture(t)
|
||||
out := t.TempDir()
|
||||
|
||||
result, err := Assemble(AssembleOptions{
|
||||
Order: []string{"sow_top", "sow_core_01"},
|
||||
EntriesDir: entriesDir,
|
||||
OutDir: out,
|
||||
GroupID: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("assemble: %v", err)
|
||||
}
|
||||
if result.Entries != 2 {
|
||||
t.Fatalf("merged %d entries, want 2 (appearance.2da is shadowed, not duplicated)", result.Entries)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(result.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read merged manifest: %v", err)
|
||||
}
|
||||
merged := sha1.Sum(data)
|
||||
if hex.EncodeToString(merged[:]) != result.SHA1 || filepath.Base(result.ManifestPath) != result.SHA1 {
|
||||
t.Errorf("manifest is not named by its own sha1: %s", result.ManifestPath)
|
||||
}
|
||||
entries, err := readManifest(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse merged manifest: %v", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.ResRef != "appearance" {
|
||||
continue
|
||||
}
|
||||
if entry.SHA1 != sha1.Sum(topBody) {
|
||||
t.Errorf("appearance.2da resolved to the wrong hak; want the earliest in --order")
|
||||
}
|
||||
if entry.SHA1 == sha1.Sum(assetBody) {
|
||||
t.Error("appearance.2da resolved to the shadowed hak")
|
||||
}
|
||||
}
|
||||
|
||||
sidecar := Sidecar{}
|
||||
body, err := os.ReadFile(result.ManifestPath + ".json")
|
||||
if err != nil {
|
||||
t.Fatalf("merged sidecar missing: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(body, &sidecar); err != nil {
|
||||
t.Fatalf("merged sidecar json: %v", err)
|
||||
}
|
||||
if sidecar.GroupID != 2 {
|
||||
t.Errorf("group_id = %d, want 2 (testing)", sidecar.GroupID)
|
||||
}
|
||||
if sidecar.SHA1 != result.SHA1 {
|
||||
t.Errorf("sidecar sha1 = %s, want %s", sidecar.SHA1, result.SHA1)
|
||||
}
|
||||
if sidecar.TotalFiles != 2 {
|
||||
t.Errorf("total_files = %d, want 2", sidecar.TotalFiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleReversedOrderPicksTheOtherHak(t *testing.T) {
|
||||
entriesDir, topBody, assetBody := emitFixture(t)
|
||||
out := t.TempDir()
|
||||
|
||||
result, err := Assemble(AssembleOptions{
|
||||
Order: []string{"sow_core_01", "sow_top"},
|
||||
EntriesDir: entriesDir,
|
||||
OutDir: out,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("assemble: %v", err)
|
||||
}
|
||||
data, _ := os.ReadFile(result.ManifestPath)
|
||||
entries, err := readManifest(data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse merged manifest: %v", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.ResRef == "appearance" && entry.SHA1 != sha1.Sum(assetBody) {
|
||||
t.Errorf("appearance.2da did not follow --order; still resolves to %x", entry.SHA1)
|
||||
}
|
||||
}
|
||||
_ = topBody
|
||||
}
|
||||
|
||||
func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) {
|
||||
entriesDir, _, _ := emitFixture(t)
|
||||
|
||||
path := filepath.Join(entriesDir, "sow_core_01.nsym.json")
|
||||
var sidecar Sidecar
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(body, &sidecar); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sidecar.CreatedWith = "crucible some-other-build"
|
||||
patched, err := marshalSidecar(sidecar)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, patched, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = Assemble(AssembleOptions{
|
||||
Order: []string{"sow_top", "sow_core_01"},
|
||||
EntriesDir: entriesDir,
|
||||
OutDir: t.TempDir(),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "emitter version mismatch") {
|
||||
t.Fatalf("assemble merged across emitter versions: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleFailsClosedOnMissingIndex(t *testing.T) {
|
||||
entriesDir, _, _ := emitFixture(t)
|
||||
_, err := Assemble(AssembleOptions{
|
||||
Order: []string{"sow_top", "sow_never_published"},
|
||||
EntriesDir: entriesDir,
|
||||
OutDir: t.TempDir(),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "sow_never_published") {
|
||||
t.Fatalf("assemble did not fail closed and name the missing artifact: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsageErrors(t *testing.T) {
|
||||
cases := [][]string{
|
||||
nil,
|
||||
{"nope"},
|
||||
{"emit"},
|
||||
{"emit", "artifact.hak"},
|
||||
{"assemble", "--entries", "x", "--out", "y"},
|
||||
{"assemble", "--order", "a", "--out", "y"},
|
||||
{"assemble", "--order", "a", "--entries", "x"},
|
||||
}
|
||||
for _, args := range cases {
|
||||
var out, errw bytes.Buffer
|
||||
if code := Run(args, &out, &errw); code != exitUsage {
|
||||
t.Errorf("Run(%v) exit=%d, want %d", args, code, exitUsage)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
// Package nwsync publishes NWSync repository data: blobs and a per-artifact
|
||||
// NSYM manifest at each artifact's birth (emit), and one merged manifest at
|
||||
// module release (assemble).
|
||||
//
|
||||
// The split exists because upstream nwn_nwsync_write wants every hak, the TLK
|
||||
// and the module present in one run on one disk, which our build hosts cannot
|
||||
// hold. Upstream stays the conformance oracle: manifests compare byte for
|
||||
// byte, blobs compare after decompression.
|
||||
package nwsync
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
exitOK = 0
|
||||
exitUsage = 64
|
||||
exitInternal = 70
|
||||
)
|
||||
|
||||
// Run executes an nwsync subcommand. args[0] is the subcommand (emit|assemble);
|
||||
// returns the process exit code.
|
||||
func Run(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
printRunUsage(stderr)
|
||||
return exitUsage
|
||||
}
|
||||
switch args[0] {
|
||||
case "emit":
|
||||
return runEmit(args[1:], stdout, stderr)
|
||||
case "assemble":
|
||||
return runAssemble(args[1:], stdout, stderr)
|
||||
case "-h", "--help", "help":
|
||||
printRunUsage(stdout)
|
||||
return exitOK
|
||||
default:
|
||||
fmt.Fprintf(stderr, "nwsync: unknown subcommand %q\n\n", args[0])
|
||||
printRunUsage(stderr)
|
||||
return exitUsage
|
||||
}
|
||||
}
|
||||
|
||||
func printRunUsage(w io.Writer) {
|
||||
fmt.Fprint(w, `usage:
|
||||
nwsync emit <artifact> --out DIR
|
||||
nwsync assemble --order NAMES --entries DIR --out DIR [--group-id N]
|
||||
|
||||
emit explodes one .hak/.erf or one loose file (the TLK) into NWSync blobs plus
|
||||
a NSYM manifest covering only that artifact. assemble merges those per-artifact
|
||||
manifests into one, reading no bulk data.
|
||||
`)
|
||||
}
|
||||
|
||||
func runEmit(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("emit", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
out := fs.String("out", "", "output directory (blobs plus the per-artifact manifest)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if fs.NArg() != 1 {
|
||||
fmt.Fprintf(stderr, "nwsync emit: exactly one artifact is required\n")
|
||||
return exitUsage
|
||||
}
|
||||
if *out == "" {
|
||||
fmt.Fprintf(stderr, "nwsync emit: --out is required\n")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
result, err := Emit(fs.Arg(0), *out)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "nwsync emit: %v\n", err)
|
||||
return exitInternal
|
||||
}
|
||||
fmt.Fprintf(stdout, "emitted %s: %d resources, %d new blobs, manifest %s\n",
|
||||
result.Name, result.Entries, result.BlobsWritten, result.ManifestPath)
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func runAssemble(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("assemble", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
order := fs.String("order", "", "comma-separated artifact names, highest priority first")
|
||||
entries := fs.String("entries", "", "directory holding the per-artifact .nsym files")
|
||||
out := fs.String("out", "", "repository root; the manifest lands in <out>/manifests")
|
||||
groupID := fs.Int("group-id", 0, "NWSync group id (1 = current, 2 = testing; 0 omits it)")
|
||||
moduleName := fs.String("module-name", "", "module name recorded in the sidecar")
|
||||
description := fs.String("description", "", "description recorded in the sidecar")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
switch {
|
||||
case *order == "":
|
||||
fmt.Fprintf(stderr, "nwsync assemble: --order is required\n")
|
||||
return exitUsage
|
||||
case *entries == "":
|
||||
fmt.Fprintf(stderr, "nwsync assemble: --entries is required\n")
|
||||
return exitUsage
|
||||
case *out == "":
|
||||
fmt.Fprintf(stderr, "nwsync assemble: --out is required\n")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
names := make([]string, 0, 8)
|
||||
for _, name := range strings.Split(*order, ",") {
|
||||
if name = strings.TrimSpace(name); name != "" {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
|
||||
result, err := Assemble(AssembleOptions{
|
||||
Order: names,
|
||||
EntriesDir: *entries,
|
||||
OutDir: *out,
|
||||
GroupID: *groupID,
|
||||
ModuleName: *moduleName,
|
||||
Description: *description,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "nwsync assemble: %v\n", err)
|
||||
return exitInternal
|
||||
}
|
||||
fmt.Fprintf(stdout, "assembled %s: %d resources from %d artifacts\n",
|
||||
result.SHA1, result.Entries, len(names))
|
||||
return exitOK
|
||||
}
|
||||
Reference in New Issue
Block a user