feat(nwsync): upload sink, key-addressed CLI, fail-closed publication marker
ci / ci (pull_request) Successful in 3m21s

Resolves the build half of sow-tools#60 and closes the CLI gap sow-tools#53's
sweep found: PR #71 shipped #53's original surface, not the one #56, #62 and
#65 settled.

- `depot.KeyStore` (`ProbeKey`/`PutReader`/`GetKey`) addresses the zone by
  object key instead of by depot sha, reusing the existing IPv4-pinned
  transport, retry and tri-state probe. The sha-addressed `Backend` now rides
  on it; no new HTTP client.
- `nwsync` gains a sink: the zone by default, a local tree with `--out DIR` as
  the conformance path. Blobs upload as they are produced, the index lands
  last, and a blob already in the zone is skipped without paying for
  compression.
- CLI is now `emit [--as NAME] [--out DIR] <artifact-key> <file>` and
  `assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>...`.
  Indexes live beside their artifact with the extension replaced, derived in
  one place. Flags may follow positionals, which cost a run during sow-tools#59.
- Fail-closed: an artifact key whose digest does not match the file is refused,
  and `assemble` refuses an artifact with no index rather than publishing a
  manifest missing a hak.

Conformance re-checked through the new CLI against upstream nwn_nwsync_write
2.1.2 on sow_vfxs_01.hak: the manifest is still byte-identical.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-30 20:16:44 +02:00
co-authored by Claude Opus 5
parent a131b25e5b
commit f2a0d6a8d1
9 changed files with 841 additions and 203 deletions
+43 -26
View File
@@ -4,18 +4,21 @@ import (
"crypto/sha1"
"encoding/json"
"fmt"
"os"
"path/filepath"
"path"
)
// AssembleOptions describes one merged manifest.
type AssembleOptions struct {
Order []string // artifact names, highest priority first
EntriesDir string // directory holding <name>.nsym and <name>.nsym.json
OutDir string // repository root; the manifest lands in <out>/manifests
GroupID int // 1 = current, 2 = testing; 0 means absent
ModuleName string
Description string
// ArtifactKeys are the depot keys of the artifacts to merge, in
// Mod_HakList order — highest priority first. Each one's index is read
// from the key beside it.
ArtifactKeys []string
TLKKey string // the TLK's key, if the manifest carries one
OutDir string // write locally instead of uploading — the conformance path
GroupID int // 1 = current, 2 = testing; 0 means absent
ModuleName string
Description string
Sink sink // test seam; nil means OutDir or the zone
}
// AssembleResult reports what one assemble run produced.
@@ -33,25 +36,43 @@ type AssembleResult struct {
// the game resolves it (upstream's resman adds haks in reverse and lets the
// last one win). Get this backwards and the wrong texture ships silently.
func Assemble(options AssembleOptions) (AssembleResult, error) {
if len(options.Order) == 0 {
return AssembleResult{}, fmt.Errorf("assemble: --order names no artifacts")
if len(options.ArtifactKeys) == 0 {
return AssembleResult{}, fmt.Errorf("assemble: no artifact keys given")
}
target, err := openSink(options.OutDir, options.Sink)
if err != nil {
return AssembleResult{}, err
}
// The TLK carries no precedence — it is not a hak and shadows nothing —
// so it merges last, after every hak has had its say.
keys := append([]string{}, options.ArtifactKeys...)
if options.TLKKey != "" {
keys = append(keys, options.TLKKey)
}
merged := make([]Entry, 0, 1024)
winner := make(map[Identity]bool, 1024)
var onDiskBytes int64
for _, name := range options.Order {
manifestPath := filepath.Join(options.EntriesDir, name+".nsym")
data, err := os.ReadFile(manifestPath)
for _, artifactKey := range keys {
key, err := resolveIndexKey(artifactKey, options.OutDir)
if err != nil {
return AssembleResult{}, fmt.Errorf("assemble: no index for %q: %w", name, err)
return AssembleResult{}, err
}
data, sidecarBody, err := target.getIndex(key)
if err != nil {
// An artifact with no index is an artifact whose emit never
// finished. Publishing a manifest without it would ship a release
// missing a hak, so this fails closed.
return AssembleResult{}, fmt.Errorf("assemble: no index for %s: %w", artifactKey, err)
}
entries, err := readManifest(data)
if err != nil {
return AssembleResult{}, fmt.Errorf("%s: %w", manifestPath, err)
return AssembleResult{}, fmt.Errorf("%s: %w", target.describe(key), err)
}
sidecar, err := readSidecar(manifestPath + ".json")
sidecar, err := parseSidecar(target.describe(key), sidecarBody)
if err != nil {
return AssembleResult{}, err
}
@@ -61,7 +82,7 @@ func Assemble(options AssembleOptions) (AssembleResult, error) {
if sidecar.EmitterVersion != emitterVersion {
return AssembleResult{}, fmt.Errorf(
"assemble: emitter version mismatch: %s was emitted by emitter %q, this is emitter %q",
name, sidecar.EmitterVersion, emitterVersion)
artifactKey, sidecar.EmitterVersion, emitterVersion)
}
// on_disk_bytes overcounts by the handful of cross-artifact
// duplicates. It is a display statistic; no dedupe pass for it.
@@ -86,26 +107,22 @@ func Assemble(options AssembleOptions) (AssembleResult, error) {
return AssembleResult{}, err
}
sha1Hex := fmt.Sprintf("%x", sha1.Sum(data))
manifestPath := filepath.Join(options.OutDir, "manifests", sha1Hex)
manifestKey := path.Join("manifests", sha1Hex)
sidecar := Sidecar{
ModuleName: options.ModuleName,
Description: options.Description,
GroupID: options.GroupID,
}
if err := writeManifestPair(manifestPath, data, merged, onDiskBytes, sidecar); err != nil {
if err := putManifestPair(target, manifestKey, data, merged, onDiskBytes, sidecar); err != nil {
return AssembleResult{}, err
}
return AssembleResult{SHA1: sha1Hex, ManifestPath: manifestPath, Entries: len(merged)}, nil
return AssembleResult{SHA1: sha1Hex, ManifestPath: target.describe(manifestKey), Entries: len(merged)}, nil
}
func readSidecar(path string) (Sidecar, error) {
data, err := os.ReadFile(path)
if err != nil {
return Sidecar{}, fmt.Errorf("assemble: missing sidecar: %w", err)
}
func parseSidecar(where string, data []byte) (Sidecar, error) {
var sidecar Sidecar
if err := json.Unmarshal(data, &sidecar); err != nil {
return Sidecar{}, fmt.Errorf("%s: %w", path, err)
return Sidecar{}, fmt.Errorf("%s: %w", where, err)
}
return sidecar, nil
}