feat(nwsync): upload sink, key-addressed CLI, fail-closed publication marker
ci / ci (pull_request) Successful in 3m21s
ci / ci (pull_request) Successful in 3m21s
Resolves the build half of sow-tools#60 and closes the CLI gap sow-tools#53's sweep found: PR #71 shipped #53's original surface, not the one #56, #62 and #65 settled. - `depot.KeyStore` (`ProbeKey`/`PutReader`/`GetKey`) addresses the zone by object key instead of by depot sha, reusing the existing IPv4-pinned transport, retry and tri-state probe. The sha-addressed `Backend` now rides on it; no new HTTP client. - `nwsync` gains a sink: the zone by default, a local tree with `--out DIR` as the conformance path. Blobs upload as they are produced, the index lands last, and a blob already in the zone is skipped without paying for compression. - CLI is now `emit [--as NAME] [--out DIR] <artifact-key> <file>` and `assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>...`. Indexes live beside their artifact with the extension replaced, derived in one place. Flags may follow positionals, which cost a run during sow-tools#59. - Fail-closed: an artifact key whose digest does not match the file is refused, and `assemble` refuses an artifact with no index rather than publishing a manifest missing a hak. Conformance re-checked through the new CLI against upstream nwn_nwsync_write 2.1.2 on sow_vfxs_01.hak: the manifest is still byte-identical. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+43
-26
@@ -4,18 +4,21 @@ import (
|
||||
"crypto/sha1"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"path"
|
||||
)
|
||||
|
||||
// AssembleOptions describes one merged manifest.
|
||||
type AssembleOptions struct {
|
||||
Order []string // artifact names, highest priority first
|
||||
EntriesDir string // directory holding <name>.nsym and <name>.nsym.json
|
||||
OutDir string // repository root; the manifest lands in <out>/manifests
|
||||
GroupID int // 1 = current, 2 = testing; 0 means absent
|
||||
ModuleName string
|
||||
Description string
|
||||
// ArtifactKeys are the depot keys of the artifacts to merge, in
|
||||
// Mod_HakList order — highest priority first. Each one's index is read
|
||||
// from the key beside it.
|
||||
ArtifactKeys []string
|
||||
TLKKey string // the TLK's key, if the manifest carries one
|
||||
OutDir string // write locally instead of uploading — the conformance path
|
||||
GroupID int // 1 = current, 2 = testing; 0 means absent
|
||||
ModuleName string
|
||||
Description string
|
||||
Sink sink // test seam; nil means OutDir or the zone
|
||||
}
|
||||
|
||||
// AssembleResult reports what one assemble run produced.
|
||||
@@ -33,25 +36,43 @@ type AssembleResult struct {
|
||||
// the game resolves it (upstream's resman adds haks in reverse and lets the
|
||||
// last one win). Get this backwards and the wrong texture ships silently.
|
||||
func Assemble(options AssembleOptions) (AssembleResult, error) {
|
||||
if len(options.Order) == 0 {
|
||||
return AssembleResult{}, fmt.Errorf("assemble: --order names no artifacts")
|
||||
if len(options.ArtifactKeys) == 0 {
|
||||
return AssembleResult{}, fmt.Errorf("assemble: no artifact keys given")
|
||||
}
|
||||
|
||||
target, err := openSink(options.OutDir, options.Sink)
|
||||
if err != nil {
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
|
||||
// The TLK carries no precedence — it is not a hak and shadows nothing —
|
||||
// so it merges last, after every hak has had its say.
|
||||
keys := append([]string{}, options.ArtifactKeys...)
|
||||
if options.TLKKey != "" {
|
||||
keys = append(keys, options.TLKKey)
|
||||
}
|
||||
|
||||
merged := make([]Entry, 0, 1024)
|
||||
winner := make(map[Identity]bool, 1024)
|
||||
var onDiskBytes int64
|
||||
|
||||
for _, name := range options.Order {
|
||||
manifestPath := filepath.Join(options.EntriesDir, name+".nsym")
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
for _, artifactKey := range keys {
|
||||
key, err := resolveIndexKey(artifactKey, options.OutDir)
|
||||
if err != nil {
|
||||
return AssembleResult{}, fmt.Errorf("assemble: no index for %q: %w", name, err)
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
data, sidecarBody, err := target.getIndex(key)
|
||||
if err != nil {
|
||||
// An artifact with no index is an artifact whose emit never
|
||||
// finished. Publishing a manifest without it would ship a release
|
||||
// missing a hak, so this fails closed.
|
||||
return AssembleResult{}, fmt.Errorf("assemble: no index for %s: %w", artifactKey, err)
|
||||
}
|
||||
entries, err := readManifest(data)
|
||||
if err != nil {
|
||||
return AssembleResult{}, fmt.Errorf("%s: %w", manifestPath, err)
|
||||
return AssembleResult{}, fmt.Errorf("%s: %w", target.describe(key), err)
|
||||
}
|
||||
sidecar, err := readSidecar(manifestPath + ".json")
|
||||
sidecar, err := parseSidecar(target.describe(key), sidecarBody)
|
||||
if err != nil {
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
@@ -61,7 +82,7 @@ func Assemble(options AssembleOptions) (AssembleResult, error) {
|
||||
if sidecar.EmitterVersion != emitterVersion {
|
||||
return AssembleResult{}, fmt.Errorf(
|
||||
"assemble: emitter version mismatch: %s was emitted by emitter %q, this is emitter %q",
|
||||
name, sidecar.EmitterVersion, emitterVersion)
|
||||
artifactKey, sidecar.EmitterVersion, emitterVersion)
|
||||
}
|
||||
// on_disk_bytes overcounts by the handful of cross-artifact
|
||||
// duplicates. It is a display statistic; no dedupe pass for it.
|
||||
@@ -86,26 +107,22 @@ func Assemble(options AssembleOptions) (AssembleResult, error) {
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
sha1Hex := fmt.Sprintf("%x", sha1.Sum(data))
|
||||
manifestPath := filepath.Join(options.OutDir, "manifests", sha1Hex)
|
||||
manifestKey := path.Join("manifests", sha1Hex)
|
||||
sidecar := Sidecar{
|
||||
ModuleName: options.ModuleName,
|
||||
Description: options.Description,
|
||||
GroupID: options.GroupID,
|
||||
}
|
||||
if err := writeManifestPair(manifestPath, data, merged, onDiskBytes, sidecar); err != nil {
|
||||
if err := putManifestPair(target, manifestKey, data, merged, onDiskBytes, sidecar); err != nil {
|
||||
return AssembleResult{}, err
|
||||
}
|
||||
return AssembleResult{SHA1: sha1Hex, ManifestPath: manifestPath, Entries: len(merged)}, nil
|
||||
return AssembleResult{SHA1: sha1Hex, ManifestPath: target.describe(manifestKey), Entries: len(merged)}, nil
|
||||
}
|
||||
|
||||
func readSidecar(path string) (Sidecar, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Sidecar{}, fmt.Errorf("assemble: missing sidecar: %w", err)
|
||||
}
|
||||
func parseSidecar(where string, data []byte) (Sidecar, error) {
|
||||
var sidecar Sidecar
|
||||
if err := json.Unmarshal(data, &sidecar); err != nil {
|
||||
return Sidecar{}, fmt.Errorf("%s: %w", path, err)
|
||||
return Sidecar{}, fmt.Errorf("%s: %w", where, err)
|
||||
}
|
||||
return sidecar, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user