ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

feat(nwsync): upload sink, key-addressed CLI, fail-closed publication marker
ci / ci (pull_request) Successful in 3m21s
Details

Browse Source 
Resolves the build half of sow-tools#60 and closes the CLI gap sow-tools#53's
sweep found: PR #71 shipped #53's original surface, not the one #56, #62 and
#65 settled.

- `depot.KeyStore` (`ProbeKey`/`PutReader`/`GetKey`) addresses the zone by
  object key instead of by depot sha, reusing the existing IPv4-pinned
  transport, retry and tri-state probe. The sha-addressed `Backend` now rides
  on it; no new HTTP client.
- `nwsync` gains a sink: the zone by default, a local tree with `--out DIR` as
  the conformance path. Blobs upload as they are produced, the index lands
  last, and a blob already in the zone is skipped without paying for
  compression.
- CLI is now `emit [--as NAME] [--out DIR] <artifact-key> <file>` and
  `assemble --group-id N [--tlk-key KEY] [--out DIR] <artifact-key>...`.
  Indexes live beside their artifact with the extension replaced, derived in
  one place. Flags may follow positionals, which cost a run during sow-tools#59.
- Fail-closed: an artifact key whose digest does not match the file is refused,
  and `assemble` refuses an artifact with no index rather than publishing a
  manifest missing a hak.

Conformance re-checked through the new CLI against upstream nwn_nwsync_write
2.1.2 on sow_vfxs_01.hak: the manifest is still byte-identical.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
archvillainette
2026-07-30 20:16:44 +02:00
co-authored by Claude Opus 5
parent a131b25e5b
commit f2a0d6a8d1
9 changed files with 841 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/nwsync/nwsync_test.go
+66 -40
View File
@@ -3,6 +3,7 @@ package nwsync
import (
"bytes"
"crypto/sha1"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
@@ -45,6 +46,30 @@ func writeHak(t *testing.T, path string, contents map[string][]byte) {
}
}
// artifactKey is the depot key a file would be published under: the sha256 of
// its bytes, hash-tree depth 2, keeping the extension.
func artifactKey(t *testing.T, path string) string {
t.Helper()
body, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(body)
digest := hex.EncodeToString(sum[:])
return "artifacts/haks/sha256/" + digest[0:2] + "/" + digest[2:4] + "/" + digest + filepath.Ext(path)
}
// emitLocal emits one artifact into a local tree, the conformance path.
func emitLocal(t *testing.T, path, out string) (EmitResult, error) {
t.Helper()
return Emit(EmitOptions{
ArtifactKey: artifactKey(t, path),
ArtifactPath: path,
As: filepath.Base(path),
OutDir: out,
})
}
func TestBlobFramingRoundTrips(t *testing.T) {
data := []byte("the quick brown fox jumps over the lazy dog, repeatedly and at length")
blob := compressBlob(data)
@@ -143,7 +168,7 @@ func TestEmitWritesBlobsAndManifest(t *testing.T) {
})
out := filepath.Join(dir, "out")
result, err := Emit(hak, out)
result, err := emitLocal(t, hak, out)
if err != nil {
t.Fatalf("emit: %v", err)
}
@@ -171,7 +196,7 @@ func TestEmitWritesBlobsAndManifest(t *testing.T) {
t.Errorf("blob decompressed to %q, want %q", got, body)
}
entries := readEmitted(t, out, "sow_test_01")
entries := readEmitted(t, result.ManifestPath)
for _, entry := range entries {
if entry.ResType == restype(t, "nss") || entry.ResType == restype(t, "ndb") || entry.ResType == restype(t, "gic") {
t.Errorf("skipped restype leaked into the manifest: %+v", entry)
@@ -200,9 +225,9 @@ func TestEmitWritesBlobsAndManifest(t *testing.T) {
}
}
func readEmitted(t *testing.T, dir, name string) []Entry {
func readEmitted(t *testing.T, indexPath string) []Entry {
t.Helper()
data, err := os.ReadFile(filepath.Join(dir, name+".nsym"))
data, err := os.ReadFile(indexPath)
if err != nil {
t.Fatalf("read emitted manifest: %v", err)
}
@@ -218,7 +243,7 @@ func TestEmitFailsClosedOnOversizeResource(t *testing.T) {
hak := filepath.Join(dir, "big.hak")
writeHak(t, hak, map[string][]byte{"huge1.tga": make([]byte, fileSizeLimit+1)})
if _, err := Emit(hak, filepath.Join(dir, "out")); err == nil {
if _, err := emitLocal(t, hak, filepath.Join(dir, "out")); err == nil {
t.Fatal("emit accepted a resource over the 15 MB limit")
}
}
@@ -230,11 +255,11 @@ func TestEmitLooseFile(t *testing.T) {
t.Fatal(err)
}
out := filepath.Join(dir, "out")
result, err := Emit(tlk, out)
result, err := emitLocal(t, tlk, out)
if err != nil {
t.Fatalf("emit tlk: %v", err)
}
entries := readEmitted(t, out, "sow_tlk")
entries := readEmitted(t, result.ManifestPath)
if len(entries) != 1 || entries[0].ResRef != "sow_tlk" || entries[0].ResType != restype(t, "tlk") {
t.Fatalf("tlk emitted as %+v", entries)
}
@@ -245,34 +270,35 @@ func TestEmitLooseFile(t *testing.T) {
// emitFixture emits two haks that share a resref, so the merge rule is
// observable: "top" holds the winning body, "assets" the shadowed one.
func emitFixture(t *testing.T) (string, []byte, []byte) {
func emitFixture(t *testing.T) (out string, keys map[string]string, topBody, assetBody []byte) {
t.Helper()
dir := t.TempDir()
topBody := []byte("2da from sow_top")
assetBody := []byte("2da from the asset hak")
topBody = []byte("2da from sow_top")
assetBody = []byte("2da from the asset hak")
writeHak(t, filepath.Join(dir, "sow_top.hak"), map[string][]byte{"appearance.2da": topBody})
writeHak(t, filepath.Join(dir, "sow_core_01.hak"), map[string][]byte{
"appearance.2da": assetBody,
"bloodstain1.tga": []byte("blood"),
})
out := filepath.Join(dir, "out")
out = filepath.Join(dir, "out")
keys = map[string]string{}
for _, name := range []string{"sow_top", "sow_core_01"} {
if _, err := Emit(filepath.Join(dir, name+".hak"), out); err != nil {
path := filepath.Join(dir, name+".hak")
keys[name] = artifactKey(t, path)
if _, err := emitLocal(t, path, out); err != nil {
t.Fatalf("emit %s: %v", name, err)
}
}
return out, topBody, assetBody
return out, keys, topBody, assetBody
}
func TestAssembleShadowsByOrder(t *testing.T) {
entriesDir, topBody, assetBody := emitFixture(t)
out := t.TempDir()
entriesDir, keys, topBody, assetBody := emitFixture(t)
result, err := Assemble(AssembleOptions{
Order: []string{"sow_top", "sow_core_01"},
EntriesDir: entriesDir,
OutDir: out,
GroupID: 2,
ArtifactKeys: []string{keys["sow_top"], keys["sow_core_01"]},
OutDir: entriesDir,
GroupID: 2,
})
if err != nil {
t.Fatalf("assemble: %v", err)
@@ -325,13 +351,11 @@ func TestAssembleShadowsByOrder(t *testing.T) {
}
func TestAssembleReversedOrderPicksTheOtherHak(t *testing.T) {
entriesDir, topBody, assetBody := emitFixture(t)
out := t.TempDir()
entriesDir, keys, topBody, assetBody := emitFixture(t)
result, err := Assemble(AssembleOptions{
Order: []string{"sow_core_01", "sow_top"},
EntriesDir: entriesDir,
OutDir: out,
ArtifactKeys: []string{keys["sow_core_01"], keys["sow_top"]},
OutDir: entriesDir,
})
if err != nil {
t.Fatalf("assemble: %v", err)
@@ -350,9 +374,13 @@ func TestAssembleReversedOrderPicksTheOtherHak(t *testing.T) {
}
func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) {
entriesDir, _, _ := emitFixture(t)
entriesDir, keys, _, _ := emitFixture(t)
path := filepath.Join(entriesDir, "sow_core_01.nsym.json")
index, err := resolveIndexKey(keys["sow_core_01"], entriesDir)
if err != nil {
t.Fatal(err)
}
path := filepath.Join(entriesDir, index+".json")
var sidecar Sidecar
body, err := os.ReadFile(path)
if err != nil {
@@ -371,9 +399,8 @@ func TestAssembleRefusesMismatchedEmitterVersions(t *testing.T) {
}
_, err = Assemble(AssembleOptions{
Order: []string{"sow_top", "sow_core_01"},
EntriesDir: entriesDir,
OutDir: t.TempDir(),
ArtifactKeys: []string{keys["sow_top"], keys["sow_core_01"]},
OutDir: entriesDir,
})
if err == nil || !strings.Contains(err.Error(), "emitter version mismatch") {
t.Fatalf("assemble merged across emitter versions: %v", err)
@@ -385,7 +412,7 @@ func TestEmitHonoursSourceDateEpoch(t *testing.T) {
dir := t.TempDir()
hak := filepath.Join(dir, "pinned.hak")
writeHak(t, hak, map[string][]byte{"one1.tga": []byte("body")})
result, err := Emit(hak, filepath.Join(dir, "out"))
result, err := emitLocal(t, hak, filepath.Join(dir, "out"))
if err != nil {
t.Fatalf("emit: %v", err)
}
@@ -414,19 +441,19 @@ func TestEmitRejectsAModule(t *testing.T) {
if err := os.WriteFile(path, out.Bytes(), 0o644); err != nil {
t.Fatal(err)
}
if _, err := Emit(path, filepath.Join(dir, "out")); err == nil {
if _, err := emitLocal(t, path, filepath.Join(dir, "out")); err == nil {
t.Fatal("emit accepted a .mod; a manifest never carries module contents")
}
}
func TestAssembleFailsClosedOnMissingIndex(t *testing.T) {
entriesDir, _, _ := emitFixture(t)
entriesDir, keys, _, _ := emitFixture(t)
missing := "artifacts/haks/sha256/00/11/" + strings.Repeat("0", 64) + ".hak"
_, err := Assemble(AssembleOptions{
Order: []string{"sow_top", "sow_never_published"},
EntriesDir: entriesDir,
OutDir: t.TempDir(),
ArtifactKeys: []string{keys["sow_top"], missing},
OutDir: entriesDir,
})
if err == nil || !strings.Contains(err.Error(), "sow_never_published") {
if err == nil || !strings.Contains(err.Error(), missing) {
t.Fatalf("assemble did not fail closed and name the missing artifact: %v", err)
}
}
@@ -436,10 +463,9 @@ func TestRunUsageErrors(t *testing.T) {
nil,
{"nope"},
{"emit"},
{"emit", "artifact.hak"},
{"assemble", "--entries", "x", "--out", "y"},
{"assemble", "--order", "a", "--out", "y"},
{"assemble", "--order", "a", "--entries", "x"},
{"emit", "artifact-key.hak"},
{"emit", "a", "b", "c"},
{"assemble", "--out", "y"},
}
for _, args := range cases {
var out, errw bytes.Buffer