fix(nwsync): declare Frame_Content_Size on every blob, and verify what is published (#87)

Closes #86. Closes #85. These land together on purpose. Fixing the encoder alone changes nothing for the blobs already in the zone, because `emit` skips whatever is already present. ## #86 — the framing fix `klauspost/compress` omits the zstd `Frame_Content_Size` field for inputs under 256 bytes, which the format permits. Reference libzstd never does, so the NWN client — which sizes its output buffer from `ZSTD_getFrameContentSize` and has therefore never met a frame without one — rejected roughly 6% of our blobs outright. Any single one stops a sync dead, so no client could complete a sync of the live manifest. No encoder option changes this, so `compressBlob` re-headers the affected frames into the shape libzstd itself emits: `Single_Segment_flag` set, `Window_Descriptor` dropped, and the freed byte spent on a one-byte `Frame_Content_Size`. Same length in, same length out, and the same descriptor byte (`0x24`) the issue recorded from libzstd. `compressBlob` then asserts its own output. An encoder upgrade that finds another way to omit the field would otherwise reproduce #86 in silence, and a blob is skipped by every later emit once written. `emitter_version` goes to `2`, so `assemble` refuses to merge an index written by the encoder that omitted the field. **Proved against the reference decoder, not just a round trip.** A real emitted 175-byte blob: ``` Frames Skips Compressed Uncompressed Ratio Check Filename 1 0 48 B 175 B 3.646 XXH64 frame.zst c59d6620d4ffd4bf3fe73df43b19b7afcfe8fea4 - <- zstd -dc | sha1sum c59d6620d4ffd4bf3fe73df43b19b7afcfe8fea4 <- the blob's own name ``` Before the fix that `Uncompressed` column was blank. ## #85 — `nwsync verify` `crucible nwsync verify <manifest-sha1>` reads a manifest and its blobs back through the **public pull zone**, with no credential, because what matters is the bytes a client is served, edge behaviour included. Every distinct blob is decompressed and hashed; failures are reported per blob as missing / malformed framing / size mismatch / hash mismatch, and the exit code is 1. - `--sample N` makes a routine check cheap against a manifest that is ~69,000 blobs and 15 GB; the default is a full sweep. - `--base URL` / `NWSYNC_PULL_BASE` overrides the public host. - The manifest is checked against its own sha1 before a single blob is fetched. - `emit --verify` applies the same check where `emit` would otherwise trust presence, and replaces a stored blob that is not what its name claims. This is what makes the #86 blobs repairable. ## Why the existing checks missed this Both new checks assert the **frame property**, not just a round trip. The conformance suite (#59) compares decompressed bytes, so a frame that decodes correctly passes regardless of its header; and the earlier zone audit decompressed 68 blobs with the `zstd` CLI, a *more* capable decoder than the client's, which certified exactly the blobs the client rejects. ## Checks `make check` and `make smoke` green. Second commit is the fixes from a two-axis review of the first. ## Not in this PR Three follow-ups, filed separately: the backfill has not been run, replacing a blob does not purge the pull-zone edge cache, and #85's runbook line belongs to `sow-platform`. 🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #87 Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
2026-07-31 22:33:12 +00:00
parent 682f920114
commit 7cc53aeb68
13 changed files with 804 additions and 22 deletions
@@ -0,0 +1,285 @@
+package nwsync
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/hex"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+// verifyFixture emits two haks and a TLK into a fake zone, assembles them, and
+// hands back a verifier reading that zone the way a client would.
+type verifyFixture struct {
+	*zoneSinkFixture
+	manifestSHA1 string
+}
+
+func newVerifyFixture(t *testing.T) *verifyFixture {
+	t.Helper()
+	zone := newZoneFixture(t)
+	dir := t.TempDir()
+	hak := filepath.Join(dir, "sow_test_01.hak")
+	// A payload under 256 bytes is the one the frame-header check exists for.
+	writeHak(t, hak, map[string][]byte{
+		"bloodstain1.tga": []byte("small"),
+		"appearance.2da":  bytes.Repeat([]byte("2DA V2.0\n"), 200),
+	})
+	tlk := filepath.Join(dir, "sow_tlk.tlk")
+	if err := os.WriteFile(tlk, []byte("TLK V3.0 payload"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	zone.emit(t, hak)
+	if _, err := Emit(EmitOptions{
+		ArtifactKey: artifactKey(t, tlk), ArtifactPath: tlk, As: "sow_tlk.tlk", Sink: zone.sink,
+	}); err != nil {
+		t.Fatalf("emit tlk: %v", err)
+	}
+	assembled, err := Assemble(AssembleOptions{
+		ArtifactKeys: []string{artifactKey(t, hak)}, TLKKey: artifactKey(t, tlk), Sink: zone.sink,
+	})
+	if err != nil {
+		t.Fatalf("assemble: %v", err)
+	}
+	return &verifyFixture{zoneSinkFixture: zone, manifestSHA1: assembled.SHA1}
+}
+
+func (f *verifyFixture) verify(t *testing.T, sample int) (VerifyResult, string, error) {
+	t.Helper()
+	var log bytes.Buffer
+	result, err := Verify(VerifyOptions{
+		ManifestSHA1: f.manifestSHA1,
+		Sample:       sample,
+		Source:       f.zone.pullZone(),
+		Log:          &log,
+	})
+	return result, log.String(), err
+}
+
+// keyOf is where a resource's blob lives, addressed by the sha1 of its
+// uncompressed bytes — the same path the client requests.
+func keyOf(body []byte) string {
+	sum := sha1.Sum(body)
+	return blobKey(hex.EncodeToString(sum[:]))
+}
+
+func TestVerifyPassesACleanZone(t *testing.T) {
+	fixture := newVerifyFixture(t)
+	result, log, err := fixture.verify(t, 0)
+	if err != nil {
+		t.Fatalf("verify: %v", err)
+	}
+	if result.Failures != 0 {
+		t.Errorf("verify reported %d failures on a clean zone: %s", result.Failures, log)
+	}
+	// A default run is a full sweep, so it must reach every blob the manifest
+	// names — not some of them.
+	if result.Checked != result.Blobs || result.Blobs == 0 {
+		t.Errorf("checked %d of %d blobs; a full sweep must check all of them", result.Checked, result.Blobs)
+	}
+}
+
+func TestVerifyReportsAMissingBlob(t *testing.T) {
+	fixture := newVerifyFixture(t)
+	key := keyOf([]byte("small"))
+	fixture.zone.mu.Lock()
+	delete(fixture.zone.objects, key)
+	fixture.zone.mu.Unlock()
+
+	result, log, err := fixture.verify(t, 0)
+	if err != nil {
+		t.Fatalf("verify: %v", err)
+	}
+	if result.Failures != 1 {
+		t.Fatalf("reported %d failures, want 1: %s", result.Failures, log)
+	}
+	if !strings.Contains(log, "missing") {
+		t.Errorf("a deleted blob was not reported as missing: %s", log)
+	}
+}
+
+func TestVerifyReportsATruncatedBlob(t *testing.T) {
+	fixture := newVerifyFixture(t)
+	key := keyOf([]byte("small"))
+	fixture.zone.mu.Lock()
+	fixture.zone.objects[key] = fixture.zone.objects[key][:blobHeaderBytes+4]
+	fixture.zone.mu.Unlock()
+
+	result, log, err := fixture.verify(t, 0)
+	if err != nil {
+		t.Fatalf("verify: %v", err)
+	}
+	if result.Failures != 1 {
+		t.Fatalf("reported %d failures, want 1: %s", result.Failures, log)
+	}
+	if !strings.Contains(log, "framing") {
+		t.Errorf("a truncated blob was not reported as malformed framing: %s", log)
+	}
+}
+
+// TestVerifyRejectsABlobWithNoDeclaredFrameContentSize is the check that #86
+// slipped past: the blob decompresses to exactly the right bytes, so a verifier
+// that only round-trips certifies it, yet the client cannot decode it.
+func TestVerifyRejectsABlobWithNoDeclaredFrameContentSize(t *testing.T) {
+	fixture := newVerifyFixture(t)
+	body := []byte("small")
+	key := keyOf(body)
+
+	fixture.zone.mu.Lock()
+	good := fixture.zone.objects[key]
+	encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1))
+	if err != nil {
+		t.Fatal(err)
+	}
+	bad := append(append([]byte{}, good[:blobHeaderBytes]...), encoder.EncodeAll(body, nil)...)
+	fixture.zone.objects[key] = bad
+	fixture.zone.mu.Unlock()
+
+	if frameDeclaresContentSize(bad[blobHeaderBytes:]) {
+		t.Fatal("the fixture blob declares a content size; it cannot exercise the check")
+	}
+	if got, err := decompressBlob(bad); err != nil || !bytes.Equal(got, body) {
+		t.Fatalf("the fixture blob must round trip, or it proves nothing: %v", err)
+	}
+
+	result, log, err := fixture.verify(t, 0)
+	if err != nil {
+		t.Fatalf("verify: %v", err)
+	}
+	if result.Failures != 1 {
+		t.Fatalf("reported %d failures, want 1: %s", result.Failures, log)
+	}
+	if !strings.Contains(log, "content size") {
+		t.Errorf("undeclared frame content size was not the reported reason: %s", log)
+	}
+}
+
+func TestVerifyReportsWrongContents(t *testing.T) {
+	fixture := newVerifyFixture(t)
+	key := keyOf([]byte("small"))
+	fixture.zone.mu.Lock()
+	// Valid framing, valid zstd, wrong bytes: only decompressing and hashing
+	// can see this, which is why Content-Length is not enough.
+	fixture.zone.objects[key] = compressBlob([]byte("wrong"))
+	fixture.zone.mu.Unlock()
+
+	result, log, err := fixture.verify(t, 0)
+	if err != nil {
+		t.Fatalf("verify: %v", err)
+	}
+	if result.Failures != 1 {
+		t.Fatalf("reported %d failures, want 1: %s", result.Failures, log)
+	}
+	if !strings.Contains(log, "hash mismatch") {
+		t.Errorf("wrong contents were not reported as a hash mismatch: %s", log)
+	}
+}
+
+func TestVerifyReportsAShortBlob(t *testing.T) {
+	fixture := newVerifyFixture(t)
+	key := keyOf([]byte("small"))
+	fixture.zone.mu.Lock()
+	// Well-formed all the way down and simply too short — the shape a killed
+	// upload leaves behind, and the one a Content-Length check would pass.
+	fixture.zone.objects[key] = compressBlob([]byte("sma"))
+	fixture.zone.mu.Unlock()
+
+	result, log, err := fixture.verify(t, 0)
+	if err != nil {
+		t.Fatalf("verify: %v", err)
+	}
+	if result.Failures != 1 {
+		t.Fatalf("reported %d failures, want 1: %s", result.Failures, log)
+	}
+	if !strings.Contains(log, "size mismatch") {
+		t.Errorf("a short blob was not reported as a size mismatch: %s", log)
+	}
+}
+
+func TestVerifyFailsWhenTheManifestIsNotTheOneAsked(t *testing.T) {
+	fixture := newVerifyFixture(t)
+	fixture.zone.mu.Lock()
+	fixture.zone.objects["manifests/"+fixture.manifestSHA1] = []byte("NSYM garbage")
+	fixture.zone.mu.Unlock()
+
+	if _, _, err := fixture.verify(t, 0); err == nil {
+		t.Fatal("verify accepted a manifest that is not the one requested")
+	}
+}
+
+func TestVerifySampleChecksFewerBlobs(t *testing.T) {
+	fixture := newVerifyFixture(t)
+	result, log, err := fixture.verify(t, 1)
+	if err != nil {
+		t.Fatalf("verify: %v", err)
+	}
+	if result.Checked != 1 {
+		t.Errorf("--sample 1 checked %d blobs, want 1: %s", result.Checked, log)
+	}
+	if result.Blobs <= result.Checked {
+		t.Errorf("sampling %d of %d blobs is not a sample", result.Checked, result.Blobs)
+	}
+}
+
+// TestEmitVerifyReplacesABlobThatIsNotItsName covers the reason #86 could not be
+// fixed by the encoder alone: emit skips whatever is already present, so every
+// blob published by the broken encoder stays broken until emit stops trusting
+// presence.
+func TestEmitVerifyReplacesABlobThatIsNotItsName(t *testing.T) {
+	fixture := newZoneFixture(t)
+	dir := t.TempDir()
+	hak := filepath.Join(dir, "sow_test_01.hak")
+	body := []byte("blood")
+	writeHak(t, hak, map[string][]byte{"bloodstain1.tga": body})
+
+	emit := func(verify bool) EmitResult {
+		t.Helper()
+		result, err := Emit(EmitOptions{
+			ArtifactKey:  artifactKey(t, hak),
+			ArtifactPath: hak,
+			Sink:         fixture.sink,
+			Verify:       verify,
+		})
+		if err != nil {
+			t.Fatalf("emit (verify=%v): %v", verify, err)
+		}
+		return result
+	}
+
+	emit(false)
+	key := keyOf(body)
+	encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1))
+	if err != nil {
+		t.Fatal(err)
+	}
+	fixture.zone.mu.Lock()
+	good := fixture.zone.objects[key]
+	fixture.zone.objects[key] = append(append([]byte{}, good[:blobHeaderBytes]...), encoder.EncodeAll(body, nil)...)
+	fixture.zone.mu.Unlock()
+
+	if plain := emit(false); plain.BlobsWritten != 0 {
+		t.Fatalf("a plain re-emit wrote %d blobs; it is supposed to trust presence", plain.BlobsWritten)
+	}
+	if verified := emit(true); verified.BlobsWritten != 1 {
+		t.Fatalf("--verify wrote %d blobs, want 1 (the bad copy must be replaced)", verified.BlobsWritten)
+	}
+
+	fixture.zone.mu.Lock()
+	repaired := fixture.zone.objects[key]
+	fixture.zone.mu.Unlock()
+	if !bytes.Equal(repaired, good) {
+		t.Error("the replaced blob is not what the current encoder produces")
+	}
+	if _, err := inspectBlob(repaired); err != nil {
+		t.Errorf("the replaced blob still fails inspection: %v", err)
+	}
+
+	// A second verifying run has nothing left to repair.
+	if again := emit(true); again.BlobsWritten != 0 {
+		t.Errorf("--verify rewrote %d good blobs", again.BlobsWritten)
+	}
+}