fix(nwsync): close the review findings on the framing fix and verify
ci / ci (pull_request) Successful in 3m38s

- compressBlob now asserts its own output declares a content size, instead of
  trusting that the only way to lose the field is the one #86 found. An encoder
  upgrade that finds another way would otherwise republish the same undecodable
  blob in silence, and a blob is skipped by every later emit once written.
- inspectBlob asserts the frame whenever one is present, rather than only when
  the payload is non-empty.
- dirSink stats instead of reading when not verifying. Reading every existing
  blob back on the default path was a plain regression, and it contradicted the
  documented promise that verifying is a repair pass, not the default.
- The manifest sha1 is now checked as hex before it is interpolated into a URL
  path, rather than only measured.
- One blobKey rule for where a blob lives, replacing three copies of the
  fanout-path expression.
- Renamed frameSizeThreshold to oneByteContentSizeCeiling, which says whose
  threshold it is.
- The read-back in zoneSink reads the storage API on purpose; said so, and
  corrected a comment that claimed a failed read-back re-uploads when it
  aborts.
- Tests derive their expected blob counts from the run instead of hard-coding
  fixture arithmetic, and the size-mismatch category has a test.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-08-01 00:26:22 +02:00
co-authored by Claude Opus 5
parent 56d4054118
commit 9b7be2c76e
6 changed files with 79 additions and 30 deletions
+35 -13
View File
@@ -60,12 +60,11 @@ func (f *verifyFixture) verify(t *testing.T, sample int) (VerifyResult, string,
return result, log.String(), err
}
// blobKey is where a resource's blob lives, addressed by the sha1 of its
// keyOf is where a resource's blob lives, addressed by the sha1 of its
// uncompressed bytes — the same path the client requests.
func blobKey(body []byte) string {
func keyOf(body []byte) string {
sum := sha1.Sum(body)
hexed := hex.EncodeToString(sum[:])
return "data/sha1/" + hexed[0:2] + "/" + hexed[2:4] + "/" + hexed
return blobKey(hex.EncodeToString(sum[:]))
}
func TestVerifyPassesACleanZone(t *testing.T) {
@@ -77,14 +76,16 @@ func TestVerifyPassesACleanZone(t *testing.T) {
if result.Failures != 0 {
t.Errorf("verify reported %d failures on a clean zone: %s", result.Failures, log)
}
if result.Checked != 3 {
t.Errorf("checked %d blobs, want 3", result.Checked)
// A default run is a full sweep, so it must reach every blob the manifest
// names — not some of them.
if result.Checked != result.Blobs || result.Blobs == 0 {
t.Errorf("checked %d of %d blobs; a full sweep must check all of them", result.Checked, result.Blobs)
}
}
func TestVerifyReportsAMissingBlob(t *testing.T) {
fixture := newVerifyFixture(t)
key := blobKey([]byte("small"))
key := keyOf([]byte("small"))
fixture.zone.mu.Lock()
delete(fixture.zone.objects, key)
fixture.zone.mu.Unlock()
@@ -103,7 +104,7 @@ func TestVerifyReportsAMissingBlob(t *testing.T) {
func TestVerifyReportsATruncatedBlob(t *testing.T) {
fixture := newVerifyFixture(t)
key := blobKey([]byte("small"))
key := keyOf([]byte("small"))
fixture.zone.mu.Lock()
fixture.zone.objects[key] = fixture.zone.objects[key][:blobHeaderBytes+4]
fixture.zone.mu.Unlock()
@@ -126,7 +127,7 @@ func TestVerifyReportsATruncatedBlob(t *testing.T) {
func TestVerifyRejectsABlobWithNoDeclaredFrameContentSize(t *testing.T) {
fixture := newVerifyFixture(t)
body := []byte("small")
key := blobKey(body)
key := keyOf(body)
fixture.zone.mu.Lock()
good := fixture.zone.objects[key]
@@ -159,7 +160,7 @@ func TestVerifyRejectsABlobWithNoDeclaredFrameContentSize(t *testing.T) {
func TestVerifyReportsWrongContents(t *testing.T) {
fixture := newVerifyFixture(t)
key := blobKey([]byte("small"))
key := keyOf([]byte("small"))
fixture.zone.mu.Lock()
// Valid framing, valid zstd, wrong bytes: only decompressing and hashing
// can see this, which is why Content-Length is not enough.
@@ -178,6 +179,27 @@ func TestVerifyReportsWrongContents(t *testing.T) {
}
}
func TestVerifyReportsAShortBlob(t *testing.T) {
fixture := newVerifyFixture(t)
key := keyOf([]byte("small"))
fixture.zone.mu.Lock()
// Well-formed all the way down and simply too short — the shape a killed
// upload leaves behind, and the one a Content-Length check would pass.
fixture.zone.objects[key] = compressBlob([]byte("sma"))
fixture.zone.mu.Unlock()
result, log, err := fixture.verify(t, 0)
if err != nil {
t.Fatalf("verify: %v", err)
}
if result.Failures != 1 {
t.Fatalf("reported %d failures, want 1: %s", result.Failures, log)
}
if !strings.Contains(log, "size mismatch") {
t.Errorf("a short blob was not reported as a size mismatch: %s", log)
}
}
func TestVerifyFailsWhenTheManifestIsNotTheOneAsked(t *testing.T) {
fixture := newVerifyFixture(t)
fixture.zone.mu.Lock()
@@ -198,8 +220,8 @@ func TestVerifySampleChecksFewerBlobs(t *testing.T) {
if result.Checked != 1 {
t.Errorf("--sample 1 checked %d blobs, want 1: %s", result.Checked, log)
}
if result.Blobs != 3 {
t.Errorf("reported %d distinct blobs, want 3", result.Blobs)
if result.Blobs <= result.Checked {
t.Errorf("sampling %d of %d blobs is not a sample", result.Checked, result.Blobs)
}
}
@@ -229,7 +251,7 @@ func TestEmitVerifyReplacesABlobThatIsNotItsName(t *testing.T) {
}
emit(false)
key := blobKey(body)
key := keyOf(body)
encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1))
if err != nil {
t.Fatal(err)