fix(nwsync): close the review findings on the framing fix and verify
ci / ci (pull_request) Successful in 3m38s
ci / ci (pull_request) Successful in 3m38s
- compressBlob now asserts its own output declares a content size, instead of trusting that the only way to lose the field is the one #86 found. An encoder upgrade that finds another way would otherwise republish the same undecodable blob in silence, and a blob is skipped by every later emit once written. - inspectBlob asserts the frame whenever one is present, rather than only when the payload is non-empty. - dirSink stats instead of reading when not verifying. Reading every existing blob back on the default path was a plain regression, and it contradicted the documented promise that verifying is a repair pass, not the default. - The manifest sha1 is now checked as hex before it is interpolated into a URL path, rather than only measured. - One blobKey rule for where a blob lives, replacing three copies of the fanout-path expression. - Renamed frameSizeThreshold to oneByteContentSizeCeiling, which says whose threshold it is. - The read-back in zoneSink reads the storage API on purpose; said so, and corrected a comment that claimed a failed read-back re-uploads when it aborts. - Tests derive their expected blob counts from the run instead of hard-coding fixture arithmetic, and the size-mismatch category has a test. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -60,12 +60,11 @@ func (f *verifyFixture) verify(t *testing.T, sample int) (VerifyResult, string,
|
||||
return result, log.String(), err
|
||||
}
|
||||
|
||||
// blobKey is where a resource's blob lives, addressed by the sha1 of its
|
||||
// keyOf is where a resource's blob lives, addressed by the sha1 of its
|
||||
// uncompressed bytes — the same path the client requests.
|
||||
func blobKey(body []byte) string {
|
||||
func keyOf(body []byte) string {
|
||||
sum := sha1.Sum(body)
|
||||
hexed := hex.EncodeToString(sum[:])
|
||||
return "data/sha1/" + hexed[0:2] + "/" + hexed[2:4] + "/" + hexed
|
||||
return blobKey(hex.EncodeToString(sum[:]))
|
||||
}
|
||||
|
||||
func TestVerifyPassesACleanZone(t *testing.T) {
|
||||
@@ -77,14 +76,16 @@ func TestVerifyPassesACleanZone(t *testing.T) {
|
||||
if result.Failures != 0 {
|
||||
t.Errorf("verify reported %d failures on a clean zone: %s", result.Failures, log)
|
||||
}
|
||||
if result.Checked != 3 {
|
||||
t.Errorf("checked %d blobs, want 3", result.Checked)
|
||||
// A default run is a full sweep, so it must reach every blob the manifest
|
||||
// names — not some of them.
|
||||
if result.Checked != result.Blobs || result.Blobs == 0 {
|
||||
t.Errorf("checked %d of %d blobs; a full sweep must check all of them", result.Checked, result.Blobs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyReportsAMissingBlob(t *testing.T) {
|
||||
fixture := newVerifyFixture(t)
|
||||
key := blobKey([]byte("small"))
|
||||
key := keyOf([]byte("small"))
|
||||
fixture.zone.mu.Lock()
|
||||
delete(fixture.zone.objects, key)
|
||||
fixture.zone.mu.Unlock()
|
||||
@@ -103,7 +104,7 @@ func TestVerifyReportsAMissingBlob(t *testing.T) {
|
||||
|
||||
func TestVerifyReportsATruncatedBlob(t *testing.T) {
|
||||
fixture := newVerifyFixture(t)
|
||||
key := blobKey([]byte("small"))
|
||||
key := keyOf([]byte("small"))
|
||||
fixture.zone.mu.Lock()
|
||||
fixture.zone.objects[key] = fixture.zone.objects[key][:blobHeaderBytes+4]
|
||||
fixture.zone.mu.Unlock()
|
||||
@@ -126,7 +127,7 @@ func TestVerifyReportsATruncatedBlob(t *testing.T) {
|
||||
func TestVerifyRejectsABlobWithNoDeclaredFrameContentSize(t *testing.T) {
|
||||
fixture := newVerifyFixture(t)
|
||||
body := []byte("small")
|
||||
key := blobKey(body)
|
||||
key := keyOf(body)
|
||||
|
||||
fixture.zone.mu.Lock()
|
||||
good := fixture.zone.objects[key]
|
||||
@@ -159,7 +160,7 @@ func TestVerifyRejectsABlobWithNoDeclaredFrameContentSize(t *testing.T) {
|
||||
|
||||
func TestVerifyReportsWrongContents(t *testing.T) {
|
||||
fixture := newVerifyFixture(t)
|
||||
key := blobKey([]byte("small"))
|
||||
key := keyOf([]byte("small"))
|
||||
fixture.zone.mu.Lock()
|
||||
// Valid framing, valid zstd, wrong bytes: only decompressing and hashing
|
||||
// can see this, which is why Content-Length is not enough.
|
||||
@@ -178,6 +179,27 @@ func TestVerifyReportsWrongContents(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyReportsAShortBlob(t *testing.T) {
|
||||
fixture := newVerifyFixture(t)
|
||||
key := keyOf([]byte("small"))
|
||||
fixture.zone.mu.Lock()
|
||||
// Well-formed all the way down and simply too short — the shape a killed
|
||||
// upload leaves behind, and the one a Content-Length check would pass.
|
||||
fixture.zone.objects[key] = compressBlob([]byte("sma"))
|
||||
fixture.zone.mu.Unlock()
|
||||
|
||||
result, log, err := fixture.verify(t, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("verify: %v", err)
|
||||
}
|
||||
if result.Failures != 1 {
|
||||
t.Fatalf("reported %d failures, want 1: %s", result.Failures, log)
|
||||
}
|
||||
if !strings.Contains(log, "size mismatch") {
|
||||
t.Errorf("a short blob was not reported as a size mismatch: %s", log)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyFailsWhenTheManifestIsNotTheOneAsked(t *testing.T) {
|
||||
fixture := newVerifyFixture(t)
|
||||
fixture.zone.mu.Lock()
|
||||
@@ -198,8 +220,8 @@ func TestVerifySampleChecksFewerBlobs(t *testing.T) {
|
||||
if result.Checked != 1 {
|
||||
t.Errorf("--sample 1 checked %d blobs, want 1: %s", result.Checked, log)
|
||||
}
|
||||
if result.Blobs != 3 {
|
||||
t.Errorf("reported %d distinct blobs, want 3", result.Blobs)
|
||||
if result.Blobs <= result.Checked {
|
||||
t.Errorf("sampling %d of %d blobs is not a sample", result.Checked, result.Blobs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,7 +251,7 @@ func TestEmitVerifyReplacesABlobThatIsNotItsName(t *testing.T) {
|
||||
}
|
||||
|
||||
emit(false)
|
||||
key := blobKey(body)
|
||||
key := keyOf(body)
|
||||
encoder, err := zstd.NewWriter(nil, zstd.WithEncoderConcurrency(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
Reference in New Issue
Block a user