diff --git a/internal/depot/remote.go b/internal/depot/remote.go index 7bdf2cf..cdd38dc 100644 --- a/internal/depot/remote.go +++ b/internal/depot/remote.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "strings" + "time" ) // NewBackend returns the backend for name ("local"|"cdn"|"bunny"). @@ -85,9 +86,26 @@ func (b *httpBackend) cdnURL(sha string) string { return fmt.Sprintf("%s/%s", b.cfg.CDNBase, BlobKey(sha)) } +// probeRetrySleep is the backoff sleeper for transient probe retries; +// tests stub it (same pattern as sweep.go's confirmSleep). +var probeRetrySleep = time.Sleep + // rangeProbe issues GET with Range: bytes=0-0 and classifies the -// response. Never HEAD (banned by spec). +// response. Never HEAD (banned by spec). Transient outcomes (transport +// error, or a status that is neither 2xx nor 404/410) retry up to 2 more +// times with linear backoff (1s, 2s) — the CDN edge throttles sustained +// sweeps; the bash this ports absorbed that with curl --retry 2. func (b *httpBackend) rangeProbe(ctx context.Context, url string, headers map[string]string) (ProbeState, bool, error) { + for attempt := 0; ; attempt++ { + state, transient, err := b.rangeProbeOnce(ctx, url, headers) + if !transient || attempt >= 2 { + return state, transient, err + } + probeRetrySleep(time.Duration(attempt+1) * time.Second) + } +} + +func (b *httpBackend) rangeProbeOnce(ctx context.Context, url string, headers map[string]string) (ProbeState, bool, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return Unconfirmed, true, err @@ -212,6 +230,7 @@ func (b *httpBackend) fetch(ctx context.Context, sha string) (*http.Response, er if resp, err := b.get(ctx, b.cdnURL(sha), nil); err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 { return resp, nil } else if err == nil { + _, _ = io.Copy(io.Discard, resp.Body) resp.Body.Close() } resp, err := b.get(ctx, b.storageURL(sha), map[string]string{"AccessKey": b.cfg.ReadKey}) @@ -219,6 +238,7 @@ func (b *httpBackend) fetch(ctx context.Context, sha string) (*http.Response, er return nil, err } if resp.StatusCode < 200 || resp.StatusCode >= 300 { + _, _ = io.Copy(io.Discard, resp.Body) resp.Body.Close() return nil, fmt.Errorf("bunny get %s: unexpected status %d", sha, resp.StatusCode) } @@ -230,6 +250,7 @@ func (b *httpBackend) fetch(ctx context.Context, sha string) (*http.Response, er return nil, err } if resp.StatusCode < 200 || resp.StatusCode >= 300 { + _, _ = io.Copy(io.Discard, resp.Body) resp.Body.Close() return nil, fmt.Errorf("cdn get %s: unexpected status %d", sha, resp.StatusCode) } diff --git a/internal/depot/remote_test.go b/internal/depot/remote_test.go index 6bb9328..36e1b17 100644 --- a/internal/depot/remote_test.go +++ b/internal/depot/remote_test.go @@ -120,6 +120,81 @@ func hostPort(srv *httptest.Server) string { return srv.URL } +// stubProbeSleep replaces probeRetrySleep for the test, recording durations. +func stubProbeSleep(t *testing.T) *[]time.Duration { + t.Helper() + var slept []time.Duration + old := probeRetrySleep + probeRetrySleep = func(d time.Duration) { slept = append(slept, d) } + t.Cleanup(func() { probeRetrySleep = old }) + return &slept +} + +func TestProbeRetriesTransientThenPresent(t *testing.T) { + slept := stubProbeSleep(t) + sha := shaOf("throttled-blob") + + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls == 1 { + w.WriteHeader(428) + return + } + w.WriteHeader(206) + })) + defer srv.Close() + + cfg := testCfg(hostPort(srv), "readkey", "writekey") + b, err := NewBackend("bunny", "", cfg) + if err != nil { + t.Fatalf("NewBackend: %v", err) + } + + state, transient, err := b.Probe(context.Background(), sha) + if err != nil { + t.Fatalf("Probe: %v", err) + } + if state != Present || transient { + t.Fatalf("expected Present/non-transient after retry, got %v/%v", state, transient) + } + if calls != 2 { + t.Fatalf("expected 2 requests (428 then 206), got %d", calls) + } + if len(*slept) != 1 || (*slept)[0] != time.Second { + t.Fatalf("expected one 1s backoff, got %v", *slept) + } +} + +func TestProbe404NoRetry(t *testing.T) { + slept := stubProbeSleep(t) + sha := shaOf("gone-blob") + + f := newFakeBunny() + srv := httptest.NewServer(f.handler()) + defer srv.Close() + + cfg := testCfg(hostPort(srv), "readkey", "writekey") + b, err := NewBackend("bunny", "", cfg) + if err != nil { + t.Fatalf("NewBackend: %v", err) + } + + state, transient, err := b.Probe(context.Background(), sha) + if err != nil { + t.Fatalf("Probe: %v", err) + } + if state != Absent || transient { + t.Fatalf("expected Absent/non-transient, got %v/%v", state, transient) + } + if got := len(f.requestsSnapshot()); got != 1 { + t.Fatalf("expected exactly 1 request for 404, got %d", got) + } + if len(*slept) != 0 { + t.Fatalf("expected zero backoffs for 404, got %v", *slept) + } +} + func TestProbeUsesRangeGetNotHead(t *testing.T) { sha := shaOf("present-blob") f := newFakeBunny() @@ -157,6 +232,7 @@ func TestProbeUsesRangeGetNotHead(t *testing.T) { } func TestProbeStates(t *testing.T) { + stubProbeSleep(t) cases := []struct { status int wantState ProbeState diff --git a/internal/depot/run_test.go b/internal/depot/run_test.go index 3578872..a2bcbda 100644 --- a/internal/depot/run_test.go +++ b/internal/depot/run_test.go @@ -105,6 +105,7 @@ func TestStatusExitCodes(t *testing.T) { }) t.Run("unconfirmed-only", func(t *testing.T) { + stubProbeSleep(t) f := newFakeBunny() sha := shaOf("flaky-blob") f.statusFor[sha] = 428