internal/depot/remote.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 // NewBackend returns the backend for name ("local"|"cdn"|"bunny").
@@ -85,9 +86,26 @@ func (b *httpBackend) cdnURL(sha string) string {
 	return fmt.Sprintf("%s/%s", b.cfg.CDNBase, BlobKey(sha))
 }
 
+// probeRetrySleep is the backoff sleeper for transient probe retries;
+// tests stub it (same pattern as sweep.go's confirmSleep).
+var probeRetrySleep = time.Sleep
+
 // rangeProbe issues GET <url> with Range: bytes=0-0 and classifies the
-// response. Never HEAD (banned by spec).
+// response. Never HEAD (banned by spec). Transient outcomes (transport
+// error, or a status that is neither 2xx nor 404/410) retry up to 2 more
+// times with linear backoff (1s, 2s) — the CDN edge throttles sustained
+// sweeps; the bash this ports absorbed that with curl --retry 2.
 func (b *httpBackend) rangeProbe(ctx context.Context, url string, headers map[string]string) (ProbeState, bool, error) {
+	for attempt := 0; ; attempt++ {
+		state, transient, err := b.rangeProbeOnce(ctx, url, headers)
+		if !transient || attempt >= 2 {
+			return state, transient, err
+		}
+		probeRetrySleep(time.Duration(attempt+1) * time.Second)
+	}
+}
+
+func (b *httpBackend) rangeProbeOnce(ctx context.Context, url string, headers map[string]string) (ProbeState, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return Unconfirmed, true, err
@@ -212,6 +230,7 @@ func (b *httpBackend) fetch(ctx context.Context, sha string) (*http.Response, er
 		if resp, err := b.get(ctx, b.cdnURL(sha), nil); err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 {
 			return resp, nil
 		} else if err == nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
 			resp.Body.Close()
 		}
 		resp, err := b.get(ctx, b.storageURL(sha), map[string]string{"AccessKey": b.cfg.ReadKey})
@@ -219,6 +238,7 @@ func (b *httpBackend) fetch(ctx context.Context, sha string) (*http.Response, er
 			return nil, err
 		}
 		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+			_, _ = io.Copy(io.Discard, resp.Body)
 			resp.Body.Close()
 			return nil, fmt.Errorf("bunny get %s: unexpected status %d", sha, resp.StatusCode)
 		}
@@ -230,6 +250,7 @@ func (b *httpBackend) fetch(ctx context.Context, sha string) (*http.Response, er
 		return nil, err
 	}
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		_, _ = io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 		return nil, fmt.Errorf("cdn get %s: unexpected status %d", sha, resp.StatusCode)
 	}

internal/depot/remote_test.go

@@ -120,6 +120,81 @@ func hostPort(srv *httptest.Server) string {
 	return srv.URL
 }
 
+// stubProbeSleep replaces probeRetrySleep for the test, recording durations.
+func stubProbeSleep(t *testing.T) *[]time.Duration {
+	t.Helper()
+	var slept []time.Duration
+	old := probeRetrySleep
+	probeRetrySleep = func(d time.Duration) { slept = append(slept, d) }
+	t.Cleanup(func() { probeRetrySleep = old })
+	return &slept
+}
+
+func TestProbeRetriesTransientThenPresent(t *testing.T) {
+	slept := stubProbeSleep(t)
+	sha := shaOf("throttled-blob")
+
+	var calls int
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls++
+		if calls == 1 {
+			w.WriteHeader(428)
+			return
+		}
+		w.WriteHeader(206)
+	}))
+	defer srv.Close()
+
+	cfg := testCfg(hostPort(srv), "readkey", "writekey")
+	b, err := NewBackend("bunny", "", cfg)
+	if err != nil {
+		t.Fatalf("NewBackend: %v", err)
+	}
+
+	state, transient, err := b.Probe(context.Background(), sha)
+	if err != nil {
+		t.Fatalf("Probe: %v", err)
+	}
+	if state != Present || transient {
+		t.Fatalf("expected Present/non-transient after retry, got %v/%v", state, transient)
+	}
+	if calls != 2 {
+		t.Fatalf("expected 2 requests (428 then 206), got %d", calls)
+	}
+	if len(*slept) != 1 || (*slept)[0] != time.Second {
+		t.Fatalf("expected one 1s backoff, got %v", *slept)
+	}
+}
+
+func TestProbe404NoRetry(t *testing.T) {
+	slept := stubProbeSleep(t)
+	sha := shaOf("gone-blob")
+
+	f := newFakeBunny()
+	srv := httptest.NewServer(f.handler())
+	defer srv.Close()
+
+	cfg := testCfg(hostPort(srv), "readkey", "writekey")
+	b, err := NewBackend("bunny", "", cfg)
+	if err != nil {
+		t.Fatalf("NewBackend: %v", err)
+	}
+
+	state, transient, err := b.Probe(context.Background(), sha)
+	if err != nil {
+		t.Fatalf("Probe: %v", err)
+	}
+	if state != Absent || transient {
+		t.Fatalf("expected Absent/non-transient, got %v/%v", state, transient)
+	}
+	if got := len(f.requestsSnapshot()); got != 1 {
+		t.Fatalf("expected exactly 1 request for 404, got %d", got)
+	}
+	if len(*slept) != 0 {
+		t.Fatalf("expected zero backoffs for 404, got %v", *slept)
+	}
+}
+
 func TestProbeUsesRangeGetNotHead(t *testing.T) {
 	sha := shaOf("present-blob")
 	f := newFakeBunny()
@@ -157,6 +232,7 @@ func TestProbeUsesRangeGetNotHead(t *testing.T) {
 }
 
 func TestProbeStates(t *testing.T) {
+	stubProbeSleep(t)
 	cases := []struct {
 		status        int
 		wantState     ProbeState

internal/depot/run_test.go

@@ -105,6 +105,7 @@ func TestStatusExitCodes(t *testing.T) {
 	})
 
 	t.Run("unconfirmed-only", func(t *testing.T) {
+		stubProbeSleep(t)
 		f := newFakeBunny()
 		sha := shaOf("flaky-blob")
 		f.statusFor[sha] = 428