feat(depot): status/push/verify/get/pull commands with exit-code contract
Adds internal/depot.Run, the stdlib-flag command surface for the five depot subcommands (0/1/2/64/70 exit contract), plus a dispatch special case that will route "crucible depot ..." and crucible-depot to it once Task 6 flips Wired:true (inert for now since Wired stays false). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,184 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestIntegrationPushStatusPull drives status/push/pull end-to-end against a
|
||||
// fake Bunny backend (real bunny HTTP path, in-memory blob store) since the
|
||||
// command surface has no local->local push (push --target must be bunny).
|
||||
func TestIntegrationPushStatusPull(t *testing.T) {
|
||||
manifestsDir := t.TempDir()
|
||||
sourceDir := t.TempDir()
|
||||
|
||||
blobs := map[string]string{
|
||||
shaOf("blob-one"): "blob-one",
|
||||
shaOf("blob-two"): "blob-two",
|
||||
shaOf("blob-three"): "blob-three",
|
||||
}
|
||||
var manifest strings.Builder
|
||||
manifest.WriteString("assets:\n")
|
||||
for sha, content := range blobs {
|
||||
manifest.WriteString(fmt.Sprintf(" - path: %s\n sha256: %s\n size: %d\n", sha, sha, len(content)))
|
||||
blobPath := filepath.Join(sourceDir, BlobKey(sha))
|
||||
if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(blobPath, []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(manifestsDir, "manifest.yml"), []byte(manifest.String()), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f := newFakeBunny()
|
||||
// Pre-seed one of the three blobs on the target so status starts with drift=2.
|
||||
var oneSHA string
|
||||
for sha := range blobs {
|
||||
oneSHA = sha
|
||||
break
|
||||
}
|
||||
f.blobs[oneSHA] = []byte(blobs[oneSHA])
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
env := map[string]string{
|
||||
"BUNNY_STORAGE_HOST": hostPort(srv),
|
||||
"BUNNY_STORAGE_READ_PASSWORD": "readkey",
|
||||
"BUNNY_STORAGE_PASSWORD": "writekey",
|
||||
"CDN_UNREACHABLE": "1", // unused; CDNBase defaults to unreachable in test cfg path via storageURL fallback
|
||||
}
|
||||
getenv := testGetenv(env)
|
||||
|
||||
// status: expect exit 1, missing=2
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"status", "--manifests", manifestsDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||
if code != 1 {
|
||||
t.Fatalf("status: expected 1, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "missing=2") {
|
||||
t.Fatalf("status: expected missing=2, got %s", out.String())
|
||||
}
|
||||
|
||||
// push: uploads the 2 missing blobs
|
||||
out.Reset()
|
||||
errb.Reset()
|
||||
code = Run([]string{"push", "--manifests", manifestsDir, "--source", sourceDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||
if code != 0 {
|
||||
t.Fatalf("push: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "uploaded=2 failed=0") {
|
||||
t.Fatalf("push: expected uploaded=2 failed=0, got %s", out.String())
|
||||
}
|
||||
|
||||
putCountAfterFirstPush := 0
|
||||
for _, r := range f.requestsSnapshot() {
|
||||
if r.Method == "PUT" {
|
||||
putCountAfterFirstPush++
|
||||
}
|
||||
}
|
||||
if putCountAfterFirstPush != 2 {
|
||||
t.Fatalf("expected 2 PUTs after first push, got %d", putCountAfterFirstPush)
|
||||
}
|
||||
|
||||
// status: now clean
|
||||
out.Reset()
|
||||
errb.Reset()
|
||||
code = Run([]string{"status", "--manifests", manifestsDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||
if code != 0 {
|
||||
t.Fatalf("status after push: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
|
||||
// push again: idempotent, no new PUTs, uploaded=0
|
||||
out.Reset()
|
||||
errb.Reset()
|
||||
code = Run([]string{"push", "--manifests", manifestsDir, "--source", sourceDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||
if code != 0 {
|
||||
t.Fatalf("second push: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "uploaded=0 failed=0") {
|
||||
t.Fatalf("second push: expected uploaded=0 failed=0, got %s", out.String())
|
||||
}
|
||||
putCountAfterSecondPush := 0
|
||||
for _, r := range f.requestsSnapshot() {
|
||||
if r.Method == "PUT" {
|
||||
putCountAfterSecondPush++
|
||||
}
|
||||
}
|
||||
if putCountAfterSecondPush != putCountAfterFirstPush {
|
||||
t.Fatalf("expected no additional PUTs on idempotent push, before=%d after=%d", putCountAfterFirstPush, putCountAfterSecondPush)
|
||||
}
|
||||
|
||||
// pull into a dest dir: one pre-populated correct file (skipped, no GET),
|
||||
// one pre-populated corrupt file (re-downloaded), one absent (downloaded).
|
||||
destDir := t.TempDir()
|
||||
shas := make([]string, 0, len(blobs))
|
||||
for sha := range blobs {
|
||||
shas = append(shas, sha)
|
||||
}
|
||||
correctSHA := shas[0]
|
||||
corruptSHA := shas[1]
|
||||
// absentSHA := shas[2] // left absent on purpose
|
||||
|
||||
correctPath := filepath.Join(destDir, BlobKey(correctSHA))
|
||||
if err := os.MkdirAll(filepath.Dir(correctPath), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(correctPath, []byte(blobs[correctSHA]), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
corruptPath := filepath.Join(destDir, BlobKey(corruptSHA))
|
||||
if err := os.MkdirAll(filepath.Dir(corruptPath), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(corruptPath, []byte("corrupted-content"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
getCountBeforePull := 0
|
||||
for _, r := range f.requestsSnapshot() {
|
||||
if r.Method == "GET" && !strings.Contains(r.Range, "0-0") {
|
||||
getCountBeforePull++
|
||||
}
|
||||
}
|
||||
|
||||
out.Reset()
|
||||
errb.Reset()
|
||||
code = Run([]string{"pull", "--manifests", manifestsDir, "--dest", destDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||
if code != 0 {
|
||||
t.Fatalf("pull: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "pulled=2 present=1") {
|
||||
t.Fatalf("pull: expected pulled=2 present=1, got %s", out.String())
|
||||
}
|
||||
|
||||
// verify the correct pre-existing file was never re-fetched with a full GET.
|
||||
fullGETsForCorrect := 0
|
||||
for _, r := range f.requestsSnapshot() {
|
||||
if r.Method == "GET" && strings.HasSuffix(r.Path, correctSHA) && r.Range != "bytes=0-0" {
|
||||
fullGETsForCorrect++
|
||||
}
|
||||
}
|
||||
if fullGETsForCorrect != 0 {
|
||||
t.Fatalf("expected no full GET for already-correct blob, got %d", fullGETsForCorrect)
|
||||
}
|
||||
|
||||
// all three blobs should now be present and correct in destDir.
|
||||
for sha, content := range blobs {
|
||||
got, err := os.ReadFile(filepath.Join(destDir, BlobKey(sha)))
|
||||
if err != nil {
|
||||
t.Fatalf("dest blob %s: %v", sha, err)
|
||||
}
|
||||
if string(got) != content {
|
||||
t.Fatalf("dest blob %s: expected %q, got %q", sha, content, string(got))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,465 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
exitOK = 0
|
||||
exitDrift = 1
|
||||
exitUnconfirmed = 2
|
||||
exitUsage = 64
|
||||
exitInternal = 70
|
||||
)
|
||||
|
||||
// Run executes a depot subcommand. args[0] is the subcommand
|
||||
// (status|push|verify|get|pull); returns the process exit code.
|
||||
func Run(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
if len(args) == 0 {
|
||||
printRunUsage(stderr)
|
||||
return exitUsage
|
||||
}
|
||||
rest := args[1:]
|
||||
switch args[0] {
|
||||
case "status":
|
||||
return runStatus(rest, stdout, stderr, getenv)
|
||||
case "push":
|
||||
return runPush(rest, stdout, stderr, getenv)
|
||||
case "verify":
|
||||
return runVerify(rest, stdout, stderr, getenv)
|
||||
case "get":
|
||||
return runGet(rest, stdout, stderr, getenv)
|
||||
case "pull":
|
||||
return runPull(rest, stdout, stderr, getenv)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "depot: unknown subcommand %q\n\n", args[0])
|
||||
printRunUsage(stderr)
|
||||
return exitUsage
|
||||
}
|
||||
}
|
||||
|
||||
func printRunUsage(w io.Writer) {
|
||||
fmt.Fprint(w, `usage:
|
||||
depot status [--manifests DIR] [--source LOCAL_DEPOT] --target bunny|cdn|local
|
||||
depot push [--manifests DIR] --source LOCAL_DEPOT --target bunny
|
||||
depot verify [--manifests DIR] --target bunny|cdn [--sample N]
|
||||
depot get <sha> <dest> --target cdn|bunny|local
|
||||
depot pull [--manifests DIR] --dest DIR --target cdn|bunny
|
||||
|
||||
--target local uses the DEPOT_DIR environment variable as the local root.
|
||||
`)
|
||||
}
|
||||
|
||||
// resolveBackend builds the named backend, resolving "local" against DEPOT_DIR.
|
||||
func resolveBackend(target string, getenv func(string) string, cfg Config) (Backend, error) {
|
||||
root := ""
|
||||
if target == "local" {
|
||||
root = getenv("DEPOT_DIR")
|
||||
if root == "" {
|
||||
return nil, fmt.Errorf("--target local requires DEPOT_DIR to be set")
|
||||
}
|
||||
}
|
||||
return NewBackend(target, root, cfg)
|
||||
}
|
||||
|
||||
func printSweepStatus(stdout io.Writer, referenced int, res SweepResult) {
|
||||
fmt.Fprintf(stdout, "referenced=%d present=%d missing=%d unconfirmed=%d\n",
|
||||
referenced, len(res.Present), len(res.Missing), len(res.Unconfirmed))
|
||||
for _, sha := range res.Missing {
|
||||
fmt.Fprintf(stdout, "missing %s\n", sha)
|
||||
}
|
||||
for _, sha := range res.Unconfirmed {
|
||||
fmt.Fprintf(stdout, "unconfirmed %s\n", sha)
|
||||
}
|
||||
}
|
||||
|
||||
func sweepExitCode(res SweepResult) int {
|
||||
if len(res.Missing) > 0 {
|
||||
return exitDrift
|
||||
}
|
||||
if len(res.Unconfirmed) > 0 {
|
||||
return exitUnconfirmed
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func shaKeys(m map[string]int64) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for sha := range m {
|
||||
out = append(out, sha)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func runStatus(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||
_ = fs.String("source", "", "local depot root (unused for status target=local; see DEPOT_DIR)")
|
||||
target := fs.String("target", "", "bunny|cdn|local")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if *target == "" {
|
||||
fmt.Fprintln(stderr, "depot status: --target is required")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
cfg := LoadConfig(getenv)
|
||||
backend, err := resolveBackend(*target, getenv, cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot status:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
shaSizes, err := ReferencedSHAs(*manifests)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot status:", err)
|
||||
return exitInternal
|
||||
}
|
||||
shas := shaKeys(shaSizes)
|
||||
|
||||
res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot status:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
printSweepStatus(stdout, len(shas), res)
|
||||
return sweepExitCode(res)
|
||||
}
|
||||
|
||||
func runPush(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
fs := flag.NewFlagSet("push", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||
source := fs.String("source", "", "local depot root to read blobs from")
|
||||
target := fs.String("target", "", "must be bunny")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if *target != "bunny" {
|
||||
fmt.Fprintln(stderr, "depot push: --target must be bunny")
|
||||
return exitUsage
|
||||
}
|
||||
if *source == "" {
|
||||
fmt.Fprintln(stderr, "depot push: --source is required")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
cfg := LoadConfig(getenv)
|
||||
backend, err := NewBackend("bunny", "", cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot push:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
shaSizes, err := ReferencedSHAs(*manifests)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot push:", err)
|
||||
return exitInternal
|
||||
}
|
||||
shas := shaKeys(shaSizes)
|
||||
|
||||
res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot push:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
toUpload := append(append([]string(nil), res.Missing...), res.Unconfirmed...)
|
||||
sort.Strings(toUpload)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
uploaded int
|
||||
failed int
|
||||
transportErr error
|
||||
)
|
||||
|
||||
work := func(sha string) {
|
||||
srcPath := filepath.Join(*source, BlobKey(sha))
|
||||
if _, statErr := os.Stat(srcPath); statErr != nil {
|
||||
mu.Lock()
|
||||
failed++
|
||||
mu.Unlock()
|
||||
fmt.Fprintf(stderr, "push: missing source blob %s\n", sha)
|
||||
return
|
||||
}
|
||||
if err := backend.Put(context.Background(), sha, srcPath); err != nil {
|
||||
mu.Lock()
|
||||
if transportErr == nil {
|
||||
transportErr = err
|
||||
}
|
||||
mu.Unlock()
|
||||
fmt.Fprintf(stderr, "push: upload %s: %v\n", sha, err)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
uploaded++
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
runWorkers(cfg.Jobs, toUpload, work)
|
||||
|
||||
fmt.Fprintf(stdout, "uploaded=%d failed=%d\n", uploaded, failed)
|
||||
|
||||
if transportErr != nil {
|
||||
return exitInternal
|
||||
}
|
||||
if failed > 0 {
|
||||
return exitDrift
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func runVerify(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
fs := flag.NewFlagSet("verify", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||
target := fs.String("target", "", "bunny|cdn")
|
||||
sample := fs.Int("sample", 3, "number of present blobs to spot-check by download (0=skip)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if *target != "bunny" && *target != "cdn" {
|
||||
fmt.Fprintln(stderr, "depot verify: --target must be bunny or cdn")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
cfg := LoadConfig(getenv)
|
||||
backend, err := NewBackend(*target, "", cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot verify:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
shaSizes, err := ReferencedSHAs(*manifests)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot verify:", err)
|
||||
return exitInternal
|
||||
}
|
||||
shas := shaKeys(shaSizes)
|
||||
|
||||
res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot verify:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
printSweepStatus(stdout, len(shas), res)
|
||||
if code := sweepExitCode(res); code != exitOK {
|
||||
return code
|
||||
}
|
||||
|
||||
if *sample <= 0 || len(res.Present) == 0 {
|
||||
return exitOK
|
||||
}
|
||||
|
||||
n := *sample
|
||||
if n > len(res.Present) {
|
||||
n = len(res.Present)
|
||||
}
|
||||
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
picks := rng.Perm(len(res.Present))[:n]
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "depot-verify-")
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot verify:", err)
|
||||
return exitInternal
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
for _, idx := range picks {
|
||||
sha := res.Present[idx]
|
||||
dest := filepath.Join(tmpDir, sha)
|
||||
if err := backend.Get(context.Background(), sha, dest); err != nil {
|
||||
fmt.Fprintf(stderr, "depot verify: sample %s: %v\n", sha, err)
|
||||
return exitDrift
|
||||
}
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
fs := flag.NewFlagSet("get", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
target := fs.String("target", "", "cdn|bunny|local")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
positional := fs.Args()
|
||||
if len(positional) != 2 {
|
||||
fmt.Fprintln(stderr, "depot get: usage: depot get <sha> <dest> --target cdn|bunny|local")
|
||||
return exitUsage
|
||||
}
|
||||
sha, dest := positional[0], positional[1]
|
||||
if !ValidSHA(sha) {
|
||||
fmt.Fprintf(stderr, "depot get: invalid sha %q\n", sha)
|
||||
return exitUsage
|
||||
}
|
||||
if *target == "" {
|
||||
fmt.Fprintln(stderr, "depot get: --target is required")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
cfg := LoadConfig(getenv)
|
||||
backend, err := resolveBackend(*target, getenv, cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot get:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
if err := backend.Get(context.Background(), sha, dest); err != nil {
|
||||
fmt.Fprintln(stderr, "depot get:", err)
|
||||
return exitInternal
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func runPull(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||
fs := flag.NewFlagSet("pull", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||
dest := fs.String("dest", "", "local directory to pull blobs into")
|
||||
target := fs.String("target", "", "cdn|bunny")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return exitUsage
|
||||
}
|
||||
if *target != "bunny" && *target != "cdn" {
|
||||
fmt.Fprintln(stderr, "depot pull: --target must be bunny or cdn")
|
||||
return exitUsage
|
||||
}
|
||||
if *dest == "" {
|
||||
fmt.Fprintln(stderr, "depot pull: --dest is required")
|
||||
return exitUsage
|
||||
}
|
||||
|
||||
cfg := LoadConfig(getenv)
|
||||
backend, err := NewBackend(*target, "", cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot pull:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
shaSizes, err := ReferencedSHAs(*manifests)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot pull:", err)
|
||||
return exitInternal
|
||||
}
|
||||
shas := shaKeys(shaSizes)
|
||||
|
||||
// Split into "already present and correct locally" (skip) vs "needs a
|
||||
// probe/download decision".
|
||||
var present int
|
||||
var toCheck []string
|
||||
for _, sha := range shas {
|
||||
destPath := filepath.Join(*dest, BlobKey(sha))
|
||||
if localFileMatchesSHA(destPath, sha) {
|
||||
present++
|
||||
continue
|
||||
}
|
||||
toCheck = append(toCheck, sha)
|
||||
}
|
||||
|
||||
// Sweep the source to distinguish "not there" (exit 1, listed) from
|
||||
// "there, download it".
|
||||
res, err := Sweep(context.Background(), backend, toCheck, cfg, io.Discard)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "depot pull:", err)
|
||||
return exitInternal
|
||||
}
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
pulled int
|
||||
anyFail bool
|
||||
)
|
||||
work := func(sha string) {
|
||||
destPath := filepath.Join(*dest, BlobKey(sha))
|
||||
if err := backend.Get(context.Background(), sha, destPath); err != nil {
|
||||
mu.Lock()
|
||||
anyFail = true
|
||||
mu.Unlock()
|
||||
fmt.Fprintf(stderr, "pull: %s: %v\n", sha, err)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
pulled++
|
||||
mu.Unlock()
|
||||
}
|
||||
runWorkers(cfg.Jobs, res.Present, work)
|
||||
|
||||
// ponytail: unconfirmed source state (probe budget exhausted / flaky
|
||||
// responses) is treated as a download failure rather than a third exit
|
||||
// path; if that proves too coarse in practice, give pull its own
|
||||
// unconfirmed accounting like status.
|
||||
if len(res.Unconfirmed) > 0 {
|
||||
anyFail = true
|
||||
for _, sha := range res.Unconfirmed {
|
||||
fmt.Fprintf(stderr, "pull: %s: unconfirmed at source\n", sha)
|
||||
}
|
||||
}
|
||||
|
||||
for _, sha := range res.Missing {
|
||||
fmt.Fprintf(stdout, "missing %s\n", sha)
|
||||
}
|
||||
|
||||
fmt.Fprintf(stdout, "pulled=%d present=%d\n", pulled, present)
|
||||
|
||||
if anyFail {
|
||||
return exitInternal
|
||||
}
|
||||
if len(res.Missing) > 0 {
|
||||
return exitDrift
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
// localFileMatchesSHA reports whether path exists and hashes to sha.
|
||||
func localFileMatchesSHA(path, sha string) bool {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return false
|
||||
}
|
||||
return fmt.Sprintf("%x", h.Sum(nil)) == sha
|
||||
}
|
||||
|
||||
// runWorkers runs work(sha) for each sha in items using up to jobs goroutines.
|
||||
func runWorkers(jobs int, items []string, work func(sha string)) {
|
||||
if jobs < 1 {
|
||||
jobs = 1
|
||||
}
|
||||
ch := make(chan string)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < jobs; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for sha := range ch {
|
||||
work(sha)
|
||||
}
|
||||
}()
|
||||
}
|
||||
for _, sha := range items {
|
||||
ch <- sha
|
||||
}
|
||||
close(ch)
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package depot
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func testGetenv(vals map[string]string) func(string) string {
|
||||
return func(k string) string { return vals[k] }
|
||||
}
|
||||
|
||||
func TestRunUsage(t *testing.T) {
|
||||
t.Run("no args", func(t *testing.T) {
|
||||
var out, errb bytes.Buffer
|
||||
code := Run(nil, &out, &errb, testGetenv(nil))
|
||||
if code != 64 {
|
||||
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown subcommand", func(t *testing.T) {
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"bogus"}, &out, &errb, testGetenv(nil))
|
||||
if code != 64 {
|
||||
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("push --target cdn rejected", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"push", "--manifests", dir, "--source", dir, "--target", "cdn"}, &out, &errb, testGetenv(nil))
|
||||
if code != 64 {
|
||||
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetInvalidSHA(t *testing.T) {
|
||||
var out, errb bytes.Buffer
|
||||
dir := t.TempDir()
|
||||
code := Run([]string{"get", "not-a-sha", dir + "/dest", "--target", "local"}, &out, &errb, testGetenv(map[string]string{"DEPOT_DIR": dir}))
|
||||
if code != 64 {
|
||||
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoKeyStatusBunnyFailsClosedNoStdin(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeManifest(t, dir, shaOf("blob-a"), 5)
|
||||
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"status", "--manifests", dir, "--target", "bunny"}, &out, &errb, testGetenv(nil))
|
||||
if code != 70 {
|
||||
t.Fatalf("expected 70, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(errb.String(), "BUNNY_STORAGE_HOST") {
|
||||
t.Fatalf("expected stderr to mention BUNNY_STORAGE_HOST, got %s", errb.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusExitCodes(t *testing.T) {
|
||||
t.Run("drift", func(t *testing.T) {
|
||||
f := newFakeBunny()
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
sha := shaOf("missing-blob")
|
||||
dir := t.TempDir()
|
||||
writeManifest(t, dir, sha, 5)
|
||||
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"status", "--manifests", dir, "--target", "bunny"}, &out, &errb,
|
||||
testGetenv(map[string]string{
|
||||
"BUNNY_STORAGE_HOST": hostPort(srv),
|
||||
"BUNNY_STORAGE_READ_PASSWORD": "readkey",
|
||||
}))
|
||||
if code != 1 {
|
||||
t.Fatalf("expected 1, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "missing="+"1") {
|
||||
t.Fatalf("expected missing=1 in output, got %s", out.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "missing "+sha) {
|
||||
t.Fatalf("expected missing line for sha, got %s", out.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unconfirmed-only", func(t *testing.T) {
|
||||
f := newFakeBunny()
|
||||
sha := shaOf("flaky-blob")
|
||||
f.statusFor[sha] = 428
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
writeManifest(t, dir, sha, 5)
|
||||
|
||||
var out, errb bytes.Buffer
|
||||
code := Run([]string{"status", "--manifests", dir, "--target", "bunny"}, &out, &errb,
|
||||
testGetenv(map[string]string{
|
||||
"BUNNY_STORAGE_HOST": hostPort(srv),
|
||||
"BUNNY_STORAGE_READ_PASSWORD": "readkey",
|
||||
"DEPOT_CONFIRM_RETRIES": "1",
|
||||
}))
|
||||
if code != 2 {
|
||||
t.Fatalf("expected 2, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||
}
|
||||
if !bytesContains(out.String(), "unconfirmed="+"1") {
|
||||
t.Fatalf("expected unconfirmed=1 in output, got %s", out.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func bytesContains(s, substr string) bool {
|
||||
return bytes.Contains([]byte(s), []byte(substr))
|
||||
}
|
||||
|
||||
func writeManifest(t *testing.T, dir, sha string, size int64) {
|
||||
t.Helper()
|
||||
content := fmt.Sprintf("assets:\n - path: foo\n sha256: %s\n size: %d\n", sha, size)
|
||||
if err := os.WriteFile(filepath.Join(dir, "manifest.yml"), []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/app"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/depot"
|
||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/menu"
|
||||
)
|
||||
|
||||
@@ -365,6 +366,12 @@ func runBuilder(name string, args []string, out, errw io.Writer) int {
|
||||
return exitOK
|
||||
}
|
||||
}
|
||||
if b.Name == "depot" && b.Wired {
|
||||
// depot parses its own subcommand (status/push/verify/get/pull) and
|
||||
// has its own richer exit contract (0/1/2/64/70), so it bypasses the
|
||||
// b.Commands/delegateLegacy routing entirely.
|
||||
return depot.Run(args, out, errw, os.Getenv)
|
||||
}
|
||||
if !b.Wired {
|
||||
// No migrated logic yet (depot): fail closed, never fake an artifact.
|
||||
fmt.Fprintf(errw, unwiredMsg, b.Name, b.Bin)
|
||||
|
||||
Reference in New Issue
Block a user