feat(depot): status/push/verify/get/pull commands with exit-code contract
Adds internal/depot.Run, the stdlib-flag command surface for the five depot subcommands (0/1/2/64/70 exit contract), plus a dispatch special case that will route "crucible depot ..." and crucible-depot to it once Task 6 flips Wired:true (inert for now since Wired stays false). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,184 @@
|
|||||||
|
package depot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestIntegrationPushStatusPull drives status/push/pull end-to-end against a
|
||||||
|
// fake Bunny backend (real bunny HTTP path, in-memory blob store) since the
|
||||||
|
// command surface has no local->local push (push --target must be bunny).
|
||||||
|
func TestIntegrationPushStatusPull(t *testing.T) {
|
||||||
|
manifestsDir := t.TempDir()
|
||||||
|
sourceDir := t.TempDir()
|
||||||
|
|
||||||
|
blobs := map[string]string{
|
||||||
|
shaOf("blob-one"): "blob-one",
|
||||||
|
shaOf("blob-two"): "blob-two",
|
||||||
|
shaOf("blob-three"): "blob-three",
|
||||||
|
}
|
||||||
|
var manifest strings.Builder
|
||||||
|
manifest.WriteString("assets:\n")
|
||||||
|
for sha, content := range blobs {
|
||||||
|
manifest.WriteString(fmt.Sprintf(" - path: %s\n sha256: %s\n size: %d\n", sha, sha, len(content)))
|
||||||
|
blobPath := filepath.Join(sourceDir, BlobKey(sha))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(blobPath, []byte(content), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(manifestsDir, "manifest.yml"), []byte(manifest.String()), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
f := newFakeBunny()
|
||||||
|
// Pre-seed one of the three blobs on the target so status starts with drift=2.
|
||||||
|
var oneSHA string
|
||||||
|
for sha := range blobs {
|
||||||
|
oneSHA = sha
|
||||||
|
break
|
||||||
|
}
|
||||||
|
f.blobs[oneSHA] = []byte(blobs[oneSHA])
|
||||||
|
srv := httptest.NewServer(f.handler())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
env := map[string]string{
|
||||||
|
"BUNNY_STORAGE_HOST": hostPort(srv),
|
||||||
|
"BUNNY_STORAGE_READ_PASSWORD": "readkey",
|
||||||
|
"BUNNY_STORAGE_PASSWORD": "writekey",
|
||||||
|
"CDN_UNREACHABLE": "1", // unused; CDNBase defaults to unreachable in test cfg path via storageURL fallback
|
||||||
|
}
|
||||||
|
getenv := testGetenv(env)
|
||||||
|
|
||||||
|
// status: expect exit 1, missing=2
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"status", "--manifests", manifestsDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("status: expected 1, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
if !bytesContains(out.String(), "missing=2") {
|
||||||
|
t.Fatalf("status: expected missing=2, got %s", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// push: uploads the 2 missing blobs
|
||||||
|
out.Reset()
|
||||||
|
errb.Reset()
|
||||||
|
code = Run([]string{"push", "--manifests", manifestsDir, "--source", sourceDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("push: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
if !bytesContains(out.String(), "uploaded=2 failed=0") {
|
||||||
|
t.Fatalf("push: expected uploaded=2 failed=0, got %s", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
putCountAfterFirstPush := 0
|
||||||
|
for _, r := range f.requestsSnapshot() {
|
||||||
|
if r.Method == "PUT" {
|
||||||
|
putCountAfterFirstPush++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if putCountAfterFirstPush != 2 {
|
||||||
|
t.Fatalf("expected 2 PUTs after first push, got %d", putCountAfterFirstPush)
|
||||||
|
}
|
||||||
|
|
||||||
|
// status: now clean
|
||||||
|
out.Reset()
|
||||||
|
errb.Reset()
|
||||||
|
code = Run([]string{"status", "--manifests", manifestsDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("status after push: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// push again: idempotent, no new PUTs, uploaded=0
|
||||||
|
out.Reset()
|
||||||
|
errb.Reset()
|
||||||
|
code = Run([]string{"push", "--manifests", manifestsDir, "--source", sourceDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("second push: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
if !bytesContains(out.String(), "uploaded=0 failed=0") {
|
||||||
|
t.Fatalf("second push: expected uploaded=0 failed=0, got %s", out.String())
|
||||||
|
}
|
||||||
|
putCountAfterSecondPush := 0
|
||||||
|
for _, r := range f.requestsSnapshot() {
|
||||||
|
if r.Method == "PUT" {
|
||||||
|
putCountAfterSecondPush++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if putCountAfterSecondPush != putCountAfterFirstPush {
|
||||||
|
t.Fatalf("expected no additional PUTs on idempotent push, before=%d after=%d", putCountAfterFirstPush, putCountAfterSecondPush)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pull into a dest dir: one pre-populated correct file (skipped, no GET),
|
||||||
|
// one pre-populated corrupt file (re-downloaded), one absent (downloaded).
|
||||||
|
destDir := t.TempDir()
|
||||||
|
shas := make([]string, 0, len(blobs))
|
||||||
|
for sha := range blobs {
|
||||||
|
shas = append(shas, sha)
|
||||||
|
}
|
||||||
|
correctSHA := shas[0]
|
||||||
|
corruptSHA := shas[1]
|
||||||
|
// absentSHA := shas[2] // left absent on purpose
|
||||||
|
|
||||||
|
correctPath := filepath.Join(destDir, BlobKey(correctSHA))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(correctPath), 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(correctPath, []byte(blobs[correctSHA]), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
corruptPath := filepath.Join(destDir, BlobKey(corruptSHA))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(corruptPath), 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(corruptPath, []byte("corrupted-content"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
getCountBeforePull := 0
|
||||||
|
for _, r := range f.requestsSnapshot() {
|
||||||
|
if r.Method == "GET" && !strings.Contains(r.Range, "0-0") {
|
||||||
|
getCountBeforePull++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out.Reset()
|
||||||
|
errb.Reset()
|
||||||
|
code = Run([]string{"pull", "--manifests", manifestsDir, "--dest", destDir, "--target", "bunny"}, &out, &errb, getenv)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("pull: expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
if !bytesContains(out.String(), "pulled=2 present=1") {
|
||||||
|
t.Fatalf("pull: expected pulled=2 present=1, got %s", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// verify the correct pre-existing file was never re-fetched with a full GET.
|
||||||
|
fullGETsForCorrect := 0
|
||||||
|
for _, r := range f.requestsSnapshot() {
|
||||||
|
if r.Method == "GET" && strings.HasSuffix(r.Path, correctSHA) && r.Range != "bytes=0-0" {
|
||||||
|
fullGETsForCorrect++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if fullGETsForCorrect != 0 {
|
||||||
|
t.Fatalf("expected no full GET for already-correct blob, got %d", fullGETsForCorrect)
|
||||||
|
}
|
||||||
|
|
||||||
|
// all three blobs should now be present and correct in destDir.
|
||||||
|
for sha, content := range blobs {
|
||||||
|
got, err := os.ReadFile(filepath.Join(destDir, BlobKey(sha)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dest blob %s: %v", sha, err)
|
||||||
|
}
|
||||||
|
if string(got) != content {
|
||||||
|
t.Fatalf("dest blob %s: expected %q, got %q", sha, content, string(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,465 @@
|
|||||||
|
package depot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math/rand"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
exitOK = 0
|
||||||
|
exitDrift = 1
|
||||||
|
exitUnconfirmed = 2
|
||||||
|
exitUsage = 64
|
||||||
|
exitInternal = 70
|
||||||
|
)
|
||||||
|
|
||||||
|
// Run executes a depot subcommand. args[0] is the subcommand
|
||||||
|
// (status|push|verify|get|pull); returns the process exit code.
|
||||||
|
func Run(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||||
|
if len(args) == 0 {
|
||||||
|
printRunUsage(stderr)
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
rest := args[1:]
|
||||||
|
switch args[0] {
|
||||||
|
case "status":
|
||||||
|
return runStatus(rest, stdout, stderr, getenv)
|
||||||
|
case "push":
|
||||||
|
return runPush(rest, stdout, stderr, getenv)
|
||||||
|
case "verify":
|
||||||
|
return runVerify(rest, stdout, stderr, getenv)
|
||||||
|
case "get":
|
||||||
|
return runGet(rest, stdout, stderr, getenv)
|
||||||
|
case "pull":
|
||||||
|
return runPull(rest, stdout, stderr, getenv)
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(stderr, "depot: unknown subcommand %q\n\n", args[0])
|
||||||
|
printRunUsage(stderr)
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func printRunUsage(w io.Writer) {
|
||||||
|
fmt.Fprint(w, `usage:
|
||||||
|
depot status [--manifests DIR] [--source LOCAL_DEPOT] --target bunny|cdn|local
|
||||||
|
depot push [--manifests DIR] --source LOCAL_DEPOT --target bunny
|
||||||
|
depot verify [--manifests DIR] --target bunny|cdn [--sample N]
|
||||||
|
depot get <sha> <dest> --target cdn|bunny|local
|
||||||
|
depot pull [--manifests DIR] --dest DIR --target cdn|bunny
|
||||||
|
|
||||||
|
--target local uses the DEPOT_DIR environment variable as the local root.
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveBackend builds the named backend, resolving "local" against DEPOT_DIR.
|
||||||
|
func resolveBackend(target string, getenv func(string) string, cfg Config) (Backend, error) {
|
||||||
|
root := ""
|
||||||
|
if target == "local" {
|
||||||
|
root = getenv("DEPOT_DIR")
|
||||||
|
if root == "" {
|
||||||
|
return nil, fmt.Errorf("--target local requires DEPOT_DIR to be set")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return NewBackend(target, root, cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
func printSweepStatus(stdout io.Writer, referenced int, res SweepResult) {
|
||||||
|
fmt.Fprintf(stdout, "referenced=%d present=%d missing=%d unconfirmed=%d\n",
|
||||||
|
referenced, len(res.Present), len(res.Missing), len(res.Unconfirmed))
|
||||||
|
for _, sha := range res.Missing {
|
||||||
|
fmt.Fprintf(stdout, "missing %s\n", sha)
|
||||||
|
}
|
||||||
|
for _, sha := range res.Unconfirmed {
|
||||||
|
fmt.Fprintf(stdout, "unconfirmed %s\n", sha)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sweepExitCode(res SweepResult) int {
|
||||||
|
if len(res.Missing) > 0 {
|
||||||
|
return exitDrift
|
||||||
|
}
|
||||||
|
if len(res.Unconfirmed) > 0 {
|
||||||
|
return exitUnconfirmed
|
||||||
|
}
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func shaKeys(m map[string]int64) []string {
|
||||||
|
out := make([]string, 0, len(m))
|
||||||
|
for sha := range m {
|
||||||
|
out = append(out, sha)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func runStatus(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||||
|
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(stderr)
|
||||||
|
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||||
|
_ = fs.String("source", "", "local depot root (unused for status target=local; see DEPOT_DIR)")
|
||||||
|
target := fs.String("target", "", "bunny|cdn|local")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if *target == "" {
|
||||||
|
fmt.Fprintln(stderr, "depot status: --target is required")
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := LoadConfig(getenv)
|
||||||
|
backend, err := resolveBackend(*target, getenv, cfg)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot status:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
|
||||||
|
shaSizes, err := ReferencedSHAs(*manifests)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot status:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
shas := shaKeys(shaSizes)
|
||||||
|
|
||||||
|
res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot status:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
|
||||||
|
printSweepStatus(stdout, len(shas), res)
|
||||||
|
return sweepExitCode(res)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runPush(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||||
|
fs := flag.NewFlagSet("push", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(stderr)
|
||||||
|
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||||
|
source := fs.String("source", "", "local depot root to read blobs from")
|
||||||
|
target := fs.String("target", "", "must be bunny")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if *target != "bunny" {
|
||||||
|
fmt.Fprintln(stderr, "depot push: --target must be bunny")
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if *source == "" {
|
||||||
|
fmt.Fprintln(stderr, "depot push: --source is required")
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := LoadConfig(getenv)
|
||||||
|
backend, err := NewBackend("bunny", "", cfg)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot push:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
|
||||||
|
shaSizes, err := ReferencedSHAs(*manifests)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot push:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
shas := shaKeys(shaSizes)
|
||||||
|
|
||||||
|
res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot push:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
|
||||||
|
toUpload := append(append([]string(nil), res.Missing...), res.Unconfirmed...)
|
||||||
|
sort.Strings(toUpload)
|
||||||
|
|
||||||
|
var (
|
||||||
|
mu sync.Mutex
|
||||||
|
uploaded int
|
||||||
|
failed int
|
||||||
|
transportErr error
|
||||||
|
)
|
||||||
|
|
||||||
|
work := func(sha string) {
|
||||||
|
srcPath := filepath.Join(*source, BlobKey(sha))
|
||||||
|
if _, statErr := os.Stat(srcPath); statErr != nil {
|
||||||
|
mu.Lock()
|
||||||
|
failed++
|
||||||
|
mu.Unlock()
|
||||||
|
fmt.Fprintf(stderr, "push: missing source blob %s\n", sha)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := backend.Put(context.Background(), sha, srcPath); err != nil {
|
||||||
|
mu.Lock()
|
||||||
|
if transportErr == nil {
|
||||||
|
transportErr = err
|
||||||
|
}
|
||||||
|
mu.Unlock()
|
||||||
|
fmt.Fprintf(stderr, "push: upload %s: %v\n", sha, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
uploaded++
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
runWorkers(cfg.Jobs, toUpload, work)
|
||||||
|
|
||||||
|
fmt.Fprintf(stdout, "uploaded=%d failed=%d\n", uploaded, failed)
|
||||||
|
|
||||||
|
if transportErr != nil {
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
if failed > 0 {
|
||||||
|
return exitDrift
|
||||||
|
}
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func runVerify(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||||
|
fs := flag.NewFlagSet("verify", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(stderr)
|
||||||
|
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||||
|
target := fs.String("target", "", "bunny|cdn")
|
||||||
|
sample := fs.Int("sample", 3, "number of present blobs to spot-check by download (0=skip)")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if *target != "bunny" && *target != "cdn" {
|
||||||
|
fmt.Fprintln(stderr, "depot verify: --target must be bunny or cdn")
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := LoadConfig(getenv)
|
||||||
|
backend, err := NewBackend(*target, "", cfg)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot verify:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
|
||||||
|
shaSizes, err := ReferencedSHAs(*manifests)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot verify:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
shas := shaKeys(shaSizes)
|
||||||
|
|
||||||
|
res, err := Sweep(context.Background(), backend, shas, cfg, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot verify:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
|
||||||
|
printSweepStatus(stdout, len(shas), res)
|
||||||
|
if code := sweepExitCode(res); code != exitOK {
|
||||||
|
return code
|
||||||
|
}
|
||||||
|
|
||||||
|
if *sample <= 0 || len(res.Present) == 0 {
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
n := *sample
|
||||||
|
if n > len(res.Present) {
|
||||||
|
n = len(res.Present)
|
||||||
|
}
|
||||||
|
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||||
|
picks := rng.Perm(len(res.Present))[:n]
|
||||||
|
|
||||||
|
tmpDir, err := os.MkdirTemp("", "depot-verify-")
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot verify:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
|
for _, idx := range picks {
|
||||||
|
sha := res.Present[idx]
|
||||||
|
dest := filepath.Join(tmpDir, sha)
|
||||||
|
if err := backend.Get(context.Background(), sha, dest); err != nil {
|
||||||
|
fmt.Fprintf(stderr, "depot verify: sample %s: %v\n", sha, err)
|
||||||
|
return exitDrift
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||||
|
fs := flag.NewFlagSet("get", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(stderr)
|
||||||
|
target := fs.String("target", "", "cdn|bunny|local")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
positional := fs.Args()
|
||||||
|
if len(positional) != 2 {
|
||||||
|
fmt.Fprintln(stderr, "depot get: usage: depot get <sha> <dest> --target cdn|bunny|local")
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
sha, dest := positional[0], positional[1]
|
||||||
|
if !ValidSHA(sha) {
|
||||||
|
fmt.Fprintf(stderr, "depot get: invalid sha %q\n", sha)
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if *target == "" {
|
||||||
|
fmt.Fprintln(stderr, "depot get: --target is required")
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := LoadConfig(getenv)
|
||||||
|
backend, err := resolveBackend(*target, getenv, cfg)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot get:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := backend.Get(context.Background(), sha, dest); err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot get:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func runPull(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||||
|
fs := flag.NewFlagSet("pull", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(stderr)
|
||||||
|
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||||
|
dest := fs.String("dest", "", "local directory to pull blobs into")
|
||||||
|
target := fs.String("target", "", "cdn|bunny")
|
||||||
|
if err := fs.Parse(args); err != nil {
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if *target != "bunny" && *target != "cdn" {
|
||||||
|
fmt.Fprintln(stderr, "depot pull: --target must be bunny or cdn")
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
if *dest == "" {
|
||||||
|
fmt.Fprintln(stderr, "depot pull: --dest is required")
|
||||||
|
return exitUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := LoadConfig(getenv)
|
||||||
|
backend, err := NewBackend(*target, "", cfg)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot pull:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
|
||||||
|
shaSizes, err := ReferencedSHAs(*manifests)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot pull:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
shas := shaKeys(shaSizes)
|
||||||
|
|
||||||
|
// Split into "already present and correct locally" (skip) vs "needs a
|
||||||
|
// probe/download decision".
|
||||||
|
var present int
|
||||||
|
var toCheck []string
|
||||||
|
for _, sha := range shas {
|
||||||
|
destPath := filepath.Join(*dest, BlobKey(sha))
|
||||||
|
if localFileMatchesSHA(destPath, sha) {
|
||||||
|
present++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
toCheck = append(toCheck, sha)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sweep the source to distinguish "not there" (exit 1, listed) from
|
||||||
|
// "there, download it".
|
||||||
|
res, err := Sweep(context.Background(), backend, toCheck, cfg, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(stderr, "depot pull:", err)
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
mu sync.Mutex
|
||||||
|
pulled int
|
||||||
|
anyFail bool
|
||||||
|
)
|
||||||
|
work := func(sha string) {
|
||||||
|
destPath := filepath.Join(*dest, BlobKey(sha))
|
||||||
|
if err := backend.Get(context.Background(), sha, destPath); err != nil {
|
||||||
|
mu.Lock()
|
||||||
|
anyFail = true
|
||||||
|
mu.Unlock()
|
||||||
|
fmt.Fprintf(stderr, "pull: %s: %v\n", sha, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
pulled++
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
runWorkers(cfg.Jobs, res.Present, work)
|
||||||
|
|
||||||
|
// ponytail: unconfirmed source state (probe budget exhausted / flaky
|
||||||
|
// responses) is treated as a download failure rather than a third exit
|
||||||
|
// path; if that proves too coarse in practice, give pull its own
|
||||||
|
// unconfirmed accounting like status.
|
||||||
|
if len(res.Unconfirmed) > 0 {
|
||||||
|
anyFail = true
|
||||||
|
for _, sha := range res.Unconfirmed {
|
||||||
|
fmt.Fprintf(stderr, "pull: %s: unconfirmed at source\n", sha)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, sha := range res.Missing {
|
||||||
|
fmt.Fprintf(stdout, "missing %s\n", sha)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(stdout, "pulled=%d present=%d\n", pulled, present)
|
||||||
|
|
||||||
|
if anyFail {
|
||||||
|
return exitInternal
|
||||||
|
}
|
||||||
|
if len(res.Missing) > 0 {
|
||||||
|
return exitDrift
|
||||||
|
}
|
||||||
|
return exitOK
|
||||||
|
}
|
||||||
|
|
||||||
|
// localFileMatchesSHA reports whether path exists and hashes to sha.
|
||||||
|
func localFileMatchesSHA(path, sha string) bool {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
h := sha256.New()
|
||||||
|
if _, err := io.Copy(h, f); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%x", h.Sum(nil)) == sha
|
||||||
|
}
|
||||||
|
|
||||||
|
// runWorkers runs work(sha) for each sha in items using up to jobs goroutines.
|
||||||
|
func runWorkers(jobs int, items []string, work func(sha string)) {
|
||||||
|
if jobs < 1 {
|
||||||
|
jobs = 1
|
||||||
|
}
|
||||||
|
ch := make(chan string)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < jobs; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for sha := range ch {
|
||||||
|
work(sha)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
for _, sha := range items {
|
||||||
|
ch <- sha
|
||||||
|
}
|
||||||
|
close(ch)
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package depot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func testGetenv(vals map[string]string) func(string) string {
|
||||||
|
return func(k string) string { return vals[k] }
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunUsage(t *testing.T) {
|
||||||
|
t.Run("no args", func(t *testing.T) {
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run(nil, &out, &errb, testGetenv(nil))
|
||||||
|
if code != 64 {
|
||||||
|
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unknown subcommand", func(t *testing.T) {
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"bogus"}, &out, &errb, testGetenv(nil))
|
||||||
|
if code != 64 {
|
||||||
|
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("push --target cdn rejected", func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"push", "--manifests", dir, "--source", dir, "--target", "cdn"}, &out, &errb, testGetenv(nil))
|
||||||
|
if code != 64 {
|
||||||
|
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetInvalidSHA(t *testing.T) {
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
dir := t.TempDir()
|
||||||
|
code := Run([]string{"get", "not-a-sha", dir + "/dest", "--target", "local"}, &out, &errb, testGetenv(map[string]string{"DEPOT_DIR": dir}))
|
||||||
|
if code != 64 {
|
||||||
|
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoKeyStatusBunnyFailsClosedNoStdin(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeManifest(t, dir, shaOf("blob-a"), 5)
|
||||||
|
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"status", "--manifests", dir, "--target", "bunny"}, &out, &errb, testGetenv(nil))
|
||||||
|
if code != 70 {
|
||||||
|
t.Fatalf("expected 70, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
if !bytesContains(errb.String(), "BUNNY_STORAGE_HOST") {
|
||||||
|
t.Fatalf("expected stderr to mention BUNNY_STORAGE_HOST, got %s", errb.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatusExitCodes(t *testing.T) {
|
||||||
|
t.Run("drift", func(t *testing.T) {
|
||||||
|
f := newFakeBunny()
|
||||||
|
srv := httptest.NewServer(f.handler())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
sha := shaOf("missing-blob")
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeManifest(t, dir, sha, 5)
|
||||||
|
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"status", "--manifests", dir, "--target", "bunny"}, &out, &errb,
|
||||||
|
testGetenv(map[string]string{
|
||||||
|
"BUNNY_STORAGE_HOST": hostPort(srv),
|
||||||
|
"BUNNY_STORAGE_READ_PASSWORD": "readkey",
|
||||||
|
}))
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("expected 1, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
if !bytesContains(out.String(), "missing="+"1") {
|
||||||
|
t.Fatalf("expected missing=1 in output, got %s", out.String())
|
||||||
|
}
|
||||||
|
if !bytesContains(out.String(), "missing "+sha) {
|
||||||
|
t.Fatalf("expected missing line for sha, got %s", out.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unconfirmed-only", func(t *testing.T) {
|
||||||
|
f := newFakeBunny()
|
||||||
|
sha := shaOf("flaky-blob")
|
||||||
|
f.statusFor[sha] = 428
|
||||||
|
srv := httptest.NewServer(f.handler())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
writeManifest(t, dir, sha, 5)
|
||||||
|
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"status", "--manifests", dir, "--target", "bunny"}, &out, &errb,
|
||||||
|
testGetenv(map[string]string{
|
||||||
|
"BUNNY_STORAGE_HOST": hostPort(srv),
|
||||||
|
"BUNNY_STORAGE_READ_PASSWORD": "readkey",
|
||||||
|
"DEPOT_CONFIRM_RETRIES": "1",
|
||||||
|
}))
|
||||||
|
if code != 2 {
|
||||||
|
t.Fatalf("expected 2, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
if !bytesContains(out.String(), "unconfirmed="+"1") {
|
||||||
|
t.Fatalf("expected unconfirmed=1 in output, got %s", out.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func bytesContains(s, substr string) bool {
|
||||||
|
return bytes.Contains([]byte(s), []byte(substr))
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeManifest(t *testing.T, dir, sha string, size int64) {
|
||||||
|
t.Helper()
|
||||||
|
content := fmt.Sprintf("assets:\n - path: foo\n sha256: %s\n size: %d\n", sha, size)
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "manifest.yml"), []byte(content), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
|
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/app"
|
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/app"
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/buildinfo"
|
||||||
|
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/depot"
|
||||||
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/menu"
|
"git.westgate.pw/ShadowsOverWestgate/sow-tools/internal/menu"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -365,6 +366,12 @@ func runBuilder(name string, args []string, out, errw io.Writer) int {
|
|||||||
return exitOK
|
return exitOK
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if b.Name == "depot" && b.Wired {
|
||||||
|
// depot parses its own subcommand (status/push/verify/get/pull) and
|
||||||
|
// has its own richer exit contract (0/1/2/64/70), so it bypasses the
|
||||||
|
// b.Commands/delegateLegacy routing entirely.
|
||||||
|
return depot.Run(args, out, errw, os.Getenv)
|
||||||
|
}
|
||||||
if !b.Wired {
|
if !b.Wired {
|
||||||
// No migrated logic yet (depot): fail closed, never fake an artifact.
|
// No migrated logic yet (depot): fail closed, never fake an artifact.
|
||||||
fmt.Fprintf(errw, unwiredMsg, b.Name, b.Bin)
|
fmt.Fprintf(errw, unwiredMsg, b.Name, b.Bin)
|
||||||
|
|||||||
Reference in New Issue
Block a user