depot: replace --target local with --out DIR #67
@@ -29,9 +29,15 @@ aliases.
|
|||||||
| `assets` | `fix-mdl` | Lowercase `.mdl` names and rewrite model identity to match. |
|
| `assets` | `fix-mdl` | Lowercase `.mdl` names and rewrite model identity to match. |
|
||||||
| `assets` | `check-dupes` | Report runtime-name (basename) collisions across dirs. |
|
| `assets` | `check-dupes` | Report runtime-name (basename) collisions across dirs. |
|
||||||
| `assets` | `clean-dupes` | Delete clean-tree files whose basename collides with primary. |
|
| `assets` | `clean-dupes` | Delete clean-tree files whose basename collides with primary. |
|
||||||
|
| `depot` | `status` | Report referenced-vs-present drift against a backend. |
|
||||||
|
| `depot` | `push` | Upload referenced-but-absent blobs from a local depot. |
|
||||||
|
| `depot` | `verify` | Existence sweep plus sampled download re-hash. |
|
||||||
|
| `depot` | `get` | Fetch one blob with sha re-verify. |
|
||||||
|
| `depot` | `pull` | Incremental verified pull of every referenced blob. |
|
||||||
|
|
||||||
`crucible-depot` remains registered but unwired. It fails closed with exit `70`
|
`depot status` and `depot get` pick their backend either with `--out DIR`, a
|
||||||
and never emits placeholder artifacts.
|
depot tree on disk, or with `--target bunny|cdn`, a remote backend. The two
|
||||||
|
flags are mutually exclusive.
|
||||||
|
|
||||||
## Hidden compatibility aliases
|
## Hidden compatibility aliases
|
||||||
|
|
||||||
@@ -49,6 +55,7 @@ but omitted from routine help and the interactive menu:
|
|||||||
| `topdata` | `build-top-package` | `build-top-package` |
|
| `topdata` | `build-top-package` | `build-top-package` |
|
||||||
| `topdata` | `compare-topdata` | `compare-topdata` |
|
| `topdata` | `compare-topdata` | `compare-topdata` |
|
||||||
| `topdata` | `convert-topdata` | `convert-topdata` |
|
| `topdata` | `convert-topdata` | `convert-topdata` |
|
||||||
|
| `depot` | `--target local` | `--out $DEPOT_DIR` on `status`/`get` |
|
||||||
| `wiki` | `build-wiki` | `build-wiki` |
|
| `wiki` | `build-wiki` | `build-wiki` |
|
||||||
| `wiki` | `deploy-wiki` | `deploy-wiki` |
|
| `wiki` | `deploy-wiki` | `deploy-wiki` |
|
||||||
|
|
||||||
|
|||||||
+46
-28
@@ -11,6 +11,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -51,13 +52,15 @@ func Run(args []string, stdout, stderr io.Writer, getenv func(string) string) in
|
|||||||
|
|
||||||
func printRunUsage(w io.Writer) {
|
func printRunUsage(w io.Writer) {
|
||||||
fmt.Fprint(w, `usage:
|
fmt.Fprint(w, `usage:
|
||||||
depot status [--manifests DIR] [--source LOCAL_DEPOT] --target bunny|cdn|local
|
depot status [--manifests DIR] --target bunny|cdn | --out DIR
|
||||||
depot push [--manifests DIR] --source LOCAL_DEPOT --target bunny
|
depot push [--manifests DIR] --source LOCAL_DEPOT --target bunny
|
||||||
depot verify [--manifests DIR] --target bunny|cdn [--sample N]
|
depot verify [--manifests DIR] --target bunny|cdn [--sample N]
|
||||||
depot get <sha> <dest> --target cdn|bunny|local
|
depot get <sha> <dest> --target cdn|bunny | --out DIR
|
||||||
depot pull [--manifests DIR] --dest DIR --target cdn|bunny
|
depot pull [--manifests DIR] --dest DIR --target cdn|bunny
|
||||||
|
|
||||||
--target local uses the DEPOT_DIR environment variable as the local root.
|
--out DIR reads and writes a depot tree on disk at DIR instead of a remote
|
||||||
|
backend. --target names a remote backend only; the two are mutually exclusive.
|
||||||
|
--target local is a deprecated alias for --out $DEPOT_DIR.
|
||||||
`)
|
`)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,16 +68,27 @@ func printRunUsage(w io.Writer) {
|
|||||||
// than internal/backend failures (exit 70).
|
// than internal/backend failures (exit 70).
|
||||||
var errUsage = errors.New("usage error")
|
var errUsage = errors.New("usage error")
|
||||||
|
|
||||||
// resolveBackend builds the named backend, resolving "local" against DEPOT_DIR.
|
// resolveBackend picks the backend for a command that can work either against a
|
||||||
func resolveBackend(target string, getenv func(string) string, cfg Config) (Backend, error) {
|
// depot tree on disk (--out DIR) or a remote backend (--target bunny|cdn).
|
||||||
root := ""
|
// "--target local" stays as a deprecated alias for --out $DEPOT_DIR so older
|
||||||
if target == "local" {
|
// callers keep working.
|
||||||
root = getenv("DEPOT_DIR")
|
func resolveBackend(out, target string, getenv func(string) string, cfg Config) (Backend, error) {
|
||||||
|
switch {
|
||||||
|
case out != "" && target != "":
|
||||||
|
return nil, fmt.Errorf("--out and --target are mutually exclusive: %w", errUsage)
|
||||||
|
case out != "":
|
||||||
|
return NewBackend("local", out, cfg)
|
||||||
|
case target == "local":
|
||||||
|
root := getenv("DEPOT_DIR")
|
||||||
if root == "" {
|
if root == "" {
|
||||||
return nil, fmt.Errorf("--target local requires DEPOT_DIR to be set: %w", errUsage)
|
return nil, fmt.Errorf("--target local requires DEPOT_DIR to be set (prefer --out DIR): %w", errUsage)
|
||||||
}
|
}
|
||||||
|
return NewBackend("local", root, cfg)
|
||||||
|
case target == "":
|
||||||
|
return nil, fmt.Errorf("--out DIR or --target bunny|cdn is required: %w", errUsage)
|
||||||
|
default:
|
||||||
|
return NewBackend(target, "", cfg)
|
||||||
}
|
}
|
||||||
return NewBackend(target, root, cfg)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// backendErrExit maps a resolveBackend error onto the exit contract.
|
// backendErrExit maps a resolveBackend error onto the exit contract.
|
||||||
@@ -119,18 +133,18 @@ func runStatus(args []string, stdout, stderr io.Writer, getenv func(string) stri
|
|||||||
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
fs := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||||
fs.SetOutput(stderr)
|
fs.SetOutput(stderr)
|
||||||
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
manifests := fs.String("manifests", "assets", "directory of *.yml manifests")
|
||||||
_ = fs.String("source", "", "local depot root (unused for status target=local; see DEPOT_DIR)")
|
source := fs.String("source", "", "deprecated and ignored (use --out DIR for a depot tree on disk)")
|
||||||
target := fs.String("target", "", "bunny|cdn|local")
|
out := fs.String("out", "", "depot tree on disk to read instead of a remote backend")
|
||||||
|
target := fs.String("target", "", "bunny|cdn")
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
if *target == "" {
|
if *source != "" {
|
||||||
fmt.Fprintln(stderr, "depot status: --target is required")
|
fmt.Fprintln(stderr, "depot status: --source is ignored; use --out DIR")
|
||||||
return exitUsage
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := LoadConfig(getenv)
|
cfg := LoadConfig(getenv)
|
||||||
backend, err := resolveBackend(*target, getenv, cfg)
|
backend, err := resolveBackend(*out, *target, getenv, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, "depot status:", err)
|
fmt.Fprintln(stderr, "depot status:", err)
|
||||||
return backendErrExit(err)
|
return backendErrExit(err)
|
||||||
@@ -309,13 +323,21 @@ func runVerify(args []string, stdout, stderr io.Writer, getenv func(string) stri
|
|||||||
func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string) int {
|
||||||
fs := flag.NewFlagSet("get", flag.ContinueOnError)
|
fs := flag.NewFlagSet("get", flag.ContinueOnError)
|
||||||
fs.SetOutput(stderr)
|
fs.SetOutput(stderr)
|
||||||
target := fs.String("target", "", "cdn|bunny|local")
|
out := fs.String("out", "", "depot tree on disk to read instead of a remote backend")
|
||||||
if err := fs.Parse(args); err != nil {
|
target := fs.String("target", "", "cdn|bunny")
|
||||||
|
// Positionals come first in the documented usage, and Go's flag package
|
||||||
|
// stops parsing at the first one, so split them off by hand.
|
||||||
|
split := 0
|
||||||
|
for split < len(args) && !strings.HasPrefix(args[split], "-") {
|
||||||
|
split++
|
||||||
|
}
|
||||||
|
positional := args[:split]
|
||||||
|
if err := fs.Parse(args[split:]); err != nil {
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
positional := fs.Args()
|
positional = append(positional, fs.Args()...)
|
||||||
if len(positional) != 2 {
|
if len(positional) != 2 {
|
||||||
fmt.Fprintln(stderr, "depot get: usage: depot get <sha> <dest> --target cdn|bunny|local")
|
fmt.Fprintln(stderr, "depot get: usage: depot get <sha> <dest> --target cdn|bunny | --out DIR")
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
sha, dest := positional[0], positional[1]
|
sha, dest := positional[0], positional[1]
|
||||||
@@ -323,13 +345,9 @@ func runGet(args []string, stdout, stderr io.Writer, getenv func(string) string)
|
|||||||
fmt.Fprintf(stderr, "depot get: invalid sha %q\n", sha)
|
fmt.Fprintf(stderr, "depot get: invalid sha %q\n", sha)
|
||||||
return exitUsage
|
return exitUsage
|
||||||
}
|
}
|
||||||
if *target == "" {
|
|
||||||
fmt.Fprintln(stderr, "depot get: --target is required")
|
|
||||||
return exitUsage
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := LoadConfig(getenv)
|
cfg := LoadConfig(getenv)
|
||||||
backend, err := resolveBackend(*target, getenv, cfg)
|
backend, err := resolveBackend(*out, *target, getenv, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, "depot get:", err)
|
fmt.Fprintln(stderr, "depot get:", err)
|
||||||
return backendErrExit(err)
|
return backendErrExit(err)
|
||||||
|
|||||||
@@ -54,6 +54,91 @@ func TestRunUsage(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOutFlag(t *testing.T) {
|
||||||
|
t.Run("status --out reads the given tree", func(t *testing.T) {
|
||||||
|
sha := shaOf("out-blob")
|
||||||
|
manifests := t.TempDir()
|
||||||
|
writeManifest(t, manifests, sha, 8)
|
||||||
|
depotDir := t.TempDir()
|
||||||
|
writeBlob(t, depotDir, sha, "out-blob")
|
||||||
|
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"status", "--manifests", manifests, "--out", depotDir}, &out, &errb, testGetenv(nil))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
if !bytesContains(out.String(), "present=1") {
|
||||||
|
t.Fatalf("expected present=1, got %s", out.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("get --out fetches from the given tree", func(t *testing.T) {
|
||||||
|
sha := shaOf("get-blob")
|
||||||
|
depotDir := t.TempDir()
|
||||||
|
writeBlob(t, depotDir, sha, "get-blob")
|
||||||
|
dest := filepath.Join(t.TempDir(), "fetched")
|
||||||
|
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"get", sha, dest, "--out", depotDir}, &out, &errb, testGetenv(nil))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("expected 0, got %d (stderr=%s)", code, errb.String())
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(dest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(got) != "get-blob" {
|
||||||
|
t.Fatalf("got %q", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("--out with any --target is rejected", func(t *testing.T) {
|
||||||
|
for _, target := range []string{"bunny", "cdn", "local"} {
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"status", "--manifests", t.TempDir(), "--out", t.TempDir(), "--target", target}, &out, &errb, testGetenv(nil))
|
||||||
|
if code != 64 {
|
||||||
|
t.Fatalf("--target %s: expected 64, got %d (stderr=%s)", target, code, errb.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("get accepts flags before the positionals", func(t *testing.T) {
|
||||||
|
sha := shaOf("flags-first-blob")
|
||||||
|
depotDir := t.TempDir()
|
||||||
|
writeBlob(t, depotDir, sha, "flags-first-blob")
|
||||||
|
dest := filepath.Join(t.TempDir(), "fetched")
|
||||||
|
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"get", "--out", depotDir, sha, dest}, &out, &errb, testGetenv(nil))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("expected 0, got %d (stderr=%s)", code, errb.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("neither --out nor --target is rejected", func(t *testing.T) {
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"status", "--manifests", t.TempDir()}, &out, &errb, testGetenv(nil))
|
||||||
|
if code != 64 {
|
||||||
|
t.Fatalf("expected 64, got %d (stderr=%s)", code, errb.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("--target local still works as a hidden alias", func(t *testing.T) {
|
||||||
|
sha := shaOf("alias-blob")
|
||||||
|
manifests := t.TempDir()
|
||||||
|
writeManifest(t, manifests, sha, 11)
|
||||||
|
depotDir := t.TempDir()
|
||||||
|
writeBlob(t, depotDir, sha, "alias-blob")
|
||||||
|
|
||||||
|
var out, errb bytes.Buffer
|
||||||
|
code := Run([]string{"status", "--manifests", manifests, "--target", "local"}, &out, &errb,
|
||||||
|
testGetenv(map[string]string{"DEPOT_DIR": depotDir}))
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("expected 0, got %d (stdout=%s stderr=%s)", code, out.String(), errb.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestGetInvalidSHA(t *testing.T) {
|
func TestGetInvalidSHA(t *testing.T) {
|
||||||
var out, errb bytes.Buffer
|
var out, errb bytes.Buffer
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
@@ -172,6 +257,17 @@ func bytesContains(s, substr string) bool {
|
|||||||
return bytes.Contains([]byte(s), []byte(substr))
|
return bytes.Contains([]byte(s), []byte(substr))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func writeBlob(t *testing.T, root, sha, content string) {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(root, BlobKey(sha))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeManifest(t *testing.T, dir, sha string, size int64) {
|
func writeManifest(t *testing.T, dir, sha string, size int64) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
content := fmt.Sprintf("assets:\n - path: foo\n sha256: %s\n size: %d\n", sha, size)
|
content := fmt.Sprintf("assets:\n - path: foo\n sha256: %s\n size: %d\n", sha, size)
|
||||||
|
|||||||
@@ -94,12 +94,12 @@ var Registry = []Builder{
|
|||||||
{
|
{
|
||||||
Name: "depot",
|
Name: "depot",
|
||||||
Bin: "crucible-depot",
|
Bin: "crucible-depot",
|
||||||
Summary: "content-addressed asset depot (local/cdn/bunny)",
|
Summary: "content-addressed asset depot (disk/cdn/bunny)",
|
||||||
Commands: []Command{
|
Commands: []Command{
|
||||||
{Name: "status", Summary: "report referenced-vs-present drift against a backend", Usage: "crucible depot status [--manifests DIR] [--source DIR] --target bunny|cdn|local"},
|
{Name: "status", Summary: "report referenced-vs-present drift against a backend", Usage: "crucible depot status [--manifests DIR] --target bunny|cdn | --out DIR"},
|
||||||
{Name: "push", Summary: "upload referenced-but-absent blobs from a local depot", Usage: "crucible depot push [--manifests DIR] --source DIR --target bunny"},
|
{Name: "push", Summary: "upload referenced-but-absent blobs from a local depot", Usage: "crucible depot push [--manifests DIR] --source DIR --target bunny"},
|
||||||
{Name: "verify", Summary: "existence sweep plus sampled download re-hash", Usage: "crucible depot verify [--manifests DIR] --target bunny|cdn [--sample N]"},
|
{Name: "verify", Summary: "existence sweep plus sampled download re-hash", Usage: "crucible depot verify [--manifests DIR] --target bunny|cdn [--sample N]"},
|
||||||
{Name: "get", Summary: "fetch one blob with sha re-verify", Usage: "crucible depot get <sha> <dest> --target cdn|bunny|local"},
|
{Name: "get", Summary: "fetch one blob with sha re-verify", Usage: "crucible depot get <sha> <dest> --target cdn|bunny | --out DIR"},
|
||||||
{Name: "pull", Summary: "incremental verified pull of every referenced blob", Usage: "crucible depot pull [--manifests DIR] --dest DIR --target cdn|bunny"},
|
{Name: "pull", Summary: "incremental verified pull of every referenced blob", Usage: "crucible depot pull [--manifests DIR] --dest DIR --target cdn|bunny"},
|
||||||
},
|
},
|
||||||
Wired: true,
|
Wired: true,
|
||||||
|
|||||||
Reference in New Issue
Block a user