From bdac6f29f07835319b490fc5009f32587a52a4c2 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Sun, 21 Jun 2026 14:50:47 +0200 Subject: [PATCH] feat(topdata): materialize Git-LFS asset bytes inside Crucible before packing Adds materializeAssetLFS + hasLFSPointerStubs helpers to top_package.go. Called at the top of packageBuiltTopData so bare CI checkouts never ship ~130-byte LFS pointer stubs. Opt-out via CRUCIBLE_SKIP_LFS env or the new --skip-lfs flag on build-topdata / build-top-package. Co-Authored-By: Claude Sonnet 4.6 --- internal/app/app.go | 16 ++++- internal/topdata/lfs_materialize_test.go | 29 +++++++++ internal/topdata/top_package.go | 82 ++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 internal/topdata/lfs_materialize_test.go diff --git a/internal/app/app.go b/internal/app/app.go index 0a606bb..e0bd1fb 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1625,6 +1625,12 @@ func runBuildTopData(ctx context) error { return err } + if opts.SkipLFS { + // ponytail: bridge --skip-lfs flag to env rather than threading a new param + // through packageBuiltTopData; CRUCIBLE_SKIP_LFS is already checked there. + os.Setenv("CRUCIBLE_SKIP_LFS", "1") //nolint:errcheck + } + console := newTopdataConsole(ctx, p, "build-topdata") spin.configure(ctx.stderr, console.spinnerEnabled) spin.start("Build Topdata: starting") @@ -1653,6 +1659,12 @@ func runBuildTopPackage(ctx context) error { return fmt.Errorf("--wiki is not supported with build-top-package; use build-topdata when wiki generation is required") } + if opts.SkipLFS { + // ponytail: bridge --skip-lfs flag to env rather than threading a new param + // through packageBuiltTopData; CRUCIBLE_SKIP_LFS is already checked there. + os.Setenv("CRUCIBLE_SKIP_LFS", "1") //nolint:errcheck + } + console := newTopdataConsole(ctx, p, "build-top-package") spin.configure(ctx.stderr, console.spinnerEnabled) spin.start("Build Top Package: starting") @@ -1674,8 +1686,10 @@ func parseBuildTopDataArgs(commandName string, args []string) (topdata.BuildAndP opts.Force = true case "--wiki": opts.BuildWiki = true + case "--skip-lfs": + opts.SkipLFS = true case "-h", "--help": - return opts, fmt.Errorf("usage: %s [--force] [--wiki]", commandName) + return opts, fmt.Errorf("usage: %s [--force] [--wiki] [--skip-lfs]", commandName) default: return opts, fmt.Errorf("unknown %s argument %q", commandName, arg) } diff --git a/internal/topdata/lfs_materialize_test.go b/internal/topdata/lfs_materialize_test.go new file mode 100644 index 0000000..089465c --- /dev/null +++ b/internal/topdata/lfs_materialize_test.go @@ -0,0 +1,29 @@ +package topdata + +import ( + "os" + "path/filepath" + "testing" +) + +func TestHasLFSPointerStubsDetectsStub(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "a"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + stub := "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 12345\n" + writeFile(t, filepath.Join(root, "a", "model.mdl"), stub) + writeFile(t, filepath.Join(root, "a", "real.txt"), "not a pointer, real bytes here") + + if !hasLFSPointerStubs(root) { + t.Fatal("expected stub detection to be true") + } +} + +func TestHasLFSPointerStubsNoStub(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "real.mdl"), "binary-ish content without a pointer header") + if hasLFSPointerStubs(root) { + t.Fatal("expected no stub detected") + } +} diff --git a/internal/topdata/top_package.go b/internal/topdata/top_package.go index 76d9a85..cf90645 100644 --- a/internal/topdata/top_package.go +++ b/internal/topdata/top_package.go @@ -1,6 +1,7 @@ package topdata import ( + "bytes" "errors" "fmt" "io" @@ -75,6 +76,7 @@ func BuildAndPackage(p *project.Project, progress func(string)) (PackageResult, type BuildAndPackageOptions struct { Force bool BuildWiki bool + SkipLFS bool } func BuildAndPackageWithOptions(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) { @@ -123,6 +125,9 @@ func packageBuiltTopData(p *project.Project, nativeResult BuildResult, progress if progress == nil { progress = func(string) {} } + if err := materializeAssetLFS(p, progress); err != nil { + return PackageResult{}, err + } outputHAK := p.TopDataPackageHAKPath() outputTLK := p.TopDataPackageTLKPath() @@ -711,3 +716,80 @@ func countCompiledFiles(dir, extension string) (int, time.Time, error) { } return count, oldest, nil } + +// materializeAssetLFS ensures the topdata asset tree holds real bytes, not +// Git-LFS pointer stubs, before the hak is packed. A bare CI checkout does not +// smudge LFS, so a direct Crucible build would otherwise ship ~130-byte stubs. +// It uses the clone's own credentials (R1) and hard-fails if stubs survive the +// pull. Set CRUCIBLE_SKIP_LFS=1 to opt out (maintain-tree regenerates the data/ +// tree and discards the package, so it must not hard-depend on LFS). +func materializeAssetLFS(p *project.Project, progress func(string)) error { + if progress == nil { + progress = func(string) {} + } + assetsDir := filepath.Join(p.TopDataSourceDir(), "assets") + info, err := os.Stat(assetsDir) + if err != nil || !info.IsDir() { + return nil // no asset tree to materialize + } + if !hasLFSPointerStubs(assetsDir) { + return nil // already real bytes (or repo does not use LFS) + } + if isTruthyEnv(os.Getenv("CRUCIBLE_SKIP_LFS")) { + progress("CRUCIBLE_SKIP_LFS set: leaving Git-LFS pointer stubs unmaterialized (package will be discarded)") + return nil + } + if _, err := gitOutput(p.Root, "lfs", "version"); err != nil { + return fmt.Errorf("asset tree has Git-LFS pointer stubs but git-lfs is unavailable; install git-lfs or enter the nix devshell: %w", err) + } + progress("Materializing Git-LFS assets (git lfs pull)...") + // A fresh clone may lack the lfs smudge/clean filters; install them locally + // first (scoped to this repo's .git/config, no host mutation). Idempotent. + if _, err := gitOutput(p.Root, "lfs", "install", "--local"); err != nil { + return fmt.Errorf("git lfs install --local failed: %w", err) + } + if _, err := gitOutput(p.Root, "lfs", "pull"); err != nil { + return fmt.Errorf("git lfs pull failed: %w", err) + } + if hasLFSPointerStubs(assetsDir) { + return fmt.Errorf("refusing to build from pointer stubs: Git-LFS pointer stubs remain under %s after git lfs pull", assetsDir) + } + return nil +} + +// hasLFSPointerStubs reports whether any small text file under root is a Git-LFS +// pointer. Real assets are larger than any pointer stub, so the size gate keeps +// this cheap on a tree of binaries. +func hasLFSPointerStubs(root string) bool { + found := false + _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || found { + return nil + } + info, ierr := d.Info() + if ierr != nil || info.Size() == 0 || info.Size() > 1024 { + return nil + } + f, oerr := os.Open(path) + if oerr != nil { + return nil + } + defer f.Close() + buf := make([]byte, 256) + n, _ := f.Read(buf) + if bytes.Contains(buf[:n], []byte("git-lfs.github.com/spec/v1")) { + found = true + } + return nil + }) + return found +} + +func isTruthyEnv(v string) bool { + switch strings.TrimSpace(strings.ToLower(v)) { + case "1", "true", "yes", "on": + return true + default: + return false + } +}