crucible build parity #14
+15
-1
@@ -1625,6 +1625,12 @@ func runBuildTopData(ctx context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if opts.SkipLFS {
|
||||||
|
// ponytail: bridge --skip-lfs flag to env rather than threading a new param
|
||||||
|
// through packageBuiltTopData; CRUCIBLE_SKIP_LFS is already checked there.
|
||||||
|
os.Setenv("CRUCIBLE_SKIP_LFS", "1") //nolint:errcheck
|
||||||
|
}
|
||||||
|
|
||||||
console := newTopdataConsole(ctx, p, "build-topdata")
|
console := newTopdataConsole(ctx, p, "build-topdata")
|
||||||
spin.configure(ctx.stderr, console.spinnerEnabled)
|
spin.configure(ctx.stderr, console.spinnerEnabled)
|
||||||
spin.start("Build Topdata: starting")
|
spin.start("Build Topdata: starting")
|
||||||
@@ -1653,6 +1659,12 @@ func runBuildTopPackage(ctx context) error {
|
|||||||
return fmt.Errorf("--wiki is not supported with build-top-package; use build-topdata when wiki generation is required")
|
return fmt.Errorf("--wiki is not supported with build-top-package; use build-topdata when wiki generation is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if opts.SkipLFS {
|
||||||
|
// ponytail: bridge --skip-lfs flag to env rather than threading a new param
|
||||||
|
// through packageBuiltTopData; CRUCIBLE_SKIP_LFS is already checked there.
|
||||||
|
os.Setenv("CRUCIBLE_SKIP_LFS", "1") //nolint:errcheck
|
||||||
|
}
|
||||||
|
|
||||||
console := newTopdataConsole(ctx, p, "build-top-package")
|
console := newTopdataConsole(ctx, p, "build-top-package")
|
||||||
spin.configure(ctx.stderr, console.spinnerEnabled)
|
spin.configure(ctx.stderr, console.spinnerEnabled)
|
||||||
spin.start("Build Top Package: starting")
|
spin.start("Build Top Package: starting")
|
||||||
@@ -1674,8 +1686,10 @@ func parseBuildTopDataArgs(commandName string, args []string) (topdata.BuildAndP
|
|||||||
opts.Force = true
|
opts.Force = true
|
||||||
case "--wiki":
|
case "--wiki":
|
||||||
opts.BuildWiki = true
|
opts.BuildWiki = true
|
||||||
|
case "--skip-lfs":
|
||||||
|
opts.SkipLFS = true
|
||||||
case "-h", "--help":
|
case "-h", "--help":
|
||||||
return opts, fmt.Errorf("usage: %s [--force] [--wiki]", commandName)
|
return opts, fmt.Errorf("usage: %s [--force] [--wiki] [--skip-lfs]", commandName)
|
||||||
default:
|
default:
|
||||||
return opts, fmt.Errorf("unknown %s argument %q", commandName, arg)
|
return opts, fmt.Errorf("unknown %s argument %q", commandName, arg)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package topdata
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHasLFSPointerStubsDetectsStub(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.MkdirAll(filepath.Join(root, "a"), 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir: %v", err)
|
||||||
|
}
|
||||||
|
stub := "version https://git-lfs.github.com/spec/v1\noid sha256:abc\nsize 12345\n"
|
||||||
|
writeFile(t, filepath.Join(root, "a", "model.mdl"), stub)
|
||||||
|
writeFile(t, filepath.Join(root, "a", "real.txt"), "not a pointer, real bytes here")
|
||||||
|
|
||||||
|
if !hasLFSPointerStubs(root) {
|
||||||
|
t.Fatal("expected stub detection to be true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHasLFSPointerStubsNoStub(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeFile(t, filepath.Join(root, "real.mdl"), "binary-ish content without a pointer header")
|
||||||
|
if hasLFSPointerStubs(root) {
|
||||||
|
t.Fatal("expected no stub detected")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package topdata
|
package topdata
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -75,6 +76,7 @@ func BuildAndPackage(p *project.Project, progress func(string)) (PackageResult,
|
|||||||
type BuildAndPackageOptions struct {
|
type BuildAndPackageOptions struct {
|
||||||
Force bool
|
Force bool
|
||||||
BuildWiki bool
|
BuildWiki bool
|
||||||
|
SkipLFS bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func BuildAndPackageWithOptions(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) {
|
func BuildAndPackageWithOptions(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) {
|
||||||
@@ -123,6 +125,9 @@ func packageBuiltTopData(p *project.Project, nativeResult BuildResult, progress
|
|||||||
if progress == nil {
|
if progress == nil {
|
||||||
progress = func(string) {}
|
progress = func(string) {}
|
||||||
}
|
}
|
||||||
|
if err := materializeAssetLFS(p, progress); err != nil {
|
||||||
|
return PackageResult{}, err
|
||||||
|
}
|
||||||
outputHAK := p.TopDataPackageHAKPath()
|
outputHAK := p.TopDataPackageHAKPath()
|
||||||
outputTLK := p.TopDataPackageTLKPath()
|
outputTLK := p.TopDataPackageTLKPath()
|
||||||
|
|
||||||
@@ -711,3 +716,80 @@ func countCompiledFiles(dir, extension string) (int, time.Time, error) {
|
|||||||
}
|
}
|
||||||
return count, oldest, nil
|
return count, oldest, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// materializeAssetLFS ensures the topdata asset tree holds real bytes, not
|
||||||
|
// Git-LFS pointer stubs, before the hak is packed. A bare CI checkout does not
|
||||||
|
// smudge LFS, so a direct Crucible build would otherwise ship ~130-byte stubs.
|
||||||
|
// It uses the clone's own credentials (R1) and hard-fails if stubs survive the
|
||||||
|
// pull. Set CRUCIBLE_SKIP_LFS=1 to opt out (maintain-tree regenerates the data/
|
||||||
|
// tree and discards the package, so it must not hard-depend on LFS).
|
||||||
|
func materializeAssetLFS(p *project.Project, progress func(string)) error {
|
||||||
|
if progress == nil {
|
||||||
|
progress = func(string) {}
|
||||||
|
}
|
||||||
|
assetsDir := filepath.Join(p.TopDataSourceDir(), "assets")
|
||||||
|
info, err := os.Stat(assetsDir)
|
||||||
|
if err != nil || !info.IsDir() {
|
||||||
|
return nil // no asset tree to materialize
|
||||||
|
}
|
||||||
|
if !hasLFSPointerStubs(assetsDir) {
|
||||||
|
return nil // already real bytes (or repo does not use LFS)
|
||||||
|
}
|
||||||
|
if isTruthyEnv(os.Getenv("CRUCIBLE_SKIP_LFS")) {
|
||||||
|
progress("CRUCIBLE_SKIP_LFS set: leaving Git-LFS pointer stubs unmaterialized (package will be discarded)")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := gitOutput(p.Root, "lfs", "version"); err != nil {
|
||||||
|
return fmt.Errorf("asset tree has Git-LFS pointer stubs but git-lfs is unavailable; install git-lfs or enter the nix devshell: %w", err)
|
||||||
|
}
|
||||||
|
progress("Materializing Git-LFS assets (git lfs pull)...")
|
||||||
|
// A fresh clone may lack the lfs smudge/clean filters; install them locally
|
||||||
|
// first (scoped to this repo's .git/config, no host mutation). Idempotent.
|
||||||
|
if _, err := gitOutput(p.Root, "lfs", "install", "--local"); err != nil {
|
||||||
|
return fmt.Errorf("git lfs install --local failed: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := gitOutput(p.Root, "lfs", "pull"); err != nil {
|
||||||
|
return fmt.Errorf("git lfs pull failed: %w", err)
|
||||||
|
}
|
||||||
|
if hasLFSPointerStubs(assetsDir) {
|
||||||
|
return fmt.Errorf("refusing to build from pointer stubs: Git-LFS pointer stubs remain under %s after git lfs pull", assetsDir)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasLFSPointerStubs reports whether any small text file under root is a Git-LFS
|
||||||
|
// pointer. Real assets are larger than any pointer stub, so the size gate keeps
|
||||||
|
// this cheap on a tree of binaries.
|
||||||
|
func hasLFSPointerStubs(root string) bool {
|
||||||
|
found := false
|
||||||
|
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||||
|
if err != nil || d.IsDir() || found {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
info, ierr := d.Info()
|
||||||
|
if ierr != nil || info.Size() == 0 || info.Size() > 1024 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
f, oerr := os.Open(path)
|
||||||
|
if oerr != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
buf := make([]byte, 256)
|
||||||
|
n, _ := f.Read(buf)
|
||||||
|
if bytes.Contains(buf[:n], []byte("git-lfs.github.com/spec/v1")) {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
func isTruthyEnv(v string) bool {
|
||||||
|
switch strings.TrimSpace(strings.ToLower(v)) {
|
||||||
|
case "1", "true", "yes", "on":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user