From 9dbb533241048ff5e9cfc1c44c5769179ba1a302 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Tue, 14 Apr 2026 11:40:38 +0200 Subject: [PATCH] Parse LFS Pointers Quickly sow-tools no longer relies on git lfs ls-files --json for planning. It now parses the checked-out LFS pointer files directly, so split sizing and content_hash use the real LFS oid and size. --- internal/pipeline/build.go | 149 +++++++++++++++++++++++++++-- internal/pipeline/pipeline_test.go | 11 ++- 2 files changed, 150 insertions(+), 10 deletions(-) diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index 23d933a..854a58c 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -54,6 +54,13 @@ type assetResource struct { Resource erf.Resource Size int64 ChangedAt time.Time + ContentID string +} + +type lfsAssetInfo struct { + OID string + Size int64 + ObjectPath string } type hakChunk struct { @@ -179,7 +186,7 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo } progressf(progress, "Collecting asset resources...") - assetResources, err := collectAssetResources(p) + assetResources, err := collectAssetResources(p, writeArchives) if err != nil { return BuildResult{}, err } @@ -556,12 +563,16 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) { } } -func collectAssetResources(p *project.Project) ([]assetResource, error) { +func collectAssetResources(p *project.Project, requireContent bool) ([]assetResource, error) { var hakResources []assetResource assetChangedAt, err := collectGitAssetChangeTimes(p) if err != nil { return nil, err } + lfsAssets, err := collectLFSAssetInfo(p) + if err != nil { + return nil, err + } assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir()) if err != nil { return nil, fmt.Errorf("resolve assets dir relative path: %w", err) @@ -570,7 +581,7 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) { for _, rel := range p.Inventory.AssetFiles { abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)) - resource, err := pathResource(abs) + resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent) if err != nil { return nil, err } @@ -585,9 +596,10 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) { } hakResources = append(hakResources, assetResource{ Rel: rel, - Resource: resource, - Size: erf.ArchiveSize([]erf.Resource{resource}), + Resource: resourceInfo.Resource, + Size: erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}), ChangedAt: changedAt, + ContentID: resourceInfo.ContentID, }) } @@ -597,6 +609,126 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) { return hakResources, nil } +func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) { + result := make(map[string]lfsAssetInfo) + for _, rel := range p.Inventory.AssetFiles { + rel = filepath.ToSlash(rel) + abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel)) + info, ok, err := parseLFSPointerFile(abs, p.Root) + if err != nil { + return nil, err + } + if ok { + result[rel] = info + } + } + + return result, nil +} + +func parseLFSPointerFile(path, repoRoot string) (lfsAssetInfo, bool, error) { + entry, err := os.Lstat(path) + if err != nil { + return lfsAssetInfo{}, false, fmt.Errorf("stat %s: %w", path, err) + } + if entry.Mode()&os.ModeSymlink != 0 || entry.Size() > 1024 { + return lfsAssetInfo{}, false, nil + } + + raw, err := os.ReadFile(path) + if err != nil { + return lfsAssetInfo{}, false, fmt.Errorf("read %s: %w", path, err) + } + text := strings.ReplaceAll(string(raw), "\r\n", "\n") + if !strings.HasPrefix(text, "version https://git-lfs.github.com/spec/v1\n") { + return lfsAssetInfo{}, false, nil + } + + info := lfsAssetInfo{} + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "oid sha256:"): + info.OID = strings.TrimSpace(strings.TrimPrefix(line, "oid sha256:")) + case strings.HasPrefix(line, "size "): + size, parseErr := strconv.ParseInt(strings.TrimSpace(strings.TrimPrefix(line, "size ")), 10, 64) + if parseErr != nil { + return lfsAssetInfo{}, false, fmt.Errorf("parse LFS pointer size in %s: %w", path, parseErr) + } + info.Size = size + } + } + if info.OID == "" || info.Size <= 0 { + return lfsAssetInfo{}, false, nil + } + if len(info.OID) >= 4 { + info.ObjectPath = filepath.Join(repoRoot, ".git", "lfs", "objects", info.OID[:2], info.OID[2:4], info.OID) + } + return info, true, nil +} + +func assetResourceFromPath(path, rel string, lfsInfo lfsAssetInfo, requireContent bool) (assetResource, error) { + if lfsInfo.OID != "" { + resource, err := lfsPathResource(path, lfsInfo, requireContent) + if err != nil { + return assetResource{}, err + } + return assetResource{ + Rel: rel, + Resource: resource, + ContentID: "lfs:" + lfsInfo.OID, + }, nil + } + + resource, err := pathResource(path) + if err != nil { + return assetResource{}, err + } + fileHash, err := fileSHA256(path) + if err != nil { + return assetResource{}, err + } + return assetResource{ + Rel: rel, + Resource: resource, + ContentID: "file:" + fileHash, + }, nil +} + +func lfsPathResource(path string, info lfsAssetInfo, requireContent bool) (erf.Resource, error) { + extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") + resourceType, ok := erf.HAKResourceTypeForExtension(extension) + if !ok { + return erf.Resource{}, fmt.Errorf("unsupported HAK resource extension %q", filepath.Ext(path)) + } + name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))) + sourcePath := "" + if requireContent { + if info.ObjectPath == "" { + return erf.Resource{}, fmt.Errorf("missing downloaded LFS object for %s", path) + } + if _, err := os.Stat(info.ObjectPath); err != nil { + return erf.Resource{}, fmt.Errorf("missing downloaded LFS object for %s: %w", path, err) + } + sourcePath = info.ObjectPath + } + return erf.Resource{ + Name: name, + Type: resourceType, + SourcePath: sourcePath, + Size: info.Size, + }, nil +} + +func fileSHA256(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read %s: %w", path, err) + } + sum := sha256.Sum256(raw) + return fmt.Sprintf("%x", sum[:]), nil +} + func collectGitAssetChangeTimes(p *project.Project) (map[string]time.Time, error) { result := make(map[string]time.Time, len(p.Inventory.AssetFiles)) if len(p.Inventory.AssetFiles) == 0 { @@ -960,9 +1092,8 @@ func chunkContentHash(chunk hakChunk) string { fmt.Fprintf(sum, "asset:%s\n", asset.Rel) fmt.Fprintf(sum, "resref:%s\n", asset.Resource.Name) fmt.Fprintf(sum, "restype:%d\n", asset.Resource.Type) - fmt.Fprintf(sum, "datasize:%d\n", len(asset.Resource.Data)) - _, _ = sum.Write(asset.Resource.Data) - _, _ = sum.Write([]byte{'\n'}) + fmt.Fprintf(sum, "datasize:%d\n", asset.Resource.Size) + fmt.Fprintf(sum, "content:%s\n", asset.ContentID) } return fmt.Sprintf("%x", sum.Sum(nil)) } @@ -1086,7 +1217,7 @@ func plannedModuleHAKOrder(p *project.Project) ([]string, error) { return append([]string(nil), manifest.ModuleHAKs...), nil } - assetResources, err := collectAssetResources(p) + assetResources, err := collectAssetResources(p, false) if err != nil { return nil, err } diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index f76a0bd..aa0ea9e 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -1831,7 +1831,7 @@ func TestPlanHAKChunksPutsNewestAssetsInLastChunk(t *testing.T) { t.Fatalf("scan: %v", err) } - assets, err := collectAssetResources(p) + assets, err := collectAssetResources(p, false) if err != nil { t.Fatalf("collect asset resources: %v", err) } @@ -2109,6 +2109,12 @@ func TestCompareFailsWhenBuildIsStale(t *testing.T) { if _, err := BuildModule(p); err != nil { t.Fatalf("build module: %v", err) } + modulePath := filepath.Join(root, "build", "testmod.mod") + moduleInfo, err := os.Stat(modulePath) + if err != nil { + t.Fatalf("stat built module: %v", err) + } + staleSourceTime := moduleInfo.ModTime().Add(2 * time.Second) mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ "file_type": "IFO ", "file_version": "V3.2", @@ -2124,6 +2130,9 @@ func TestCompareFailsWhenBuildIsStale(t *testing.T) { } } `) + if err := os.Chtimes(filepath.Join(root, "src", "module", "module.ifo.json"), staleSourceTime, staleSourceTime); err != nil { + t.Fatalf("chtimes updated source: %v", err) + } if err := p.Scan(); err != nil { t.Fatalf("rescan: %v", err) }