Parse LFS Pointers Quickly

sow-tools no longer relies on git lfs ls-files --json for planning. It now parses the checked-out LFS pointer files directly, so split sizing and content_hash use the real LFS oid and size.
2026-04-14 11:40:38 +02:00
parent bf7c844f2e
commit 9dbb533241
2 changed files with 150 additions and 10 deletions
@@ -54,6 +54,13 @@ type assetResource struct {
 	Resource  erf.Resource
 	Size      int64
 	ChangedAt time.Time
+	ContentID string
+}
+
+type lfsAssetInfo struct {
+	OID        string
+	Size       int64
+	ObjectPath string
 }

 type hakChunk struct {
@@ -179,7 +186,7 @@ func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bo
 	}

 	progressf(progress, "Collecting asset resources...")
-	assetResources, err := collectAssetResources(p)
+	assetResources, err := collectAssetResources(p, writeArchives)
 	if err != nil {
 		return BuildResult{}, err
 	}
@@ -556,12 +563,16 @@ func fieldStringValueForScripts(value gff.Value) (string, bool) {
 	}
 }

-func collectAssetResources(p *project.Project) ([]assetResource, error) {
+func collectAssetResources(p *project.Project, requireContent bool) ([]assetResource, error) {
 	var hakResources []assetResource
 	assetChangedAt, err := collectGitAssetChangeTimes(p)
 	if err != nil {
 		return nil, err
 	}
+	lfsAssets, err := collectLFSAssetInfo(p)
+	if err != nil {
+		return nil, err
+	}
 	assetsRelPath, err := filepath.Rel(p.Root, p.AssetsDir())
 	if err != nil {
 		return nil, fmt.Errorf("resolve assets dir relative path: %w", err)
@@ -570,7 +581,7 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) {

 	for _, rel := range p.Inventory.AssetFiles {
 		abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
-		resource, err := pathResource(abs)
+		resourceInfo, err := assetResourceFromPath(abs, filepath.ToSlash(rel), lfsAssets[filepath.ToSlash(rel)], requireContent)
 		if err != nil {
 			return nil, err
 		}
@@ -585,9 +596,10 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) {
 		}
 		hakResources = append(hakResources, assetResource{
 			Rel:       rel,
-			Resource:  resource,
-			Size:      erf.ArchiveSize([]erf.Resource{resource}),
+			Resource:  resourceInfo.Resource,
+			Size:      erf.ArchiveSize([]erf.Resource{resourceInfo.Resource}),
 			ChangedAt: changedAt,
+			ContentID: resourceInfo.ContentID,
 		})
 	}

@@ -597,6 +609,126 @@ func collectAssetResources(p *project.Project) ([]assetResource, error) {
 	return hakResources, nil
 }

+func collectLFSAssetInfo(p *project.Project) (map[string]lfsAssetInfo, error) {
+	result := make(map[string]lfsAssetInfo)
+	for _, rel := range p.Inventory.AssetFiles {
+		rel = filepath.ToSlash(rel)
+		abs := filepath.Join(p.AssetsDir(), filepath.FromSlash(rel))
+		info, ok, err := parseLFSPointerFile(abs, p.Root)
+		if err != nil {
+			return nil, err
+		}
+		if ok {
+			result[rel] = info
+		}
+	}
+
+	return result, nil
+}
+
+func parseLFSPointerFile(path, repoRoot string) (lfsAssetInfo, bool, error) {
+	entry, err := os.Lstat(path)
+	if err != nil {
+		return lfsAssetInfo{}, false, fmt.Errorf("stat %s: %w", path, err)
+	}
+	if entry.Mode()&os.ModeSymlink != 0 || entry.Size() > 1024 {
+		return lfsAssetInfo{}, false, nil
+	}
+
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return lfsAssetInfo{}, false, fmt.Errorf("read %s: %w", path, err)
+	}
+	text := strings.ReplaceAll(string(raw), "\r\n", "\n")
+	if !strings.HasPrefix(text, "version https://git-lfs.github.com/spec/v1\n") {
+		return lfsAssetInfo{}, false, nil
+	}
+
+	info := lfsAssetInfo{}
+	for _, line := range strings.Split(text, "\n") {
+		line = strings.TrimSpace(line)
+		switch {
+		case strings.HasPrefix(line, "oid sha256:"):
+			info.OID = strings.TrimSpace(strings.TrimPrefix(line, "oid sha256:"))
+		case strings.HasPrefix(line, "size "):
+			size, parseErr := strconv.ParseInt(strings.TrimSpace(strings.TrimPrefix(line, "size ")), 10, 64)
+			if parseErr != nil {
+				return lfsAssetInfo{}, false, fmt.Errorf("parse LFS pointer size in %s: %w", path, parseErr)
+			}
+			info.Size = size
+		}
+	}
+	if info.OID == "" || info.Size <= 0 {
+		return lfsAssetInfo{}, false, nil
+	}
+	if len(info.OID) >= 4 {
+		info.ObjectPath = filepath.Join(repoRoot, ".git", "lfs", "objects", info.OID[:2], info.OID[2:4], info.OID)
+	}
+	return info, true, nil
+}
+
+func assetResourceFromPath(path, rel string, lfsInfo lfsAssetInfo, requireContent bool) (assetResource, error) {
+	if lfsInfo.OID != "" {
+		resource, err := lfsPathResource(path, lfsInfo, requireContent)
+		if err != nil {
+			return assetResource{}, err
+		}
+		return assetResource{
+			Rel:       rel,
+			Resource:  resource,
+			ContentID: "lfs:" + lfsInfo.OID,
+		}, nil
+	}
+
+	resource, err := pathResource(path)
+	if err != nil {
+		return assetResource{}, err
+	}
+	fileHash, err := fileSHA256(path)
+	if err != nil {
+		return assetResource{}, err
+	}
+	return assetResource{
+		Rel:       rel,
+		Resource:  resource,
+		ContentID: "file:" + fileHash,
+	}, nil
+}
+
+func lfsPathResource(path string, info lfsAssetInfo, requireContent bool) (erf.Resource, error) {
+	extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
+	resourceType, ok := erf.HAKResourceTypeForExtension(extension)
+	if !ok {
+		return erf.Resource{}, fmt.Errorf("unsupported HAK resource extension %q", filepath.Ext(path))
+	}
+	name := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)))
+	sourcePath := ""
+	if requireContent {
+		if info.ObjectPath == "" {
+			return erf.Resource{}, fmt.Errorf("missing downloaded LFS object for %s", path)
+		}
+		if _, err := os.Stat(info.ObjectPath); err != nil {
+			return erf.Resource{}, fmt.Errorf("missing downloaded LFS object for %s: %w", path, err)
+		}
+		sourcePath = info.ObjectPath
+	}
+	return erf.Resource{
+		Name:       name,
+		Type:       resourceType,
+		SourcePath: sourcePath,
+		Size:       info.Size,
+	}, nil
+}
+
+func fileSHA256(path string) (string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("read %s: %w", path, err)
+	}
+	sum := sha256.Sum256(raw)
+	return fmt.Sprintf("%x", sum[:]), nil
+}
+
 func collectGitAssetChangeTimes(p *project.Project) (map[string]time.Time, error) {
 	result := make(map[string]time.Time, len(p.Inventory.AssetFiles))
 	if len(p.Inventory.AssetFiles) == 0 {
@@ -960,9 +1092,8 @@ func chunkContentHash(chunk hakChunk) string {
 		fmt.Fprintf(sum, "asset:%s\n", asset.Rel)
 		fmt.Fprintf(sum, "resref:%s\n", asset.Resource.Name)
 		fmt.Fprintf(sum, "restype:%d\n", asset.Resource.Type)
-		fmt.Fprintf(sum, "datasize:%d\n", len(asset.Resource.Data))
-		_, _ = sum.Write(asset.Resource.Data)
-		_, _ = sum.Write([]byte{'\n'})
+		fmt.Fprintf(sum, "datasize:%d\n", asset.Resource.Size)
+		fmt.Fprintf(sum, "content:%s\n", asset.ContentID)
 	}
 	return fmt.Sprintf("%x", sum.Sum(nil))
 }
@@ -1086,7 +1217,7 @@ func plannedModuleHAKOrder(p *project.Project) ([]string, error) {
 		return append([]string(nil), manifest.ModuleHAKs...), nil
 	}

-	assetResources, err := collectAssetResources(p)
+	assetResources, err := collectAssetResources(p, false)
 	if err != nil {
 		return nil, err
 	}