sow-tools/internal/topdata/autogen.go

package topdata

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"time"

	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)

const autogenManifestCacheMaxAge = time.Hour

type autogenManifest struct {
	ID          string                 `json:"id"`
	Repo        string                 `json:"repo"`
	Ref         string                 `json:"ref"`
	GeneratedAt string                 `json:"generated_at"`
	Entries     []autogenManifestEntry `json:"entries"`
}

type autogenManifestEntry struct {
	Source    string `json:"source"`
	Group     string `json:"group,omitempty"`
	Subgroup  string `json:"subgroup,omitempty"`
	ModelStem string `json:"model_stem,omitempty"`
	RowID     int    `json:"row_id,omitempty"`
}

type autogenManifestAssetMetadata struct {
	DownloadURL string
	UpdatedAt   time.Time
}

type ProducedAutogenManifest struct {
	ID         string
	AssetName  string
	OutputPath string
	Payload    []byte
}

func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDataset, progress func(string)) ([]nativeCollectedDataset, error) {
	if len(p.Config.Autogen.Consumers) == 0 {
		return collected, nil
	}
	if progress == nil {
		progress = func(string) {}
	}

	result := append([]nativeCollectedDataset(nil), collected...)
	for _, consumer := range p.Config.Autogen.Consumers {
		if !autogenConsumerTargetsCollectedDataset(result, consumer) {
			continue
		}
		entries, err := resolveAutogenConsumerEntries(p, consumer, progress)
		if err != nil {
			if consumer.Optional && isIgnorableOptionalAutogenError(err) {
				if progress != nil {
					progress(fmt.Sprintf("Skipping optional autogen consumer %s: %v", consumer.ID, err))
				}
				continue
			}
			return nil, err
		}
		if len(entries) == 0 {
			continue
		}
		switch consumer.Mode {
		case "parts_rows":
			result = augmentWithAutogeneratedParts(result, autogenPartsInventory(entries))
		case "head_visualeffects":
			result, err = augmentWithAutogeneratedHeadVisualeffects(result, entries, consumer)
			if err != nil {
				return nil, err
			}
		default:
			return nil, fmt.Errorf("unsupported autogen consumer mode %q", consumer.Mode)
		}
	}
	return result, nil
}

func autogenConsumerTargetsCollectedDataset(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig) bool {
	for _, dataset := range collected {
		switch consumer.Mode {
		case "parts_rows":
			if isAutogenEligiblePartsDataset(dataset) {
				return true
			}
		case "head_visualeffects":
			if dataset.Dataset.Name == "visualeffects" {
				return true
			}
		case "cachedmodels_rows":
			if dataset.Dataset.Name == "cachedmodels" {
				return true
			}
		}
	}
	return false
}

func isIgnorableOptionalAutogenError(err error) bool {
	if err == nil {
		return false
	}
	message := err.Error()
	return strings.Contains(message, "HTTP 404") ||
		strings.Contains(message, "not found") ||
		strings.Contains(message, "entries is empty")
}

func resolveAutogenConsumerEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) {
	overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer)
	if err != nil {
		return nil, err
	}
	if overrideRoot != "" {
		if progress != nil {
			progress(fmt.Sprintf("Scanning local autogen override for %s from %s...", consumer.ID, overrideRoot))
		}
		return scanLocalAutogenEntries(overrideRoot, consumer)
	}
	return resolveReleasedAutogenManifestEntries(p, consumer, progress)
}

func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.AutogenConsumerConfig) (string, error) {
	root := strings.TrimSpace(consumer.LocalOverrideRoot)
	if root != "" {
		if filepath.IsAbs(root) {
			return root, nil
		}
		return filepath.Join(p.Root, root), nil
	}

	spec := strings.TrimSpace(p.Config.TopData.Assets)
	if spec == "" {
		return "", nil
	}
	if looksLikeGitRepoSpec(spec) {
		return "", fmt.Errorf("topdata.assets no longer supports git repo specs for autogen discovery; use a local path override or the default released manifest")
	}
	path := p.TopDataAssetsDir()
	if path == "" {
		return "", nil
	}
	if _, err := os.Stat(path); err != nil {
		return "", fmt.Errorf("configured topdata.assets path %s is not accessible: %w", path, err)
	}
	candidate := filepath.Join(path, consumer.Root)
	info, err := os.Stat(candidate)
	if err != nil {
		if os.IsNotExist(err) {
			return "", nil
		}
		return "", fmt.Errorf("configured topdata.assets override root %s is not accessible: %w", candidate, err)
	}
	if !info.IsDir() {
		return "", fmt.Errorf("configured topdata.assets override root %s is not a directory", candidate)
	}
	return path, nil
}

func scanLocalAutogenEntries(overrideRoot string, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
	scanRoot := filepath.Join(overrideRoot, consumer.Root)
	return scanConfiguredAutogenEntries(scanRoot, consumer.Include, consumer.Derive)
}

func scanConfiguredAutogenEntries(scanRoot string, include []string, derive project.AutogenDeriveConfig) ([]autogenManifestEntry, error) {
	info, err := os.Stat(scanRoot)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("stat autogen override root %s: %w", scanRoot, err)
	}
	if !info.IsDir() {
		return nil, fmt.Errorf("autogen override root %s is not a directory", scanRoot)
	}

	var entries []autogenManifestEntry
	err = filepath.Walk(scanRoot, func(path string, info os.FileInfo, walkErr error) error {
		if walkErr != nil {
			return walkErr
		}
		if info.IsDir() {
			return nil
		}
		rel, err := filepath.Rel(scanRoot, path)
		if err != nil {
			return err
		}
		rel = filepath.ToSlash(rel)
		if !matchesAutogenInclude(rel, include) {
			return nil
		}
		entry, ok := deriveAutogenManifestEntry(rel, derive)
		if !ok {
			return nil
		}
		entries = append(entries, entry)
		return nil
	})
	if err != nil {
		return nil, fmt.Errorf("scan autogen override %s: %w", scanRoot, err)
	}

	slices.SortFunc(entries, func(a, b autogenManifestEntry) int {
		return strings.Compare(a.Source, b.Source)
	})
	return entries, nil
}

func ProduceAutogenManifests(p *project.Project) ([]ProducedAutogenManifest, error) {
	if len(p.Config.Autogen.Producers) == 0 {
		return nil, nil
	}

	assetsRoot := strings.TrimSpace(p.AssetsDir())
	if assetsRoot == "" {
		return nil, fmt.Errorf("autogen producers require paths.assets to be configured")
	}
	info, err := os.Stat(assetsRoot)
	if err != nil {
		return nil, fmt.Errorf("stat assets root %s: %w", assetsRoot, err)
	}
	if !info.IsDir() {
		return nil, fmt.Errorf("assets root %s is not a directory", assetsRoot)
	}

	repo, ref := autogenManifestSourceMetadata(p.Root)
	results := make([]ProducedAutogenManifest, 0, len(p.Config.Autogen.Producers))
	for _, producer := range p.Config.Autogen.Producers {
		scanRoot := filepath.Join(assetsRoot, producer.Root)
		entries, err := scanConfiguredAutogenEntries(scanRoot, producer.Include, producer.Derive)
		if err != nil {
			return nil, fmt.Errorf("scan autogen producer %s: %w", producer.ID, err)
		}
		if len(entries) == 0 {
			return nil, fmt.Errorf("autogen producer %s discovered no matching entries under %s", producer.ID, scanRoot)
		}
		payload, err := formatAutogenManifest(p.Root, producer, repo, ref, entries)
		if err != nil {
			return nil, fmt.Errorf("format autogen producer %s manifest: %w", producer.ID, err)
		}
		payload = append(payload, '\n')
		results = append(results, ProducedAutogenManifest{
			ID:         producer.ID,
			AssetName:  producer.Manifest.AssetName,
			OutputPath: filepath.Join(p.BuildDir(), producer.Manifest.AssetName),
			Payload:    payload,
		})
	}
	return results, nil
}

func autogenManifestSourceMetadata(root string) (repo string, ref string) {
	if remote, err := gitOutput(root, "remote", "get-url", "origin"); err == nil {
		if spec, ok := parseRepoSpec(strings.TrimSpace(remote)); ok {
			repo = spec.Owner + "/" + spec.Repo
		}
	}
	if head, err := gitOutput(root, "rev-parse", "HEAD"); err == nil {
		ref = strings.TrimSpace(head)
	}
	return repo, ref
}

func matchesAutogenInclude(rel string, include []string) bool {
	rel = filepath.ToSlash(rel)
	for _, pattern := range include {
		pattern = filepath.ToSlash(strings.TrimSpace(pattern))
		if pattern == "" {
			continue
		}
		switch {
		case pattern == "**/*.mdl":
			if strings.HasSuffix(strings.ToLower(rel), ".mdl") {
				return true
			}
		case strings.HasSuffix(pattern, "/**/*.mdl"):
			prefix := strings.TrimSuffix(pattern, "/**/*.mdl")
			if strings.HasPrefix(rel, prefix+"/") && strings.HasSuffix(strings.ToLower(rel), ".mdl") {
				return true
			}
		case strings.HasSuffix(pattern, "/*.mdl"):
			prefix := strings.TrimSuffix(pattern, "/*.mdl")
			dir := filepath.ToSlash(filepath.Dir(rel))
			if dir == prefix && strings.HasSuffix(strings.ToLower(rel), ".mdl") {
				return true
			}
		case rel == pattern:
			return true
		}
	}
	return false
}

func deriveAutogenManifestEntry(rel string, derive project.AutogenDeriveConfig) (autogenManifestEntry, bool) {
	rel = filepath.ToSlash(rel)
	stem := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))
	entry := autogenManifestEntry{
		Source:    rel,
		ModelStem: stem,
	}
	parts := strings.Split(rel, "/")
	if derive.GroupFrom == "first_path_segment" && len(parts) > 1 {
		entry.Group = parts[0]
		if len(parts) > 2 {
			entry.Subgroup = strings.Join(parts[1:len(parts)-1], "/")
		}
	}

	switch derive.Kind {
	case "trailing_numeric_suffix":
		rowID, ok := extractTrailingNumericSuffix(stem)
		if !ok {
			return autogenManifestEntry{}, false
		}
		entry.RowID = rowID
	case "model_stem":
	default:
		return autogenManifestEntry{}, false
	}

	return entry, true
}

func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) {
	cachePath := p.AutogenCachePath(consumer.Manifest.CacheName)
	if strings.TrimSpace(os.Getenv(p.AutogenRefreshEnv())) == "" {
		manifest, fresh, err := readFreshAutogenManifestCache(cachePath, p.AutogenCacheMaxAge())
		if err != nil {
			if progress != nil {
				progress(fmt.Sprintf("Ignoring cached autogen manifest %s: %v", consumer.Manifest.CacheName, err))
			}
		} else if fresh {
			if current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, manifest); err != nil {
				if progress != nil {
					progress(fmt.Sprintf("Using cached released autogen manifest %s (remote check failed: %v)...", cachePath, err))
				}
				return manifest.Entries, nil
			} else if current {
				if progress != nil {
					progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath))
				}
				return manifest.Entries, nil
			} else if progress != nil {
				progress(fmt.Sprintf("Refreshing released autogen manifest %s; remote asset changed at %s.", consumer.Manifest.CacheName, remoteTime.Format(time.RFC3339)))
			}
		}
	}

	spec, err := deriveSowAssetsRepoSpec(p)
	if err != nil {
		return nil, err
	}
	manifestURL, err := resolveAutogenManifestAssetURL(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName)
	if err != nil {
		if consumer.Mode == "parts_rows" {
			return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1))
		}
		return nil, err
	}
	if progress != nil {
		progress(fmt.Sprintf("Fetching released autogen manifest from %s...", manifestURL))
	}
	manifest, err := fetchAutogenManifest(manifestURL)
	if err != nil {
		if consumer.Mode == "parts_rows" {
			return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1))
		}
		return nil, err
	}
	if manifest.ID != "" && manifest.ID != consumer.Producer {
		return nil, fmt.Errorf("autogen manifest %s declared id %q, expected %q", consumer.Manifest.AssetName, manifest.ID, consumer.Producer)
	}
	if err := writeAutogenManifestCache(cachePath, manifest); err != nil {
		return nil, err
	}
	return manifest.Entries, nil
}

func releasedAutogenManifestCacheCurrent(p *project.Project, consumer project.AutogenConsumerConfig, cachePath string, manifest *autogenManifest) (bool, time.Time, error) {
	spec, err := deriveSowAssetsRepoSpec(p)
	if err != nil {
		return false, time.Time{}, err
	}
	asset, err := resolveAutogenManifestAssetMetadata(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName)
	if err != nil {
		return false, time.Time{}, err
	}
	if asset.UpdatedAt.IsZero() {
		return true, time.Time{}, nil
	}
	return !asset.UpdatedAt.After(localAutogenManifestComparableTime(cachePath, manifest)), asset.UpdatedAt, nil
}

func localAutogenManifestComparableTime(cachePath string, manifest *autogenManifest) time.Time {
	best := time.Time{}
	if info, err := os.Stat(cachePath); err == nil {
		best = info.ModTime()
	}
	if generatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(manifest.GeneratedAt)); err == nil && generatedAt.After(best) {
		best = generatedAt
	}
	return best
}

func newestReleasedAutogenManifestInput(p *project.Project, consumer project.AutogenConsumerConfig, now time.Time, cachePath string) (time.Time, string, error) {
	if strings.TrimSpace(os.Getenv(p.AutogenRefreshEnv())) != "" {
		return now, cachePath, nil
	}

	info, err := os.Stat(cachePath)
	if err != nil {
		if os.IsNotExist(err) {
			return now, cachePath, nil
		}
		return time.Time{}, "", fmt.Errorf("stat autogen manifest cache %s: %w", cachePath, err)
	}
	if autogenManifestCacheMaxAge > 0 && now.Sub(info.ModTime()) > autogenManifestCacheMaxAge {
		return now, cachePath, nil
	}

	raw, err := os.ReadFile(cachePath)
	if err != nil {
		return info.ModTime(), cachePath, nil
	}
	var manifest autogenManifest
	if err := json.Unmarshal(raw, &manifest); err != nil {
		return info.ModTime(), cachePath, nil
	}

	current, remoteTime, err := releasedAutogenManifestCacheCurrent(p, consumer, cachePath, &manifest)
	if err != nil || current || remoteTime.IsZero() {
		return info.ModTime(), cachePath, nil
	}
	return remoteTime, consumer.Manifest.AssetName, nil
}

func resolveAutogenManifestAssetURL(spec giteaRepoSpec, releaseTag, assetName string) (string, error) {
	asset, err := resolveAutogenManifestAssetMetadata(spec, releaseTag, assetName)
	if err != nil {
		return "", err
	}
	return asset.DownloadURL, nil
}

func resolveAutogenManifestAssetMetadata(spec giteaRepoSpec, releaseTag, assetName string) (autogenManifestAssetMetadata, error) {
	apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), releasePathEscape(spec.Owner), releasePathEscape(spec.Repo), releasePathEscape(releaseTag))
	var release struct {
		Assets []struct {
			Name               string `json:"name"`
			BrowserDownloadURL string `json:"browser_download_url"`
			DownloadURL        string `json:"download_url"`
			UpdatedAt          string `json:"updated_at"`
		} `json:"assets"`
	}
	if err := fetchJSON(apiURL, &release); err != nil {
		return autogenManifestAssetMetadata{}, fmt.Errorf("fetch autogen manifest release metadata: %w", err)
	}
	for _, asset := range release.Assets {
		if asset.Name != assetName {
			continue
		}
		metadata := autogenManifestAssetMetadata{}
		if updatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(asset.UpdatedAt)); err == nil {
			metadata.UpdatedAt = updatedAt
		}
		if asset.DownloadURL != "" {
			metadata.DownloadURL = asset.DownloadURL
			return metadata, nil
		}
		if asset.BrowserDownloadURL != "" {
			metadata.DownloadURL = asset.BrowserDownloadURL
			return metadata, nil
		}
	}
	return autogenManifestAssetMetadata{}, fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag)
}

func releasePathEscape(value string) string {
	replacer := strings.NewReplacer("/", "%2F")
	return replacer.Replace(value)
}

func fetchAutogenManifest(manifestURL string) (*autogenManifest, error) {
	var manifest autogenManifest
	if err := fetchJSON(manifestURL, &manifest); err != nil {
		return nil, fmt.Errorf("download autogen manifest: %w", err)
	}
	if len(manifest.Entries) == 0 {
		var legacy partsManifest
		if err := fetchJSON(manifestURL, &legacy); err == nil && len(legacy.Categories) > 0 {
			return legacyPartsManifestToAutogen(&legacy), nil
		}
	}
	if len(manifest.Entries) == 0 {
		return nil, fmt.Errorf("download autogen manifest: entries is empty")
	}
	return &manifest, nil
}

func readFreshAutogenManifestCache(path string, maxAge time.Duration) (*autogenManifest, bool, error) {
	info, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, false, nil
		}
		return nil, false, err
	}
	if maxAge > 0 && time.Since(info.ModTime()) > maxAge {
		return nil, false, nil
	}
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, false, err
	}
	var manifest autogenManifest
	if err := json.Unmarshal(raw, &manifest); err != nil {
		return nil, false, err
	}
	if len(manifest.Entries) == 0 {
		var legacy partsManifest
		if err := json.Unmarshal(raw, &legacy); err == nil && len(legacy.Categories) > 0 {
			return legacyPartsManifestToAutogen(&legacy), true, nil
		}
	}
	if len(manifest.Entries) == 0 {
		return nil, false, fmt.Errorf("entries is empty")
	}
	return &manifest, true, nil
}

func legacyPartsManifestToAutogen(legacy *partsManifest) *autogenManifest {
	entries := make([]autogenManifestEntry, 0)
	for _, category := range supportedPartCategories {
		for _, rowID := range legacy.Categories[category] {
			entries = append(entries, autogenManifestEntry{
				Group: category,
				RowID: rowID,
			})
		}
	}
	slices.SortFunc(entries, func(a, b autogenManifestEntry) int {
		if cmp := strings.Compare(a.Group, b.Group); cmp != 0 {
			return cmp
		}
		return a.RowID - b.RowID
	})
	return &autogenManifest{
		ID:          "parts",
		Repo:        legacy.Repo,
		Ref:         legacy.Ref,
		GeneratedAt: legacy.GeneratedAt,
		Entries:     entries,
	}
}

func writeAutogenManifestCache(path string, manifest *autogenManifest) error {
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return fmt.Errorf("create autogen manifest cache dir: %w", err)
	}
	payload, err := json.MarshalIndent(manifest, "", "  ")
	if err != nil {
		return fmt.Errorf("marshal autogen manifest cache: %w", err)
	}
	payload = append(payload, '\n')
	current, err := os.ReadFile(path)
	if err == nil && bytes.Equal(current, payload) {
		now := time.Now()
		if err := os.Chtimes(path, now, now); err != nil {
			return fmt.Errorf("refresh autogen manifest cache timestamp: %w", err)
		}
		return nil
	}
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("read autogen manifest cache: %w", err)
	}
	tmpPath := path + ".tmp"
	if err := os.WriteFile(tmpPath, payload, 0o644); err != nil {
		return fmt.Errorf("write autogen manifest cache: %w", err)
	}
	if err := os.Rename(tmpPath, path); err != nil {
		return fmt.Errorf("replace autogen manifest cache: %w", err)
	}
	return nil
}

func autogenPartsInventory(entries []autogenManifestEntry) map[string]map[int]struct{} {
	result := make(map[string]map[int]struct{}, len(supportedPartCategories))
	for _, category := range supportedPartCategories {
		result[category] = map[int]struct{}{}
	}
	for _, entry := range entries {
		if !isSupportedPartCategory(entry.Group) || entry.RowID <= 0 {
			continue
		}
		result[entry.Group][entry.RowID] = struct{}{}
	}
	return result
}

func augmentWithAutogeneratedCachedModels(collected []nativeCollectedDataset, entries []autogenManifestEntry) ([]nativeCollectedDataset, error) {
	if len(entries) == 0 {
		return collected, nil
	}

	result := append([]nativeCollectedDataset(nil), collected...)
	for i, dataset := range result {
		if dataset.Dataset.Name != "cachedmodels" {
			continue
		}
		if !slices.Contains(dataset.Columns, "Model") {
			return nil, fmt.Errorf("cachedmodels dataset must include Model column for cachedmodels_rows autogen")
		}

		rows := make([]map[string]any, len(dataset.Rows), len(dataset.Rows)+len(entries))
		seenModels := make(map[string]struct{}, len(dataset.Rows)+len(entries))
		nextID := 0
		for index, row := range dataset.Rows {
			cloned := cloneRowMap(row)
			rows[index] = cloned
			if rowID, ok := cloned["id"].(int); ok && rowID >= nextID {
				nextID = rowID + 1
			}
			if model, ok := cloned["Model"].(string); ok {
				model = strings.TrimSpace(model)
				if model != "" && model != nullValue {
					seenModels[strings.ToLower(model)] = struct{}{}
				}
			}
		}

		for _, entry := range entries {
			model := strings.TrimSpace(entry.ModelStem)
			if model == "" {
				continue
			}
			key := strings.ToLower(model)
			if _, exists := seenModels[key]; exists {
				continue
			}
			rows = append(rows, map[string]any{
				"id":    nextID,
				"Model": model,
			})
			seenModels[key] = struct{}{}
			nextID++
		}

		result[i].Rows = rows
	}
	return result, nil
}

func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDataset, entries []autogenManifestEntry, consumer project.AutogenConsumerConfig) ([]nativeCollectedDataset, error) {
	if len(entries) == 0 {
		return collected, nil
	}

	policy := resolveHeadVisualeffectsPolicy(consumer)
	result := append([]nativeCollectedDataset(nil), collected...)
	for i, dataset := range result {
		if dataset.Dataset.Name != "visualeffects" {
			continue
		}

		rows := make([]map[string]any, len(dataset.Rows))
		rowByKey := make(map[string]map[string]any, len(dataset.Rows))
		historicalLockData := make(map[string]int, len(dataset.LockData))
		for key, rowID := range dataset.LockData {
			historicalLockData[key] = rowID
		}
		if dataset.Dataset.LockPath != "" {
			if existingLockData, err := loadLockfile(dataset.Dataset.LockPath); err == nil {
				for key, rowID := range existingLockData {
					if _, ok := historicalLockData[key]; !ok {
						historicalLockData[key] = rowID
					}
				}
			}
		}
		usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(historicalLockData))
		lockData := make(map[string]int, len(dataset.LockData))
		for _, rowID := range historicalLockData {
			usedIDs[rowID] = struct{}{}
		}
		for key, rowID := range dataset.LockData {
			lockData[key] = rowID
		}
		for index, row := range dataset.Rows {
			cloned := cloneRowMap(row)
			rows[index] = cloned
			if key, ok := cloned["key"].(string); ok && key != "" {
				rowByKey[key] = cloned
			}
			if rowID, ok := cloned["id"].(int); ok {
				usedIDs[rowID] = struct{}{}
			}
		}

		nextID := nextAvailableAutogenID(usedIDs)
		allocateNextID := func() int {
			rowID := nextID
			usedIDs[rowID] = struct{}{}
			nextID = nextAvailableAutogenID(usedIDs)
			return rowID
		}

		for _, entry := range entries {
			key, label, modelStem, groupPolicy, ok := headVisualeffectIdentity(dataset.Dataset.Name, entry, policy)
			if !ok {
				continue
			}
			if existing, exists := rowByKey[key]; exists {
				applyDiscoveredHeadVisualeffectDefaults(existing, dataset.Columns, modelStem, label, policy, groupPolicy)
				continue
			}

			rowID, ok := lockData[key]
			if !ok {
				if preservedRowID, preserved := historicalLockData[key]; preserved {
					rowID = preservedRowID
				} else {
					rowID = allocateNextID()
				}
				lockData[key] = rowID
			}
			newRow := createDefaultHeadVisualeffectRow(dataset.Columns, rowID, key, label, modelStem, policy, groupPolicy)
			rows = append(rows, newRow)
			rowByKey[key] = newRow
		}

		slices.SortFunc(rows, func(a, b map[string]any) int {
			return a["id"].(int) - b["id"].(int)
		})
		result[i].Rows = rows
		result[i].LockData = lockData
		result[i].LockModified = !lockDataEqual(dataset.LockData, lockData)
	}
	return result, nil
}

func nextAvailableAutogenID(used map[int]struct{}) int {
	for rowID := 0; ; rowID++ {
		if _, ok := used[rowID]; !ok {
			return rowID
		}
	}
}

type headVisualeffectsPolicy struct {
	Groups             map[string]headVisualeffectsGroupPolicy
	StripModelPrefixes []string
	KeyFormat          string
	LabelFormat        string
	ModelColumn        string
	RowDefaults        map[string]string
}

type headVisualeffectsGroupPolicy struct {
	Prefix       string
	ModelColumns []string
	RowDefaults  map[string]string
}

func resolveHeadVisualeffectsPolicy(consumer project.AutogenConsumerConfig) headVisualeffectsPolicy {
	policy := defaultHeadVisualeffectsPolicy()
	cfg := consumer.HeadVisualeffects
	for group, groupCfg := range cfg.Groups {
		group = strings.TrimSpace(group)
		prefix := strings.TrimSpace(groupCfg.Prefix)
		if group == "" || prefix == "" {
			continue
		}
		groupPolicy := headVisualeffectsGroupPolicy{
			Prefix:       prefix,
			ModelColumns: normalizeHeadVisualeffectsModelColumns(groupCfg.ModelColumn, groupCfg.ModelColumns),
			RowDefaults:  normalizeHeadVisualeffectsRowDefaults(groupCfg.RowDefaults),
		}
		policy.Groups[group] = groupPolicy
	}
	if len(cfg.StripModelPrefixes) > 0 {
		policy.StripModelPrefixes = append([]string(nil), cfg.StripModelPrefixes...)
	}
	if strings.TrimSpace(cfg.KeyFormat) != "" {
		policy.KeyFormat = strings.TrimSpace(cfg.KeyFormat)
	}
	if strings.TrimSpace(cfg.LabelFormat) != "" {
		policy.LabelFormat = strings.TrimSpace(cfg.LabelFormat)
	}
	if strings.TrimSpace(cfg.ModelColumn) != "" {
		policy.ModelColumn = strings.TrimSpace(cfg.ModelColumn)
	}
	mergeStringMap(policy.RowDefaults, normalizeHeadVisualeffectsRowDefaults(cfg.RowDefaults))
	return policy
}

func defaultHeadVisualeffectsPolicy() headVisualeffectsPolicy {
	return headVisualeffectsPolicy{
		Groups: map[string]headVisualeffectsGroupPolicy{
			"head_accessories": {Prefix: "headaccessory"},
			"head_decorations": {Prefix: "headdecoration"},
			"head_features":    {Prefix: "headfeature"},
		},
		StripModelPrefixes: []string{"hfx_", "vfx_"},
		KeyFormat:          "{dataset}:{prefix}_{stem}",
		LabelFormat:        "{prefix}_{stem_upper}",
		ModelColumn:        "Imp_HeadCon_Node",
		RowDefaults: map[string]string{
			"Type_FD":          "D",
			"OrientWithGround": "0",
			"OrientWithObject": "1",
		},
	}
}

func normalizeHeadVisualeffectsModelColumns(modelColumn string, modelColumns []string) []string {
	var result []string
	if column := strings.TrimSpace(modelColumn); column != "" {
		result = append(result, column)
	}
	for _, column := range modelColumns {
		column = strings.TrimSpace(column)
		if column == "" {
			continue
		}
		if !slices.Contains(result, column) {
			result = append(result, column)
		}
	}
	return result
}

func normalizeHeadVisualeffectsRowDefaults(defaults map[string]string) map[string]string {
	result := map[string]string{}
	for column, value := range defaults {
		column = strings.TrimSpace(column)
		if column == "" {
			continue
		}
		result[column] = value
	}
	return result
}

func mergeStringMap(target map[string]string, source map[string]string) {
	for key, value := range source {
		target[key] = value
	}
}

func headVisualeffectIdentity(dataset string, entry autogenManifestEntry, policy headVisualeffectsPolicy) (string, string, string, headVisualeffectsGroupPolicy, bool) {
	group, ok := policy.Groups[entry.Group]
	if !ok || strings.TrimSpace(entry.ModelStem) == "" {
		return "", "", "", headVisualeffectsGroupPolicy{}, false
	}
	modelStem := strings.TrimSpace(entry.ModelStem)
	stem := modelStem
	for _, prefix := range policy.StripModelPrefixes {
		prefix = strings.TrimSpace(prefix)
		if prefix == "" {
			continue
		}
		stem = strings.TrimPrefix(stem, prefix)
	}
	stem = strings.TrimSpace(stem)
	if stem == "" {
		return "", "", "", headVisualeffectsGroupPolicy{}, false
	}
	values := map[string]string{
		"dataset":    dataset,
		"group":      entry.Group,
		"prefix":     group.Prefix,
		"stem":       stem,
		"stem_upper": strings.ToUpper(stem),
		"model_stem": modelStem,
	}
	key := expandHeadVisualeffectsFormat(policy.KeyFormat, values)
	label := expandHeadVisualeffectsFormat(policy.LabelFormat, values)
	if strings.TrimSpace(key) == "" || strings.TrimSpace(label) == "" {
		return "", "", "", headVisualeffectsGroupPolicy{}, false
	}
	return key, label, modelStem, group, true
}

func expandHeadVisualeffectsFormat(format string, values map[string]string) string {
	result := format
	for key, value := range values {
		result = strings.ReplaceAll(result, "{"+key+"}", value)
	}
	return result
}

func createDefaultHeadVisualeffectRow(columns []string, rowID int, key, label, modelStem string, policy headVisualeffectsPolicy, group headVisualeffectsGroupPolicy) map[string]any {
	row := map[string]any{
		"id":  rowID,
		"key": key,
	}
	for _, column := range columns {
		row[column] = nullValue
	}
	row["Label"] = label
	for column, value := range policy.RowDefaults {
		row[column] = value
	}
	for column, value := range group.RowDefaults {
		row[column] = value
	}
	for _, column := range headVisualeffectModelColumns(policy, group) {
		row[column] = modelStem
	}
	return row
}

func applyDiscoveredHeadVisualeffectDefaults(row map[string]any, columns []string, modelStem, label string, policy headVisualeffectsPolicy, group headVisualeffectsGroupPolicy) {
	if isNullLikeValue(row["Label"]) {
		row["Label"] = label
	}
	for column, value := range policy.RowDefaults {
		if isNullLikeValue(row[column]) {
			row[column] = value
		}
	}
	for column, value := range group.RowDefaults {
		if isNullLikeValue(row[column]) {
			row[column] = value
		}
	}
	for _, column := range headVisualeffectModelColumns(policy, group) {
		if isNullLikeValue(row[column]) {
			row[column] = modelStem
		}
	}
	for _, column := range columns {
		if _, ok := row[column]; !ok {
			row[column] = nullValue
		}
	}
}

func headVisualeffectModelColumns(policy headVisualeffectsPolicy, group headVisualeffectsGroupPolicy) []string {
	if len(group.ModelColumns) > 0 {
		return group.ModelColumns
	}
	if strings.TrimSpace(policy.ModelColumn) == "" {
		return nil
	}
	return []string{policy.ModelColumn}
}

func formatAutogenManifest(root string, producer project.AutogenProducerConfig, repo, ref string, entries []autogenManifestEntry) ([]byte, error) {
	manifest := autogenManifest{
		ID:          producer.ID,
		Repo:        repo,
		Ref:         ref,
		GeneratedAt: time.Now().UTC().Format(time.RFC3339),
		Entries:     entries,
	}
	return json.MarshalIndent(manifest, "", "  ")
}

func parseAutogenManifestEntryList(raw []byte) ([]autogenManifestEntry, error) {
	var manifest autogenManifest
	if err := json.Unmarshal(raw, &manifest); err != nil {
		return nil, err
	}
	return manifest.Entries, nil
}

func normalizeAutogenNumericValue(value any) string {
	switch typed := value.(type) {
	case string:
		return typed
	case int:
		return strconv.Itoa(typed)
	case float64:
		return strconv.Itoa(int(typed))
	default:
		return fmt.Sprintf("%v", value)
	}
}