Head Accessories AutoGen Configuration

internal/project/project.go

@@ -34,6 +34,7 @@ type Config struct {
 	HAKs    []HAKConfig   `json:"haks"`
 	TopData TopDataConfig `json:"topdata"`
 	Extract ExtractConfig `json:"extract"`
+	Autogen AutogenConfig `json:"autogen"`
 }
 
 type ModuleConfig struct {
@@ -72,6 +73,43 @@ type ExtractConfig struct {
 	DeleteModuleArchiveAfterSuccess bool     `json:"delete_module_archive_after_success,omitempty"`
 }
 
+type AutogenConfig struct {
+	Producers []AutogenProducerConfig `json:"producers"`
+	Consumers []AutogenConsumerConfig `json:"consumers"`
+}
+
+type AutogenProducerConfig struct {
+	ID       string                `json:"id"`
+	Root     string                `json:"root"`
+	Include  []string              `json:"include"`
+	Derive   AutogenDeriveConfig   `json:"derive"`
+	Manifest AutogenManifestConfig `json:"manifest"`
+}
+
+type AutogenConsumerConfig struct {
+	ID                string                `json:"id"`
+	Producer          string                `json:"producer"`
+	Dataset           string                `json:"dataset"`
+	Mode              string                `json:"mode"`
+	Optional          bool                  `json:"optional,omitempty"`
+	Root              string                `json:"root"`
+	Include           []string              `json:"include"`
+	Derive            AutogenDeriveConfig   `json:"derive"`
+	Manifest          AutogenManifestConfig `json:"manifest"`
+	LocalOverrideRoot string                `json:"local_override_root,omitempty"`
+}
+
+type AutogenDeriveConfig struct {
+	Kind      string `json:"kind"`
+	GroupFrom string `json:"group_from,omitempty"`
+}
+
+type AutogenManifestConfig struct {
+	ReleaseTag string `json:"release_tag"`
+	AssetName  string `json:"asset_name"`
+	CacheName  string `json:"cache_name"`
+}
+
 type Inventory struct {
 	SourceFiles []string
 	ScriptFiles []string
@@ -141,6 +179,7 @@ func (p *Project) ValidateLayout() error {
 			failures = append(failures, err)
 		}
 	}
+	failures = append(failures, validateAutogenConfig(p.Config.Autogen)...)
 
 	for index, hak := range p.Config.HAKs {
 		if strings.TrimSpace(hak.Name) == "" {
@@ -447,6 +486,100 @@ func defaultConfig() Config {
 	}
 }
 
+func validateAutogenConfig(cfg AutogenConfig) []error {
+	var failures []error
+	producerIDs := map[string]struct{}{}
+	consumerIDs := map[string]struct{}{}
+
+	for index, producer := range cfg.Producers {
+		fieldPrefix := fmt.Sprintf("autogen.producers[%d]", index)
+		id := strings.TrimSpace(producer.ID)
+		if id == "" {
+			failures = append(failures, fmt.Errorf("%s.id is required", fieldPrefix))
+		} else {
+			if _, exists := producerIDs[id]; exists {
+				failures = append(failures, fmt.Errorf("%s.id %q is duplicated", fieldPrefix, id))
+			}
+			producerIDs[id] = struct{}{}
+		}
+		if strings.TrimSpace(producer.Root) == "" {
+			failures = append(failures, fmt.Errorf("%s.root is required", fieldPrefix))
+		}
+		if len(producer.Include) == 0 {
+			failures = append(failures, fmt.Errorf("%s.include must contain at least one glob", fieldPrefix))
+		}
+		failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", producer.Derive)...)
+		failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", producer.Manifest)...)
+	}
+
+	for index, consumer := range cfg.Consumers {
+		fieldPrefix := fmt.Sprintf("autogen.consumers[%d]", index)
+		id := strings.TrimSpace(consumer.ID)
+		if id == "" {
+			failures = append(failures, fmt.Errorf("%s.id is required", fieldPrefix))
+		} else {
+			if _, exists := consumerIDs[id]; exists {
+				failures = append(failures, fmt.Errorf("%s.id %q is duplicated", fieldPrefix, id))
+			}
+			consumerIDs[id] = struct{}{}
+		}
+		if strings.TrimSpace(consumer.Producer) == "" {
+			failures = append(failures, fmt.Errorf("%s.producer is required", fieldPrefix))
+		}
+		if strings.TrimSpace(consumer.Dataset) == "" {
+			failures = append(failures, fmt.Errorf("%s.dataset is required", fieldPrefix))
+		}
+		switch strings.TrimSpace(consumer.Mode) {
+		case "parts_rows", "head_visualeffects":
+		case "":
+			failures = append(failures, fmt.Errorf("%s.mode is required", fieldPrefix))
+		default:
+			failures = append(failures, fmt.Errorf("%s.mode %q is not supported", fieldPrefix, consumer.Mode))
+		}
+		if strings.TrimSpace(consumer.Root) == "" {
+			failures = append(failures, fmt.Errorf("%s.root is required", fieldPrefix))
+		}
+		if len(consumer.Include) == 0 {
+			failures = append(failures, fmt.Errorf("%s.include must contain at least one glob", fieldPrefix))
+		}
+		failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", consumer.Derive)...)
+		failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", consumer.Manifest)...)
+	}
+
+	return failures
+}
+
+func validateAutogenDeriveConfig(fieldPrefix string, cfg AutogenDeriveConfig) []error {
+	var failures []error
+	switch strings.TrimSpace(cfg.Kind) {
+	case "trailing_numeric_suffix", "model_stem":
+	case "":
+		failures = append(failures, fmt.Errorf("%s.kind is required", fieldPrefix))
+	default:
+		failures = append(failures, fmt.Errorf("%s.kind %q is not supported", fieldPrefix, cfg.Kind))
+	}
+	switch strings.TrimSpace(cfg.GroupFrom) {
+	case "", "first_path_segment":
+	default:
+		failures = append(failures, fmt.Errorf("%s.group_from %q is not supported", fieldPrefix, cfg.GroupFrom))
+	}
+	return failures
+}
+
+func validateAutogenManifestConfig(fieldPrefix string, cfg AutogenManifestConfig) []error {
+	var failures []error
+	if strings.TrimSpace(cfg.ReleaseTag) == "" {
+		failures = append(failures, fmt.Errorf("%s.release_tag is required", fieldPrefix))
+	}
+	if err := validateOutputFileName(fieldPrefix+".asset_name", cfg.AssetName, ".json"); err != nil {
+		failures = append(failures, err)
+	}
+	if err := validateOutputFileName(fieldPrefix+".cache_name", cfg.CacheName, ".json"); err != nil {
+		failures = append(failures, err)
+	}
+	return failures
+}
+
 func scanDir(root string, include func(path string) bool) ([]string, []string, error) {
 	var files []string
 	extSet := map[string]struct{}{}

internal/project/project_test.go

@@ -196,6 +196,52 @@ func TestValidateLayoutRejectsInvalidTopDataPackageOutputNames(t *testing.T) {
 	}
 }
 
+func TestValidateLayoutRejectsInvalidAutogenConsumerConfig(t *testing.T) {
+	root := t.TempDir()
+	mkdirAll(t, filepath.Join(root, "src"))
+	mkdirAll(t, filepath.Join(root, "build"))
+
+	proj := &Project{
+		Root: root,
+		Config: Config{
+			Module: ModuleConfig{Name: "Test", ResRef: "test"},
+			Paths:  PathConfig{Source: "src", Build: "build"},
+			Autogen: AutogenConfig{
+				Consumers: []AutogenConsumerConfig{
+					{
+						ID:       "head_visualeffects",
+						Producer: "head_visualeffects",
+						Dataset:  "visualeffects",
+						Mode:     "unsupported",
+						Root:     "vfxs",
+						Include:  []string{"head_accessories/**/*.mdl"},
+						Derive:   AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"},
+						Manifest: AutogenManifestConfig{
+							ReleaseTag: "head-vfx-manifest-current",
+							AssetName:  "nested/sow-head-vfx-manifest.json",
+							CacheName:  "sow-head-vfx-manifest.txt",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	err := proj.ValidateLayout()
+	if err == nil {
+		t.Fatal("expected autogen validation error")
+	}
+	if !strings.Contains(err.Error(), "autogen.consumers[0].mode") {
+		t.Fatalf("expected autogen mode validation error, got %v", err)
+	}
+	if !strings.Contains(err.Error(), "autogen.consumers[0].manifest.asset_name") {
+		t.Fatalf("expected autogen asset_name validation error, got %v", err)
+	}
+	if !strings.Contains(err.Error(), "autogen.consumers[0].manifest.cache_name") {
+		t.Fatalf("expected autogen cache_name validation error, got %v", err)
+	}
+}
+
 func TestScanAllowsMissingAssetsDir(t *testing.T) {
 	root := t.TempDir()
 	mkdirAll(t, filepath.Join(root, "src"))

internal/topdata/autogen.go

@@ -0,0 +1,616 @@
+package topdata
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+const autogenManifestCacheMaxAge = time.Hour
+
+type autogenManifest struct {
+	ID          string                 `json:"id"`
+	Repo        string                 `json:"repo"`
+	Ref         string                 `json:"ref"`
+	GeneratedAt string                 `json:"generated_at"`
+	Entries     []autogenManifestEntry `json:"entries"`
+}
+
+type autogenManifestEntry struct {
+	Source    string `json:"source"`
+	Group     string `json:"group,omitempty"`
+	Subgroup  string `json:"subgroup,omitempty"`
+	ModelStem string `json:"model_stem,omitempty"`
+	RowID     int    `json:"row_id,omitempty"`
+}
+
+func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDataset, progress func(string)) ([]nativeCollectedDataset, error) {
+	if len(p.Config.Autogen.Consumers) == 0 {
+		return collected, nil
+	}
+	if progress == nil {
+		progress = func(string) {}
+	}
+
+	result := append([]nativeCollectedDataset(nil), collected...)
+	for _, consumer := range p.Config.Autogen.Consumers {
+		if !autogenConsumerTargetsCollectedDataset(result, consumer) {
+			continue
+		}
+		entries, err := resolveAutogenConsumerEntries(p, consumer, progress)
+		if err != nil {
+			if consumer.Optional && isIgnorableOptionalAutogenError(err) {
+				if progress != nil {
+					progress(fmt.Sprintf("Skipping optional autogen consumer %s: %v", consumer.ID, err))
+				}
+				continue
+			}
+			return nil, err
+		}
+		if len(entries) == 0 {
+			continue
+		}
+		switch consumer.Mode {
+		case "parts_rows":
+			result = augmentWithAutogeneratedParts(result, autogenPartsInventory(entries))
+		case "head_visualeffects":
+			result, err = augmentWithAutogeneratedHeadVisualeffects(result, entries)
+			if err != nil {
+				return nil, err
+			}
+		default:
+			return nil, fmt.Errorf("unsupported autogen consumer mode %q", consumer.Mode)
+		}
+	}
+	return result, nil
+}
+
+func autogenConsumerTargetsCollectedDataset(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig) bool {
+	for _, dataset := range collected {
+		switch consumer.Mode {
+		case "parts_rows":
+			if isAutogenEligiblePartsDataset(dataset) {
+				return true
+			}
+		case "head_visualeffects":
+			if dataset.Dataset.Name == "visualeffects" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func isIgnorableOptionalAutogenError(err error) bool {
+	if err == nil {
+		return false
+	}
+	message := err.Error()
+	return strings.Contains(message, "HTTP 404") ||
+		strings.Contains(message, "not found") ||
+		strings.Contains(message, "entries is empty")
+}
+
+func resolveAutogenConsumerEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) {
+	overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer)
+	if err != nil {
+		return nil, err
+	}
+	if overrideRoot != "" {
+		if progress != nil {
+			progress(fmt.Sprintf("Scanning local autogen override for %s from %s...", consumer.ID, overrideRoot))
+		}
+		return scanLocalAutogenEntries(overrideRoot, consumer)
+	}
+	return resolveReleasedAutogenManifestEntries(p, consumer, progress)
+}
+
+func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.AutogenConsumerConfig) (string, error) {
+	root := strings.TrimSpace(consumer.LocalOverrideRoot)
+	if root != "" {
+		if filepath.IsAbs(root) {
+			return root, nil
+		}
+		return filepath.Join(p.Root, root), nil
+	}
+
+	spec := strings.TrimSpace(p.Config.TopData.Assets)
+	if spec == "" {
+		return "", nil
+	}
+	if looksLikeGitRepoSpec(spec) {
+		return "", fmt.Errorf("topdata.assets no longer supports git repo specs for autogen discovery; use a local path override or the default released manifest")
+	}
+	path := p.TopDataAssetsDir()
+	if path == "" {
+		return "", nil
+	}
+	if _, err := os.Stat(path); err != nil {
+		return "", fmt.Errorf("configured topdata.assets path %s is not accessible: %w", path, err)
+	}
+	candidate := filepath.Join(path, consumer.Root)
+	info, err := os.Stat(candidate)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", nil
+		}
+		return "", fmt.Errorf("configured topdata.assets override root %s is not accessible: %w", candidate, err)
+	}
+	if !info.IsDir() {
+		return "", fmt.Errorf("configured topdata.assets override root %s is not a directory", candidate)
+	}
+	return path, nil
+}
+
+func scanLocalAutogenEntries(overrideRoot string, consumer project.AutogenConsumerConfig) ([]autogenManifestEntry, error) {
+	scanRoot := filepath.Join(overrideRoot, consumer.Root)
+	info, err := os.Stat(scanRoot)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("stat autogen override root %s: %w", scanRoot, err)
+	}
+	if !info.IsDir() {
+		return nil, fmt.Errorf("autogen override root %s is not a directory", scanRoot)
+	}
+
+	var entries []autogenManifestEntry
+	err = filepath.Walk(scanRoot, func(path string, info os.FileInfo, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		if info.IsDir() {
+			return nil
+		}
+		rel, err := filepath.Rel(scanRoot, path)
+		if err != nil {
+			return err
+		}
+		rel = filepath.ToSlash(rel)
+		if !matchesAutogenInclude(rel, consumer.Include) {
+			return nil
+		}
+		entry, ok := deriveAutogenManifestEntry(rel, consumer.Derive)
+		if !ok {
+			return nil
+		}
+		entries = append(entries, entry)
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("scan autogen override %s: %w", scanRoot, err)
+	}
+
+	slices.SortFunc(entries, func(a, b autogenManifestEntry) int {
+		return strings.Compare(a.Source, b.Source)
+	})
+	return entries, nil
+}
+
+func matchesAutogenInclude(rel string, include []string) bool {
+	rel = filepath.ToSlash(rel)
+	for _, pattern := range include {
+		pattern = filepath.ToSlash(strings.TrimSpace(pattern))
+		if pattern == "" {
+			continue
+		}
+		switch {
+		case pattern == "**/*.mdl":
+			if strings.HasSuffix(strings.ToLower(rel), ".mdl") {
+				return true
+			}
+		case strings.HasSuffix(pattern, "/**/*.mdl"):
+			prefix := strings.TrimSuffix(pattern, "/**/*.mdl")
+			if strings.HasPrefix(rel, prefix+"/") && strings.HasSuffix(strings.ToLower(rel), ".mdl") {
+				return true
+			}
+		case strings.HasSuffix(pattern, "/*.mdl"):
+			prefix := strings.TrimSuffix(pattern, "/*.mdl")
+			dir := filepath.ToSlash(filepath.Dir(rel))
+			if dir == prefix && strings.HasSuffix(strings.ToLower(rel), ".mdl") {
+				return true
+			}
+		case rel == pattern:
+			return true
+		}
+	}
+	return false
+}
+
+func deriveAutogenManifestEntry(rel string, derive project.AutogenDeriveConfig) (autogenManifestEntry, bool) {
+	rel = filepath.ToSlash(rel)
+	stem := strings.TrimSuffix(filepath.Base(rel), filepath.Ext(rel))
+	entry := autogenManifestEntry{
+		Source:    rel,
+		ModelStem: stem,
+	}
+	parts := strings.Split(rel, "/")
+	if derive.GroupFrom == "first_path_segment" && len(parts) > 1 {
+		entry.Group = parts[0]
+		if len(parts) > 2 {
+			entry.Subgroup = strings.Join(parts[1:len(parts)-1], "/")
+		}
+	}
+
+	switch derive.Kind {
+	case "trailing_numeric_suffix":
+		rowID, ok := extractTrailingNumericSuffix(stem)
+		if !ok {
+			return autogenManifestEntry{}, false
+		}
+		entry.RowID = rowID
+	case "model_stem":
+	default:
+		return autogenManifestEntry{}, false
+	}
+
+	return entry, true
+}
+
+func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) {
+	cachePath := filepath.Join(p.Root, ".cache", consumer.Manifest.CacheName)
+	if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) == "" {
+		manifest, fresh, err := readFreshAutogenManifestCache(cachePath, autogenManifestCacheMaxAge)
+		if err != nil {
+			if progress != nil {
+				progress(fmt.Sprintf("Ignoring cached autogen manifest %s: %v", consumer.Manifest.CacheName, err))
+			}
+		} else if fresh {
+			if progress != nil {
+				progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath))
+			}
+			return manifest.Entries, nil
+		}
+	}
+
+	spec, err := deriveSowAssetsRepoSpec(p.Root)
+	if err != nil {
+		return nil, err
+	}
+	manifestURL, err := resolveAutogenManifestAssetURL(spec, consumer.Manifest.ReleaseTag, consumer.Manifest.AssetName)
+	if err != nil {
+		if consumer.Mode == "parts_rows" {
+			return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1))
+		}
+		return nil, err
+	}
+	if progress != nil {
+		progress(fmt.Sprintf("Fetching released autogen manifest from %s...", manifestURL))
+	}
+	manifest, err := fetchAutogenManifest(manifestURL)
+	if err != nil {
+		if consumer.Mode == "parts_rows" {
+			return nil, fmt.Errorf("%s", strings.Replace(err.Error(), "autogen manifest", "parts manifest", 1))
+		}
+		return nil, err
+	}
+	if manifest.ID != "" && manifest.ID != consumer.Producer {
+		return nil, fmt.Errorf("autogen manifest %s declared id %q, expected %q", consumer.Manifest.AssetName, manifest.ID, consumer.Producer)
+	}
+	if err := writeAutogenManifestCache(cachePath, manifest); err != nil {
+		return nil, err
+	}
+	return manifest.Entries, nil
+}
+
+func resolveAutogenManifestAssetURL(spec giteaRepoSpec, releaseTag, assetName string) (string, error) {
+	apiURL := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases/tags/%s", strings.TrimRight(spec.BaseURL, "/"), releasePathEscape(spec.Owner), releasePathEscape(spec.Repo), releasePathEscape(releaseTag))
+	var release struct {
+		Assets []struct {
+			Name               string `json:"name"`
+			BrowserDownloadURL string `json:"browser_download_url"`
+			DownloadURL        string `json:"download_url"`
+		} `json:"assets"`
+	}
+	if err := fetchJSON(apiURL, &release); err != nil {
+		return "", fmt.Errorf("fetch autogen manifest release metadata: %w", err)
+	}
+	for _, asset := range release.Assets {
+		if asset.Name != assetName {
+			continue
+		}
+		if asset.DownloadURL != "" {
+			return asset.DownloadURL, nil
+		}
+		if asset.BrowserDownloadURL != "" {
+			return asset.BrowserDownloadURL, nil
+		}
+	}
+	return "", fmt.Errorf("autogen manifest asset %s not found in release %s/%s:%s", assetName, spec.Owner, spec.Repo, releaseTag)
+}
+
+func releasePathEscape(value string) string {
+	replacer := strings.NewReplacer("/", "%2F")
+	return replacer.Replace(value)
+}
+
+func fetchAutogenManifest(manifestURL string) (*autogenManifest, error) {
+	var manifest autogenManifest
+	if err := fetchJSON(manifestURL, &manifest); err != nil {
+		return nil, fmt.Errorf("download autogen manifest: %w", err)
+	}
+	if len(manifest.Entries) == 0 {
+		var legacy partsManifest
+		if err := fetchJSON(manifestURL, &legacy); err == nil && len(legacy.Categories) > 0 {
+			return legacyPartsManifestToAutogen(&legacy), nil
+		}
+	}
+	if len(manifest.Entries) == 0 {
+		return nil, fmt.Errorf("download autogen manifest: entries is empty")
+	}
+	return &manifest, nil
+}
+
+func readFreshAutogenManifestCache(path string, maxAge time.Duration) (*autogenManifest, bool, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, false, nil
+		}
+		return nil, false, err
+	}
+	if maxAge > 0 && time.Since(info.ModTime()) > maxAge {
+		return nil, false, nil
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, false, err
+	}
+	var manifest autogenManifest
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		return nil, false, err
+	}
+	if len(manifest.Entries) == 0 {
+		var legacy partsManifest
+		if err := json.Unmarshal(raw, &legacy); err == nil && len(legacy.Categories) > 0 {
+			return legacyPartsManifestToAutogen(&legacy), true, nil
+		}
+	}
+	if len(manifest.Entries) == 0 {
+		return nil, false, fmt.Errorf("entries is empty")
+	}
+	return &manifest, true, nil
+}
+
+func legacyPartsManifestToAutogen(legacy *partsManifest) *autogenManifest {
+	entries := make([]autogenManifestEntry, 0)
+	for _, category := range supportedPartCategories {
+		for _, rowID := range legacy.Categories[category] {
+			entries = append(entries, autogenManifestEntry{
+				Group: category,
+				RowID: rowID,
+			})
+		}
+	}
+	slices.SortFunc(entries, func(a, b autogenManifestEntry) int {
+		if cmp := strings.Compare(a.Group, b.Group); cmp != 0 {
+			return cmp
+		}
+		return a.RowID - b.RowID
+	})
+	return &autogenManifest{
+		ID:          "parts",
+		Repo:        legacy.Repo,
+		Ref:         legacy.Ref,
+		GeneratedAt: legacy.GeneratedAt,
+		Entries:     entries,
+	}
+}
+
+func writeAutogenManifestCache(path string, manifest *autogenManifest) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("create autogen manifest cache dir: %w", err)
+	}
+	payload, err := json.MarshalIndent(manifest, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal autogen manifest cache: %w", err)
+	}
+	payload = append(payload, '\n')
+	current, err := os.ReadFile(path)
+	if err == nil && bytes.Equal(current, payload) {
+		now := time.Now()
+		if err := os.Chtimes(path, now, now); err != nil {
+			return fmt.Errorf("refresh autogen manifest cache timestamp: %w", err)
+		}
+		return nil
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("read autogen manifest cache: %w", err)
+	}
+	tmpPath := path + ".tmp"
+	if err := os.WriteFile(tmpPath, payload, 0o644); err != nil {
+		return fmt.Errorf("write autogen manifest cache: %w", err)
+	}
+	if err := os.Rename(tmpPath, path); err != nil {
+		return fmt.Errorf("replace autogen manifest cache: %w", err)
+	}
+	return nil
+}
+
+func autogenPartsInventory(entries []autogenManifestEntry) map[string]map[int]struct{} {
+	result := make(map[string]map[int]struct{}, len(supportedPartCategories))
+	for _, category := range supportedPartCategories {
+		result[category] = map[int]struct{}{}
+	}
+	for _, entry := range entries {
+		if !isSupportedPartCategory(entry.Group) || entry.RowID <= 0 {
+			continue
+		}
+		result[entry.Group][entry.RowID] = struct{}{}
+	}
+	return result
+}
+
+func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDataset, entries []autogenManifestEntry) ([]nativeCollectedDataset, error) {
+	if len(entries) == 0 {
+		return collected, nil
+	}
+
+	result := append([]nativeCollectedDataset(nil), collected...)
+	for i, dataset := range result {
+		if dataset.Dataset.Name != "visualeffects" {
+			continue
+		}
+
+		rows := make([]map[string]any, len(dataset.Rows))
+		rowByKey := make(map[string]map[string]any, len(dataset.Rows))
+		usedIDs := make(map[int]struct{}, len(dataset.Rows)+len(dataset.LockData))
+		lockData := make(map[string]int, len(dataset.LockData))
+		for key, rowID := range dataset.LockData {
+			lockData[key] = rowID
+			usedIDs[rowID] = struct{}{}
+		}
+		for index, row := range dataset.Rows {
+			cloned := cloneRowMap(row)
+			rows[index] = cloned
+			if key, ok := cloned["key"].(string); ok && key != "" {
+				rowByKey[key] = cloned
+			}
+			if rowID, ok := cloned["id"].(int); ok {
+				usedIDs[rowID] = struct{}{}
+			}
+		}
+
+		nextID := nextAvailableAutogenID(usedIDs)
+		allocateNextID := func() int {
+			rowID := nextID
+			usedIDs[rowID] = struct{}{}
+			nextID = nextAvailableAutogenID(usedIDs)
+			return rowID
+		}
+
+		for _, entry := range entries {
+			key, label, ok := headVisualeffectIdentity(entry)
+			if !ok {
+				continue
+			}
+			if existing, exists := rowByKey[key]; exists {
+				applyDiscoveredHeadVisualeffectDefaults(existing, dataset.Columns, entry.ModelStem, label)
+				continue
+			}
+
+			rowID, ok := lockData[key]
+			if !ok {
+				rowID = allocateNextID()
+				lockData[key] = rowID
+			}
+			newRow := createDefaultHeadVisualeffectRow(dataset.Columns, rowID, key, label, entry.ModelStem)
+			rows = append(rows, newRow)
+			rowByKey[key] = newRow
+		}
+
+		slices.SortFunc(rows, func(a, b map[string]any) int {
+			return a["id"].(int) - b["id"].(int)
+		})
+		result[i].Rows = rows
+		result[i].LockData = lockData
+		result[i].LockModified = !lockDataEqual(dataset.LockData, lockData)
+	}
+	return result, nil
+}
+
+func nextAvailableAutogenID(used map[int]struct{}) int {
+	for rowID := 0; ; rowID++ {
+		if _, ok := used[rowID]; !ok {
+			return rowID
+		}
+	}
+}
+
+func headVisualeffectIdentity(entry autogenManifestEntry) (string, string, bool) {
+	groupPrefix := map[string]string{
+		"head_accessories": "headaccessory",
+		"head_features":    "headfeature",
+	}
+	prefix, ok := groupPrefix[entry.Group]
+	if !ok || strings.TrimSpace(entry.ModelStem) == "" {
+		return "", "", false
+	}
+	stem := strings.TrimPrefix(entry.ModelStem, "hfx_")
+	stem = strings.TrimPrefix(stem, "vfx_")
+	stem = strings.TrimSpace(stem)
+	if stem == "" {
+		return "", "", false
+	}
+	return "visualeffects:" + prefix + "_" + stem, prefix + "_" + strings.ToUpper(stem), true
+}
+
+func createDefaultHeadVisualeffectRow(columns []string, rowID int, key, label, modelStem string) map[string]any {
+	row := map[string]any{
+		"id":  rowID,
+		"key": key,
+	}
+	for _, column := range columns {
+		row[column] = nullValue
+	}
+	row["Label"] = label
+	row["Type_FD"] = "D"
+	row["OrientWithGround"] = "0"
+	row["Imp_HeadCon_Node"] = modelStem
+	row["OrientWithObject"] = "1"
+	return row
+}
+
+func applyDiscoveredHeadVisualeffectDefaults(row map[string]any, columns []string, modelStem, label string) {
+	if isNullLikeValue(row["Label"]) {
+		row["Label"] = label
+	}
+	if isNullLikeValue(row["Type_FD"]) {
+		row["Type_FD"] = "D"
+	}
+	if isNullLikeValue(row["OrientWithGround"]) {
+		row["OrientWithGround"] = "0"
+	}
+	if isNullLikeValue(row["Imp_HeadCon_Node"]) {
+		row["Imp_HeadCon_Node"] = modelStem
+	}
+	if isNullLikeValue(row["OrientWithObject"]) {
+		row["OrientWithObject"] = "1"
+	}
+	for _, column := range columns {
+		if _, ok := row[column]; !ok {
+			row[column] = nullValue
+		}
+	}
+}
+
+func formatAutogenManifest(root string, producer project.AutogenProducerConfig, repo, ref string, entries []autogenManifestEntry) ([]byte, error) {
+	manifest := autogenManifest{
+		ID:          producer.ID,
+		Repo:        repo,
+		Ref:         ref,
+		GeneratedAt: time.Now().UTC().Format(time.RFC3339),
+		Entries:     entries,
+	}
+	return json.MarshalIndent(manifest, "", "  ")
+}
+
+func parseAutogenManifestEntryList(raw []byte) ([]autogenManifestEntry, error) {
+	var manifest autogenManifest
+	if err := json.Unmarshal(raw, &manifest); err != nil {
+		return nil, err
+	}
+	return manifest.Entries, nil
+}
+
+func normalizeAutogenNumericValue(value any) string {
+	switch typed := value.(type) {
+	case string:
+		return typed
+	case int:
+		return strconv.Itoa(typed)
+	case float64:
+		return strconv.Itoa(int(typed))
+	default:
+		return fmt.Sprintf("%v", value)
+	}
+}

internal/topdata/native.go

@@ -284,16 +284,9 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
 			}
 		}
 	}
-	if hasCollectedPartsDatasets(collected) {
-		progress("Resolving released parts manifest...")
-		autogenerated, err := resolveAutogeneratedPartsInventory(p, progress)
-		if err != nil {
-			return BuildResult{}, err
-		}
-		if autogenerated != nil {
-			collected = augmentWithAutogeneratedParts(collected, autogenerated)
-			progress("Augmented parts datasets with discovered models")
-		}
+	collected, err = applyAutogenConsumers(p, collected, progress)
+	if err != nil {
+		return BuildResult{}, err
 	}
 	collected, err = applyPartOverrides(sourceDir, collected)
 	if err != nil {

internal/topdata/parts_discovery.go

@@ -60,6 +60,19 @@ func isPartsDataset(datasetName string) bool {
 	return false
 }
 
+func isAutogenEligiblePartsDataset(dataset nativeCollectedDataset) bool {
+	if !isPartsDataset(dataset.Dataset.Name) {
+		return false
+	}
+	for _, row := range dataset.Rows {
+		key, ok := row["key"].(string)
+		if ok && strings.TrimSpace(key) != "" {
+			return false
+		}
+	}
+	return true
+}
+
 // extractPartCategory extracts the part category from a dataset name
 // e.g., "parts/belt" -> "belt", "parts/bicep" -> "bicep"
 func extractPartCategory(datasetName string) string {
@@ -430,6 +443,9 @@ func augmentWithAutogeneratedParts(collected []nativeCollectedDataset, autogener
 	copy(result, collected)
 
 	for i, dataset := range collected {
+		if !isAutogenEligiblePartsDataset(dataset) {
+			continue
+		}
 		category := partAssetCategoryForDataset(dataset.Dataset.Name)
 		if !isSupportedPartCategory(category) {
 			continue

internal/topdata/topdata_test.go

@@ -5479,6 +5479,121 @@ with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f:
 	}
 }
 
+func TestApplyAutogenConsumersAugmentsPartsFromLocalOverride(t *testing.T) {
+	root := testProjectRoot(t)
+	overrideRoot := filepath.Join(root, "autogen-assets")
+	mkdirAll(t, filepath.Join(overrideRoot, "part", "belt"))
+	writeFile(t, filepath.Join(overrideRoot, "part", "belt", "pfa0_belt018.mdl"), "mdl\n")
+
+	p := testProject(root)
+	p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{
+		{
+			ID:                "parts",
+			Producer:          "parts",
+			Dataset:           "parts",
+			Mode:              "parts_rows",
+			Root:              "part",
+			Include:           []string{"**/*.mdl"},
+			Derive:            project.AutogenDeriveConfig{Kind: "trailing_numeric_suffix", GroupFrom: "first_path_segment"},
+			Manifest:          project.AutogenManifestConfig{ReleaseTag: "parts-manifest-current", AssetName: "sow-parts-manifest.json", CacheName: "sow-parts-manifest.json"},
+			LocalOverrideRoot: "autogen-assets",
+		},
+	}
+
+	collected := []nativeCollectedDataset{
+		{
+			Dataset: nativeDataset{Name: "parts/belt"},
+			Columns: []string{"COSTMODIFIER", "ACBONUS"},
+			Rows: []map[string]any{
+				{"id": 0, "COSTMODIFIER": "0", "ACBONUS": "0.00"},
+			},
+		},
+	}
+
+	got, err := applyAutogenConsumers(p, collected, nil)
+	if err != nil {
+		t.Fatalf("applyAutogenConsumers failed: %v", err)
+	}
+	if len(got) != 1 || len(got[0].Rows) != 2 {
+		t.Fatalf("expected autogenerated part row, got %#v", got)
+	}
+	row := got[0].Rows[1]
+	if row["id"] != 18 || row["COSTMODIFIER"] != "0" || row["ACBONUS"] != "0.00" {
+		t.Fatalf("unexpected autogenerated part row: %#v", row)
+	}
+}
+
+func TestApplyAutogenConsumersAugmentsHeadVisualeffectsFromLocalOverride(t *testing.T) {
+	root := testProjectRoot(t)
+	overrideRoot := filepath.Join(root, "autogen-assets")
+	mkdirAll(t, filepath.Join(overrideRoot, "vfxs", "head_accessories"))
+	mkdirAll(t, filepath.Join(overrideRoot, "vfxs", "head_features", "hair"))
+	writeFile(t, filepath.Join(overrideRoot, "vfxs", "head_accessories", "hfx_bandana.mdl"), "mdl\n")
+	writeFile(t, filepath.Join(overrideRoot, "vfxs", "head_features", "hair", "hfx_hair_bangs.mdl"), "mdl\n")
+
+	p := testProject(root)
+	p.Config.Autogen.Consumers = []project.AutogenConsumerConfig{
+		{
+			ID:                "head_visualeffects",
+			Producer:          "head_visualeffects",
+			Dataset:           "visualeffects",
+			Mode:              "head_visualeffects",
+			Root:              "vfxs",
+			Include:           []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"},
+			Derive:            project.AutogenDeriveConfig{Kind: "model_stem", GroupFrom: "first_path_segment"},
+			Manifest:          project.AutogenManifestConfig{ReleaseTag: "head-vfx-manifest-current", AssetName: "sow-head-vfx-manifest.json", CacheName: "sow-head-vfx-manifest.json"},
+			LocalOverrideRoot: "autogen-assets",
+		},
+	}
+
+	collected := []nativeCollectedDataset{
+		{
+			Dataset: nativeDataset{Name: "visualeffects", Kind: nativeDatasetBase},
+			Columns: []string{"Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "OrientWithObject"},
+			Rows:    nil,
+			LockData: map[string]int{
+				"visualeffects:existing": 17,
+			},
+		},
+	}
+
+	got, err := applyAutogenConsumers(p, collected, nil)
+	if err != nil {
+		t.Fatalf("applyAutogenConsumers failed: %v", err)
+	}
+	if len(got) != 1 || len(got[0].Rows) != 2 {
+		t.Fatalf("expected autogenerated visualeffects rows, got %#v", got)
+	}
+
+	rowByKey := map[string]map[string]any{}
+	for _, row := range got[0].Rows {
+		rowByKey[row["key"].(string)] = row
+	}
+
+	bandana := rowByKey["visualeffects:headaccessory_bandana"]
+	if bandana == nil {
+		t.Fatalf("expected head accessory row, got %#v", got[0].Rows)
+	}
+	if bandana["Label"] != "headaccessory_BANDANA" || bandana["Imp_HeadCon_Node"] != "hfx_bandana" || bandana["OrientWithObject"] != "1" {
+		t.Fatalf("unexpected head accessory defaults: %#v", bandana)
+	}
+
+	hair := rowByKey["visualeffects:headfeature_hair_bangs"]
+	if hair == nil {
+		t.Fatalf("expected head feature row, got %#v", got[0].Rows)
+	}
+	if hair["Label"] != "headfeature_HAIR_BANGS" || hair["Imp_HeadCon_Node"] != "hfx_hair_bangs" || hair["Type_FD"] != "D" {
+		t.Fatalf("unexpected head feature defaults: %#v", hair)
+	}
+
+	if _, ok := got[0].LockData["visualeffects:headaccessory_bandana"]; !ok {
+		t.Fatalf("expected head accessory lock id, got %#v", got[0].LockData)
+	}
+	if _, ok := got[0].LockData["visualeffects:headfeature_hair_bangs"]; !ok {
+		t.Fatalf("expected head feature lock id, got %#v", got[0].LockData)
+	}
+}
+
 func TestNormalizeProjectImportsLegacyLoadscreens(t *testing.T) {
 	root := testProjectRoot(t)
 	mkdirAll(t, filepath.Join(root, "topdata", "data"))
@@ -9028,6 +9143,27 @@ func TestBuildNativeWithReleasedPartsManifest(t *testing.T) {
 				Source: "topdata",
 				Build:  "build/topdata",
 			},
+			Autogen: project.AutogenConfig{
+				Consumers: []project.AutogenConsumerConfig{
+					{
+						ID:       "parts",
+						Producer: "parts",
+						Dataset:  "parts",
+						Mode:     "parts_rows",
+						Root:     "part",
+						Include:  []string{"**/*.mdl"},
+						Derive: project.AutogenDeriveConfig{
+							Kind:      "trailing_numeric_suffix",
+							GroupFrom: "first_path_segment",
+						},
+						Manifest: project.AutogenManifestConfig{
+							ReleaseTag: "parts-manifest-current",
+							AssetName:  "sow-parts-manifest.json",
+							CacheName:  "sow-parts-manifest.json",
+						},
+					},
+				},
+			},
 		},
 	}
 
@@ -9105,6 +9241,27 @@ func TestBuildNativeFailsWhenReleasedPartsManifestUnavailable(t *testing.T) {
 				Source: "topdata",
 				Build:  "build/topdata",
 			},
+			Autogen: project.AutogenConfig{
+				Consumers: []project.AutogenConsumerConfig{
+					{
+						ID:       "parts",
+						Producer: "parts",
+						Dataset:  "parts",
+						Mode:     "parts_rows",
+						Root:     "part",
+						Include:  []string{"**/*.mdl"},
+						Derive: project.AutogenDeriveConfig{
+							Kind:      "trailing_numeric_suffix",
+							GroupFrom: "first_path_segment",
+						},
+						Manifest: project.AutogenManifestConfig{
+							ReleaseTag: "parts-manifest-current",
+							AssetName:  "sow-parts-manifest.json",
+							CacheName:  "sow-parts-manifest.json",
+						},
+					},
+				},
+			},
 		},
 	}
 
@@ -9564,6 +9721,45 @@ func testProject(root string) *project.Project {
 				ReferenceBuilder: "reference",
 				Assets:           ".",
 			},
+			Autogen: project.AutogenConfig{
+				Consumers: []project.AutogenConsumerConfig{
+					{
+						ID:       "parts",
+						Producer: "parts",
+						Dataset:  "parts",
+						Mode:     "parts_rows",
+						Root:     "part",
+						Include:  []string{"**/*.mdl"},
+						Derive: project.AutogenDeriveConfig{
+							Kind:      "trailing_numeric_suffix",
+							GroupFrom: "first_path_segment",
+						},
+						Manifest: project.AutogenManifestConfig{
+							ReleaseTag: "parts-manifest-current",
+							AssetName:  "sow-parts-manifest.json",
+							CacheName:  "sow-parts-manifest.json",
+						},
+					},
+					{
+						ID:       "head_visualeffects",
+						Producer: "head_visualeffects",
+						Dataset:  "visualeffects",
+						Mode:     "head_visualeffects",
+						Optional: true,
+						Root:     "vfxs",
+						Include:  []string{"head_accessories/**/*.mdl", "head_features/**/*.mdl"},
+						Derive: project.AutogenDeriveConfig{
+							Kind:      "model_stem",
+							GroupFrom: "first_path_segment",
+						},
+						Manifest: project.AutogenManifestConfig{
+							ReleaseTag: "head-vfx-manifest-current",
+							AssetName:  "sow-head-vfx-manifest.json",
+							CacheName:  "sow-head-vfx-manifest.json",
+						},
+					},
+				},
+			},
 		},
 	}
 }