Migrate parts data to assets repo

internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md

@@ -2,15 +2,15 @@
 
 ## Status Snapshot
 
-Current state as of 2026-05-13:
+Current state as of 2026-05-17:
 
-- Implemented.
-- The toolkit resolves released parts manifests through the configured autogen
-  release source and covers this behavior with `parts_manifest_test.go` plus
-  broader native topdata tests.
-- Remaining work is operational hardening only: keep manifest contracts stable
-  across `assets/` releases and maintain regression coverage as autogen config
-  evolves.
+- Superseded for current Shadows Over Westgate builds.
+- The module no longer consumes a released parts manifest. Parts data is owned by
+  `assets/`, generated as HAK-side 2DA assets from `assets/topdata/data/parts`
+  plus a direct scan of `assets/content/part/**/*.mdl`, and packaged in
+  `sow_part`.
+- This file remains as historical context for the legacy manifest-backed
+  consumer path.
 
 ## Objective
 
@@ -22,14 +22,14 @@ This behavior is required for parity with the current asset set and must not dep
 
 ## Source of Truth
 
-The native builder must obtain existing model information from the released
-`sow-parts-manifest.json` artifact published by the **`sow-assets` repository**.
+The superseded manifest-backed native builder obtained existing model
+information from a released artifact published by the **`sow-assets` repository**.
 
 Resolution rules:
 
 - if `topdata.assets` is configured to a local path, use that exact path
 - otherwise derive the **`sow-assets` repository** from the module repo's Gitea origin
-  and fetch only the released manifest from the `parts-manifest-current` release
+  and fetch only the configured released manifest
 
 Normal builds must not clone or scan the `sow-assets` repo tree. They may cache the
 downloaded manifest locally for reuse, but the source of truth remains the released
@@ -122,9 +122,9 @@ That means:
 
 ## Acceptance Criteria
 
-Implementation is correct only if all of the following are true:
+The superseded implementation was correct only if all of the following were true:
 
-1. The builder fetches the released `sow-parts-manifest.json` artifact for part models.
+1. The builder fetches the configured released manifest artifact for part models.
 2. Only the supported part categories are considered.
 3. Manifest IDs derived from trailing numeric suffixes generate corresponding 2DA rows.
 4. The numeric suffix is used as the row ID.

internal/topdata/generated_assets.go

@@ -0,0 +1,194 @@
+package topdata
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+type Generated2DAAsset struct {
+	Rel        string
+	SourcePath string
+}
+
+func BuildGenerated2DAAssets(p *project.Project, progress func(string)) ([]Generated2DAAsset, error) {
+	if progress == nil {
+		progress = func(string) {}
+	}
+	configs := p.EffectiveConfig().Generated.TopData2DA
+	if len(configs) == 0 {
+		return nil, nil
+	}
+
+	results := make([]Generated2DAAsset, 0)
+	for _, cfg := range configs {
+		progress(fmt.Sprintf("Building generated 2DA assets for %s...", cfg.ID))
+		generated, err := buildGenerated2DAAssetGroup(p, cfg, progress)
+		if err != nil {
+			return nil, err
+		}
+		results = append(results, generated...)
+	}
+	return results, nil
+}
+
+func buildGenerated2DAAssetGroup(p *project.Project, cfg project.GeneratedTopData2DAConfig, progress func(string)) ([]Generated2DAAsset, error) {
+	sourceDir := filepath.Join(p.Root, filepath.FromSlash(strings.TrimSpace(cfg.Source)))
+	dataDir := filepath.Join(sourceDir, "data")
+	outputDir := filepath.Join(p.Root, filepath.FromSlash(strings.TrimSpace(cfg.Output)))
+
+	datasets, err := discoverNativeDatasets(dataDir)
+	if err != nil {
+		return nil, fmt.Errorf("discover generated topdata 2DA datasets %s: %w", cfg.ID, err)
+	}
+	selected := make([]nativeDataset, 0, len(datasets))
+	for _, dataset := range datasets {
+		if generatedDatasetIncluded(dataset.Name, cfg.IncludeDatasets) {
+			selected = append(selected, dataset)
+		}
+	}
+	if len(selected) == 0 {
+		return nil, fmt.Errorf("generated topdata 2DA config %s matched no datasets under %s", cfg.ID, dataDir)
+	}
+
+	collected := make([]nativeCollectedDataset, 0, len(selected))
+	for _, dataset := range selected {
+		collectedDataset, err := collectNativeDataset(dataset)
+		if err != nil {
+			return nil, err
+		}
+		collected = append(collected, collectedDataset)
+	}
+
+	consumer := cfg.Autogen
+	consumer.LocalOverrideRoot = p.AssetsDir()
+	collected, err = applyAutogenConsumersForGeneratedAssets(collected, consumer, progress)
+	if err != nil {
+		return nil, err
+	}
+	collected, err = applyPartOverrides(sourceDir, collected)
+	if err != nil {
+		return nil, err
+	}
+	if err := rejectGenerated2DATLKValues(cfg.ID, collected); err != nil {
+		return nil, err
+	}
+
+	tableRegistry, err := newResolvedTableRegistry(collected)
+	if err != nil {
+		return nil, err
+	}
+	keyToID, rowByKey := generated2DAGlobalRows(collected)
+
+	if err := os.RemoveAll(outputDir); err != nil {
+		return nil, fmt.Errorf("clean generated topdata 2DA output %s: %w", outputDir, err)
+	}
+	if err := os.MkdirAll(outputDir, 0o755); err != nil {
+		return nil, fmt.Errorf("create generated topdata 2DA output %s: %w", outputDir, err)
+	}
+
+	results := make([]Generated2DAAsset, 0, len(collected))
+	for _, dataset := range collected {
+		compiled, err := resolveNativeDataset(dataset, keyToID, rowByKey, tableRegistry, nil)
+		if err != nil {
+			return nil, err
+		}
+		outputPath := filepath.Join(outputDir, dataset.Dataset.OutputName)
+		if err := write2DA(compiled, outputPath, dataset.Dataset.Kind == nativeDatasetBase); err != nil {
+			return nil, err
+		}
+		results = append(results, Generated2DAAsset{
+			Rel:        filepath.ToSlash(filepath.Join(cfg.PackageRoot, dataset.Dataset.OutputName)),
+			SourcePath: outputPath,
+		})
+	}
+	return results, nil
+}
+
+func applyAutogenConsumersForGeneratedAssets(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig, progress func(string)) ([]nativeCollectedDataset, error) {
+	if strings.TrimSpace(consumer.Mode) == "" {
+		return collected, nil
+	}
+	if progress == nil {
+		progress = func(string) {}
+	}
+	if !autogenConsumerTargetsCollectedDataset(collected, consumer) {
+		return collected, nil
+	}
+	if progress != nil {
+		progress(fmt.Sprintf("Scanning local generated 2DA autogen input for %s from %s...", consumer.ID, consumer.LocalOverrideRoot))
+	}
+	entries, err := scanLocalAutogenEntries(consumer.LocalOverrideRoot, consumer)
+	if err != nil {
+		return nil, err
+	}
+	switch consumer.Mode {
+	case "parts_rows":
+		return augmentWithAutogeneratedParts(collected, autogenPartsInventory(entries)), nil
+	default:
+		return nil, fmt.Errorf("unsupported generated topdata 2DA autogen mode %q", consumer.Mode)
+	}
+}
+
+func generatedDatasetIncluded(name string, patterns []string) bool {
+	name = filepath.ToSlash(strings.TrimSpace(name))
+	for _, pattern := range patterns {
+		pattern = filepath.ToSlash(strings.TrimSpace(pattern))
+		if pattern == "" {
+			continue
+		}
+		if pattern == name {
+			return true
+		}
+		if strings.HasSuffix(pattern, "/**") {
+			prefix := strings.TrimSuffix(pattern, "/**")
+			if name == prefix || strings.HasPrefix(name, prefix+"/") {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func generated2DAGlobalRows(collected []nativeCollectedDataset) (map[string]int, map[string]map[string]any) {
+	keyToID := map[string]int{}
+	rowByKey := map[string]map[string]any{}
+	for _, dataset := range collected {
+		for key, rowID := range dataset.LockData {
+			keyToID[key] = rowID
+		}
+		for _, row := range dataset.Rows {
+			key, _ := row["key"].(string)
+			if key == "" {
+				continue
+			}
+			if rowID, ok := row["id"].(int); ok {
+				keyToID[key] = rowID
+				rowByKey[key] = row
+			}
+		}
+	}
+	return keyToID, rowByKey
+}
+
+func rejectGenerated2DATLKValues(configID string, collected []nativeCollectedDataset) error {
+	for _, dataset := range collected {
+		for _, row := range dataset.Rows {
+			for field, value := range row {
+				if field == "id" || field == "key" {
+					continue
+				}
+				allowBare := columnMatchesSpec(dataset.Dataset.Spec, field)
+				if _, ok, err := parseTLKPayload(value, allowBare); err != nil {
+					return fmt.Errorf("generated topdata 2DA config %s dataset %s field %s has invalid TLK payload: %w", configID, dataset.Dataset.Name, field, err)
+				} else if ok {
+					return fmt.Errorf("generated topdata 2DA config %s dataset %s field %s uses TLK-backed text; 2DA-only generated asset builds do not generate TLK output", configID, dataset.Dataset.Name, field)
+				}
+			}
+		}
+	}
+	return nil
+}

internal/topdata/parts_discovery.go

@@ -364,8 +364,7 @@ func applyPartOverrides(sourceDir string, collected []nativeCollectedDataset) ([
 		if category == "" {
 			continue
 		}
-		overridePath := filepath.Join(sourceDir, "data", "parts", "overrides", category+".json")
-		overrides, err := loadPartOverrides(overridePath)
+		overrides, err := loadPartOverridesForCategory(sourceDir, category)
 		if err != nil {
 			return nil, err
 		}
@@ -386,11 +385,11 @@ func applyPartOverrides(sourceDir string, collected []nativeCollectedDataset) ([
 		for index, override := range overrides {
 			rawID, ok := override["id"]
 			if !ok {
-				return nil, fmt.Errorf("%s: override %d is missing id", overridePath, index)
+				return nil, fmt.Errorf("parts/%s override %d is missing id", category, index)
 			}
 			rowID, err := asInt(rawID)
 			if err != nil {
-				return nil, fmt.Errorf("%s: override %d id is not numeric", overridePath, index)
+				return nil, fmt.Errorf("parts/%s override %d id is not numeric", category, index)
 			}
 			row, ok := rowByID[rowID]
 			if !ok {
@@ -416,6 +415,75 @@ func applyPartOverrides(sourceDir string, collected []nativeCollectedDataset) ([
 	return result, nil
 }
 
+func loadPartOverridesForCategory(sourceDir, category string) ([]map[string]any, error) {
+	overrides := []map[string]any{}
+
+	legacyPath := filepath.Join(sourceDir, "data", "parts", "overrides", category+".json")
+	legacyOverrides, err := loadPartOverrides(legacyPath)
+	if err != nil {
+		return nil, err
+	}
+	overrides = append(overrides, legacyOverrides...)
+
+	moduleDir := filepath.Join(sourceDir, "data", "parts", "modules")
+	modulePaths, err := partModuleOverridePaths(moduleDir, category)
+	if err != nil {
+		return nil, err
+	}
+	for _, path := range modulePaths {
+		moduleOverrides, err := loadPartOverrides(path)
+		if err != nil {
+			return nil, err
+		}
+		overrides = append(overrides, moduleOverrides...)
+	}
+	return overrides, nil
+}
+
+func partModuleOverridePaths(moduleDir, category string) ([]string, error) {
+	info, err := os.Stat(moduleDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	if !info.IsDir() {
+		return nil, fmt.Errorf("parts module override path %s is not a directory", moduleDir)
+	}
+
+	var paths []string
+	err = filepath.WalkDir(moduleDir, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		if strings.ToLower(filepath.Ext(path)) != ".json" {
+			return nil
+		}
+		if partModuleOverrideMatchesCategory(path, category) {
+			paths = append(paths, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	slices.Sort(paths)
+	return paths, nil
+}
+
+func partModuleOverrideMatchesCategory(path, category string) bool {
+	stem := strings.TrimSuffix(strings.ToLower(filepath.Base(path)), strings.ToLower(filepath.Ext(path)))
+	category = strings.ToLower(strings.TrimSpace(category))
+	return stem == category ||
+		strings.HasPrefix(stem, category+"_") ||
+		strings.HasSuffix(stem, "_"+category) ||
+		strings.Contains(stem, "_"+category+"_")
+}
+
 func normalizePartOverrideValue(value any) any {
 	switch typed := value.(type) {
 	case float64: