Migrate parts data to assets repo

2026-05-17 13:46:09 +02:00
parent faf1459dba
commit d7704c5f68
9 changed files with 652 additions and 48 deletions
@@ -0,0 +1,194 @@
+package topdata
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+type Generated2DAAsset struct {
+	Rel        string
+	SourcePath string
+}
+
+func BuildGenerated2DAAssets(p *project.Project, progress func(string)) ([]Generated2DAAsset, error) {
+	if progress == nil {
+		progress = func(string) {}
+	}
+	configs := p.EffectiveConfig().Generated.TopData2DA
+	if len(configs) == 0 {
+		return nil, nil
+	}
+
+	results := make([]Generated2DAAsset, 0)
+	for _, cfg := range configs {
+		progress(fmt.Sprintf("Building generated 2DA assets for %s...", cfg.ID))
+		generated, err := buildGenerated2DAAssetGroup(p, cfg, progress)
+		if err != nil {
+			return nil, err
+		}
+		results = append(results, generated...)
+	}
+	return results, nil
+}
+
+func buildGenerated2DAAssetGroup(p *project.Project, cfg project.GeneratedTopData2DAConfig, progress func(string)) ([]Generated2DAAsset, error) {
+	sourceDir := filepath.Join(p.Root, filepath.FromSlash(strings.TrimSpace(cfg.Source)))
+	dataDir := filepath.Join(sourceDir, "data")
+	outputDir := filepath.Join(p.Root, filepath.FromSlash(strings.TrimSpace(cfg.Output)))
+
+	datasets, err := discoverNativeDatasets(dataDir)
+	if err != nil {
+		return nil, fmt.Errorf("discover generated topdata 2DA datasets %s: %w", cfg.ID, err)
+	}
+	selected := make([]nativeDataset, 0, len(datasets))
+	for _, dataset := range datasets {
+		if generatedDatasetIncluded(dataset.Name, cfg.IncludeDatasets) {
+			selected = append(selected, dataset)
+		}
+	}
+	if len(selected) == 0 {
+		return nil, fmt.Errorf("generated topdata 2DA config %s matched no datasets under %s", cfg.ID, dataDir)
+	}
+
+	collected := make([]nativeCollectedDataset, 0, len(selected))
+	for _, dataset := range selected {
+		collectedDataset, err := collectNativeDataset(dataset)
+		if err != nil {
+			return nil, err
+		}
+		collected = append(collected, collectedDataset)
+	}
+
+	consumer := cfg.Autogen
+	consumer.LocalOverrideRoot = p.AssetsDir()
+	collected, err = applyAutogenConsumersForGeneratedAssets(collected, consumer, progress)
+	if err != nil {
+		return nil, err
+	}
+	collected, err = applyPartOverrides(sourceDir, collected)
+	if err != nil {
+		return nil, err
+	}
+	if err := rejectGenerated2DATLKValues(cfg.ID, collected); err != nil {
+		return nil, err
+	}
+
+	tableRegistry, err := newResolvedTableRegistry(collected)
+	if err != nil {
+		return nil, err
+	}
+	keyToID, rowByKey := generated2DAGlobalRows(collected)
+
+	if err := os.RemoveAll(outputDir); err != nil {
+		return nil, fmt.Errorf("clean generated topdata 2DA output %s: %w", outputDir, err)
+	}
+	if err := os.MkdirAll(outputDir, 0o755); err != nil {
+		return nil, fmt.Errorf("create generated topdata 2DA output %s: %w", outputDir, err)
+	}
+
+	results := make([]Generated2DAAsset, 0, len(collected))
+	for _, dataset := range collected {
+		compiled, err := resolveNativeDataset(dataset, keyToID, rowByKey, tableRegistry, nil)
+		if err != nil {
+			return nil, err
+		}
+		outputPath := filepath.Join(outputDir, dataset.Dataset.OutputName)
+		if err := write2DA(compiled, outputPath, dataset.Dataset.Kind == nativeDatasetBase); err != nil {
+			return nil, err
+		}
+		results = append(results, Generated2DAAsset{
+			Rel:        filepath.ToSlash(filepath.Join(cfg.PackageRoot, dataset.Dataset.OutputName)),
+			SourcePath: outputPath,
+		})
+	}
+	return results, nil
+}
+
+func applyAutogenConsumersForGeneratedAssets(collected []nativeCollectedDataset, consumer project.AutogenConsumerConfig, progress func(string)) ([]nativeCollectedDataset, error) {
+	if strings.TrimSpace(consumer.Mode) == "" {
+		return collected, nil
+	}
+	if progress == nil {
+		progress = func(string) {}
+	}
+	if !autogenConsumerTargetsCollectedDataset(collected, consumer) {
+		return collected, nil
+	}
+	if progress != nil {
+		progress(fmt.Sprintf("Scanning local generated 2DA autogen input for %s from %s...", consumer.ID, consumer.LocalOverrideRoot))
+	}
+	entries, err := scanLocalAutogenEntries(consumer.LocalOverrideRoot, consumer)
+	if err != nil {
+		return nil, err
+	}
+	switch consumer.Mode {
+	case "parts_rows":
+		return augmentWithAutogeneratedParts(collected, autogenPartsInventory(entries)), nil
+	default:
+		return nil, fmt.Errorf("unsupported generated topdata 2DA autogen mode %q", consumer.Mode)
+	}
+}
+
+func generatedDatasetIncluded(name string, patterns []string) bool {
+	name = filepath.ToSlash(strings.TrimSpace(name))
+	for _, pattern := range patterns {
+		pattern = filepath.ToSlash(strings.TrimSpace(pattern))
+		if pattern == "" {
+			continue
+		}
+		if pattern == name {
+			return true
+		}
+		if strings.HasSuffix(pattern, "/**") {
+			prefix := strings.TrimSuffix(pattern, "/**")
+			if name == prefix || strings.HasPrefix(name, prefix+"/") {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func generated2DAGlobalRows(collected []nativeCollectedDataset) (map[string]int, map[string]map[string]any) {
+	keyToID := map[string]int{}
+	rowByKey := map[string]map[string]any{}
+	for _, dataset := range collected {
+		for key, rowID := range dataset.LockData {
+			keyToID[key] = rowID
+		}
+		for _, row := range dataset.Rows {
+			key, _ := row["key"].(string)
+			if key == "" {
+				continue
+			}
+			if rowID, ok := row["id"].(int); ok {
+				keyToID[key] = rowID
+				rowByKey[key] = row
+			}
+		}
+	}
+	return keyToID, rowByKey
+}
+
+func rejectGenerated2DATLKValues(configID string, collected []nativeCollectedDataset) error {
+	for _, dataset := range collected {
+		for _, row := range dataset.Rows {
+			for field, value := range row {
+				if field == "id" || field == "key" {
+					continue
+				}
+				allowBare := columnMatchesSpec(dataset.Dataset.Spec, field)
+				if _, ok, err := parseTLKPayload(value, allowBare); err != nil {
+					return fmt.Errorf("generated topdata 2DA config %s dataset %s field %s has invalid TLK payload: %w", configID, dataset.Dataset.Name, field, err)
+				} else if ok {
+					return fmt.Errorf("generated topdata 2DA config %s dataset %s field %s uses TLK-backed text; 2DA-only generated asset builds do not generate TLK output", configID, dataset.Dataset.Name, field)
+				}
+			}
+		}
+	}
+	return nil
+}