Topdata Migration (#2)

Native topdata migration summary - Replaced the legacy 2dabuilder runtime path with the native topdata builder. - Native build, validate, compare, and convert flows now cover the canonical topdata pipeline. - compare-topdata is now a native self-check; legacy reference-builder runtime usage was removed. - Parts generation follows the native asset-scan contract and uses sow-assets via project asset resolution. - Generated feat families, class-feat injects, masterfeat/successor expansion, and dataset-derived feat generation were brought to parity and regression-covered. - Remaining mirrored datasets were migrated into native-owned canonical data while preserving lock IDs. - normalize-topdata bridge behavior and migration-era ownership markers were removed. - Documentation was rewritten around the native workflow and active contracts were cleaned up. - Final hardening pass removed stale validator assumptions and cleaned ignored/generated artifacts for PR readiness. Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/2 Co-authored-by: vickydotbat <vickydotbat@tutamail.com> Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
2026-04-09 07:14:01 +00:00
parent 99f9d29d7a
commit dc93feb176
1402 changed files with 20955 additions and 3405 deletions
@@ -0,0 +1,428 @@
+package topdata
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// supportedPartCategories defines the part types that should be auto-discovered
+// from the sow-assets repository.
+var supportedPartCategories = []string{
+	"belt",
+	"bicep",
+	"chest",
+	"foot",
+	"forearm",
+	"leg",
+	"neck",
+	"pelvis",
+	"robe",
+	"shin",
+	"shoulder",
+}
+
+var partDatasetToAssetCategory = map[string]string{
+	"belt":     "belt",
+	"bicep":    "bicep",
+	"chest":    "chest",
+	"foot":     "foot",
+	"forearm":  "forearm",
+	"legs":     "leg",
+	"neck":     "neck",
+	"pelvis":   "pelvis",
+	"robe":     "robe",
+	"shin":     "shin",
+	"shoulder": "shoulder",
+}
+
+// trailingNumberRegex matches trailing digits at the end of a string
+var trailingNumberRegex = regexp.MustCompile(`(\d+)$`)
+
+// isPartsDataset checks if a dataset name corresponds to a parts dataset
+func isPartsDataset(datasetName string) bool {
+	// Check if the dataset is under parts/ directory
+	if strings.HasPrefix(datasetName, "parts/") {
+		return true
+	}
+	// Check if it's exactly a parts file (e.g., "parts/belt")
+	parts := strings.Split(datasetName, "/")
+	if len(parts) == 2 && parts[0] == "parts" {
+		return true
+	}
+	return false
+}
+
+// extractPartCategory extracts the part category from a dataset name
+// e.g., "parts/belt" -> "belt", "parts/bicep" -> "bicep"
+func extractPartCategory(datasetName string) string {
+	parts := strings.Split(datasetName, "/")
+	if len(parts) >= 2 {
+		return parts[len(parts)-1]
+	}
+	return ""
+}
+
+func partAssetCategoryForDataset(datasetName string) string {
+	category := extractPartCategory(datasetName)
+	if mapped, ok := partDatasetToAssetCategory[category]; ok {
+		return mapped
+	}
+	return category
+}
+
+// isSupportedPartCategory checks if the given category is in the supported list
+func isSupportedPartCategory(category string) bool {
+	for _, c := range supportedPartCategories {
+		if c == category {
+			return true
+		}
+	}
+	return false
+}
+
+// extractTrailingNumericSuffix extracts the trailing numeric suffix from a filename stem
+// e.g., "pfa0_belt018" -> 18, "pfa0_belt171" -> 171
+// Returns 0 and false if no trailing numeric suffix is found
+func extractTrailingNumericSuffix(filenameStem string) (int, bool) {
+	matches := trailingNumberRegex.FindStringSubmatch(filenameStem)
+	if len(matches) < 2 {
+		return 0, false
+	}
+	num, err := strconv.Atoi(matches[1])
+	if err != nil {
+		return 0, false
+	}
+	return num, true
+}
+
+// DiscoveredPartModel represents a discovered model file with its row ID
+type DiscoveredPartModel struct {
+	RowID    int
+	Filename string
+}
+
+// DiscoverPartModels scans the assets/part directory for .mdl files
+// and returns a map of rowID -> default row data for the given category.
+// The assetsDir should be the root assets directory (containing the part/ subdirectory).
+func DiscoverPartModels(assetsDir string, category string) (map[int]*DiscoveredPartModel, error) {
+	result := make(map[int]*DiscoveredPartModel)
+
+	if !isSupportedPartCategory(category) {
+		return result, nil
+	}
+
+	partRoot, err := resolvePartsRoot(assetsDir)
+	if err != nil {
+		return nil, err
+	}
+	if partRoot == "" {
+		return result, nil
+	}
+
+	// Construct the path to the part category directory
+	partDir := filepath.Join(partRoot, category)
+
+	// Check if the directory exists
+	info, err := os.Stat(partDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return result, nil
+		}
+		return nil, fmt.Errorf("failed to access part directory %s: %w", partDir, err)
+	}
+	if !info.IsDir() {
+		return nil, fmt.Errorf("part directory %s is not a directory", partDir)
+	}
+
+	// Walk the directory and find .mdl files
+	err = filepath.Walk(partDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		// Only process .mdl files
+		if !strings.HasSuffix(strings.ToLower(info.Name()), ".mdl") {
+			return nil
+		}
+
+		// Extract the filename stem (without extension)
+		filenameStem := strings.TrimSuffix(info.Name(), filepath.Ext(info.Name()))
+
+		// Extract trailing numeric suffix
+		rowID, hasSuffix := extractTrailingNumericSuffix(filenameStem)
+		if !hasSuffix {
+			return nil
+		}
+
+		// Store the discovered model
+		if existing, ok := result[rowID]; ok {
+			// If we already have this row ID, keep the first one found (alphabetically)
+			if path < existing.Filename {
+				result[rowID] = &DiscoveredPartModel{
+					RowID:    rowID,
+					Filename: path,
+				}
+			}
+		} else {
+			result[rowID] = &DiscoveredPartModel{
+				RowID:    rowID,
+				Filename: path,
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to scan part directory %s: %w", partDir, err)
+	}
+
+	return result, nil
+}
+
+// CreateDefaultPartRow creates a default row for a discovered part model
+func CreateDefaultPartRow(rowID int) map[string]any {
+	return map[string]any{
+		"id":           rowID,
+		"COSTMODIFIER": "0",
+		"ACBONUS":      "0.00",
+	}
+}
+
+func isUnsetPartValue(value any) bool {
+	switch typed := value.(type) {
+	case nil:
+		return true
+	case string:
+		return typed == "" || typed == "****"
+	}
+	return false
+}
+
+func resolvePartsRoot(root string) (string, error) {
+	if strings.TrimSpace(root) == "" {
+		return "", nil
+	}
+	candidates := []string{
+		filepath.Join(root, "assets", "parts"),
+		filepath.Join(root, "assets", "part"),
+		filepath.Join(root, "parts"),
+		filepath.Join(root, "part"),
+	}
+	for _, candidate := range candidates {
+		info, err := os.Stat(candidate)
+		if err == nil && info.IsDir() {
+			return candidate, nil
+		}
+		if err != nil && !os.IsNotExist(err) {
+			return "", fmt.Errorf("failed to access parts root %s: %w", candidate, err)
+		}
+	}
+	return "", nil
+}
+
+// scanAutogeneratedParts scans the assets/part directory for all supported part categories
+// and returns a map of category -> set of row IDs that should be auto-generated.
+// The assetsDir parameter should be the path to the assets directory that contains the part/ subdirectory.
+func scanAutogeneratedParts(assetsDir string) (map[string]map[int]struct{}, error) {
+	if assetsDir == "" {
+		return nil, nil
+	}
+
+	result := make(map[string]map[int]struct{})
+
+	for _, category := range supportedPartCategories {
+		discovered, err := DiscoverPartModels(assetsDir, category)
+		if err != nil {
+			return nil, err
+		}
+		// Always include the category, even if no IDs were discovered
+		ids := make(map[int]struct{}, len(discovered))
+		for rowID := range discovered {
+			ids[rowID] = struct{}{}
+		}
+		result[category] = ids
+	}
+
+	return result, nil
+}
+
+func loadPartOverrides(path string) ([]map[string]any, error) {
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	obj, err := loadJSONObject(path)
+	if err != nil {
+		return nil, err
+	}
+	rawOverrides, ok := obj["overrides"]
+	if !ok {
+		return nil, nil
+	}
+	overrideList, ok := rawOverrides.([]any)
+	if !ok {
+		return nil, fmt.Errorf("%s: overrides must be an array", path)
+	}
+	overrides := make([]map[string]any, 0, len(overrideList))
+	for index, rawOverride := range overrideList {
+		override, ok := rawOverride.(map[string]any)
+		if !ok {
+			return nil, fmt.Errorf("%s: override %d must be an object", path, index)
+		}
+		overrides = append(overrides, override)
+	}
+	return overrides, nil
+}
+
+func applyPartOverrides(sourceDir string, collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) {
+	result := make([]nativeCollectedDataset, len(collected))
+	copy(result, collected)
+	for i, dataset := range result {
+		if !strings.HasPrefix(dataset.Dataset.Name, "parts/") {
+			continue
+		}
+		category := extractPartCategory(dataset.Dataset.Name)
+		if category == "" {
+			continue
+		}
+		overridePath := filepath.Join(sourceDir, "data", "parts", "overrides", category+".json")
+		overrides, err := loadPartOverrides(overridePath)
+		if err != nil {
+			return nil, err
+		}
+		if len(overrides) == 0 {
+			continue
+		}
+
+		rows := make([]map[string]any, len(dataset.Rows))
+		rowByID := make(map[int]map[string]any, len(dataset.Rows))
+		for index, row := range dataset.Rows {
+			cloned := cloneRowMap(row)
+			rows[index] = cloned
+			if rowID, ok := cloned["id"].(int); ok {
+				rowByID[rowID] = cloned
+			}
+		}
+
+		for index, override := range overrides {
+			rawID, ok := override["id"]
+			if !ok {
+				return nil, fmt.Errorf("%s: override %d is missing id", overridePath, index)
+			}
+			rowID, err := asInt(rawID)
+			if err != nil {
+				return nil, fmt.Errorf("%s: override %d id is not numeric", overridePath, index)
+			}
+			row, ok := rowByID[rowID]
+			if !ok {
+				row = CreateDefaultPartRow(rowID)
+				rows = append(rows, row)
+				rowByID[rowID] = row
+			}
+			for field, value := range override {
+				if field == "id" {
+					continue
+				}
+				row[field] = normalizePartOverrideValue(value)
+			}
+		}
+
+		slices.SortFunc(rows, func(a, b map[string]any) int {
+			idA := a["id"].(int)
+			idB := b["id"].(int)
+			return idA - idB
+		})
+		result[i].Rows = rows
+	}
+	return result, nil
+}
+
+func normalizePartOverrideValue(value any) any {
+	switch typed := value.(type) {
+	case float64:
+		if typed == float64(int(typed)) {
+			return strconv.Itoa(int(typed))
+		}
+	case int:
+		return strconv.Itoa(typed)
+	case json.Number:
+		return typed.String()
+	}
+	return value
+}
+
+// augmentWithAutogeneratedParts applies repository-backed discovery defaults.
+// Missing discovered IDs are added with default values. Existing discovered rows
+// retain authored values unless they still use placeholder values, in which case
+// discovery activates them with engine-visible defaults.
+func augmentWithAutogeneratedParts(collected []nativeCollectedDataset, autogenerated map[string]map[int]struct{}) []nativeCollectedDataset {
+	if len(autogenerated) == 0 {
+		return collected
+	}
+
+	result := make([]nativeCollectedDataset, len(collected))
+	copy(result, collected)
+
+	for i, dataset := range collected {
+		category := partAssetCategoryForDataset(dataset.Dataset.Name)
+		if !isSupportedPartCategory(category) {
+			continue
+		}
+
+		ids, ok := autogenerated[category]
+		if !ok || len(ids) == 0 {
+			continue
+		}
+
+		// Build a map of existing row IDs.
+		existingRows := make(map[int]map[string]any, len(dataset.Rows))
+		for _, row := range dataset.Rows {
+			if id, ok := row["id"].(int); ok {
+				existingRows[id] = row
+			}
+		}
+
+		// Add rows for discovered IDs that don't already exist
+		newRows := make([]map[string]any, 0, len(dataset.Rows))
+		newRows = append(newRows, dataset.Rows...)
+
+		for rowID := range ids {
+			if row, exists := existingRows[rowID]; exists {
+				if isUnsetPartValue(row["COSTMODIFIER"]) {
+					row["COSTMODIFIER"] = "0"
+				}
+				if isUnsetPartValue(row["ACBONUS"]) {
+					row["ACBONUS"] = "0.00"
+				}
+				continue
+			}
+			if _, exists := existingRows[rowID]; !exists {
+				newRow := CreateDefaultPartRow(rowID)
+				newRows = append(newRows, newRow)
+			}
+		}
+
+		// Sort rows by ID
+		slices.SortFunc(newRows, func(a, b map[string]any) int {
+			idA := a["id"].(int)
+			idB := b["id"].(int)
+			return idA - idB
+		})
+
+		result[i].Rows = newRows
+	}
+
+	return result
+}