Topdata Migration (#2)

Native topdata migration summary - Replaced the legacy 2dabuilder runtime path with the native topdata builder. - Native build, validate, compare, and convert flows now cover the canonical topdata pipeline. - compare-topdata is now a native self-check; legacy reference-builder runtime usage was removed. - Parts generation follows the native asset-scan contract and uses sow-assets via project asset resolution. - Generated feat families, class-feat injects, masterfeat/successor expansion, and dataset-derived feat generation were brought to parity and regression-covered. - Remaining mirrored datasets were migrated into native-owned canonical data while preserving lock IDs. - normalize-topdata bridge behavior and migration-era ownership markers were removed. - Documentation was rewritten around the native workflow and active contracts were cleaned up. - Final hardening pass removed stale validator assumptions and cleaned ignored/generated artifacts for PR readiness. Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/2 Co-authored-by: vickydotbat <vickydotbat@tutamail.com> Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
2026-04-09 07:14:01 +00:00
parent 99f9d29d7a
commit dc93feb176
1402 changed files with 20955 additions and 3405 deletions
@@ -0,0 +1,223 @@
+package topdata
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+)
+
+var portraitColumns = []string{"BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"}
+
+func parseRowMetadata(raw any) (map[string]any, error) {
+	if raw == nil {
+		return nil, nil
+	}
+	obj, ok := raw.(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("meta must be an object")
+	}
+	if len(obj) == 0 {
+		return map[string]any{}, nil
+	}
+
+	meta := make(map[string]any, len(obj))
+	for key, value := range obj {
+		switch normalizeMetadataKey(key) {
+		case "family":
+			family, err := parseFamilyMetadata(value)
+			if err != nil {
+				return nil, fmt.Errorf("meta.family: %w", err)
+			}
+			meta["family"] = family
+		case "wiki":
+			wiki, err := parseWikiMetadata(value)
+			if err != nil {
+				return nil, fmt.Errorf("meta.wiki: %w", err)
+			}
+			if len(wiki) > 0 {
+				meta["wiki"] = wiki
+			}
+		case "portrait":
+			portrait, err := parsePortraitMetadata(value)
+			if err != nil {
+				return nil, fmt.Errorf("meta.portrait: %w", err)
+			}
+			meta["portrait"] = portrait
+		default:
+			return nil, fmt.Errorf("unknown metadata key %q", key)
+		}
+	}
+	return meta, nil
+}
+
+func parsePortraitMetadata(raw any) (map[string]any, error) {
+	obj, ok := raw.(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("must be an object")
+	}
+	portrait := map[string]any{}
+	for key, value := range obj {
+		switch normalizeMetadataKey(key) {
+		case "resref":
+			text, ok := value.(string)
+			if !ok || strings.TrimSpace(text) == "" {
+				return nil, fmt.Errorf("resref must be a non-empty string")
+			}
+			portrait["resref"] = text
+		case "sex", "race", "inanimate_type", "plot", "low_gore":
+			portrait[normalizeMetadataKey(key)] = value
+		default:
+			return nil, fmt.Errorf("unknown field %q", key)
+		}
+	}
+	if _, ok := portrait["resref"]; !ok {
+		return nil, fmt.Errorf("resref is required")
+	}
+	return portrait, nil
+}
+
+func normalizeMetadataKey(key string) string {
+	replacer := strings.NewReplacer("-", "_", " ", "_")
+	return strings.ToLower(replacer.Replace(strings.TrimSpace(key)))
+}
+
+func portraitMetaFromRow(row map[string]any) (map[string]any, bool) {
+	rawMeta, ok := lookupField(row, "meta")
+	if !ok {
+		return nil, false
+	}
+	meta, ok := rawMeta.(map[string]any)
+	if !ok {
+		return nil, false
+	}
+	rawPortrait, ok := meta["portrait"]
+	if !ok {
+		return nil, false
+	}
+	portrait, ok := rawPortrait.(map[string]any)
+	if !ok {
+		return nil, false
+	}
+	return portrait, true
+}
+
+func mergePortraitMetadata(collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) {
+	portraitIndex := -1
+	for i, dataset := range collected {
+		if dataset.Dataset.OutputName == "portraits.2da" || dataset.Dataset.Name == "portraits" {
+			portraitIndex = i
+			break
+		}
+	}
+	if portraitIndex < 0 {
+		return collected, nil
+	}
+
+	portraits := collected[portraitIndex]
+	if len(portraits.Columns) == 0 {
+		portraits.Columns = append([]string(nil), portraitColumns...)
+	}
+
+	rowByResref := map[string]map[string]any{}
+	usedIDs := map[int]struct{}{}
+	for _, row := range portraits.Rows {
+		rowID, ok := row["id"].(int)
+		if ok {
+			usedIDs[rowID] = struct{}{}
+		}
+		resref, _ := lookupField(row, "BaseResRef")
+		if text, ok := resref.(string); ok {
+			rowByResref[text] = row
+		}
+	}
+	for _, rowID := range portraits.LockData {
+		usedIDs[rowID] = struct{}{}
+	}
+	nextID := nextAvailableID(usedIDs)
+	lockModified := false
+
+	for _, dataset := range collected {
+		for _, row := range dataset.Rows {
+			portrait, ok := portraitMetaFromRow(row)
+			if !ok {
+				continue
+			}
+			resref := portrait["resref"].(string)
+			target, ok := rowByResref[resref]
+			if !ok {
+				lockKey := "portraits:auto:" + resref
+				rowID, found := portraits.LockData[lockKey]
+				if !found {
+					rowID = nextID
+					portraits.LockData[lockKey] = rowID
+					lockModified = true
+					usedIDs[rowID] = struct{}{}
+					nextID = nextAvailableID(usedIDs)
+				}
+				target = map[string]any{
+					"id":  rowID,
+					"key": lockKey,
+				}
+				for _, column := range portraits.Columns {
+					target[column] = nullValue
+				}
+				target["BaseResRef"] = resref
+				portraits.Rows = append(portraits.Rows, target)
+				rowByResref[resref] = target
+			}
+			if err := applyPortraitContribution(target, portrait, rowIdentity(dataset.Dataset.Name, row)); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	if lockModified && portraits.Dataset.LockPath != "" {
+		if err := saveLockfile(portraits.Dataset.LockPath, portraits.LockData); err != nil {
+			return nil, fmt.Errorf("dataset %s: %w", portraits.Dataset.Name, err)
+		}
+	}
+
+	slices.SortFunc(portraits.Rows, func(a, b map[string]any) int {
+		return a["id"].(int) - b["id"].(int)
+	})
+	collected[portraitIndex] = portraits
+	return collected, nil
+}
+
+func applyPortraitContribution(target map[string]any, portrait map[string]any, source string) error {
+	mapping := map[string]string{
+		"resref":         "BaseResRef",
+		"sex":            "Sex",
+		"race":           "Race",
+		"inanimate_type": "InanimateType",
+		"plot":           "Plot",
+		"low_gore":       "LowGore",
+	}
+	for key, value := range portrait {
+		column, ok := mapping[normalizeMetadataKey(key)]
+		if !ok {
+			continue
+		}
+		current, _ := lookupField(target, column)
+		if current == nil || current == nullValue {
+			target[column] = value
+			continue
+		}
+		if normalizeComparableValue(current) == normalizeComparableValue(value) {
+			continue
+		}
+		return fmt.Errorf("portrait %q conflicts for column %s between existing value %v and contributor %s value %v", portrait["resref"], column, current, source, value)
+	}
+	return nil
+}
+
+func normalizeComparableValue(value any) string {
+	switch typed := value.(type) {
+	case nil:
+		return nullValue
+	case string:
+		return typed
+	default:
+		return format2DAValue(typed)
+	}
+}