Topdata Migration (#2)

Native topdata migration summary - Replaced the legacy 2dabuilder runtime path with the native topdata builder. - Native build, validate, compare, and convert flows now cover the canonical topdata pipeline. - compare-topdata is now a native self-check; legacy reference-builder runtime usage was removed. - Parts generation follows the native asset-scan contract and uses sow-assets via project asset resolution. - Generated feat families, class-feat injects, masterfeat/successor expansion, and dataset-derived feat generation were brought to parity and regression-covered. - Remaining mirrored datasets were migrated into native-owned canonical data while preserving lock IDs. - normalize-topdata bridge behavior and migration-era ownership markers were removed. - Documentation was rewritten around the native workflow and active contracts were cleaned up. - Final hardening pass removed stale validator assumptions and cleaned ignored/generated artifacts for PR readiness. Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/2 Co-authored-by: vickydotbat <vickydotbat@tutamail.com> Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
2026-04-09 07:14:01 +00:00
parent 99f9d29d7a
commit dc93feb176
1402 changed files with 20955 additions and 3405 deletions
@@ -0,0 +1,530 @@
+package topdata
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+type convertOptions struct {
+	Namespace     string
+	KeyFields     []string
+	CollisionMode string
+	BaseDialog    string
+	Type          string
+}
+
+func RunConvertCommand(args []string, stdout io.Writer) error {
+	if len(args) == 0 || args[0] == "--help" || args[0] == "-h" {
+		printConvertUsage(stdout)
+		return nil
+	}
+	switch args[0] {
+	case "2da-to-json":
+		if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
+			_, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-json [flags] <input.2da> <output.json>")
+			return nil
+		}
+		opts, input, output, err := parseConvertArgs(args[1:], false)
+		if err != nil {
+			return err
+		}
+		data, err := parse2DAFile(input)
+		if err != nil {
+			return err
+		}
+		result, err := convert2DAToJSON(data, output, opts)
+		if err != nil {
+			return err
+		}
+		return writeJSONFile(output, result)
+	case "2da-to-module":
+		if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
+			_, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-module --namespace <dataset> [--type entries|override] [flags] <input.2da> [output.json]")
+			return nil
+		}
+		opts, input, output, err := parseConvertArgs(args[1:], true)
+		if err != nil {
+			return err
+		}
+		data, err := parse2DAFile(input)
+		if err != nil {
+			return err
+		}
+		if opts.Type == "override" {
+			result := map[string]any{"overrides": toOverridesRows(data.Rows)}
+			if err := writeJSONFile(output, result); err != nil {
+				return err
+			}
+			_, _ = fmt.Fprintf(stdout, "converted overrides: %d\n", len(result["overrides"].([]map[string]any)))
+			return nil
+		}
+		result, skipped, err := convert2DAToModule(data, output, opts)
+		if err != nil {
+			return err
+		}
+		if err := writeJSONFile(output, result); err != nil {
+			return err
+		}
+		if skipped > 0 {
+			_, _ = fmt.Fprintf(stdout, "converted entries: %d, skipped unkeyed: %d\n", len(result["entries"].(map[string]any)), skipped)
+			return nil
+		}
+		_, _ = fmt.Fprintf(stdout, "converted entries: %d\n", len(result["entries"].(map[string]any)))
+		return nil
+	case "json-to-2da":
+		if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
+			_, _ = fmt.Fprintln(stdout, "usage: convert-topdata json-to-2da <input.json> <output.2da>")
+			return nil
+		}
+		if len(args) != 3 {
+			return errors.New("usage: convert-topdata json-to-2da <input.json> <output.2da>")
+		}
+		data, err := readCanonicalJSON(args[1])
+		if err != nil {
+			return err
+		}
+		return write2DAFile(data, args[2])
+	default:
+		return fmt.Errorf("unknown convert-topdata subcommand %q", args[0])
+	}
+}
+
+func printConvertUsage(stdout io.Writer) {
+	_, _ = fmt.Fprintln(stdout, "usage: convert-topdata <2da-to-json|2da-to-module|json-to-2da> ...")
+	_, _ = fmt.Fprintln(stdout, "")
+	_, _ = fmt.Fprintln(stdout, "subcommands:")
+	_, _ = fmt.Fprintln(stdout, "  2da-to-json   Convert a 2DA file into canonical JSON rows")
+	_, _ = fmt.Fprintln(stdout, "  2da-to-module Convert a 2DA file into module entries or override JSON")
+	_, _ = fmt.Fprintln(stdout, "  json-to-2da   Convert canonical JSON rows into a 2DA file")
+}
+
+type parsed2DA struct {
+	Columns []string         `json:"columns"`
+	Rows    []map[string]any `json:"rows"`
+}
+
+func parseConvertArgs(args []string, allowFormat bool) (convertOptions, string, string, error) {
+	opts := convertOptions{
+		CollisionMode: "prompt",
+		Type:          "entries",
+	}
+	positional := make([]string, 0, 2)
+	for index := 0; index < len(args); index++ {
+		arg := args[index]
+		switch arg {
+		case "--namespace":
+			index++
+			if index >= len(args) {
+				return opts, "", "", errors.New("--namespace requires a value")
+			}
+			opts.Namespace = args[index]
+		case "--key-field":
+			index++
+			if index >= len(args) {
+				return opts, "", "", errors.New("--key-field requires a value")
+			}
+			opts.KeyFields = append(opts.KeyFields, args[index])
+		case "--collision":
+			index++
+			if index >= len(args) {
+				return opts, "", "", errors.New("--collision requires a value")
+			}
+			opts.CollisionMode = args[index]
+		case "--base-dialog":
+			index++
+			if index >= len(args) {
+				return opts, "", "", errors.New("--base-dialog requires a value")
+			}
+			opts.BaseDialog = args[index]
+		case "--format", "--type":
+			if !allowFormat {
+				return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), arg)
+			}
+			index++
+			if index >= len(args) {
+				return opts, "", "", fmt.Errorf("%s requires a value", arg)
+			}
+			opts.Type = args[index]
+		default:
+			if strings.HasPrefix(arg, "--") {
+				return opts, "", "", fmt.Errorf("unknown flag %q", arg)
+			}
+			positional = append(positional, arg)
+		}
+	}
+	if allowFormat {
+		if strings.TrimSpace(opts.Namespace) == "" {
+			return opts, "", "", errors.New("2da-to-module requires --namespace")
+		}
+		if len(positional) < 1 || len(positional) > 2 {
+			return opts, "", "", errors.New("usage: convert-topdata 2da-to-module --namespace <dataset> [--type entries|override] [flags] <input.2da> [output.json]")
+		}
+	} else if len(positional) != 2 {
+		return opts, "", "", errors.New("usage: convert-topdata 2da-to-json [flags] <input.2da> <output.json>")
+	}
+	if opts.CollisionMode == "prompt" {
+		opts.CollisionMode = "suffix"
+	}
+	if opts.CollisionMode != "suffix" && opts.CollisionMode != "error" {
+		return opts, "", "", fmt.Errorf("unsupported collision mode %q", opts.CollisionMode)
+	}
+	if allowFormat {
+		switch opts.Type {
+		case "entry":
+			opts.Type = "entries"
+		case "overrides":
+			opts.Type = "override"
+		}
+		if opts.Type != "entries" && opts.Type != "override" {
+			return opts, "", "", fmt.Errorf("unsupported module type %q", opts.Type)
+		}
+		output := ""
+		if len(positional) == 2 {
+			output = positional[1]
+		} else {
+			resolved, err := defaultModuleOutputPath(positional[0], opts)
+			if err != nil {
+				return opts, "", "", err
+			}
+			output = resolved
+		}
+		return opts, positional[0], output, nil
+	}
+	return opts, positional[0], positional[1], nil
+}
+
+func positionalSafe(args []string) string {
+	if len(args) == 0 {
+		return "command"
+	}
+	return args[0]
+}
+
+func parse2DAFile(path string) (parsed2DA, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return parsed2DA{}, err
+	}
+	lines := strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n")
+	trimmed := make([]string, 0, len(lines))
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line != "" {
+			trimmed = append(trimmed, line)
+		}
+	}
+	if len(trimmed) < 2 || !strings.HasPrefix(trimmed[0], "2DA") {
+		return parsed2DA{}, errors.New("invalid 2DA file")
+	}
+	columns := split2DALine(trimmed[1])
+	rows := make([]map[string]any, 0, max(0, len(trimmed)-2))
+	for _, line := range trimmed[2:] {
+		fields := split2DALine(line)
+		if len(fields) == 0 {
+			continue
+		}
+		rowID, err := strconv.Atoi(fields[0])
+		if err != nil {
+			return parsed2DA{}, fmt.Errorf("invalid 2DA row id %q", fields[0])
+		}
+		row := map[string]any{"id": rowID}
+		for index, column := range columns {
+			if index+1 < len(fields) {
+				row[column] = smartConvertScalar(fields[index+1])
+				continue
+			}
+			row[column] = nullValue
+		}
+		rows = append(rows, row)
+	}
+	return parsed2DA{Columns: columns, Rows: rows}, nil
+}
+
+func split2DALine(line string) []string {
+	fields := make([]string, 0)
+	var current strings.Builder
+	inQuotes := false
+	for _, ch := range line {
+		switch {
+		case ch == '"':
+			inQuotes = !inQuotes
+		case !inQuotes && (ch == '\t' || ch == ' '):
+			if current.Len() > 0 {
+				fields = append(fields, current.String())
+				current.Reset()
+			}
+		default:
+			current.WriteRune(ch)
+		}
+	}
+	if current.Len() > 0 {
+		fields = append(fields, current.String())
+	}
+	return fields
+}
+
+func smartConvertScalar(value string) any {
+	if value == "" {
+		return ""
+	}
+	if parsed, err := strconv.Atoi(value); err == nil {
+		return parsed
+	}
+	return value
+}
+
+func convert2DAToJSON(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) {
+	rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts)
+	if err != nil {
+		return nil, err
+	}
+	outRows := make([]map[string]any, 0, len(rows))
+	for _, row := range rows {
+		ordered := map[string]any{"id": row["id"]}
+		if key, ok := row["key"].(string); ok && key != "" {
+			ordered["key"] = key
+		}
+		for _, column := range data.Columns {
+			ordered[column] = row[column]
+		}
+		outRows = append(outRows, ordered)
+	}
+	return map[string]any{"columns": data.Columns, "rows": outRows}, nil
+}
+
+func convert2DAToModule(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, int, error) {
+	rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts)
+	if err != nil {
+		return nil, 0, err
+	}
+	entries := map[string]any{}
+	skipped := 0
+	for _, row := range rows {
+		key, _ := row["key"].(string)
+		if key == "" {
+			skipped++
+			continue
+		}
+		entry := map[string]any{}
+		for _, column := range data.Columns {
+			value := row[column]
+			if format2DAValue(value) == nullValue {
+				continue
+			}
+			entry[column] = value
+		}
+		entries[key] = entry
+	}
+	return map[string]any{"entries": entries}, skipped, nil
+}
+
+func toOverridesRows(rows []map[string]any) []map[string]any {
+	out := make([]map[string]any, 0, len(rows))
+	for _, row := range rows {
+		override := map[string]any{"id": row["id"]}
+		keys := make([]string, 0, len(row))
+		for key := range row {
+			if key == "id" || key == "key" {
+				continue
+			}
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+		for _, key := range keys {
+			if format2DAValue(row[key]) == nullValue {
+				continue
+			}
+			override[key] = row[key]
+		}
+		out = append(out, override)
+	}
+	return out
+}
+
+func assignConvertedRowKeys(rows []map[string]any, outputPath string, opts convertOptions) ([]map[string]any, error) {
+	namespace, profiles := inferConvertNamespace(outputPath, opts.Namespace)
+	out := make([]map[string]any, 0, len(rows))
+	used := map[string]struct{}{}
+	for _, row := range rows {
+		cloned := cloneRowMap(row)
+		if namespace == "" {
+			out = append(out, cloned)
+			continue
+		}
+		key := convertedRowKey(cloned, namespace, profiles, opts.KeyFields)
+		if key == "" {
+			out = append(out, cloned)
+			continue
+		}
+		if _, ok := used[key]; ok {
+			if opts.CollisionMode == "error" {
+				return nil, fmt.Errorf("duplicate generated key %q", key)
+			}
+			key = fmt.Sprintf("%s_%v", key, cloned["id"])
+		}
+		used[key] = struct{}{}
+		cloned["key"] = key
+		out = append(out, cloned)
+	}
+	return out, nil
+}
+
+type convertKeyProfile struct {
+	Field string
+	Mode  string
+}
+
+func inferConvertNamespace(outputPath, override string) (string, []convertKeyProfile) {
+	if strings.TrimSpace(override) != "" {
+		return strings.TrimSpace(override), nil
+	}
+	normalized := filepath.ToSlash(outputPath)
+	prefixes := []struct {
+		Prefix    string
+		Namespace string
+		Profiles  []convertKeyProfile
+	}{
+		{"data/itemprops/costtables/index/", "itemprops:costtable_index", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
+		{"data/itemprops/paramtables/index/", "itemprops:paramtable_index", []convertKeyProfile{{"Name", "literal"}, {"Lable", "literal"}, {"TableResRef", "literal"}}},
+		{"data/itemprops/defs/", "itempropdef", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
+		{"data/itemprops/sets/", "itemprops", []convertKeyProfile{{"StringRef", "literal"}, {"Label", "literal"}}},
+		{"data/appearance/", "appearance", []convertKeyProfile{{"STRING_REF", "literal"}, {"LABEL", "literal"}}},
+		{"data/baseitems/", "baseitems", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
+		{"data/feat/", "feat", []convertKeyProfile{{"FEAT", "literal"}, {"LABEL", "literal"}, {"Label", "literal"}}},
+		{"data/spells/", "spells", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
+		{"data/skills/", "skills", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
+		{"data/portraits/", "portraits", []convertKeyProfile{{"BaseResRef", "literal"}, {"Label", "literal"}}},
+		{"data/racialtypes/", "racialtypes", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
+		{"data/damagetypes/", "damagetypes", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
+	}
+	for _, prefix := range prefixes {
+		if strings.Contains(normalized, prefix.Prefix) {
+			return prefix.Namespace, prefix.Profiles
+		}
+	}
+	return "", nil
+}
+
+var convertKeyWhitespace = regexp.MustCompile(`\s+`)
+var convertKeyCleaner = regexp.MustCompile(`[^a-z0-9_]`)
+
+func convertedRowKey(row map[string]any, namespace string, profiles []convertKeyProfile, preferred []string) string {
+	candidates := make([]convertKeyProfile, 0, len(preferred)+len(profiles))
+	for _, field := range preferred {
+		candidates = append(candidates, convertKeyProfile{Field: field, Mode: "literal"})
+	}
+	candidates = append(candidates, profiles...)
+	if len(candidates) == 0 {
+		candidates = append(candidates,
+			convertKeyProfile{Field: "LABEL", Mode: "literal"},
+			convertKeyProfile{Field: "Label", Mode: "literal"},
+			convertKeyProfile{Field: "Name", Mode: "literal"},
+		)
+	}
+	for _, candidate := range candidates {
+		value, ok := row[candidate.Field]
+		if !ok {
+			continue
+		}
+		text := strings.TrimSpace(format2DAValue(value))
+		if text == "" || text == nullValue {
+			continue
+		}
+		normalized := normalizeConvertedKeyText(text)
+		if normalized == "" {
+			continue
+		}
+		return namespace + ":" + normalized
+	}
+	return ""
+}
+
+func normalizeConvertedKeyText(text string) string {
+	text = strings.TrimSpace(strings.ToLower(text))
+	text = convertKeyWhitespace.ReplaceAllString(text, "_")
+	text = convertKeyCleaner.ReplaceAllString(text, "")
+	text = strings.Trim(text, "_")
+	return text
+}
+
+func defaultModuleOutputPath(input string, opts convertOptions) (string, error) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	root, err := project.FindRoot(cwd)
+	if err != nil {
+		return "", errors.New("2da-to-module requires an explicit output path when not run inside a project root")
+	}
+	p, err := project.Load(root)
+	if err != nil {
+		return "", err
+	}
+	if !p.HasTopData() {
+		return "", errors.New("2da-to-module requires topdata to be configured when inferring an output path")
+	}
+	filename := strings.TrimSuffix(filepath.Base(input), filepath.Ext(input)) + ".json"
+	modulesDir := filepath.Join(p.TopDataSourceDir(), "data", filepath.FromSlash(opts.Namespace), "modules")
+	return filepath.Join(modulesDir, filename), nil
+}
+
+func readCanonicalJSON(path string) (parsed2DA, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return parsed2DA{}, err
+	}
+	var payload struct {
+		Columns []string         `json:"columns"`
+		Rows    []map[string]any `json:"rows"`
+	}
+	if err := json.Unmarshal(raw, &payload); err != nil {
+		return parsed2DA{}, err
+	}
+	return parsed2DA{Columns: payload.Columns, Rows: payload.Rows}, nil
+}
+
+func writeJSONFile(path string, payload any) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." {
+		return err
+	}
+	raw, err := json.MarshalIndent(payload, "", "  ")
+	if err != nil {
+		return err
+	}
+	raw = append(raw, '\n')
+	return os.WriteFile(path, raw, 0o644)
+}
+
+func write2DAFile(data parsed2DA, path string) error {
+	rows := make([]map[string]any, 0, len(data.Rows))
+	for _, row := range data.Rows {
+		cloned := cloneRowMap(row)
+		if rowID, err := asInt(cloned["id"]); err == nil {
+			cloned["id"] = rowID
+		}
+		rows = append(rows, cloned)
+	}
+	sort.Slice(rows, func(i, j int) bool {
+		left, _ := asInt(rows[i]["id"])
+		right, _ := asInt(rows[j]["id"])
+		return left < right
+	})
+	table := map[string]any{
+		"columns": data.Columns,
+		"rows":    rows,
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." {
+		return err
+	}
+	return write2DA(table, path, false)
+}