sow-tools/internal/topdata/convert.go

package topdata

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)

type convertOptions struct {
	Namespace     string
	KeyFields     []string
	CollisionMode string
	BaseDialog    string
	Type          string
	Name          string
}

func RunConvertCommand(args []string, stdout io.Writer) error {
	if len(args) == 0 || args[0] == "--help" || args[0] == "-h" {
		printConvertUsage(stdout)
		return nil
	}
	switch args[0] {
	case "2da-to-json":
		if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
			_, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-json [flags] <input.2da> <output.json>")
			return nil
		}
		opts, input, output, err := parseConvertArgs(args[1:], false)
		if err != nil {
			return err
		}
		data, err := parse2DAFile(input)
		if err != nil {
			return err
		}
		result, err := convert2DAToJSON(data, output, opts)
		if err != nil {
			return err
		}
		return writeJSONFile(output, result)
	case "2da-to-module":
		if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
			_, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-module [--namespace <dataset>] [--name <module-name>] [--type entries|override] [flags] <input.2da> [output.json]")
			return nil
		}
		opts, input, output, err := parseConvertArgs(args[1:], true)
		if err != nil {
			return err
		}
		data, err := parse2DAFile(input)
		if err != nil {
			return err
		}
		if opts.Type == "override" {
			result := map[string]any{"overrides": toOverridesRows(data.Rows)}
			if err := writeJSONFile(output, result); err != nil {
				return err
			}
			_, _ = fmt.Fprintf(stdout, "converted overrides: %d\n", len(result["overrides"].([]map[string]any)))
			return nil
		}
		result, err := convert2DAToModule(data, output, opts)
		if err != nil {
			return err
		}
		if err := writeJSONFile(output, result); err != nil {
			return err
		}
		_, _ = fmt.Fprintf(stdout, "converted entries: %d\n", len(result["entries"].(map[string]any)))
		return nil
	case "json-to-2da":
		if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
			_, _ = fmt.Fprintln(stdout, "usage: convert-topdata json-to-2da <input.json> <output.2da>")
			return nil
		}
		if len(args) != 3 {
			return errors.New("usage: convert-topdata json-to-2da <input.json> <output.2da>")
		}
		data, err := readCanonicalJSON(args[1])
		if err != nil {
			return err
		}
		return write2DAFile(data, args[2])
	default:
		return fmt.Errorf("unknown convert-topdata subcommand %q", args[0])
	}
}

func printConvertUsage(stdout io.Writer) {
	_, _ = fmt.Fprintln(stdout, "usage: convert-topdata <2da-to-json|2da-to-module|json-to-2da> ...")
	_, _ = fmt.Fprintln(stdout, "")
	_, _ = fmt.Fprintln(stdout, "subcommands:")
	_, _ = fmt.Fprintln(stdout, "  2da-to-json   Convert a 2DA file into canonical JSON rows")
	_, _ = fmt.Fprintln(stdout, "  2da-to-module Convert a 2DA file into module entries or override JSON")
	_, _ = fmt.Fprintln(stdout, "  json-to-2da   Convert canonical JSON rows into a 2DA file")
	_, _ = fmt.Fprintln(stdout, "")
	_, _ = fmt.Fprintln(stdout, "2da-to-module notes:")
	_, _ = fmt.Fprintln(stdout, "  - --namespace is optional when topdata/templates/config.json declares it for the input table")
	_, _ = fmt.Fprintln(stdout, "  - --name controls inferred output naming: add_<namespace>_<name>.json")
	_, _ = fmt.Fprintln(stdout, "  - unkeyed rows now fail conversion instead of being skipped")
}

type parsed2DA struct {
	Columns []string         `json:"columns"`
	Rows    []map[string]any `json:"rows"`
}

func parseConvertArgs(args []string, allowFormat bool) (convertOptions, string, string, error) {
	opts := convertOptions{
		CollisionMode: "error",
		Type:          "entries",
	}
	positional := make([]string, 0, 2)
	for index := 0; index < len(args); index++ {
		arg := args[index]
		switch arg {
		case "--namespace":
			index++
			if index >= len(args) {
				return opts, "", "", errors.New("--namespace requires a value")
			}
			opts.Namespace = args[index]
		case "--key-field":
			index++
			if index >= len(args) {
				return opts, "", "", errors.New("--key-field requires a value")
			}
			opts.KeyFields = append(opts.KeyFields, args[index])
		case "--name":
			index++
			if index >= len(args) {
				return opts, "", "", errors.New("--name requires a value")
			}
			opts.Name = args[index]
		case "--collision":
			index++
			if index >= len(args) {
				return opts, "", "", errors.New("--collision requires a value")
			}
			opts.CollisionMode = args[index]
		case "--base-dialog":
			index++
			if index >= len(args) {
				return opts, "", "", errors.New("--base-dialog requires a value")
			}
			opts.BaseDialog = args[index]
		case "--format", "--type":
			if !allowFormat {
				return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), arg)
			}
			index++
			if index >= len(args) {
				return opts, "", "", fmt.Errorf("%s requires a value", arg)
			}
			opts.Type = args[index]
		default:
			if strings.HasPrefix(arg, "--") {
				return opts, "", "", fmt.Errorf("unknown flag %q", arg)
			}
			positional = append(positional, arg)
		}
	}
	if allowFormat {
		if len(positional) < 1 || len(positional) > 2 {
			return opts, "", "", errors.New("usage: convert-topdata 2da-to-module [--namespace <dataset>] [--name <module-name>] [--type entries|override] [flags] <input.2da> [output.json]")
		}
	} else if len(positional) != 2 {
		return opts, "", "", errors.New("usage: convert-topdata 2da-to-json [flags] <input.2da> <output.json>")
	}
	if opts.CollisionMode != "suffix" && opts.CollisionMode != "error" {
		return opts, "", "", fmt.Errorf("unsupported collision mode %q", opts.CollisionMode)
	}
	if allowFormat {
		ctx, err := resolveConvertContext(positional[0])
		if err != nil {
			return opts, "", "", err
		}
		if strings.TrimSpace(opts.Namespace) == "" {
			opts.Namespace = ctx.Template.Namespace
		}
		if len(opts.KeyFields) == 0 && len(ctx.Template.KeyFields) > 0 {
			opts.KeyFields = append(opts.KeyFields, ctx.Template.KeyFields...)
		}
		if strings.TrimSpace(opts.Namespace) == "" {
			return opts, "", "", errors.New("2da-to-module requires --namespace or a topdata/templates/config.json entry for the input table")
		}
		switch opts.Type {
		case "entry":
			opts.Type = "entries"
		case "overrides":
			opts.Type = "override"
		}
		if opts.Type != "entries" && opts.Type != "override" {
			return opts, "", "", fmt.Errorf("unsupported module type %q", opts.Type)
		}
		output := ""
		if len(positional) == 2 {
			output = positional[1]
		} else {
			resolved, err := defaultModuleOutputPath(positional[0], opts, ctx.Project)
			if err != nil {
				return opts, "", "", err
			}
			output = resolved
		}
		return opts, positional[0], output, nil
	}
	ctx, err := resolveConvertContext(positional[0])
	if err != nil {
		return opts, "", "", err
	}
	if strings.TrimSpace(opts.Namespace) == "" {
		opts.Namespace = ctx.Template.Namespace
	}
	if len(opts.KeyFields) == 0 && len(ctx.Template.KeyFields) > 0 {
		opts.KeyFields = append(opts.KeyFields, ctx.Template.KeyFields...)
	}
	return opts, positional[0], positional[1], nil
}

func positionalSafe(args []string) string {
	if len(args) == 0 {
		return "command"
	}
	return args[0]
}

func parse2DAFile(path string) (parsed2DA, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return parsed2DA{}, fmt.Errorf("read %s: %w", path, err)
	}
	lines := strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n")
	trimmed := make([]string, 0, len(lines))
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line != "" {
			trimmed = append(trimmed, line)
		}
	}
	if len(trimmed) < 2 || !strings.HasPrefix(trimmed[0], "2DA") {
		return parsed2DA{}, fmt.Errorf("%s: invalid 2DA file", path)
	}
	columns := split2DALine(trimmed[1])
	if len(columns) == 0 {
		return parsed2DA{}, fmt.Errorf("%s: missing 2DA columns", path)
	}
	rows := make([]map[string]any, 0, max(0, len(trimmed)-2))
	seenIDs := map[int]struct{}{}
	for _, line := range trimmed[2:] {
		fields := split2DALine(line)
		if len(fields) == 0 {
			continue
		}
		rowID, err := strconv.Atoi(fields[0])
		if err != nil {
			return parsed2DA{}, fmt.Errorf("%s: invalid 2DA row id %q", path, fields[0])
		}
		if len(fields)-1 > len(columns) {
			return parsed2DA{}, fmt.Errorf("%s: row %d has %d values but only %d columns are declared", path, rowID, len(fields)-1, len(columns))
		}
		if _, ok := seenIDs[rowID]; ok {
			return parsed2DA{}, fmt.Errorf("%s: duplicate 2DA row id %d", path, rowID)
		}
		seenIDs[rowID] = struct{}{}
		row := map[string]any{"id": rowID}
		for index, column := range columns {
			if index+1 < len(fields) {
				row[column] = smartConvertScalar(fields[index+1])
				continue
			}
			row[column] = nullValue
		}
		rows = append(rows, row)
	}
	return parsed2DA{Columns: columns, Rows: rows}, nil
}

func split2DALine(line string) []string {
	fields := make([]string, 0)
	var current strings.Builder
	inQuotes := false
	for _, ch := range line {
		switch {
		case ch == '"':
			inQuotes = !inQuotes
		case !inQuotes && (ch == '\t' || ch == ' '):
			if current.Len() > 0 {
				fields = append(fields, current.String())
				current.Reset()
			}
		default:
			current.WriteRune(ch)
		}
	}
	if current.Len() > 0 {
		fields = append(fields, current.String())
	}
	return fields
}

func smartConvertScalar(value string) any {
	if value == "" {
		return ""
	}
	if parsed, err := strconv.Atoi(value); err == nil {
		return parsed
	}
	return value
}

func convert2DAToJSON(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) {
	rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts)
	if err != nil {
		return nil, err
	}
	outRows := make([]map[string]any, 0, len(rows))
	for _, row := range rows {
		ordered := map[string]any{"id": row["id"]}
		if key, ok := row["key"].(string); ok && key != "" {
			ordered["key"] = key
		}
		for _, column := range data.Columns {
			ordered[column] = row[column]
		}
		outRows = append(outRows, ordered)
	}
	return map[string]any{"columns": data.Columns, "rows": outRows}, nil
}

func convert2DAToModule(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) {
	rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts)
	if err != nil {
		return nil, err
	}
	entries := map[string]any{}
	missing := []string{}
	for _, row := range rows {
		key, _ := row["key"].(string)
		if key == "" {
			missing = append(missing, strconv.Itoa(row["id"].(int)))
			continue
		}
		entry := map[string]any{}
		for _, column := range data.Columns {
			value := row[column]
			if format2DAValue(value) == nullValue {
				continue
			}
			entry[column] = value
		}
		entries[key] = entry
	}
	if len(missing) > 0 {
		return nil, fmt.Errorf("unable to generate keys for row ids %s; declare --key-field or configure key_fields in topdata/templates/config.json", strings.Join(missing, ", "))
	}
	return map[string]any{"entries": entries}, nil
}

func toOverridesRows(rows []map[string]any) []map[string]any {
	out := make([]map[string]any, 0, len(rows))
	for _, row := range rows {
		override := map[string]any{"id": row["id"]}
		keys := make([]string, 0, len(row))
		for key := range row {
			if key == "id" || key == "key" {
				continue
			}
			keys = append(keys, key)
		}
		sort.Strings(keys)
		for _, key := range keys {
			if format2DAValue(row[key]) == nullValue {
				continue
			}
			override[key] = row[key]
		}
		out = append(out, override)
	}
	return out
}

func assignConvertedRowKeys(rows []map[string]any, outputPath string, opts convertOptions) ([]map[string]any, error) {
	namespace, profiles := inferConvertNamespace(outputPath, opts.Namespace)
	out := make([]map[string]any, 0, len(rows))
	used := map[string]struct{}{}
	for _, row := range rows {
		cloned := cloneRowMap(row)
		if namespace == "" {
			out = append(out, cloned)
			continue
		}
		key := convertedRowKey(cloned, namespace, profiles, opts.KeyFields)
		if key == "" {
			out = append(out, cloned)
			continue
		}
		if _, ok := used[key]; ok {
			if opts.CollisionMode == "error" {
				return nil, fmt.Errorf("duplicate generated key %q", key)
			}
			key = fmt.Sprintf("%s_%v", key, cloned["id"])
		}
		used[key] = struct{}{}
		cloned["key"] = key
		out = append(out, cloned)
	}
	return out, nil
}

type convertKeyProfile struct {
	Field string
	Mode  string
}

func inferConvertNamespace(outputPath, override string) (string, []convertKeyProfile) {
	if strings.TrimSpace(override) != "" {
		return strings.TrimSpace(override), nil
	}
	normalized := filepath.ToSlash(outputPath)
	prefixes := []struct {
		Prefix    string
		Namespace string
		Profiles  []convertKeyProfile
	}{
		{"data/itemprops/costtables/index/", "itemprops:costtable_index", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
		{"data/itemprops/paramtables/index/", "itemprops:paramtable_index", []convertKeyProfile{{"Name", "literal"}, {"Lable", "literal"}, {"TableResRef", "literal"}}},
		{"data/itemprops/defs/", "itempropdef", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
		{"data/itemprops/sets/", "itemprops", []convertKeyProfile{{"StringRef", "literal"}, {"Label", "literal"}}},
		{"data/appearance/", "appearance", []convertKeyProfile{{"STRING_REF", "literal"}, {"LABEL", "literal"}}},
		{"data/baseitems/", "baseitems", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
		{"data/feat/", "feat", []convertKeyProfile{{"FEAT", "literal"}, {"LABEL", "literal"}, {"Label", "literal"}}},
		{"data/spells/", "spells", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
		{"data/skills/", "skills", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
		{"data/portraits/", "portraits", []convertKeyProfile{{"BaseResRef", "literal"}, {"Label", "literal"}}},
		{"data/racialtypes/", "racialtypes", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
		{"data/damagetypes/", "damagetypes", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}},
	}
	for _, prefix := range prefixes {
		if strings.Contains(normalized, prefix.Prefix) {
			return prefix.Namespace, prefix.Profiles
		}
	}
	return "", nil
}

var convertKeyWhitespace = regexp.MustCompile(`\s+`)
var convertKeyCleaner = regexp.MustCompile(`[^a-z0-9_]`)

func convertedRowKey(row map[string]any, namespace string, profiles []convertKeyProfile, preferred []string) string {
	candidates := make([]convertKeyProfile, 0, len(preferred)+len(profiles))
	for _, field := range preferred {
		candidates = append(candidates, convertKeyProfile{Field: field, Mode: "literal"})
	}
	candidates = append(candidates, profiles...)
	if len(candidates) == 0 {
		candidates = append(candidates,
			convertKeyProfile{Field: "LABEL", Mode: "literal"},
			convertKeyProfile{Field: "Label", Mode: "literal"},
			convertKeyProfile{Field: "Name", Mode: "literal"},
		)
	}
	for _, candidate := range candidates {
		value, ok := row[candidate.Field]
		if !ok {
			continue
		}
		text := strings.TrimSpace(format2DAValue(value))
		if text == "" || text == nullValue {
			continue
		}
		normalized := normalizeConvertedKeyText(text)
		if normalized == "" {
			continue
		}
		return namespace + ":" + normalized
	}
	return ""
}

func normalizeConvertedKeyText(text string) string {
	text = strings.TrimSpace(strings.ToLower(text))
	text = convertKeyWhitespace.ReplaceAllString(text, "_")
	text = convertKeyCleaner.ReplaceAllString(text, "")
	text = strings.Trim(text, "_")
	return text
}

type convertTemplateTable struct {
	Namespace string   `json:"namespace"`
	KeyFields []string `json:"key_fields"`
}

type convertTemplateConfig struct {
	Tables map[string]convertTemplateTable `json:"tables"`
}

type convertContext struct {
	Project  *project.Project
	Template convertTemplateTable
}

func resolveConvertContext(input string) (convertContext, error) {
	cwd, err := os.Getwd()
	if err != nil {
		return convertContext{}, err
	}
	root, err := project.FindRoot(cwd)
	if err != nil {
		return convertContext{}, nil
	}
	p, err := project.Load(root)
	if err != nil {
		return convertContext{}, err
	}
	if !p.HasTopData() {
		return convertContext{Project: p}, nil
	}
	table, err := loadTemplateTableConfig(p, input)
	if err != nil {
		return convertContext{}, err
	}
	return convertContext{
		Project:  p,
		Template: table,
	}, nil
}

func loadTemplateTableConfig(p *project.Project, input string) (convertTemplateTable, error) {
	templatesDir := filepath.Join(p.TopDataSourceDir(), "templates")
	inputAbs, err := filepath.Abs(input)
	if err != nil {
		return convertTemplateTable{}, err
	}
	templatesAbs, err := filepath.Abs(templatesDir)
	if err != nil {
		return convertTemplateTable{}, err
	}
	rel, err := filepath.Rel(templatesAbs, inputAbs)
	if err != nil || strings.HasPrefix(rel, "..") || rel == "." {
		return convertTemplateTable{}, nil
	}

	configPath := filepath.Join(templatesDir, "config.json")
	raw, err := os.ReadFile(configPath)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return convertTemplateTable{}, nil
		}
		return convertTemplateTable{}, fmt.Errorf("read %s: %w", configPath, err)
	}

	var config convertTemplateConfig
	if err := json.Unmarshal(raw, &config); err != nil {
		return convertTemplateTable{}, fmt.Errorf("parse %s: %w", configPath, err)
	}
	if len(config.Tables) == 0 {
		return convertTemplateTable{}, nil
	}

	rel = filepath.ToSlash(rel)
	base := filepath.Base(rel)
	stem := strings.TrimSuffix(base, filepath.Ext(base))
	for _, key := range []string{rel, base, stem} {
		if table, ok := config.Tables[key]; ok {
			return table, nil
		}
	}
	return convertTemplateTable{}, nil
}

func defaultModuleOutputPath(input string, opts convertOptions, p *project.Project) (string, error) {
	if p == nil {
		return "", errors.New("2da-to-module requires an explicit output path when not run inside a project root")
	}
	if !p.HasTopData() {
		return "", errors.New("2da-to-module requires topdata to be configured when inferring an output path")
	}
	filename, err := defaultModuleFileName(input, opts, p)
	if err != nil {
		return "", err
	}
	modulesDir := filepath.Join(p.TopDataSourceDir(), "data", filepath.FromSlash(opts.Namespace), "modules")
	return filepath.Join(modulesDir, filename), nil
}

func defaultModuleFileName(input string, opts convertOptions, p *project.Project) (string, error) {
	if strings.TrimSpace(opts.Name) != "" {
		name := normalizeConvertedKeyText(opts.Name)
		if name == "" {
			return "", fmt.Errorf("invalid --name %q", opts.Name)
		}
		namespace := normalizeModuleFilenameNamespace(opts.Namespace)
		return fmt.Sprintf("add_%s_%s.json", namespace, name), nil
	}
	if isTemplateInput(p, input) {
		return "", errors.New("2da-to-module requires --name when converting from topdata/templates without an explicit output path")
	}
	return strings.TrimSuffix(filepath.Base(input), filepath.Ext(input)) + ".json", nil
}

func normalizeModuleFilenameNamespace(namespace string) string {
	namespace = strings.TrimSpace(strings.ToLower(namespace))
	namespace = strings.ReplaceAll(namespace, "/", "_")
	namespace = strings.ReplaceAll(namespace, "\\", "_")
	namespace = convertKeyWhitespace.ReplaceAllString(namespace, "_")
	namespace = convertKeyCleaner.ReplaceAllString(namespace, "_")
	namespace = strings.Trim(namespace, "_")
	if namespace == "" {
		return "module"
	}
	return namespace
}

func isTemplateInput(p *project.Project, input string) bool {
	if p == nil || !p.HasTopData() {
		return false
	}
	templatesAbs, err := filepath.Abs(filepath.Join(p.TopDataSourceDir(), "templates"))
	if err != nil {
		return false
	}
	inputAbs, err := filepath.Abs(input)
	if err != nil {
		return false
	}
	rel, err := filepath.Rel(templatesAbs, inputAbs)
	if err != nil {
		return false
	}
	return rel != "." && !strings.HasPrefix(rel, "..")
}

func readCanonicalJSON(path string) (parsed2DA, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return parsed2DA{}, err
	}
	var payload struct {
		Columns []string         `json:"columns"`
		Rows    []map[string]any `json:"rows"`
	}
	if err := json.Unmarshal(raw, &payload); err != nil {
		return parsed2DA{}, err
	}
	return parsed2DA{Columns: payload.Columns, Rows: payload.Rows}, nil
}

func writeJSONFile(path string, payload any) error {
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." {
		return err
	}
	raw, err := json.MarshalIndent(payload, "", "  ")
	if err != nil {
		return err
	}
	raw = append(raw, '\n')
	return os.WriteFile(path, raw, 0o644)
}

func write2DAFile(data parsed2DA, path string) error {
	rows := make([]map[string]any, 0, len(data.Rows))
	for _, row := range data.Rows {
		cloned := cloneRowMap(row)
		if rowID, err := asInt(cloned["id"]); err == nil {
			cloned["id"] = rowID
		}
		rows = append(rows, cloned)
	}
	sort.Slice(rows, func(i, j int) bool {
		left, _ := asInt(rows[i]["id"])
		right, _ := asInt(rows[j]["id"])
		return left < right
	})
	table := map[string]any{
		"columns": data.Columns,
		"rows":    rows,
	}
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." {
		return err
	}
	return write2DA(table, path, false)
}