package topdata import ( "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) type convertOptions struct { Namespace string KeyFields []string CollisionMode string BaseDialog string Type string } func RunConvertCommand(args []string, stdout io.Writer) error { if len(args) == 0 || args[0] == "--help" || args[0] == "-h" { printConvertUsage(stdout) return nil } switch args[0] { case "2da-to-json": if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") { _, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-json [flags] ") return nil } opts, input, output, err := parseConvertArgs(args[1:], false) if err != nil { return err } data, err := parse2DAFile(input) if err != nil { return err } result, err := convert2DAToJSON(data, output, opts) if err != nil { return err } return writeJSONFile(output, result) case "2da-to-module": if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") { _, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-module --namespace [--type entries|override] [flags] [output.json]") return nil } opts, input, output, err := parseConvertArgs(args[1:], true) if err != nil { return err } data, err := parse2DAFile(input) if err != nil { return err } if opts.Type == "override" { result := map[string]any{"overrides": toOverridesRows(data.Rows)} if err := writeJSONFile(output, result); err != nil { return err } _, _ = fmt.Fprintf(stdout, "converted overrides: %d\n", len(result["overrides"].([]map[string]any))) return nil } result, skipped, err := convert2DAToModule(data, output, opts) if err != nil { return err } if err := writeJSONFile(output, result); err != nil { return err } if skipped > 0 { _, _ = fmt.Fprintf(stdout, "converted entries: %d, skipped unkeyed: %d\n", len(result["entries"].(map[string]any)), skipped) return nil } _, _ = fmt.Fprintf(stdout, "converted entries: %d\n", len(result["entries"].(map[string]any))) return nil case "json-to-2da": if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") { _, _ = fmt.Fprintln(stdout, "usage: convert-topdata json-to-2da ") return nil } if len(args) != 3 { return errors.New("usage: convert-topdata json-to-2da ") } data, err := readCanonicalJSON(args[1]) if err != nil { return err } return write2DAFile(data, args[2]) default: return fmt.Errorf("unknown convert-topdata subcommand %q", args[0]) } } func printConvertUsage(stdout io.Writer) { _, _ = fmt.Fprintln(stdout, "usage: convert-topdata <2da-to-json|2da-to-module|json-to-2da> ...") _, _ = fmt.Fprintln(stdout, "") _, _ = fmt.Fprintln(stdout, "subcommands:") _, _ = fmt.Fprintln(stdout, " 2da-to-json Convert a 2DA file into canonical JSON rows") _, _ = fmt.Fprintln(stdout, " 2da-to-module Convert a 2DA file into module entries or override JSON") _, _ = fmt.Fprintln(stdout, " json-to-2da Convert canonical JSON rows into a 2DA file") } type parsed2DA struct { Columns []string `json:"columns"` Rows []map[string]any `json:"rows"` } func parseConvertArgs(args []string, allowFormat bool) (convertOptions, string, string, error) { opts := convertOptions{ CollisionMode: "prompt", Type: "entries", } positional := make([]string, 0, 2) for index := 0; index < len(args); index++ { arg := args[index] switch arg { case "--namespace": index++ if index >= len(args) { return opts, "", "", errors.New("--namespace requires a value") } opts.Namespace = args[index] case "--key-field": index++ if index >= len(args) { return opts, "", "", errors.New("--key-field requires a value") } opts.KeyFields = append(opts.KeyFields, args[index]) case "--collision": index++ if index >= len(args) { return opts, "", "", errors.New("--collision requires a value") } opts.CollisionMode = args[index] case "--base-dialog": index++ if index >= len(args) { return opts, "", "", errors.New("--base-dialog requires a value") } opts.BaseDialog = args[index] case "--format", "--type": if !allowFormat { return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), arg) } index++ if index >= len(args) { return opts, "", "", fmt.Errorf("%s requires a value", arg) } opts.Type = args[index] default: if strings.HasPrefix(arg, "--") { return opts, "", "", fmt.Errorf("unknown flag %q", arg) } positional = append(positional, arg) } } if allowFormat { if strings.TrimSpace(opts.Namespace) == "" { return opts, "", "", errors.New("2da-to-module requires --namespace") } if len(positional) < 1 || len(positional) > 2 { return opts, "", "", errors.New("usage: convert-topdata 2da-to-module --namespace [--type entries|override] [flags] [output.json]") } } else if len(positional) != 2 { return opts, "", "", errors.New("usage: convert-topdata 2da-to-json [flags] ") } if opts.CollisionMode == "prompt" { opts.CollisionMode = "suffix" } if opts.CollisionMode != "suffix" && opts.CollisionMode != "error" { return opts, "", "", fmt.Errorf("unsupported collision mode %q", opts.CollisionMode) } if allowFormat { switch opts.Type { case "entry": opts.Type = "entries" case "overrides": opts.Type = "override" } if opts.Type != "entries" && opts.Type != "override" { return opts, "", "", fmt.Errorf("unsupported module type %q", opts.Type) } output := "" if len(positional) == 2 { output = positional[1] } else { resolved, err := defaultModuleOutputPath(positional[0], opts) if err != nil { return opts, "", "", err } output = resolved } return opts, positional[0], output, nil } return opts, positional[0], positional[1], nil } func positionalSafe(args []string) string { if len(args) == 0 { return "command" } return args[0] } func parse2DAFile(path string) (parsed2DA, error) { raw, err := os.ReadFile(path) if err != nil { return parsed2DA{}, err } lines := strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") trimmed := make([]string, 0, len(lines)) for _, line := range lines { line = strings.TrimSpace(line) if line != "" { trimmed = append(trimmed, line) } } if len(trimmed) < 2 || !strings.HasPrefix(trimmed[0], "2DA") { return parsed2DA{}, errors.New("invalid 2DA file") } columns := split2DALine(trimmed[1]) rows := make([]map[string]any, 0, max(0, len(trimmed)-2)) for _, line := range trimmed[2:] { fields := split2DALine(line) if len(fields) == 0 { continue } rowID, err := strconv.Atoi(fields[0]) if err != nil { return parsed2DA{}, fmt.Errorf("invalid 2DA row id %q", fields[0]) } row := map[string]any{"id": rowID} for index, column := range columns { if index+1 < len(fields) { row[column] = smartConvertScalar(fields[index+1]) continue } row[column] = nullValue } rows = append(rows, row) } return parsed2DA{Columns: columns, Rows: rows}, nil } func split2DALine(line string) []string { fields := make([]string, 0) var current strings.Builder inQuotes := false for _, ch := range line { switch { case ch == '"': inQuotes = !inQuotes case !inQuotes && (ch == '\t' || ch == ' '): if current.Len() > 0 { fields = append(fields, current.String()) current.Reset() } default: current.WriteRune(ch) } } if current.Len() > 0 { fields = append(fields, current.String()) } return fields } func smartConvertScalar(value string) any { if value == "" { return "" } if parsed, err := strconv.Atoi(value); err == nil { return parsed } return value } func convert2DAToJSON(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) { rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts) if err != nil { return nil, err } outRows := make([]map[string]any, 0, len(rows)) for _, row := range rows { ordered := map[string]any{"id": row["id"]} if key, ok := row["key"].(string); ok && key != "" { ordered["key"] = key } for _, column := range data.Columns { ordered[column] = row[column] } outRows = append(outRows, ordered) } return map[string]any{"columns": data.Columns, "rows": outRows}, nil } func convert2DAToModule(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, int, error) { rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts) if err != nil { return nil, 0, err } entries := map[string]any{} skipped := 0 for _, row := range rows { key, _ := row["key"].(string) if key == "" { skipped++ continue } entry := map[string]any{} for _, column := range data.Columns { value := row[column] if format2DAValue(value) == nullValue { continue } entry[column] = value } entries[key] = entry } return map[string]any{"entries": entries}, skipped, nil } func toOverridesRows(rows []map[string]any) []map[string]any { out := make([]map[string]any, 0, len(rows)) for _, row := range rows { override := map[string]any{"id": row["id"]} keys := make([]string, 0, len(row)) for key := range row { if key == "id" || key == "key" { continue } keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { if format2DAValue(row[key]) == nullValue { continue } override[key] = row[key] } out = append(out, override) } return out } func assignConvertedRowKeys(rows []map[string]any, outputPath string, opts convertOptions) ([]map[string]any, error) { namespace, profiles := inferConvertNamespace(outputPath, opts.Namespace) out := make([]map[string]any, 0, len(rows)) used := map[string]struct{}{} for _, row := range rows { cloned := cloneRowMap(row) if namespace == "" { out = append(out, cloned) continue } key := convertedRowKey(cloned, namespace, profiles, opts.KeyFields) if key == "" { out = append(out, cloned) continue } if _, ok := used[key]; ok { if opts.CollisionMode == "error" { return nil, fmt.Errorf("duplicate generated key %q", key) } key = fmt.Sprintf("%s_%v", key, cloned["id"]) } used[key] = struct{}{} cloned["key"] = key out = append(out, cloned) } return out, nil } type convertKeyProfile struct { Field string Mode string } func inferConvertNamespace(outputPath, override string) (string, []convertKeyProfile) { if strings.TrimSpace(override) != "" { return strings.TrimSpace(override), nil } normalized := filepath.ToSlash(outputPath) prefixes := []struct { Prefix string Namespace string Profiles []convertKeyProfile }{ {"data/itemprops/costtables/index/", "itemprops:costtable_index", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}}, {"data/itemprops/paramtables/index/", "itemprops:paramtable_index", []convertKeyProfile{{"Name", "literal"}, {"Lable", "literal"}, {"TableResRef", "literal"}}}, {"data/itemprops/defs/", "itempropdef", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}}, {"data/itemprops/sets/", "itemprops", []convertKeyProfile{{"StringRef", "literal"}, {"Label", "literal"}}}, {"data/appearance/", "appearance", []convertKeyProfile{{"STRING_REF", "literal"}, {"LABEL", "literal"}}}, {"data/baseitems/", "baseitems", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}}, {"data/feat/", "feat", []convertKeyProfile{{"FEAT", "literal"}, {"LABEL", "literal"}, {"Label", "literal"}}}, {"data/spells/", "spells", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}}, {"data/skills/", "skills", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}}, {"data/portraits/", "portraits", []convertKeyProfile{{"BaseResRef", "literal"}, {"Label", "literal"}}}, {"data/racialtypes/", "racialtypes", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}}, {"data/damagetypes/", "damagetypes", []convertKeyProfile{{"Name", "literal"}, {"Label", "literal"}}}, } for _, prefix := range prefixes { if strings.Contains(normalized, prefix.Prefix) { return prefix.Namespace, prefix.Profiles } } return "", nil } var convertKeyWhitespace = regexp.MustCompile(`\s+`) var convertKeyCleaner = regexp.MustCompile(`[^a-z0-9_]`) func convertedRowKey(row map[string]any, namespace string, profiles []convertKeyProfile, preferred []string) string { candidates := make([]convertKeyProfile, 0, len(preferred)+len(profiles)) for _, field := range preferred { candidates = append(candidates, convertKeyProfile{Field: field, Mode: "literal"}) } candidates = append(candidates, profiles...) if len(candidates) == 0 { candidates = append(candidates, convertKeyProfile{Field: "LABEL", Mode: "literal"}, convertKeyProfile{Field: "Label", Mode: "literal"}, convertKeyProfile{Field: "Name", Mode: "literal"}, ) } for _, candidate := range candidates { value, ok := row[candidate.Field] if !ok { continue } text := strings.TrimSpace(format2DAValue(value)) if text == "" || text == nullValue { continue } normalized := normalizeConvertedKeyText(text) if normalized == "" { continue } return namespace + ":" + normalized } return "" } func normalizeConvertedKeyText(text string) string { text = strings.TrimSpace(strings.ToLower(text)) text = convertKeyWhitespace.ReplaceAllString(text, "_") text = convertKeyCleaner.ReplaceAllString(text, "") text = strings.Trim(text, "_") return text } func defaultModuleOutputPath(input string, opts convertOptions) (string, error) { cwd, err := os.Getwd() if err != nil { return "", err } root, err := project.FindRoot(cwd) if err != nil { return "", errors.New("2da-to-module requires an explicit output path when not run inside a project root") } p, err := project.Load(root) if err != nil { return "", err } if !p.HasTopData() { return "", errors.New("2da-to-module requires topdata to be configured when inferring an output path") } filename := strings.TrimSuffix(filepath.Base(input), filepath.Ext(input)) + ".json" modulesDir := filepath.Join(p.TopDataSourceDir(), "data", filepath.FromSlash(opts.Namespace), "modules") return filepath.Join(modulesDir, filename), nil } func readCanonicalJSON(path string) (parsed2DA, error) { raw, err := os.ReadFile(path) if err != nil { return parsed2DA{}, err } var payload struct { Columns []string `json:"columns"` Rows []map[string]any `json:"rows"` } if err := json.Unmarshal(raw, &payload); err != nil { return parsed2DA{}, err } return parsed2DA{Columns: payload.Columns, Rows: payload.Rows}, nil } func writeJSONFile(path string, payload any) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." { return err } raw, err := json.MarshalIndent(payload, "", " ") if err != nil { return err } raw = append(raw, '\n') return os.WriteFile(path, raw, 0o644) } func write2DAFile(data parsed2DA, path string) error { rows := make([]map[string]any, 0, len(data.Rows)) for _, row := range data.Rows { cloned := cloneRowMap(row) if rowID, err := asInt(cloned["id"]); err == nil { cloned["id"] = rowID } rows = append(rows, cloned) } sort.Slice(rows, func(i, j int) bool { left, _ := asInt(rows[i]["id"]) right, _ := asInt(rows[j]["id"]) return left < right }) table := map[string]any{ "columns": data.Columns, "rows": rows, } if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." { return err } return write2DA(table, path, false) }