Phase 5: scaffold Crucible (sow-tools) — dispatcher + builder shims, container, PR-first CI, fail-closed (D11, D19).
This commit is contained in:
@@ -1,888 +0,0 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type convertOptions struct {
|
||||
Namespace string
|
||||
KeyFields []string
|
||||
CollisionMode string
|
||||
BaseDialog string
|
||||
Type string
|
||||
Name string
|
||||
FilenamePrefixes map[string]string
|
||||
}
|
||||
|
||||
func RunConvertCommand(args []string, stdout io.Writer) error {
|
||||
if len(args) == 0 || args[0] == "--help" || args[0] == "-h" {
|
||||
printConvertUsage(stdout)
|
||||
return nil
|
||||
}
|
||||
switch args[0] {
|
||||
case "2da-to-json":
|
||||
if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
|
||||
_, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-json [flags] <input.2da> <output.json>")
|
||||
return nil
|
||||
}
|
||||
opts, input, output, err := parseConvertArgs(args[1:], false, "suffix")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := parse2DAFile(input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := convert2DAToJSON(data, output, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeJSONFile(output, result)
|
||||
case "2da-to-module":
|
||||
if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
|
||||
_, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-module [--namespace <dataset>] [--name <module-name>] [--type entries|override] [flags] <input.2da> [output.json]")
|
||||
return nil
|
||||
}
|
||||
opts, input, output, err := parseConvertArgs(args[1:], true, "error")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := parse2DAFile(input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.Type == "override" {
|
||||
result := map[string]any{"overrides": toOverridesRows(data.Rows)}
|
||||
if err := writeJSONFile(output, result); err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout, "converted overrides: %d\n", len(result["overrides"].([]map[string]any)))
|
||||
return nil
|
||||
}
|
||||
result, err := convert2DAToModule(data, output, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeJSONFile(output, result); err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = fmt.Fprintf(stdout, "converted entries: %d\n", len(result["entries"].(map[string]any)))
|
||||
return nil
|
||||
case "json-to-2da":
|
||||
if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") {
|
||||
_, _ = fmt.Fprintln(stdout, "usage: convert-topdata json-to-2da <input.json> <output.2da>")
|
||||
return nil
|
||||
}
|
||||
if len(args) != 3 {
|
||||
return errors.New("usage: convert-topdata json-to-2da <input.json> <output.2da>")
|
||||
}
|
||||
data, err := readCanonicalJSON(args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return write2DAFile(data, args[2])
|
||||
default:
|
||||
return fmt.Errorf("unknown convert-topdata subcommand %q", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func printConvertUsage(stdout io.Writer) {
|
||||
_, _ = fmt.Fprintln(stdout, "usage: convert-topdata <2da-to-json|2da-to-module|json-to-2da> ...")
|
||||
_, _ = fmt.Fprintln(stdout, "")
|
||||
_, _ = fmt.Fprintln(stdout, "subcommands:")
|
||||
_, _ = fmt.Fprintln(stdout, " 2da-to-json Convert a 2DA file into canonical JSON rows")
|
||||
_, _ = fmt.Fprintln(stdout, " 2da-to-module Convert a 2DA file into module entries or override JSON")
|
||||
_, _ = fmt.Fprintln(stdout, " json-to-2da Convert canonical JSON rows into a 2DA file")
|
||||
_, _ = fmt.Fprintln(stdout, "")
|
||||
_, _ = fmt.Fprintln(stdout, "2da-to-module notes:")
|
||||
_, _ = fmt.Fprintln(stdout, " - --namespace is optional when topdata/templates/config.yaml declares it for the input table")
|
||||
_, _ = fmt.Fprintln(stdout, " - --name controls inferred output naming: <prefix>_<namespace>_<name>.json")
|
||||
_, _ = fmt.Fprintln(stdout, " - topdata/templates/config.yaml module_output.filename_prefixes can set entries/override prefixes")
|
||||
_, _ = fmt.Fprintln(stdout, " - flags accept both --flag value and --flag=value forms")
|
||||
_, _ = fmt.Fprintln(stdout, " - unkeyed rows now fail conversion instead of being skipped")
|
||||
}
|
||||
|
||||
type parsed2DA struct {
|
||||
Columns []string `json:"columns"`
|
||||
Rows []map[string]any `json:"rows"`
|
||||
}
|
||||
|
||||
func parseConvertArgs(args []string, allowFormat bool, defaultCollision string) (convertOptions, string, string, error) {
|
||||
opts := convertOptions{
|
||||
CollisionMode: defaultCollision,
|
||||
Type: "entries",
|
||||
}
|
||||
positional := make([]string, 0, 2)
|
||||
for index := 0; index < len(args); index++ {
|
||||
arg := args[index]
|
||||
switch arg {
|
||||
case "--namespace":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", errors.New("--namespace requires a value")
|
||||
}
|
||||
opts.Namespace = args[index]
|
||||
case "--key-field":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", errors.New("--key-field requires a value")
|
||||
}
|
||||
opts.KeyFields = append(opts.KeyFields, args[index])
|
||||
case "--name":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", errors.New("--name requires a value")
|
||||
}
|
||||
opts.Name = args[index]
|
||||
case "--collision":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", errors.New("--collision requires a value")
|
||||
}
|
||||
opts.CollisionMode = args[index]
|
||||
case "--base-dialog":
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", errors.New("--base-dialog requires a value")
|
||||
}
|
||||
opts.BaseDialog = args[index]
|
||||
case "--format", "--type":
|
||||
if !allowFormat {
|
||||
return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), arg)
|
||||
}
|
||||
index++
|
||||
if index >= len(args) {
|
||||
return opts, "", "", fmt.Errorf("%s requires a value", arg)
|
||||
}
|
||||
opts.Type = args[index]
|
||||
default:
|
||||
if value, ok := parseInlineFlagValue(arg, "--namespace"); ok {
|
||||
opts.Namespace = value
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--key-field"); ok {
|
||||
opts.KeyFields = append(opts.KeyFields, value)
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--name"); ok {
|
||||
opts.Name = value
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--collision"); ok {
|
||||
opts.CollisionMode = value
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--base-dialog"); ok {
|
||||
opts.BaseDialog = value
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--format"); ok {
|
||||
if !allowFormat {
|
||||
return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), "--format")
|
||||
}
|
||||
opts.Type = value
|
||||
continue
|
||||
}
|
||||
if value, ok := parseInlineFlagValue(arg, "--type"); ok {
|
||||
if !allowFormat {
|
||||
return opts, "", "", fmt.Errorf("%s does not accept %s", positionalSafe(args), "--type")
|
||||
}
|
||||
opts.Type = value
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(arg, "--") {
|
||||
return opts, "", "", fmt.Errorf("unknown flag %q", arg)
|
||||
}
|
||||
positional = append(positional, arg)
|
||||
}
|
||||
}
|
||||
if allowFormat {
|
||||
if len(positional) < 1 || len(positional) > 2 {
|
||||
return opts, "", "", errors.New("usage: convert-topdata 2da-to-module [--namespace <dataset>] [--name <module-name>] [--type entries|override] [flags] <input.2da> [output.json]")
|
||||
}
|
||||
} else if len(positional) != 2 {
|
||||
return opts, "", "", errors.New("usage: convert-topdata 2da-to-json [flags] <input.2da> <output.json>")
|
||||
}
|
||||
if opts.CollisionMode != "suffix" && opts.CollisionMode != "error" {
|
||||
return opts, "", "", fmt.Errorf("unsupported collision mode %q", opts.CollisionMode)
|
||||
}
|
||||
if allowFormat {
|
||||
output := ""
|
||||
if len(positional) == 2 {
|
||||
output = positional[1]
|
||||
}
|
||||
ctx, err := resolveConvertContext(positional[0])
|
||||
if err != nil {
|
||||
return opts, "", "", err
|
||||
}
|
||||
if ctx.Project != nil {
|
||||
if outputCtx, err := resolveConvertOutputContext(ctx.Project, output); err != nil {
|
||||
return opts, "", "", err
|
||||
} else if ctx.Template.Namespace == "" && len(ctx.Template.KeyFields) == 0 {
|
||||
ctx.Template = outputCtx
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(opts.Namespace) == "" {
|
||||
opts.Namespace = ctx.Template.Namespace
|
||||
}
|
||||
if len(opts.KeyFields) == 0 && len(ctx.Template.KeyFields) > 0 {
|
||||
opts.KeyFields = append(opts.KeyFields, ctx.Template.KeyFields...)
|
||||
}
|
||||
if len(opts.FilenamePrefixes) == 0 && len(ctx.Config.ModuleOutput.FilenamePrefixes) > 0 {
|
||||
opts.FilenamePrefixes = cloneStringMap(ctx.Config.ModuleOutput.FilenamePrefixes)
|
||||
}
|
||||
if strings.TrimSpace(opts.Namespace) == "" {
|
||||
return opts, "", "", errors.New("2da-to-module requires --namespace or a topdata/templates/config.yaml entry for the input table")
|
||||
}
|
||||
switch opts.Type {
|
||||
case "entry":
|
||||
opts.Type = "entries"
|
||||
case "overrides":
|
||||
opts.Type = "override"
|
||||
}
|
||||
if opts.Type != "entries" && opts.Type != "override" {
|
||||
return opts, "", "", fmt.Errorf("unsupported module type %q", opts.Type)
|
||||
}
|
||||
if output == "" {
|
||||
resolved, err := defaultModuleOutputPath(positional[0], opts, ctx.Project)
|
||||
if err != nil {
|
||||
return opts, "", "", err
|
||||
}
|
||||
output = resolved
|
||||
}
|
||||
return opts, positional[0], output, nil
|
||||
}
|
||||
ctx, err := resolveConvertContext(positional[0])
|
||||
if err != nil {
|
||||
return opts, "", "", err
|
||||
}
|
||||
if ctx.Project != nil {
|
||||
if outputCtx, err := resolveConvertOutputContext(ctx.Project, positional[1]); err != nil {
|
||||
return opts, "", "", err
|
||||
} else if ctx.Template.Namespace == "" && len(ctx.Template.KeyFields) == 0 {
|
||||
ctx.Template = outputCtx
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(opts.Namespace) == "" {
|
||||
opts.Namespace = ctx.Template.Namespace
|
||||
}
|
||||
if len(opts.KeyFields) == 0 && len(ctx.Template.KeyFields) > 0 {
|
||||
opts.KeyFields = append(opts.KeyFields, ctx.Template.KeyFields...)
|
||||
}
|
||||
return opts, positional[0], positional[1], nil
|
||||
}
|
||||
|
||||
func parseInlineFlagValue(arg, name string) (string, bool) {
|
||||
prefix := name + "="
|
||||
if !strings.HasPrefix(arg, prefix) {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(arg[len(prefix):]), true
|
||||
}
|
||||
|
||||
func positionalSafe(args []string) string {
|
||||
if len(args) == 0 {
|
||||
return "command"
|
||||
}
|
||||
return args[0]
|
||||
}
|
||||
|
||||
func parse2DAFile(path string) (parsed2DA, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return parsed2DA{}, fmt.Errorf("read %s: %w", path, err)
|
||||
}
|
||||
lines := strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n")
|
||||
trimmed := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" {
|
||||
trimmed = append(trimmed, line)
|
||||
}
|
||||
}
|
||||
if len(trimmed) < 2 || !strings.HasPrefix(trimmed[0], "2DA") {
|
||||
return parsed2DA{}, fmt.Errorf("%s: invalid 2DA file", path)
|
||||
}
|
||||
columns := split2DALine(trimmed[1])
|
||||
if len(columns) == 0 {
|
||||
return parsed2DA{}, fmt.Errorf("%s: missing 2DA columns", path)
|
||||
}
|
||||
rows := make([]map[string]any, 0, max(0, len(trimmed)-2))
|
||||
seenIDs := map[int]struct{}{}
|
||||
for _, line := range trimmed[2:] {
|
||||
fields := split2DALine(line)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
rowID, err := strconv.Atoi(fields[0])
|
||||
if err != nil {
|
||||
return parsed2DA{}, fmt.Errorf("%s: invalid 2DA row id %q", path, fields[0])
|
||||
}
|
||||
if len(fields)-1 > len(columns) {
|
||||
return parsed2DA{}, fmt.Errorf("%s: row %d has %d values but only %d columns are declared", path, rowID, len(fields)-1, len(columns))
|
||||
}
|
||||
if _, ok := seenIDs[rowID]; ok {
|
||||
return parsed2DA{}, fmt.Errorf("%s: duplicate 2DA row id %d", path, rowID)
|
||||
}
|
||||
seenIDs[rowID] = struct{}{}
|
||||
row := map[string]any{"id": rowID}
|
||||
for index, column := range columns {
|
||||
if index+1 < len(fields) {
|
||||
row[column] = smartConvertScalar(fields[index+1])
|
||||
continue
|
||||
}
|
||||
row[column] = nullValue
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return parsed2DA{Columns: columns, Rows: rows}, nil
|
||||
}
|
||||
|
||||
func split2DALine(line string) []string {
|
||||
fields := make([]string, 0)
|
||||
var current strings.Builder
|
||||
inQuotes := false
|
||||
for _, ch := range line {
|
||||
switch {
|
||||
case ch == '"':
|
||||
inQuotes = !inQuotes
|
||||
case !inQuotes && (ch == '\t' || ch == ' '):
|
||||
if current.Len() > 0 {
|
||||
fields = append(fields, current.String())
|
||||
current.Reset()
|
||||
}
|
||||
default:
|
||||
current.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
if current.Len() > 0 {
|
||||
fields = append(fields, current.String())
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func smartConvertScalar(value string) any {
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
if parsed, err := strconv.Atoi(value); err == nil {
|
||||
return parsed
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func convert2DAToJSON(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) {
|
||||
rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outRows := make([]map[string]any, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
ordered := map[string]any{"id": row["id"]}
|
||||
if key, ok := row["key"].(string); ok && key != "" {
|
||||
ordered["key"] = key
|
||||
}
|
||||
for _, column := range data.Columns {
|
||||
ordered[column] = row[column]
|
||||
}
|
||||
outRows = append(outRows, ordered)
|
||||
}
|
||||
return map[string]any{"columns": data.Columns, "rows": outRows}, nil
|
||||
}
|
||||
|
||||
func convert2DAToModule(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) {
|
||||
rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries := map[string]any{}
|
||||
missing := []string{}
|
||||
for _, row := range rows {
|
||||
key, _ := row["key"].(string)
|
||||
if key == "" {
|
||||
missing = append(missing, strconv.Itoa(row["id"].(int)))
|
||||
continue
|
||||
}
|
||||
entry := map[string]any{}
|
||||
for _, column := range data.Columns {
|
||||
value := row[column]
|
||||
if format2DAValue(value) == nullValue {
|
||||
continue
|
||||
}
|
||||
entry[column] = value
|
||||
}
|
||||
entries[key] = entry
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return nil, fmt.Errorf("unable to generate keys for row ids %s; declare --key-field or configure key_fields in topdata/templates/config.yaml", strings.Join(missing, ", "))
|
||||
}
|
||||
return map[string]any{"entries": entries}, nil
|
||||
}
|
||||
|
||||
func toOverridesRows(rows []map[string]any) []map[string]any {
|
||||
out := make([]map[string]any, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
override := map[string]any{"id": row["id"]}
|
||||
keys := make([]string, 0, len(row))
|
||||
for key := range row {
|
||||
if key == "id" || key == "key" {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
if format2DAValue(row[key]) == nullValue {
|
||||
continue
|
||||
}
|
||||
override[key] = row[key]
|
||||
}
|
||||
out = append(out, override)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assignConvertedRowKeys(rows []map[string]any, outputPath string, opts convertOptions) ([]map[string]any, error) {
|
||||
namespace := strings.TrimSpace(opts.Namespace)
|
||||
out := make([]map[string]any, 0, len(rows))
|
||||
used := map[string]struct{}{}
|
||||
rowIDsByKey := map[string]int{}
|
||||
for _, row := range rows {
|
||||
cloned := cloneRowMap(row)
|
||||
if namespace == "" {
|
||||
out = append(out, cloned)
|
||||
continue
|
||||
}
|
||||
candidates := convertedRowKeyCandidates(cloned, opts.KeyFields)
|
||||
if len(candidates) == 0 {
|
||||
out = append(out, cloned)
|
||||
continue
|
||||
}
|
||||
key := ""
|
||||
for _, candidate := range candidates {
|
||||
candidateKey := namespace + ":" + candidate
|
||||
if _, ok := used[candidateKey]; ok {
|
||||
continue
|
||||
}
|
||||
key = candidateKey
|
||||
break
|
||||
}
|
||||
if key == "" {
|
||||
key = namespace + ":" + candidates[0]
|
||||
if opts.CollisionMode == "error" {
|
||||
return nil, fmt.Errorf("duplicate generated key %q for row %v (already used by row %d)", key, cloned["id"], rowIDsByKey[key])
|
||||
}
|
||||
key = fmt.Sprintf("%s_%v", key, cloned["id"])
|
||||
}
|
||||
used[key] = struct{}{}
|
||||
rowID, _ := asInt(cloned["id"])
|
||||
rowIDsByKey[key] = rowID
|
||||
cloned["key"] = key
|
||||
out = append(out, cloned)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var convertKeyWhitespace = regexp.MustCompile(`\s+`)
|
||||
var convertKeyCleaner = regexp.MustCompile(`[^a-z0-9_]`)
|
||||
|
||||
func convertedRowKeyCandidates(row map[string]any, preferred []string) []string {
|
||||
fields := preferred
|
||||
if len(fields) == 0 {
|
||||
fields = []string{"LABEL", "Label", "Name"}
|
||||
} else {
|
||||
for _, field := range fields {
|
||||
value, ok := row[field]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
text := strings.TrimSpace(format2DAValue(value))
|
||||
if text == "" || text == nullValue {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
values := make([]string, 0, len(fields))
|
||||
seen := map[string]struct{}{}
|
||||
candidates := make([]string, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
value, ok := row[field]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
text := strings.TrimSpace(format2DAValue(value))
|
||||
if text == "" || text == nullValue {
|
||||
continue
|
||||
}
|
||||
normalized := normalizeConvertedKeyText(text)
|
||||
if normalized == "" {
|
||||
continue
|
||||
}
|
||||
values = append(values, normalized)
|
||||
joined := strings.Join(values, "_")
|
||||
if _, ok := seen[joined]; ok {
|
||||
continue
|
||||
}
|
||||
seen[joined] = struct{}{}
|
||||
candidates = append(candidates, joined)
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func normalizeConvertedKeyText(text string) string {
|
||||
text = strings.TrimSpace(strings.ToLower(text))
|
||||
text = convertKeyWhitespace.ReplaceAllString(text, "_")
|
||||
text = convertKeyCleaner.ReplaceAllString(text, "")
|
||||
text = strings.Trim(text, "_")
|
||||
return text
|
||||
}
|
||||
|
||||
type convertTemplateTable struct {
|
||||
Namespace string `json:"namespace" yaml:"namespace"`
|
||||
KeyFields []string `json:"key_fields" yaml:"key_fields"`
|
||||
}
|
||||
|
||||
type convertTemplateConfig struct {
|
||||
ModuleOutput convertModuleOutputConfig `json:"module_output" yaml:"module_output"`
|
||||
Tables map[string]convertTemplateTable `json:"tables" yaml:"tables"`
|
||||
}
|
||||
|
||||
type convertModuleOutputConfig struct {
|
||||
FilenamePrefixes map[string]string `json:"filename_prefixes" yaml:"filename_prefixes"`
|
||||
}
|
||||
|
||||
type convertContext struct {
|
||||
Project *project.Project
|
||||
Config convertTemplateConfig
|
||||
Template convertTemplateTable
|
||||
}
|
||||
|
||||
func resolveConvertContext(input string) (convertContext, error) {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
return convertContext{}, err
|
||||
}
|
||||
root, err := project.FindRoot(cwd)
|
||||
if err != nil {
|
||||
return convertContext{}, nil
|
||||
}
|
||||
p, err := project.Load(root)
|
||||
if err != nil {
|
||||
return convertContext{}, err
|
||||
}
|
||||
if !p.HasTopData() {
|
||||
return convertContext{Project: p}, nil
|
||||
}
|
||||
config, err := loadConvertTemplateConfig(p)
|
||||
if err != nil {
|
||||
return convertContext{}, err
|
||||
}
|
||||
table, err := templateTableConfig(config, p, input)
|
||||
if err != nil {
|
||||
return convertContext{}, err
|
||||
}
|
||||
return convertContext{
|
||||
Project: p,
|
||||
Config: config,
|
||||
Template: table,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func loadTemplateTableConfig(p *project.Project, input string) (convertTemplateTable, error) {
|
||||
config, err := loadConvertTemplateConfig(p)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
return templateTableConfig(config, p, input)
|
||||
}
|
||||
|
||||
func templateTableConfig(config convertTemplateConfig, p *project.Project, input string) (convertTemplateTable, error) {
|
||||
if len(config.Tables) == 0 {
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
templatesDir := filepath.Join(p.TopDataSourceDir(), "templates")
|
||||
inputAbs, err := filepath.Abs(input)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
templatesAbs, err := filepath.Abs(templatesDir)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
rel, err := filepath.Rel(templatesAbs, inputAbs)
|
||||
if err != nil || strings.HasPrefix(rel, "..") || rel == "." {
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
|
||||
rel = filepath.ToSlash(rel)
|
||||
base := filepath.Base(rel)
|
||||
stem := strings.TrimSuffix(base, filepath.Ext(base))
|
||||
for _, key := range []string{rel, base, stem} {
|
||||
if table, ok := config.Tables[key]; ok {
|
||||
return table, nil
|
||||
}
|
||||
}
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
|
||||
func loadConvertTemplateConfig(p *project.Project) (convertTemplateConfig, error) {
|
||||
templatesDir := filepath.Join(p.TopDataSourceDir(), "templates")
|
||||
for _, candidate := range []string{"config.yaml", "config.yml", "config.json"} {
|
||||
configPath := filepath.Join(templatesDir, candidate)
|
||||
raw, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
return convertTemplateConfig{}, fmt.Errorf("read %s: %w", configPath, err)
|
||||
}
|
||||
|
||||
var config convertTemplateConfig
|
||||
switch filepath.Ext(configPath) {
|
||||
case ".yaml", ".yml":
|
||||
if err := yaml.Unmarshal(raw, &config); err != nil {
|
||||
return convertTemplateConfig{}, fmt.Errorf("parse %s: %w", configPath, err)
|
||||
}
|
||||
default:
|
||||
if err := json.Unmarshal(raw, &config); err != nil {
|
||||
return convertTemplateConfig{}, fmt.Errorf("parse %s: %w", configPath, err)
|
||||
}
|
||||
}
|
||||
if err := validateConvertTemplateConfig(config); err != nil {
|
||||
return convertTemplateConfig{}, fmt.Errorf("validate %s: %w", configPath, err)
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
return convertTemplateConfig{}, nil
|
||||
}
|
||||
|
||||
func validateConvertTemplateConfig(config convertTemplateConfig) error {
|
||||
for moduleType, prefix := range config.ModuleOutput.FilenamePrefixes {
|
||||
switch moduleType {
|
||||
case "entries", "override":
|
||||
default:
|
||||
return fmt.Errorf("unsupported module_output.filename_prefixes key %q; expected entries or override", moduleType)
|
||||
}
|
||||
if normalizeModuleFilenamePrefix(prefix) == "" {
|
||||
return fmt.Errorf("module_output.filename_prefixes.%s must contain at least one filename-safe character", moduleType)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveConvertOutputContext(p *project.Project, output string) (convertTemplateTable, error) {
|
||||
if p == nil || !p.HasTopData() || strings.TrimSpace(output) == "" {
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
config, err := loadConvertTemplateConfig(p)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
if len(config.Tables) == 0 {
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
dataDir := filepath.Join(p.TopDataSourceDir(), "data")
|
||||
dataAbs, err := filepath.Abs(dataDir)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
outputAbs, err := filepath.Abs(output)
|
||||
if err != nil {
|
||||
return convertTemplateTable{}, err
|
||||
}
|
||||
rel, err := filepath.Rel(dataAbs, outputAbs)
|
||||
if err != nil || rel == "." || strings.HasPrefix(rel, "..") {
|
||||
return convertTemplateTable{}, nil
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
|
||||
bestMatch := ""
|
||||
bestTable := convertTemplateTable{}
|
||||
for _, table := range config.Tables {
|
||||
namespacePath := namespaceToPath(table.Namespace)
|
||||
if namespacePath == "" {
|
||||
continue
|
||||
}
|
||||
if rel != namespacePath && !strings.HasPrefix(rel, namespacePath+"/") {
|
||||
continue
|
||||
}
|
||||
if len(namespacePath) <= len(bestMatch) {
|
||||
continue
|
||||
}
|
||||
bestMatch = namespacePath
|
||||
bestTable = table
|
||||
}
|
||||
return bestTable, nil
|
||||
}
|
||||
|
||||
func namespaceToPath(namespace string) string {
|
||||
namespace = strings.TrimSpace(strings.ToLower(namespace))
|
||||
if namespace == "" {
|
||||
return ""
|
||||
}
|
||||
namespace = strings.ReplaceAll(namespace, "\\", "/")
|
||||
namespace = strings.ReplaceAll(namespace, ":", "/")
|
||||
namespace = convertKeyWhitespace.ReplaceAllString(namespace, "_")
|
||||
namespace = strings.Trim(namespace, "/")
|
||||
return namespace
|
||||
}
|
||||
|
||||
func defaultModuleOutputPath(input string, opts convertOptions, p *project.Project) (string, error) {
|
||||
if p == nil {
|
||||
return "", errors.New("2da-to-module requires an explicit output path when not run inside a project root")
|
||||
}
|
||||
if !p.HasTopData() {
|
||||
return "", errors.New("2da-to-module requires topdata to be configured when inferring an output path")
|
||||
}
|
||||
filename, err := defaultModuleFileName(input, opts, p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
modulesDir := filepath.Join(p.TopDataSourceDir(), "data", filepath.FromSlash(opts.Namespace), "modules")
|
||||
return filepath.Join(modulesDir, filename), nil
|
||||
}
|
||||
|
||||
func defaultModuleFileName(input string, opts convertOptions, p *project.Project) (string, error) {
|
||||
if strings.TrimSpace(opts.Name) != "" {
|
||||
name := normalizeConvertedKeyText(opts.Name)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("invalid --name %q", opts.Name)
|
||||
}
|
||||
namespace := normalizeModuleFilenameNamespace(opts.Namespace)
|
||||
prefix := moduleFilenamePrefix(opts)
|
||||
return fmt.Sprintf("%s_%s_%s.json", prefix, namespace, name), nil
|
||||
}
|
||||
if isTemplateInput(p, input) {
|
||||
return "", errors.New("2da-to-module requires --name when converting from topdata/templates without an explicit output path")
|
||||
}
|
||||
return strings.TrimSuffix(filepath.Base(input), filepath.Ext(input)) + ".json", nil
|
||||
}
|
||||
|
||||
func moduleFilenamePrefix(opts convertOptions) string {
|
||||
moduleType := strings.TrimSpace(opts.Type)
|
||||
if moduleType == "entry" {
|
||||
moduleType = "entries"
|
||||
}
|
||||
if moduleType == "overrides" {
|
||||
moduleType = "override"
|
||||
}
|
||||
if configured := strings.TrimSpace(opts.FilenamePrefixes[moduleType]); configured != "" {
|
||||
if prefix := normalizeModuleFilenamePrefix(configured); prefix != "" {
|
||||
return prefix
|
||||
}
|
||||
}
|
||||
return "add"
|
||||
}
|
||||
|
||||
func normalizeModuleFilenamePrefix(prefix string) string {
|
||||
prefix = strings.TrimSpace(strings.ToLower(prefix))
|
||||
prefix = strings.ReplaceAll(prefix, "/", "_")
|
||||
prefix = strings.ReplaceAll(prefix, "\\", "_")
|
||||
prefix = convertKeyWhitespace.ReplaceAllString(prefix, "_")
|
||||
prefix = convertKeyCleaner.ReplaceAllString(prefix, "_")
|
||||
prefix = strings.Trim(prefix, "_")
|
||||
return prefix
|
||||
}
|
||||
|
||||
func normalizeModuleFilenameNamespace(namespace string) string {
|
||||
namespace = strings.TrimSpace(strings.ToLower(namespace))
|
||||
namespace = strings.ReplaceAll(namespace, "/", "_")
|
||||
namespace = strings.ReplaceAll(namespace, "\\", "_")
|
||||
namespace = convertKeyWhitespace.ReplaceAllString(namespace, "_")
|
||||
namespace = convertKeyCleaner.ReplaceAllString(namespace, "_")
|
||||
namespace = strings.Trim(namespace, "_")
|
||||
if namespace == "" {
|
||||
return "module"
|
||||
}
|
||||
return namespace
|
||||
}
|
||||
|
||||
func isTemplateInput(p *project.Project, input string) bool {
|
||||
if p == nil || !p.HasTopData() {
|
||||
return false
|
||||
}
|
||||
templatesAbs, err := filepath.Abs(filepath.Join(p.TopDataSourceDir(), "templates"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
inputAbs, err := filepath.Abs(input)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
rel, err := filepath.Rel(templatesAbs, inputAbs)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel != "." && !strings.HasPrefix(rel, "..")
|
||||
}
|
||||
|
||||
func readCanonicalJSON(path string) (parsed2DA, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return parsed2DA{}, err
|
||||
}
|
||||
var payload struct {
|
||||
Columns []string `json:"columns"`
|
||||
Rows []map[string]any `json:"rows"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
return parsed2DA{}, err
|
||||
}
|
||||
return parsed2DA{Columns: payload.Columns, Rows: payload.Rows}, nil
|
||||
}
|
||||
|
||||
func writeJSONFile(path string, payload any) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." {
|
||||
return err
|
||||
}
|
||||
raw, err := json.MarshalIndent(payload, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw = append(raw, '\n')
|
||||
return os.WriteFile(path, raw, 0o644)
|
||||
}
|
||||
|
||||
func write2DAFile(data parsed2DA, path string) error {
|
||||
rows := make([]map[string]any, 0, len(data.Rows))
|
||||
for _, row := range data.Rows {
|
||||
cloned := cloneRowMap(row)
|
||||
if rowID, err := asInt(cloned["id"]); err == nil {
|
||||
cloned["id"] = rowID
|
||||
}
|
||||
rows = append(rows, cloned)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
left, _ := asInt(rows[i]["id"])
|
||||
right, _ := asInt(rows[j]["id"])
|
||||
return left < right
|
||||
})
|
||||
table := map[string]any{
|
||||
"columns": data.Columns,
|
||||
"rows": rows,
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil && filepath.Dir(path) != "." {
|
||||
return err
|
||||
}
|
||||
return write2DA(table, path, false, -1)
|
||||
}
|
||||
|
||||
func cloneStringMap(in map[string]string) map[string]string {
|
||||
out := make(map[string]string, len(in))
|
||||
for key, value := range in {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user