sow-tools/internal/topdata/native.go

package topdata

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"

	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)

const nullValue = "****"
const internalPortraitModuleAuthoredKey = "__topdata_portrait_module_authored"

type nativeDataset struct {
	Kind             nativeDatasetKind
	Name             string
	RootPath         string
	BasePath         string
	LockPath         string
	ModulesDir       string
	GeneratedDir     string
	OutputName       string
	Spec             datasetTextSpec
	Columns          []string
	CompareReference bool
}

type nativeDatasetKind string

const (
	nativeDatasetBase  nativeDatasetKind = "base"
	nativeDatasetPlain nativeDatasetKind = "plain"
)

type nativeCollectedDataset struct {
	Dataset  nativeDataset
	Columns  []string
	Rows     []map[string]any
	LockData map[string]int
	TableKey string
}

type resolvedTable struct {
	Key         string
	DatasetName string
	OutputName  string
	OutputStem  string
	Columns     []string
	Rows        []map[string]any
	LockData    map[string]int
	Kind        nativeDatasetKind
}

type resolvedTableRegistry struct {
	stemByKey  map[string]string
	tableByKey map[string]resolvedTable
}

func newResolvedTableRegistry(collected []nativeCollectedDataset) (resolvedTableRegistry, error) {
	registry := resolvedTableRegistry{
		stemByKey:  map[string]string{},
		tableByKey: map[string]resolvedTable{},
	}
	for _, collectedDataset := range collected {
		if collectedDataset.TableKey == "" {
			continue
		}
		key := collectedDataset.TableKey
		if existing, ok := registry.tableByKey[key]; ok {
			return resolvedTableRegistry{}, fmt.Errorf(
				"duplicate table key %q for %s (%s) conflicts with %s (%s)",
				key,
				collectedDataset.Dataset.Name,
				collectedDataset.Dataset.OutputName,
				existing.DatasetName,
				existing.OutputName,
			)
		}
		rows := make([]map[string]any, 0, len(collectedDataset.Rows))
		for _, row := range collectedDataset.Rows {
			rows = append(rows, cloneRowMap(row))
		}
		lockData := map[string]int{}
		for lockKey, rowID := range collectedDataset.LockData {
			lockData[lockKey] = rowID
		}
		registryTable := resolvedTable{
			Key:         key,
			DatasetName: collectedDataset.Dataset.Name,
			OutputName:  collectedDataset.Dataset.OutputName,
			OutputStem:  outputStem(collectedDataset.Dataset.OutputName),
			Columns:     append([]string(nil), collectedDataset.Columns...),
			Rows:        rows,
			LockData:    lockData,
			Kind:        collectedDataset.Dataset.Kind,
		}
		registry.stemByKey[key] = registryTable.OutputStem
		registry.tableByKey[key] = registryTable
	}
	return registry, nil
}

func (r resolvedTableRegistry) resolveTableByKey(key string) (resolvedTable, bool) {
	table, ok := r.tableByKey[key]
	return table, ok
}

func cloneRowMap(row map[string]any) map[string]any {
	if row == nil {
		return nil
	}
	cloned := make(map[string]any, len(row))
	for key, value := range row {
		cloned[key] = cloneAuthoringValue(value)
	}
	return cloned
}

type legacyTLKData struct {
	Language string
	Entries  map[string]tlkEntryData
	Lock     map[string]int
}

func Build(p *project.Project, progress func(string)) (BuildResult, error) {
	return BuildNative(p, progress)
}

func BuildNative(p *project.Project, progress func(string)) (BuildResult, error) {
	if progress == nil {
		progress = func(string) {}
	}
	if !p.HasTopData() {
		return BuildResult{}, errors.New("topdata is not configured for this project")
	}

	report := ValidateProject(p)
	if report.HasErrors() {
		summary := report.SummaryLines(3)
		if len(summary) > 0 {
			return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s): %s", report.ErrorCount(), summary[0])
		}
		return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount())
	}
	return buildNativeUnchecked(p, progress)
}

func buildNativeUnchecked(p *project.Project, progress func(string)) (BuildResult, error) {
	if progress == nil {
		progress = func(string) {}
	}
	sourceDir := p.TopDataSourceDir()
	dataDir := filepath.Join(sourceDir, "data")
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		return BuildResult{}, err
	}
	registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
	if err != nil {
		return BuildResult{}, err
	}
	if len(datasets) == 0 && len(registryDatasets) == 0 {
		return BuildResult{}, fmt.Errorf("no canonical topdata datasets were found in topdata/data")
	}

	compiler, err := newTLKCompiler(sourceDir, nil)
	if err != nil {
		return BuildResult{}, err
	}

	buildDir := p.TopDataBuildDir()
	output2DA := filepath.Join(buildDir, "2da")
	outputTLK := filepath.Join(buildDir, "tlk")
	progress("Preparing native topdata build directories...")
	if err := os.RemoveAll(buildDir); err != nil {
		return BuildResult{}, fmt.Errorf("clean topdata build dir: %w", err)
	}
	if err := os.MkdirAll(output2DA, 0o755); err != nil {
		return BuildResult{}, fmt.Errorf("create 2da output dir: %w", err)
	}
	if err := os.MkdirAll(outputTLK, 0o755); err != nil {
		return BuildResult{}, fmt.Errorf("create tlk output dir: %w", err)
	}

	progress("Building native topdata datasets...")
	collected := make([]nativeCollectedDataset, 0, len(datasets)+len(registryDatasets))
	globalKeyToID := map[string]int{}
	globalRowByKey := map[string]map[string]any{}
	for _, dataset := range datasets {
		collectedDataset, err := collectNativeDataset(dataset)
		if err != nil {
			return BuildResult{}, err
		}
		collected = append(collected, collectedDataset)
		for key, rowID := range collectedDataset.LockData {
			globalKeyToID[key] = rowID
		}
		for _, row := range collectedDataset.Rows {
			if key, ok := row["key"].(string); ok && key != "" {
				globalKeyToID[key] = row["id"].(int)
				globalRowByKey[key] = row
			}
		}
	}
	for _, collectedDataset := range registryDatasets {
		collected = append(collected, collectedDataset)
		for key, rowID := range collectedDataset.LockData {
			globalKeyToID[key] = rowID
		}
		for _, row := range collectedDataset.Rows {
			if key, ok := row["key"].(string); ok && key != "" {
				globalKeyToID[key] = row["id"].(int)
				globalRowByKey[key] = row
			}
		}
	}

	if hasCollectedPartsDatasets(collected) {
		progress("Resolving released parts manifest...")
		autogenerated, err := resolveAutogeneratedPartsInventory(p, progress)
		if err != nil {
			return BuildResult{}, err
		}
		if autogenerated != nil {
			collected = augmentWithAutogeneratedParts(collected, autogenerated)
			progress("Augmented parts datasets with discovered models")
		}
	}
	collected, err = applyPartOverrides(sourceDir, collected)
	if err != nil {
		return BuildResult{}, err
	}

	tableRegistry, err := newResolvedTableRegistry(collected)
	if err != nil {
		return BuildResult{}, err
	}
	collected, err = mergePortraitMetadata(collected)
	if err != nil {
		return BuildResult{}, err
	}
	globalKeyToID = map[string]int{}
	globalRowByKey = map[string]map[string]any{}
	for _, collectedDataset := range collected {
		for key, rowID := range collectedDataset.LockData {
			globalKeyToID[key] = rowID
		}
		for _, row := range collectedDataset.Rows {
			if key, ok := row["key"].(string); ok && key != "" {
				globalKeyToID[key] = row["id"].(int)
				globalRowByKey[key] = row
			}
		}
	}
	tableRegistry, err = newResolvedTableRegistry(collected)
	if err != nil {
		return BuildResult{}, err
	}
	supplementalKeyToID, err := collectSupplementalLockIDs(sourceDir, "")
	if err != nil {
		return BuildResult{}, err
	}
	for key, rowID := range supplementalKeyToID {
		if _, ok := globalKeyToID[key]; ok {
			continue
		}
		globalKeyToID[key] = rowID
	}

	groupStats := nativeCompileGroupStats(collected)
	currentGroup := ""
	for _, dataset := range collected {
		group := nativeCompileGroup(dataset.Dataset.Name)
		if group != currentGroup {
			currentGroup = group
			stats := groupStats[group]
			progress(fmt.Sprintf(
				"Compiling %s datasets (%d output tables from %d source fragments)...",
				group,
				stats.OutputTables,
				stats.SourceFragments,
			))
		}
		compiled, err := resolveNativeDataset(dataset, globalKeyToID, globalRowByKey, tableRegistry, compiler)
		if err != nil {
			return BuildResult{}, err
		}
		if err := write2DA(compiled, filepath.Join(output2DA, dataset.Dataset.OutputName), dataset.Dataset.Kind == nativeDatasetBase); err != nil {
			return BuildResult{}, err
		}
	}

	filesTLK, err := compiler.finish(outputTLK)
	if err != nil {
		return BuildResult{}, err
	}
	files2DA, err := countFiles(output2DA)
	if err != nil {
		return BuildResult{}, err
	}

	return BuildResult{
		Mode:         "native",
		Output2DADir: output2DA,
		OutputTLKDir: outputTLK,
		Files2DA:     files2DA,
		FilesTLK:     filesTLK,
	}, nil
}

func nativeCompileGroup(datasetName string) string {
	head, _, ok := strings.Cut(datasetName, "/")
	if !ok {
		return datasetName
	}
	switch head {
	case "itemprops", "racialtypes", "damagetypes", "parts", "classes":
		return head
	default:
		return head
	}
}

type nativeCompileGroupStat struct {
	OutputTables    int
	SourceFragments int
}

func nativeCompileGroupStats(collected []nativeCollectedDataset) map[string]nativeCompileGroupStat {
	stats := make(map[string]nativeCompileGroupStat)
	for _, dataset := range collected {
		group := nativeCompileGroup(dataset.Dataset.Name)
		groupStat := stats[group]
		groupStat.OutputTables++
		groupStat.SourceFragments += nativeDatasetSourceFragments(dataset.Dataset)
		stats[group] = groupStat
	}
	return stats
}

func nativeDatasetSourceFragments(dataset nativeDataset) int {
	fragments := 1
	if dataset.Kind == nativeDatasetBase {
		fragments += countModuleFiles(dataset.ModulesDir)
		fragments += countModuleFiles(dataset.GeneratedDir)
	}
	return fragments
}

func countModuleFiles(dir string) int {
	if strings.TrimSpace(dir) == "" {
		return 0
	}
	paths, err := collectModulePaths(dir)
	if err != nil {
		return 0
	}
	return len(paths)
}

func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
	var datasets []nativeDataset
	err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if !d.IsDir() {
			return nil
		}
		if d.Name() == "modules" || d.Name() == "generated" {
			return filepath.SkipDir
		}

		rel, err := filepath.Rel(dataDir, path)
		if err != nil {
			return err
		}
		if rel == "." {
			return nil
		}
		if filepath.ToSlash(rel) == "parts/overrides" {
			return filepath.SkipDir
		}
		if filepath.ToSlash(rel) == itempropsRegistryDirName || filepath.ToSlash(rel) == damagetypesRegistryDirName || filepath.ToSlash(rel) == racialtypesRegistryDirName {
			return filepath.SkipDir
		}

		basePath := filepath.Join(path, "base.json")
		if _, err := os.Stat(basePath); err == nil {
			baseData, err := loadJSONObject(basePath)
			if err != nil {
				return err
			}
			outputName, _ := baseData["output"].(string)
			if strings.TrimSpace(outputName) == "" {
				outputName = filepath.Base(path) + ".2da"
			}
			datasets = append(datasets, nativeDataset{
				Kind:             nativeDatasetBase,
				Name:             filepath.ToSlash(rel),
				RootPath:         path,
				BasePath:         basePath,
				LockPath:         filepath.Join(path, "lock.json"),
				ModulesDir:       filepath.Join(path, "modules"),
				GeneratedDir:     filepath.Join(path, "generated"),
				OutputName:       outputName,
				Spec:             specForDataset(filepath.ToSlash(rel)),
				CompareReference: optionalBool(baseData["compare_reference"], true),
			})
			return nil
		}

		entries, err := os.ReadDir(path)
		if err != nil {
			return err
		}
		for _, entry := range entries {
			if entry.IsDir() {
				continue
			}
			name := entry.Name()
			if strings.HasSuffix(strings.ToLower(name), ".json") && name != "lock.json" {
				filePath := filepath.Join(path, name)
				tableData, err := loadJSONObject(filePath)
				if err != nil {
					return err
				}
				outputName, _ := tableData["output"].(string)
				if strings.TrimSpace(outputName) == "" {
					outputName = strings.TrimSuffix(name, filepath.Ext(name)) + ".2da"
				}
				datasets = append(datasets, nativeDataset{
					Kind:             nativeDatasetPlain,
					Name:             filepath.ToSlash(filepath.Join(rel, strings.TrimSuffix(name, filepath.Ext(name)))),
					RootPath:         path,
					BasePath:         filePath,
					LockPath:         filepath.Join(path, "lock.json"),
					OutputName:       outputName,
					Spec:             specForDataset(filepath.ToSlash(filepath.Join(rel, strings.TrimSuffix(name, filepath.Ext(name))))),
					CompareReference: optionalBool(tableData["compare_reference"], true),
				})
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	slices.SortFunc(datasets, func(a, b nativeDataset) int {
		return strings.Compare(a.Name, b.Name)
	})
	return datasets, nil
}

func discoverNativeOutputCatalog(dataDir string) (map[string]string, error) {
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		return nil, err
	}
	registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
	if err != nil {
		return nil, err
	}
	catalog := make(map[string]string, len(datasets)+len(registryDatasets))
	for _, dataset := range datasets {
		if !dataset.CompareReference {
			continue
		}
		catalog[dataset.OutputName] = dataset.Name
	}
	for _, dataset := range registryDatasets {
		catalog[dataset.Dataset.OutputName] = dataset.Dataset.Name
	}
	return catalog, nil
}

func missingNativeOutputs(projectOutputs, referenceOutputs map[string]string) []string {
	missing := make([]string, 0)
	for outputName, datasetName := range referenceOutputs {
		if _, ok := projectOutputs[outputName]; ok {
			continue
		}
		missing = append(missing, fmt.Sprintf("%s (%s)", outputName, datasetName))
	}
	slices.Sort(missing)
	return missing
}

func collectNativeDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
	switch dataset.Kind {
	case nativeDatasetBase:
		return collectBaseDataset(dataset)
	case nativeDatasetPlain:
		return collectPlainDataset(dataset)
	default:
		return nativeCollectedDataset{}, fmt.Errorf("unsupported native dataset kind %q", dataset.Kind)
	}
}

func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
	baseData, err := loadJSONObject(dataset.BasePath)
	if err != nil {
		return nativeCollectedDataset{}, err
	}

	columns, err := parseColumns(baseData, dataset.Name)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	rawBaseRows, ok := baseData["rows"].([]any)
	if !ok {
		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: rows must be an array", dataset.Name)
	}
	baseRowKeys := map[string]struct{}{}
	for _, raw := range rawBaseRows {
		rowObj, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		key, ok := rowObj["key"].(string)
		if !ok || key == "" {
			continue
		}
		baseRowKeys[key] = struct{}{}
	}

	lockData, err := loadLockfile(dataset.LockPath)
	if err != nil {
		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
	}

	rows := make([]map[string]any, 0, len(rawBaseRows))
	rowByID := map[int]map[string]any{}
	rowByKey := map[string]map[string]any{}
	usedIDs := map[int]struct{}{}
	lockModified := false

	for _, rowID := range lockData {
		usedIDs[rowID] = struct{}{}
	}
	for index, raw := range rawBaseRows {
		rowObj, ok := raw.(map[string]any)
		if !ok {
			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d must be an object", dataset.Name, index)
		}
		row, err := canonicalizeBaseRow(dataset, columns, rowObj, index)
		if err != nil {
			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
		}
		rows = append(rows, row)
		rowID := row["id"].(int)
		rowByID[rowID] = row
		if key, ok := row["key"].(string); ok && key != "" {
			rowByKey[key] = row
			if lockData[key] != rowID {
				lockData[key] = rowID
				lockModified = true
			}
		}
		usedIDs[rowID] = struct{}{}
	}

	nextID := nextAvailableID(usedIDs)
	applyModuleData := func(sourceLabel string, moduleData map[string]any, allowPortraitMetadataInjection bool) error {
		if rawEntries, ok := moduleData["entries"]; ok {
			entries, ok := rawEntries.(map[string]any)
			if !ok {
				return fmt.Errorf("dataset %s: entries in %s must be an object", dataset.Name, sourceLabel)
			}
			for _, key := range sortedKeys(entries) {
				rawEntry, ok := entries[key].(map[string]any)
				if !ok {
					return fmt.Errorf("dataset %s: entry %q in %s must be an object", dataset.Name, key, sourceLabel)
				}
				if existing, ok := rowByKey[key]; ok {
					candidate := map[string]any{}
					for field, value := range existing {
						candidate[field] = value
					}
					for field, value := range rawEntry {
						if field == "id" || field == "key" || field == "_tlk" {
							continue
						}
						if isMetadataField(field) {
							meta, err := parseRowMetadata(value)
							if err != nil {
								return fmt.Errorf("dataset %s: %w", dataset.Name, err)
							}
							if meta != nil {
								existingMeta, _ := parseExistingMetadata(candidate)
								candidate["meta"] = mergeMetadataMaps(existingMeta, meta)
							}
							continue
						}
						if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
							if err := setLegacyWikiMetadata(candidate, field, value); err != nil {
								return fmt.Errorf("dataset %s: %w", dataset.Name, err)
							}
							continue
						}
						columnName, ok := canonicalColumn(columns, field)
						if !ok {
							return fmt.Errorf("dataset %s: entry field %q is not a known column", dataset.Name, field)
						}
						candidate[columnName] = value
					}
					expanded, err := expandRowAuthoringSugar(dataset, columns, rawEntry, candidate)
					if err != nil {
						return err
					}
					keyChanged := updateOverrideRowKey(existing, expanded, rowByKey, lockData)
					if keyChanged {
						lockModified = true
					}
					for field, value := range expanded {
						if field == "id" || field == "_tlk" {
							continue
						}
						existing[field] = value
					}
					if allowPortraitMetadataInjection && rowHasPortraitMetadata(existing) {
						existing[internalPortraitModuleAuthoredKey] = true
					}
					continue
				}
				rowID, ok := lockData[key]
				if !ok {
					rowID = nextID
					lockData[key] = rowID
					lockModified = true
					usedIDs[rowID] = struct{}{}
					nextID = nextAvailableID(usedIDs)
				}
				row, err := canonicalizeEntry(dataset, columns, rowID, key, rawEntry)
				if err != nil {
					return fmt.Errorf("dataset %s: %w", dataset.Name, err)
				}
				rows = append(rows, row)
				rowByID[rowID] = row
				rowByKey[key] = row
				if allowPortraitMetadataInjection && rowHasPortraitMetadata(row) {
					row[internalPortraitModuleAuthoredKey] = true
				}
			}
		}

		if rawOverrides, ok := moduleData["overrides"]; ok {
			overrideList, ok := rawOverrides.([]any)
			if !ok {
				return fmt.Errorf("dataset %s: overrides in %s must be an array", dataset.Name, sourceLabel)
			}
			for index, rawOverride := range overrideList {
				override, ok := rawOverride.(map[string]any)
				if !ok {
					return fmt.Errorf("dataset %s: override %d in %s must be an object", dataset.Name, index, sourceLabel)
				}
				row, err := resolveOverrideTarget(dataset.Name, override, rowByID, rowByKey)
				if err != nil {
					rawKey, hasKey := override["key"].(string)
					rowID := 0
					hasRowID := false
					if rawID, hasID := override["id"]; hasID {
						parsedID, parseErr := asInt(rawID)
						if parseErr != nil {
							return fmt.Errorf("dataset %s: override id %v is not numeric", dataset.Name, rawID)
						}
						rowID = parsedID
						hasRowID = true
					}
					if hasKey && rawKey != "" {
						if lockedID, hasLockedID := lockData[rawKey]; hasLockedID {
							rowID = lockedID
							hasRowID = true
						} else {
							if !hasRowID {
								rowID = nextID
								hasRowID = true
								nextID = nextAvailableID(usedIDs)
							}
							lockData[rawKey] = rowID
							lockModified = true
						}
					}
					if !hasRowID {
						return err
					}
					usedIDs[rowID] = struct{}{}
					row = map[string]any{"id": rowID}
					if hasKey && rawKey != "" {
						row["key"] = rawKey
					}
					rows = append(rows, row)
					rowByID[rowID] = row
					if hasKey && rawKey != "" {
						rowByKey[rawKey] = row
					}
				}
				candidate := map[string]any{}
				for field, value := range row {
					candidate[field] = value
				}
				for field, value := range override {
					if field == "id" || field == "key" || field == "_tlk" {
						continue
					}
					if isMetadataField(field) {
						meta, err := parseRowMetadata(value)
						if err != nil {
							return fmt.Errorf("dataset %s: %w", dataset.Name, err)
						}
						if meta != nil {
							existingMeta, _ := parseExistingMetadata(candidate)
							candidate["meta"] = mergeMetadataMaps(existingMeta, meta)
						}
						continue
					}
					if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
						if err := setLegacyWikiMetadata(candidate, field, value); err != nil {
							return fmt.Errorf("dataset %s: %w", dataset.Name, err)
						}
						continue
					}
					columnName, ok := canonicalColumn(columns, field)
					if !ok {
						if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
							continue
						}
						return fmt.Errorf("dataset %s: override field %q is not a known column", dataset.Name, field)
					}
					candidate[columnName] = value
				}
				if rawKey, ok := override["key"]; ok {
					candidate["key"] = rawKey
				}
				expanded, err := expandRowAuthoringSugar(dataset, columns, override, candidate)
				if err != nil {
					return err
				}
				keyChanged := updateOverrideRowKey(row, expanded, rowByKey, lockData)
				if keyChanged {
					lockModified = true
				}
				for field, value := range expanded {
					if field == "id" || field == "_tlk" {
						continue
					}
					row[field] = value
				}
				if allowPortraitMetadataInjection && rowHasPortraitMetadata(row) {
					row[internalPortraitModuleAuthoredKey] = true
				}
			}
		}
		return nil
	}

	generatedModules, err := collectGeneratedModules(dataset, lockData)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	for _, module := range generatedModules {
		if err := applyModuleData(module.Name, module.Data, false); err != nil {
			return nativeCollectedDataset{}, err
		}
	}

	modulePaths, err := collectModulePaths(dataset.ModulesDir)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	for _, path := range modulePaths {
		moduleData, err := loadJSONObject(path)
		if err != nil {
			return nativeCollectedDataset{}, err
		}
		if err := applyModuleData(path, moduleData, true); err != nil {
			return nativeCollectedDataset{}, err
		}
	}

	if dataset.Name == "spells" {
		normalizeCollectedSpellImpactScripts(rows, baseRowKeys)
	}

	if lockModified {
		if err := saveLockfile(dataset.LockPath, lockData); err != nil {
			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
		}
	}

	rows = dedupeCollectedRows(rows)
	slices.SortFunc(rows, func(a, b map[string]any) int {
		return a["id"].(int) - b["id"].(int)
	})
	return nativeCollectedDataset{
		Dataset:  dataset,
		Columns:  columns,
		Rows:     rows,
		LockData: lockData,
	}, nil
}

func dedupeCollectedRows(rows []map[string]any) []map[string]any {
	if len(rows) <= 1 {
		return rows
	}
	seen := make(map[string]int, len(rows))
	out := make([]map[string]any, 0, len(rows))
	for _, row := range rows {
		rowID, _ := row["id"].(int)
		rowKey, _ := row["key"].(string)
		if rowKey == "" {
			out = append(out, row)
			continue
		}
		identity := strconv.Itoa(rowID) + "\x00" + rowKey
		if index, ok := seen[identity]; ok {
			out[index] = row
			continue
		}
		seen[identity] = len(out)
		out = append(out, row)
	}
	return out
}

type nativeGeneratedModule struct {
	Name string
	Data map[string]any
}

func collectGeneratedModules(dataset nativeDataset, lockData map[string]int) ([]nativeGeneratedModule, error) {
	if dataset.Name != "feat" {
		return nil, nil
	}
	ctx, err := newFeatGeneratedContext(dataset, lockData)
	if err != nil {
		return nil, err
	}
	paths, err := collectModulePaths(dataset.GeneratedDir)
	if err != nil {
		return nil, err
	}
	modules := make([]nativeGeneratedModule, 0, len(paths))
	for _, path := range paths {
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, err
		}
		family, _ := obj["family"].(string)
		switch family {
		case "skill_focus",
			"greater_skill_focus",
			"weapon_focus",
			"weapon_specialization",
			"greater_weapon_focus",
			"greater_weapon_specialization",
			"improved_critical",
			"overwhelming_critical",
			"proficiencies",
			"special_attacks":
			moduleData, err := buildFeatGeneratedModule(path, obj, ctx)
			if err != nil {
				return nil, err
			}
			modules = append(modules, nativeGeneratedModule{
				Name: path,
				Data: moduleData,
			})
		case "":
			return nil, fmt.Errorf("dataset %s: generated file %s is missing family", dataset.Name, path)
		default:
			return nil, fmt.Errorf("dataset %s: generated file %s uses unsupported family %q", dataset.Name, path, family)
		}
	}
	racialtypesModule, err := buildRacialtypesSkillAffinityModule(ctx)
	if err != nil {
		return nil, err
	}
	if racialtypesModule != nil {
		modules = append(modules, nativeGeneratedModule{
			Name: filepath.Join(dataset.RootPath, "generated", "skill_affinity<racialtypes>"),
			Data: racialtypesModule,
		})
	}
	return modules, nil
}

type featGeneratedContext struct {
	sourceDir      string
	dataDir        string
	lockData       map[string]int
	supplementalID map[string]int
	tlkStateKeys   map[string]struct{}
	existingFeat   map[string]struct{}
	datasets       map[string]nativeDataset
	rowsByDataset  map[string]map[string]map[string]any
}

type compactSourceOverride = map[string]map[string]any

type featWeaponFamilySpec struct {
	FamilyKey         string
	SourceColumn      string
	LabelPrefix       string
	ConstantPrefix    string
	NamePrefix        string
	AutoPrereqFamily1 string
	AutoPrereqFamily2 string
}

type affinityGrant struct {
	featKey string
	races   []string
}

var featWeaponFamilies = map[string]featWeaponFamilySpec{
	"weapon_focus": {
		SourceColumn:   "WeaponFocusFeat",
		FamilyKey:      "weaponfocus",
		LabelPrefix:    "FEAT_WEAPON_FOCUS",
		ConstantPrefix: "FEAT_WEAPON_FOCUS",
		NamePrefix:     "Weapon Focus",
	},
	"weapon_specialization": {
		SourceColumn:      "WeaponSpecializationFeat",
		FamilyKey:         "weaponspecialization",
		LabelPrefix:       "FEAT_WEAPON_SPECIALIZATION",
		ConstantPrefix:    "FEAT_WEAPON_SPECIALIZATION",
		NamePrefix:        "Weapon Specialization",
		AutoPrereqFamily1: "weaponfocus",
	},
	"greater_weapon_focus": {
		SourceColumn:      "EpicWeaponFocusFeat",
		FamilyKey:         "greaterweaponfocus",
		LabelPrefix:       "FEAT_GREATER_WEAPON_FOCUS",
		ConstantPrefix:    "FEAT_GREATER_WEAPON_FOCUS",
		NamePrefix:        "Greater Weapon Focus",
		AutoPrereqFamily1: "weaponfocus",
	},
	"greater_weapon_specialization": {
		SourceColumn:      "EpicWeaponSpecializationFeat",
		FamilyKey:         "greaterweaponspecialization",
		LabelPrefix:       "FEAT_GREATER_WEAPON_SPECIALIZATION",
		ConstantPrefix:    "FEAT_GREATER_WEAPON_SPECIALIZATION",
		NamePrefix:        "Greater Weapon Specialization",
		AutoPrereqFamily1: "weaponspecialization",
		AutoPrereqFamily2: "greaterweaponfocus",
	},
	"improved_critical": {
		SourceColumn:      "WeaponImprovedCriticalFeat",
		FamilyKey:         "improvedcritical",
		LabelPrefix:       "FEAT_IMPROVED_CRITICAL",
		ConstantPrefix:    "FEAT_IMPROVED_CRITICAL",
		NamePrefix:        "Improved Critical",
		AutoPrereqFamily1: "weaponfocus",
	},
	"overwhelming_critical": {
		SourceColumn:      "EpicWeaponOverwhelmingCriticalFeat",
		FamilyKey:         "overwhelmingcritical",
		LabelPrefix:       "FEAT_EPIC_OVERWHELMING_CRITICAL",
		ConstantPrefix:    "FEAT_EPIC_OVERWHELMING_CRITICAL",
		NamePrefix:        "Overwhelming Critical",
		AutoPrereqFamily2: "improvedcritical",
	},
}

func newFeatGeneratedContext(dataset nativeDataset, lockData map[string]int) (*featGeneratedContext, error) {
	sourceDir := filepath.Dir(filepath.Dir(dataset.RootPath))
	dataDir := filepath.Join(sourceDir, "data")
	allDatasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		return nil, err
	}
	datasetMap := make(map[string]nativeDataset, len(allDatasets))
	for _, item := range allDatasets {
		datasetMap[item.Name] = item
	}
	supplementalID, err := collectSupplementalLockIDs(sourceDir, "")
	if err != nil {
		return nil, err
	}
	tlkState, err := loadTLKState(filepath.Join(sourceDir, tlkStateFile))
	if err != nil {
		return nil, err
	}
	tlkStateKeys := make(map[string]struct{}, len(tlkState.Entries))
	for key := range tlkState.Entries {
		tlkStateKeys[key] = struct{}{}
	}
	lockCopy := make(map[string]int, len(lockData))
	for key, value := range lockData {
		lockCopy[key] = value
	}
	existingFeat, err := collectExistingFeatKeys(filepath.Join(dataDir, "feat"))
	if err != nil {
		return nil, err
	}
	return &featGeneratedContext{
		sourceDir:      sourceDir,
		dataDir:        dataDir,
		lockData:       lockCopy,
		supplementalID: supplementalID,
		tlkStateKeys:   tlkStateKeys,
		existingFeat:   existingFeat,
		datasets:       datasetMap,
		rowsByDataset:  map[string]map[string]map[string]any{},
	}, nil
}

func collectExistingFeatKeys(featDir string) (map[string]struct{}, error) {
	keys := map[string]struct{}{}
	baseObj, err := loadJSONObject(filepath.Join(featDir, "base.json"))
	if err != nil {
		return nil, err
	}
	if rawRows, ok := baseObj["rows"].([]any); ok {
		for _, raw := range rawRows {
			row, ok := raw.(map[string]any)
			if !ok {
				continue
			}
			if key, ok := row["key"].(string); ok && key != "" {
				keys[key] = struct{}{}
			}
		}
	}
	modulePaths, err := collectModulePaths(filepath.Join(featDir, "modules"))
	if err != nil {
		return nil, err
	}
	for _, path := range modulePaths {
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, err
		}
		if rawEntries, ok := obj["entries"].(map[string]any); ok {
			for key := range rawEntries {
				if key != "" {
					keys[key] = struct{}{}
				}
			}
		}
		if rawOverrides, ok := obj["overrides"].([]any); ok {
			for _, raw := range rawOverrides {
				override, ok := raw.(map[string]any)
				if !ok {
					continue
				}
				if key, ok := override["key"].(string); ok && key != "" {
					keys[key] = struct{}{}
				}
			}
		}
	}
	return keys, nil
}

func (c *featGeneratedContext) featKeyExists(key string) bool {
	_, ok := c.existingFeat[key]
	return ok
}

func (c *featGeneratedContext) familyHasExistingRows(familyKey string) bool {
	prefix := "feat:" + familyKey + "_"
	for key := range c.existingFeat {
		if strings.HasPrefix(key, prefix) {
			return true
		}
	}
	return false
}

func (c *featGeneratedContext) datasetRows(name string) (map[string]map[string]any, error) {
	if rows, ok := c.rowsByDataset[name]; ok {
		return rows, nil
	}
	dataset, ok := c.datasets[name]
	if !ok {
		return nil, fmt.Errorf("generated feat family requires canonical dataset %q", name)
	}
	collected, err := collectNativeDataset(dataset)
	if err != nil {
		return nil, err
	}
	rows := make(map[string]map[string]any, len(collected.Rows))
	for _, row := range collected.Rows {
		if key, ok := row["key"].(string); ok && key != "" {
			rows[key] = row
		}
	}
	c.rowsByDataset[name] = rows
	return rows, nil
}

func (c *featGeneratedContext) featID(key string, explicit any) (int, bool, error) {
	if explicit != nil && explicit != nullValue {
		parsed, err := featSourceID(explicit)
		if err != nil {
			return 0, false, err
		}
		if parsed > 0 {
			return parsed, true, nil
		}
	}
	if value, ok := c.lockData[key]; ok && value > 0 {
		return value, true, nil
	}
	if value, ok := c.supplementalID[key]; ok && value > 0 {
		return value, true, nil
	}
	return 0, false, nil
}

func (c *featGeneratedContext) preferredGeneratedFeatKey(familyKey, sourceSlug string) string {
	exact := "feat:" + familyKey + "_" + sourceSlug
	target := normalizeKeyIdentity(sourceSlug)
	for _, candidates := range []map[string]int{c.lockData, c.supplementalID} {
		matches := make([]string, 0)
		for key, value := range candidates {
			if value <= 0 {
				continue
			}
			if !strings.HasPrefix(key, "feat:"+familyKey+"_") {
				continue
			}
			suffix := strings.TrimPrefix(key, "feat:"+familyKey+"_")
			if normalizeKeyIdentity(suffix) == target {
				matches = append(matches, key)
			}
		}
		if best, ok := preferredExpandedKey(matches); ok {
			return best
		}
	}
	for _, candidates := range []map[string]int{c.lockData, c.supplementalID} {
		if value, ok := candidates[exact]; ok && value > 0 {
			return exact
		}
	}
	stateMatches := make([]string, 0)
	for key := range c.tlkStateKeys {
		if !strings.HasPrefix(key, "feat:"+familyKey+"_") {
			continue
		}
		suffix := strings.TrimPrefix(key, "feat:"+familyKey+"_")
		suffix = strings.TrimSuffix(strings.TrimSuffix(suffix, ".name"), ".description")
		if normalizeKeyIdentity(suffix) == target {
			stateMatches = append(stateMatches, "feat:"+familyKey+"_"+suffix)
		}
	}
	if best, ok := preferredExpandedKey(stateMatches); ok {
		return best
	}
	return exact
}

func preferredExpandedKey(matches []string) (string, bool) {
	if len(matches) == 0 {
		return "", false
	}
	best := matches[0]
	for _, candidate := range matches[1:] {
		best = betterExpandedKey(best, candidate)
	}
	return best, true
}

func betterExpandedKey(a, b string) string {
	aUnderscores := strings.Count(a, "_")
	bUnderscores := strings.Count(b, "_")
	if aUnderscores != bUnderscores {
		if aUnderscores < bUnderscores {
			return a
		}
		return b
	}
	if len(a) != len(b) {
		if len(a) < len(b) {
			return a
		}
		return b
	}
	if a <= b {
		return a
	}
	return b
}

func buildFeatGeneratedModule(path string, obj map[string]any, ctx *featGeneratedContext) (map[string]any, error) {
	family, _ := obj["family"].(string)
	switch family {
	case "skill_focus", "greater_skill_focus":
		return buildSkillFocusGeneratedModule(path, obj, ctx)
	case "weapon_focus",
		"weapon_specialization",
		"greater_weapon_focus",
		"greater_weapon_specialization",
		"improved_critical",
		"overwhelming_critical":
		return buildWeaponFamilyGeneratedModule(path, obj, ctx)
	case "proficiencies", "special_attacks":
		return buildLegacyFeatGeneratedModule(path, obj)
	default:
		return nil, fmt.Errorf("generated feat file %s uses unsupported family %q", path, family)
	}
}

func buildLegacyFeatGeneratedModule(path string, obj map[string]any) (map[string]any, error) {
	module := map[string]any{}
	if rawEntries, ok := obj["entries"]; ok {
		entries, ok := rawEntries.(map[string]any)
		if !ok {
			return nil, fmt.Errorf("generated feat file %s entries must be an object", path)
		}
		module["entries"] = entries
	}
	if rawOverrides, ok := obj["overrides"]; ok {
		overrides, ok := rawOverrides.([]any)
		if !ok {
			return nil, fmt.Errorf("generated feat file %s overrides must be an array", path)
		}
		module["overrides"] = overrides
	}
	if len(module) == 0 {
		return nil, fmt.Errorf("generated feat file %s must contain entries and/or overrides", path)
	}
	return module, nil
}

func buildSkillFocusGeneratedModule(path string, obj map[string]any, ctx *featGeneratedContext) (map[string]any, error) {
	spec, err := parseFamilyExpansionSpec(path, obj)
	if err != nil {
		return nil, err
	}
	overrides, err := parseCompactSourceOverrides(obj["overrides"])
	if err != nil {
		return nil, fmt.Errorf("generated feat file %s: %w", path, err)
	}
	if spec.ChildSource.Dataset != "skills" {
		return nil, fmt.Errorf("generated feat file %s: child_source.dataset must be skills", path)
	}
	skillRows, err := ctx.datasetRows("skills")
	if err != nil {
		return nil, err
	}

	labelPrefix := "FEAT_SKILL_FOCUS"
	namePrefix := "Skill Focus"
	if spec.Family == "greater_skill_focus" {
		labelPrefix = "FEAT_GREATER_SKILL_FOCUS"
		namePrefix = "Greater Skill Focus"
	}

	overrideList := make([]any, 0)
	familyAllowlist := ctx.familyHasExistingRows(spec.FamilyKey)
	for _, skillKey := range sortedStringMapKeys(skillRows) {
		row := skillRows[skillKey]
		if spec.ChildSource.Predicate == "accessible" && isHiddenSkill(row) {
			continue
		}
		slug := strings.TrimPrefix(skillKey, "skills:")
		featKey := ctx.preferredGeneratedFeatKey(spec.FamilyKey, slug)
		childToken := childTokenFromExpandedKey(featKey, spec.FamilyKey)
		labelSuffix := upperSnake(displayNameForSkill(row))
		rowID, hasID, err := ctx.featID(featKey, nil)
		if err != nil {
			return nil, fmt.Errorf("generated feat file %s: %w", path, err)
		}
		override := map[string]any{
			"key":             featKey,
			"MASTERFEAT":      map[string]any{"id": spec.Template},
			"REQSKILL":        map[string]any{"id": skillKey},
			"TOOLSCATEGORIES": "2",
			"meta":            familyMetadata(spec.FamilyKey, childToken, skillKey, spec.Template),
		}
		if hasID {
			override["id"] = rowID
		}
		if familyAllowlist && !ctx.featKeyExists(featKey) {
			continue
		}
		if !ctx.featKeyExists(featKey) {
			override["LABEL"] = labelPrefix + "_" + labelSuffix
			override["FEAT"] = map[string]any{
				"tlk": map[string]any{
					"key":  featKey + ".name",
					"text": fmt.Sprintf("%s (%s)", namePrefix, displayNameForSkill(row)),
				},
			}
			override["DESCRIPTION"] = map[string]any{"field": "DESCRIPTION", "ref": spec.Template}
			override["ICON"] = map[string]any{"field": "ICON", "ref": spec.Template}
			override["CRValue"] = map[string]any{"field": "CRValue", "ref": spec.Template}
			override["ReqSkillMinRanks"] = map[string]any{"field": "ReqSkillMinRanks", "ref": spec.Template}
			override["Constant"] = labelPrefix + "_" + labelSuffix
			override["TOOLSCATEGORIES"] = "2"
			override["ALLCLASSESCANUSE"] = map[string]any{"field": "ALLCLASSESCANUSE", "ref": spec.Template}
			override["PreReqEpic"] = map[string]any{"field": "PreReqEpic", "ref": spec.Template}
			override["ReqAction"] = map[string]any{"field": "ReqAction", "ref": spec.Template}
		}
		mergeGeneratedOverride(override, overrides[skillKey])
		overrideList = append(overrideList, override)
	}
	return map[string]any{"overrides": overrideList}, nil
}

func buildWeaponFamilyGeneratedModule(path string, obj map[string]any, ctx *featGeneratedContext) (map[string]any, error) {
	fileSpec, err := parseFamilyExpansionSpec(path, obj)
	if err != nil {
		return nil, err
	}
	spec, ok := featWeaponFamilies[fileSpec.Family]
	if !ok {
		return nil, fmt.Errorf("generated feat file %s uses unsupported weapon family %q", path, fileSpec.Family)
	}
	if fileSpec.ChildSource.Dataset != "baseitems" {
		return nil, fmt.Errorf("generated feat file %s: child_source.dataset must be baseitems", path)
	}
	if fileSpec.ChildSource.Column == "" {
		return nil, fmt.Errorf("generated feat file %s: child_source.column must be set", path)
	}
	overrides, err := parseCompactSourceOverrides(obj["overrides"])
	if err != nil {
		return nil, fmt.Errorf("generated feat file %s: %w", path, err)
	}
	baseitemRows, err := ctx.datasetRows("baseitems")
	if err != nil {
		return nil, err
	}

	overrideList := make([]any, 0)
	for _, baseitemKey := range sortedStringMapKeys(baseitemRows) {
		row := baseitemRows[baseitemKey]
		rawID, ok := lookupField(row, fileSpec.ChildSource.Column)
		if !ok || isNullishValue(rawID) {
			continue
		}
		slug := strings.TrimPrefix(baseitemKey, "baseitems:")
		featKey, rowID, hasID, err := ctx.resolveWeaponFeatIdentity(fileSpec.FamilyKey, slug, rawID)
		if err != nil {
			return nil, fmt.Errorf("generated feat file %s: %w", path, err)
		}
		childToken := childTokenFromExpandedKey(featKey, fileSpec.FamilyKey)
		labelSuffix := upperSnake(displayNameForBaseitem(row))
		override := map[string]any{
			"key":             featKey,
			"MASTERFEAT":      map[string]any{"id": fileSpec.Template},
			"TOOLSCATEGORIES": "1",
			"meta":            familyMetadata(fileSpec.FamilyKey, childToken, baseitemKey, fileSpec.Template),
		}
		if hasID {
			override["id"] = rowID
		}
		if !ctx.featKeyExists(featKey) {
			override["LABEL"] = spec.LabelPrefix + "_" + labelSuffix
			override["FEAT"] = map[string]any{
				"tlk": map[string]any{
					"key":  featKey + ".name",
					"text": fmt.Sprintf("%s (%s)", spec.NamePrefix, displayNameForBaseitem(row)),
				},
			}
			override["DESCRIPTION"] = map[string]any{"field": "DESCRIPTION", "ref": fileSpec.Template}
			override["ICON"] = map[string]any{"field": "ICON", "ref": fileSpec.Template}
			override["MINATTACKBONUS"] = map[string]any{"field": "MINATTACKBONUS", "ref": fileSpec.Template}
			override["ALLCLASSESCANUSE"] = map[string]any{"field": "ALLCLASSESCANUSE", "ref": fileSpec.Template}
			override["PreReqEpic"] = map[string]any{"field": "PreReqEpic", "ref": fileSpec.Template}
			override["ReqAction"] = map[string]any{"field": "ReqAction", "ref": fileSpec.Template}
			override["TOOLSCATEGORIES"] = "1"
			override["CRValue"] = map[string]any{"field": "CRValue", "ref": fileSpec.Template}
			override["MinLevel"] = map[string]any{"field": "MinLevel", "ref": fileSpec.Template}
			override["MinLevelClass"] = map[string]any{"field": "MinLevelClass", "ref": fileSpec.Template}
			override["MINSTR"] = map[string]any{"field": "MINSTR", "ref": fileSpec.Template}
			override["Constant"] = spec.ConstantPrefix + "_" + labelSuffix
		}
		if prereqKey, ok := weaponFamilyPrereqKey(ctx, row, slug, spec.AutoPrereqFamily1); ok {
			override["PREREQFEAT1"] = map[string]any{"id": prereqKey}
		}
		if prereqKey, ok := weaponFamilyPrereqKey(ctx, row, slug, spec.AutoPrereqFamily2); ok {
			override["PREREQFEAT2"] = map[string]any{"id": prereqKey}
		}
		mergeGeneratedOverride(override, overrides[baseitemKey])
		overrideList = append(overrideList, override)
	}
	return map[string]any{"overrides": overrideList}, nil
}

func buildRacialtypesSkillAffinityModule(ctx *featGeneratedContext) (map[string]any, error) {
	racesDir := filepath.Join(ctx.dataDir, "racialtypes", "registry", "races")
	entries, err := os.ReadDir(racesDir)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return nil, nil
		}
		return nil, err
	}
	grants := map[string]*affinityGrant{}
	for _, entry := range entries {
		if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
			continue
		}
		path := filepath.Join(racesDir, entry.Name())
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, err
		}
		raceName := displayNameForRaceFile(obj)
		rawFeats, ok := obj["feats"].([]any)
		if !ok {
			continue
		}
		for _, rawFeat := range rawFeats {
			featObj, ok := rawFeat.(map[string]any)
			if !ok {
				continue
			}
			featKey, ok := extractRaceFeatKey(featObj)
			if !ok || !isSkillAffinityFeatKey(featKey) {
				continue
			}
			item := grants[featKey]
			if item == nil {
				item = &affinityGrant{featKey: featKey}
				grants[featKey] = item
			}
			item.races = append(item.races, raceName)
		}
	}
	if len(grants) == 0 {
		return nil, nil
	}
	skillRows, err := ctx.datasetRows("skills")
	if err != nil {
		return nil, err
	}
	overrideList := make([]any, 0, len(grants))
	for _, featKey := range sortedGrantKeys(grants) {
		grant := grants[featKey]
		skillKey, bonus, err := parseSkillAffinityKey(featKey)
		if err != nil {
			return nil, err
		}
		skillKey = resolveAffinitySkillKey(skillRows, skillKey)
		skillRow, ok := skillRows[skillKey]
		skillName := affinitySkillDisplayName(featKey, skillKey)
		if ok {
			skillName = displayNameForSkill(skillRow)
		}
		rowID, hasID, err := ctx.featID(featKey, nil)
		if err != nil {
			return nil, err
		}
		races := dedupeAndSort(grant.races)
		title := affinityTitleFromKey(featKey, skillName)
		description := affinityDescription(skillName, bonus, races)
		label := affinityLabel(featKey)
		parent := "skillaffinity"
		if bonus == 1 {
			parent = "partialskillaffinity"
		}
		child := childTokenFromExpandedKey(featKey, parent)
		override := map[string]any{
			"key":   featKey,
			"LABEL": label,
			"FEAT": map[string]any{
				"tlk": map[string]any{
					"key":  featKey + ".name",
					"text": title,
				},
			},
			"DESCRIPTION": map[string]any{
				"tlk": map[string]any{
					"key":  featKey + ".description",
					"text": description,
				},
			},
			"ICON":             "ife_racial",
			"ALLCLASSESCANUSE": "0",
			"TOOLSCATEGORIES":  "6",
			"PreReqEpic":       "0",
			"ReqAction":        "1",
			"Constant":         label,
			"meta":             familyMetadata(parent, child, skillKey, ""),
		}
		if hasID {
			override["id"] = rowID
		}
		overrideList = append(overrideList, override)
	}
	return map[string]any{"overrides": overrideList}, nil
}

func affinitySkillDisplayName(featKey, skillKey string) string {
	slug := strings.TrimPrefix(skillKey, "skills:")
	if slug == "" {
		slug = strings.TrimPrefix(strings.TrimPrefix(featKey, "feat:skillaffinity"), "feat:partialskillaffinity")
	}
	parts := strings.Fields(strings.ReplaceAll(strings.ReplaceAll(slug, "_", " "), "-", " "))
	for i, part := range parts {
		if part == "" {
			continue
		}
		parts[i] = strings.ToUpper(part[:1]) + part[1:]
	}
	return strings.Join(parts, " ")
}

func requiredStringField(obj map[string]any, field string) (string, error) {
	raw, ok := obj[field]
	if !ok {
		return "", fmt.Errorf("%s is required", field)
	}
	text, ok := raw.(string)
	if !ok || strings.TrimSpace(text) == "" {
		return "", fmt.Errorf("%s must be a non-empty string", field)
	}
	return strings.TrimSpace(text), nil
}

func parseCompactSourceOverrides(raw any) (compactSourceOverride, error) {
	if raw == nil {
		return compactSourceOverride{}, nil
	}
	obj, ok := raw.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("overrides must be an object keyed by source dataset key")
	}
	result := make(compactSourceOverride, len(obj))
	for key, value := range obj {
		item, ok := value.(map[string]any)
		if !ok {
			return nil, fmt.Errorf("override %q must be an object", key)
		}
		result[key] = item
	}
	return result, nil
}

func sortedStringMapKeys[T any](items map[string]T) []string {
	keys := make([]string, 0, len(items))
	for key := range items {
		keys = append(keys, key)
	}
	slices.Sort(keys)
	return keys
}

func mergeGeneratedOverride(dst map[string]any, src map[string]any) {
	if len(src) == 0 {
		return
	}
	if name, ok := src["name"].(string); ok && strings.TrimSpace(name) != "" {
		key, _ := dst["key"].(string)
		dst["FEAT"] = map[string]any{
			"tlk": map[string]any{
				"key":  key + ".name",
				"text": strings.TrimSpace(name),
			},
		}
	}
	if description, ok := src["description_text"].(string); ok && strings.TrimSpace(description) != "" {
		key, _ := dst["key"].(string)
		dst["DESCRIPTION"] = map[string]any{
			"tlk": map[string]any{
				"key":  key + ".description",
				"text": strings.TrimSpace(description),
			},
		}
	}
	if icon, ok := src["icon"]; ok {
		dst["ICON"] = icon
	}
	for field, value := range src {
		switch field {
		case "name", "description_text", "icon":
			continue
		default:
			dst[field] = value
		}
	}
}

func isHiddenSkill(row map[string]any) bool {
	value, ok := lookupField(row, "HideFromLevelUp")
	if !ok {
		return false
	}
	switch typed := value.(type) {
	case string:
		return strings.TrimSpace(typed) == "1"
	case int:
		return typed == 1
	case float64:
		return typed == 1
	default:
		return false
	}
}

func displayNameForSkill(row map[string]any) string {
	if text, ok := displayTextFromValue(row["Name"]); ok {
		return text
	}
	if label, ok := row["Label"].(string); ok && strings.TrimSpace(label) != "" {
		return normalizeDisplayName(strings.TrimSpace(label))
	}
	if key, ok := row["key"].(string); ok {
		return displayNameFromSlug(strings.TrimPrefix(key, "skills:"))
	}
	return "Unknown Skill"
}

func displayNameForBaseitem(row map[string]any) string {
	if text, ok := displayTextFromValue(row["Name"]); ok {
		return text
	}
	if label, ok := row["label"].(string); ok && strings.TrimSpace(label) != "" {
		return normalizeDisplayName(strings.TrimSpace(label))
	}
	if key, ok := row["key"].(string); ok {
		return displayNameFromSlug(strings.TrimPrefix(key, "baseitems:"))
	}
	return "Unknown Weapon"
}

func displayNameForRaceFile(obj map[string]any) string {
	core, ok := obj["core"].(map[string]any)
	if ok {
		if text, ok := displayTextFromValue(core["Name"]); ok {
			return text
		}
		if label, ok := core["Label"].(string); ok && strings.TrimSpace(label) != "" {
			return normalizeDisplayName(strings.TrimSpace(label))
		}
	}
	if key, ok := obj["key"].(string); ok {
		return displayNameFromSlug(strings.TrimPrefix(key, "racialtypes:"))
	}
	return "Unknown Race"
}

func displayTextFromValue(value any) (string, bool) {
	switch typed := value.(type) {
	case string:
		if strings.TrimSpace(typed) == "" || isNumericText(typed) || typed == nullValue {
			return "", false
		}
		return normalizeDisplayName(strings.TrimSpace(typed)), true
	case map[string]any:
		if tlk, ok := typed["tlk"].(map[string]any); ok {
			if text, ok := tlk["text"].(string); ok && strings.TrimSpace(text) != "" {
				return strings.TrimSpace(text), true
			}
		}
	}
	return "", false
}

func isNumericText(text string) bool {
	if strings.TrimSpace(text) == "" {
		return false
	}
	for _, r := range text {
		if r < '0' || r > '9' {
			return false
		}
	}
	return true
}

func displayNameFromSlug(slug string) string {
	text := strings.TrimSpace(slug)
	text = strings.TrimPrefix(text, "knowledge")
	if strings.HasPrefix(slug, "knowledge") {
		remainder := strings.TrimPrefix(slug, "knowledge")
		if remainder == "" {
			return "Knowledge"
		}
		return "Knowledge (" + normalizeDisplayName(remainder) + ")"
	}
	if strings.HasPrefix(slug, "craft_") {
		remainder := strings.TrimPrefix(slug, "craft_")
		return "Craft (" + normalizeDisplayName(remainder) + ")"
	}
	return normalizeDisplayName(text)
}

func normalizeDisplayName(text string) string {
	text = strings.ReplaceAll(text, "_", " ")
	text = splitCamelText(text)
	parts := strings.Fields(text)
	for index, part := range parts {
		lower := strings.ToLower(part)
		switch lower {
		case "of", "and", "vs":
			parts[index] = lower
		default:
			runes := []rune(lower)
			if len(runes) == 0 {
				continue
			}
			runes[0] = []rune(strings.ToUpper(string(runes[0])))[0]
			parts[index] = string(runes)
		}
	}
	return strings.Join(parts, " ")
}

func splitCamelText(text string) string {
	var out []rune
	var prev rune
	for index, r := range text {
		if index > 0 && isCamelBoundary(prev, r) {
			out = append(out, ' ')
		}
		out = append(out, r)
		prev = r
	}
	return string(out)
}

func isCamelBoundary(prev, current rune) bool {
	return prev >= 'a' && prev <= 'z' && current >= 'A' && current <= 'Z'
}

func upperSnake(text string) string {
	normalized := normalizeDisplayName(text)
	normalized = strings.ReplaceAll(normalized, "(", " ")
	normalized = strings.ReplaceAll(normalized, ")", " ")
	normalized = strings.ReplaceAll(normalized, "-", " ")
	normalized = strings.Join(strings.Fields(normalized), "_")
	return strings.ToUpper(normalized)
}

func isNullishValue(value any) bool {
	switch typed := value.(type) {
	case nil:
		return true
	case string:
		return strings.TrimSpace(typed) == "" || typed == nullValue
	default:
		return false
	}
}

func featSourceID(value any) (int, error) {
	if isNullishValue(value) {
		return 0, nil
	}
	switch typed := value.(type) {
	case int:
		return typed, nil
	case float64:
		return int(typed), nil
	case string:
		return asInt(typed)
	default:
		return 0, fmt.Errorf("cannot derive feat id from %T", value)
	}
}

func (c *featGeneratedContext) resolveWeaponFeatIdentity(familyKey, slug string, rawSource any) (string, int, bool, error) {
	if rawSource != nil && rawSource != nullValue {
		if parsedID, err := featSourceID(rawSource); err == nil && parsedID > 0 {
			if existingKey, ok := c.preferredFeatKeyForID(familyKey, parsedID); ok {
				return existingKey, parsedID, true, nil
			}
		}
	}
	if rawMap, ok := rawSource.(map[string]any); ok {
		if ref, ok := rawMap["id"].(string); ok && strings.HasPrefix(ref, "feat:") {
			rowID, hasID, err := c.featID(ref, nil)
			return ref, rowID, hasID, err
		}
	}
	featKey := c.preferredGeneratedFeatKey(familyKey, slug)
	rowID, hasID, err := c.featID(featKey, rawSource)
	return featKey, rowID, hasID, err
}

func weaponFamilyPrereqKey(ctx *featGeneratedContext, baseitemRow map[string]any, slug, familyKey string) (string, bool) {
	if familyKey == "" {
		return "", false
	}
	column := weaponFamilySourceColumn(familyKey)
	if column == "" {
		return "", false
	}
	rawSource, ok := lookupField(baseitemRow, column)
	if !ok || isNullishValue(rawSource) {
		return "", false
	}
	featKey, _, _, err := ctx.resolveWeaponFeatIdentity(familyKey, slug, rawSource)
	if err != nil {
		return "", false
	}
	return featKey, true
}

func weaponFamilySourceColumn(familyKey string) string {
	for _, spec := range featWeaponFamilies {
		if spec.FamilyKey == familyKey {
			return spec.SourceColumn
		}
	}
	return ""
}

func (c *featGeneratedContext) preferredFeatKeyForID(familyKey string, rowID int) (string, bool) {
	prefix := "feat:" + familyKey
	type candidateBuckets struct {
		familyExisting []string
		otherExisting  []string
		familyLegacy   []string
		otherLegacy    []string
		familyOther    []string
		otherOther     []string
	}

	buckets := candidateBuckets{}
	appendMatches := func(candidates map[string]int, legacy bool) {
		for key, candidateID := range candidates {
			if candidateID != rowID {
				continue
			}
			inFamily := strings.HasPrefix(key, prefix)
			exists := c.featKeyExists(key)
			switch {
			case exists && inFamily:
				buckets.familyExisting = append(buckets.familyExisting, key)
			case exists:
				buckets.otherExisting = append(buckets.otherExisting, key)
			case legacy && inFamily:
				buckets.familyLegacy = append(buckets.familyLegacy, key)
			case legacy:
				buckets.otherLegacy = append(buckets.otherLegacy, key)
			case inFamily:
				buckets.familyOther = append(buckets.familyOther, key)
			default:
				buckets.otherOther = append(buckets.otherOther, key)
			}
		}
	}

	appendMatches(c.lockData, false)
	appendMatches(c.supplementalID, true)

	for _, group := range [][]string{
		buckets.familyExisting,
		buckets.otherExisting,
		buckets.otherLegacy,
		buckets.familyLegacy,
		buckets.familyOther,
		buckets.otherOther,
	} {
		if best, ok := preferredExpandedKey(group); ok {
			return best, true
		}
	}
	return "", false
}

func extractRaceFeatKey(obj map[string]any) (string, bool) {
	value, ok := obj["FeatIndex"].(map[string]any)
	if !ok {
		return "", false
	}
	key, ok := value["id"].(string)
	if !ok || !strings.HasPrefix(key, "feat:") {
		return "", false
	}
	return key, true
}

func isSkillAffinityFeatKey(key string) bool {
	return strings.HasPrefix(key, "feat:skillaffinity") || strings.HasPrefix(key, "feat:partialskillaffinity")
}

func parseSkillAffinityKey(featKey string) (string, int, error) {
	switch {
	case strings.HasPrefix(featKey, "feat:partialskillaffinity"):
		return "skills:" + strings.TrimPrefix(featKey, "feat:partialskillaffinity"), 1, nil
	case strings.HasPrefix(featKey, "feat:skillaffinity"):
		return "skills:" + strings.TrimPrefix(featKey, "feat:skillaffinity"), 2, nil
	default:
		return "", 0, fmt.Errorf("unsupported skill affinity key %q", featKey)
	}
}

func resolveAffinitySkillKey(skillRows map[string]map[string]any, skillKey string) string {
	if _, ok := skillRows[skillKey]; ok {
		return skillKey
	}
	target := normalizeKeyIdentity(strings.TrimPrefix(skillKey, "skills:"))
	for key := range skillRows {
		if normalizeKeyIdentity(strings.TrimPrefix(key, "skills:")) == target {
			return key
		}
	}
	return skillKey
}

func normalizeKeyIdentity(text string) string {
	text = strings.ToLower(strings.TrimSpace(text))
	text = strings.ReplaceAll(text, "_", "")
	text = strings.ReplaceAll(text, " ", "")
	return text
}

func affinityTitleFromKey(featKey, skillName string) string {
	if strings.HasPrefix(featKey, "feat:partialskillaffinity") {
		return fmt.Sprintf("Partial Skill Affinity (%s)", skillName)
	}
	return fmt.Sprintf("Skill Affinity (%s)", skillName)
}

func affinityDescription(skillName string, bonus int, races []string) string {
	return fmt.Sprintf("Gain a +%d racial bonus to %s checks. Prerequisites: %s.", bonus, skillName, strings.Join(races, ", "))
}

func affinityLabel(featKey string) string {
	skillKey, _, err := parseSkillAffinityKey(featKey)
	if err != nil {
		return "FEAT_UNKNOWN_SKILL_AFFINITY"
	}
	skillSlug := strings.TrimPrefix(skillKey, "skills:")
	if strings.HasPrefix(featKey, "feat:partialskillaffinity") {
		return "FEAT_PARTIAL_SKILL_AFFINITY_" + upperSnake(displayNameFromSlug(skillSlug))
	}
	return "FEAT_SKILL_AFFINITY_" + upperSnake(displayNameFromSlug(skillSlug))
}

func dedupeAndSort(items []string) []string {
	seen := map[string]struct{}{}
	out := make([]string, 0, len(items))
	for _, item := range items {
		item = strings.TrimSpace(item)
		if item == "" {
			continue
		}
		if _, ok := seen[item]; ok {
			continue
		}
		seen[item] = struct{}{}
		out = append(out, item)
	}
	slices.Sort(out)
	return out
}

func sortedGrantKeys(items map[string]*affinityGrant) []string {
	keys := make([]string, 0, len(items))
	for key := range items {
		keys = append(keys, key)
	}
	slices.Sort(keys)
	return keys
}

func collectPlainDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
	tableData, err := loadJSONObject(dataset.BasePath)
	if err != nil {
		return nativeCollectedDataset{}, err
	}

	columns, err := parseColumns(tableData, dataset.Name)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	rawRows, ok := tableData["rows"].([]any)
	if !ok {
		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: rows must be an array", dataset.Name)
	}

	lockData, err := loadLockfile(dataset.LockPath)
	if err != nil {
		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
	}
	usedIDs := map[int]struct{}{}
	for _, rowID := range lockData {
		usedIDs[rowID] = struct{}{}
	}

	rows := make([]map[string]any, 0, len(rawRows))
	lockModified := false
	explicitID := make([]bool, 0, len(rawRows))
	for index, raw := range rawRows {
		rowObj, ok := raw.(map[string]any)
		if !ok {
			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d must be an object", dataset.Name, index)
		}
		row, err := canonicalizeBaseRow(dataset, columns, rowObj, index)
		if err != nil {
			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
		}
		rows = append(rows, row)
		_, hasExplicitID := rowObj["id"]
		explicitID = append(explicitID, hasExplicitID)
		if hasExplicitID {
			usedIDs[row["id"].(int)] = struct{}{}
		}
	}

	nextID := nextAvailableID(usedIDs)
	for index, row := range rows {
		key, hasKey := row["key"].(string)
		if hasKey && key != "" {
			if lockedID, ok := lockData[key]; ok {
				row["id"] = lockedID
				continue
			}
			if explicitID[index] {
				lockData[key] = row["id"].(int)
			} else {
				row["id"] = nextID
				lockData[key] = nextID
				usedIDs[nextID] = struct{}{}
				nextID = nextAvailableID(usedIDs)
			}
			lockModified = true
			continue
		}
		if !explicitID[index] {
			row["id"] = index
		}
	}

	if lockModified {
		if err := saveLockfile(dataset.LockPath, lockData); err != nil {
			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
		}
	}

	slices.SortFunc(rows, func(a, b map[string]any) int {
		return a["id"].(int) - b["id"].(int)
	})
	tableKey, _ := tableData["key"].(string)
	return nativeCollectedDataset{
		Dataset:  dataset,
		Columns:  columns,
		Rows:     rows,
		LockData: lockData,
		TableKey: tableKey,
	}, nil
}

func resolveNativeDataset(dataset nativeCollectedDataset, keyToID map[string]int, globalRowByKey map[string]map[string]any, tableRegistry resolvedTableRegistry, compiler *tlkCompiler) (map[string]any, error) {
	rows := dataset.Rows
	if strings.HasPrefix(filepath.ToSlash(dataset.Dataset.Name), "classes/feats/") {
		classKey := "classes:" + dataset.Dataset.Name[strings.LastIndex(dataset.Dataset.Name, "/")+1:]
		featSuccessors := buildFeatSuccessorsIndex(globalRowByKey, keyToID)
		classSkills := buildClassSkillsIndex(tableRegistry, classKey)
		expanded, err := expandClassesFeatRows(rows, keyToID, globalRowByKey, featSuccessors, classSkills, globalRowByKey)
		if err != nil {
			return nil, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
		}
		rows = expanded
	}

	dataset.Dataset.Columns = dataset.Columns
	resolver := valueResolver{
		dataset:       dataset.Dataset,
		keyToID:       keyToID,
		rowByKey:      globalRowByKey,
		tableRegistry: tableRegistry,
		compiler:      compiler,
		cache:         map[string]map[string]tlkCompiledValue{},
	}

	resolvedRows := make([]map[string]any, 0, len(rows))
	for _, row := range rows {
		resolved, err := resolver.resolveRow(row, nil)
		if err != nil {
			return nil, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
		}
		resolvedRows = append(resolvedRows, resolved)
	}
	return map[string]any{
		"columns": dataset.Columns,
		"rows":    resolvedRows,
	}, nil
}

var (
	classFeatGlobalRows = []map[string]any{
		{"FeatIndex": map[string]any{"id": "feat:combatexpertise"}, "List": "0", "GrantedOnLevel": "-1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:powerattack"}, "List": "0", "GrantedOnLevel": "-1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:specialattacks"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:throw"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:grapple"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:offensivefighting"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:defensivefighting"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:horsemenu"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
	}
	classFeatClassSkillShorthandRows = []map[string]any{
		{"FeatIndex": map[string]any{"id": "masterfeats:skillfocus", "filter": "classskills"}, "List": "1", "GrantedOnLevel": "-1", "OnMenu": "0"},
		{"FeatIndex": map[string]any{"id": "masterfeats:greaterskillfocus", "filter": "classskills"}, "List": "1", "GrantedOnLevel": "12", "OnMenu": "0"},
	}
)

func expandClassesFeatRows(rows []map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, featSuccessors map[string]string, classSkills map[string]bool, allRowByKey map[string]map[string]any) ([]map[string]any, error) {

	injected := make([]map[string]any, 0, len(classFeatGlobalRows)+len(classFeatClassSkillShorthandRows))
	existingRefIDs := make(map[string]struct{}, len(rows))
	for _, row := range rows {
		featRef, ok := row["FeatIndex"].(map[string]any)
		if !ok {
			continue
		}
		refID, _ := featRef["id"].(string)
		if refID == "" {
			continue
		}
		existingRefIDs[refID] = struct{}{}
	}

	for _, row := range classFeatGlobalRows {
		featKey := row["FeatIndex"].(map[string]any)["id"].(string)
		if _, exists := existingRefIDs[featKey]; exists {
			continue
		}
		if _, exists := allRowByKey[featKey]; exists {
			cloned := deepCopyValue(row).(map[string]any)
			if label := lookupReferencedFeatLabel(featKey, allRowByKey); label != "" {
				cloned["FeatLabel"] = label
			}
			injected = append(injected, cloned)
		}
	}

	hasClassSkills := classSkills != nil && len(classSkills) > 0
	for _, row := range classFeatClassSkillShorthandRows {
		if !hasClassSkills {
			continue
		}
		refID := row["FeatIndex"].(map[string]any)["id"].(string)
		if _, exists := existingRefIDs[refID]; exists {
			continue
		}
		rowItems, err := expandClassesFeatRow(row, keyToID, rowByKey, featSuccessors, classSkills)
		if err != nil {
			return nil, err
		}
		if len(rowItems) > 0 {
			injected = append(injected, rowItems...)
		}
	}

	expanded := make([]map[string]any, 0, len(rows))
	for _, row := range rows {
		rowItems, err := expandClassesFeatRow(row, keyToID, rowByKey, featSuccessors, classSkills)
		if err != nil {
			return nil, err
		}
		expanded = append(expanded, rowItems...)
	}

	combined := make([]map[string]any, 0, len(injected)+len(expanded))
	combined = append(combined, injected...)
	combined = append(combined, expanded...)

	for index, row := range combined {
		row["id"] = index
	}
	return combined, nil
}

func expandClassesFeatRow(row map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, featSuccessors map[string]string, classSkills map[string]bool) ([]map[string]any, error) {
	featRef, ok := row["FeatIndex"].(map[string]any)
	if !ok {
		return []map[string]any{row}, nil
	}
	refKey, _ := featRef["id"].(string)
	filterMode, _ := featRef["filter"].(string)

	switch {
	case strings.HasPrefix(refKey, "feat:"):
		if filterMode == "successors" {
			return expandFeatSuccessors(refKey, row, featSuccessors, rowByKey)
		}
		cloned := deepCopyValue(row).(map[string]any)
		if label := lookupReferencedFeatLabel(refKey, rowByKey); label != "" && isNullLikeValue(cloned["FeatLabel"]) {
			cloned["FeatLabel"] = label
		}
		return []map[string]any{cloned}, nil
	case strings.HasPrefix(refKey, "masterfeats:"):
		masterfeatKey := refKey
		children, err := expandMasterfeatChildren(masterfeatKey, row, keyToID, rowByKey)
		if err != nil {
			return nil, err
		}
		if filterMode == "classskills" {
			if classSkills == nil || len(classSkills) == 0 {
				return []map[string]any{}, nil
			}
			filtered := make([]map[string]any, 0, len(children))
			for _, child := range children {
				featRef, ok := child["FeatIndex"].(map[string]any)
				if !ok {
					continue
				}
				featKey, _ := featRef["id"].(string)
				featRow := rowByKey[featKey]
				if featRow == nil {
					continue
				}
				reqSkillKey := resolveReqSkill(featRow, keyToID)
				if reqSkillKey == "" {
					continue
				}
				if classSkills[reqSkillKey] {
					filtered = append(filtered, child)
				}
			}
			if len(filtered) == 0 {
				return []map[string]any{}, nil
			}
			return filtered, nil
		}
		if len(children) == 0 {
			return []map[string]any{row}, nil
		}
		return children, nil
	default:
		return []map[string]any{row}, nil
	}
}

func expandMasterfeatChildren(masterfeatKey string, row map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any) ([]map[string]any, error) {
	if _, ok := keyToID[masterfeatKey]; !ok {
		return nil, fmt.Errorf("unknown masterfeat reference %q", masterfeatKey)
	}
	type match struct {
		key string
		id  int
	}
	matches := make([]match, 0)
	for key, featRow := range rowByKey {
		if !strings.HasPrefix(key, "feat:") {
			continue
		}
		if !rowReferencesMasterfeat(featRow, masterfeatKey, keyToID) {
			continue
		}
		featID, ok := keyToID[key]
		if !ok {
			continue
		}
		matches = append(matches, match{key: key, id: featID})
	}
	slices.SortFunc(matches, func(a, b match) int {
		return a.id - b.id
	})
	expanded := make([]map[string]any, 0, len(matches))
	for _, item := range matches {
		cloned := deepCopyValue(row).(map[string]any)
		cloned["FeatIndex"] = map[string]any{"id": item.key}
		if label := lookupReferencedFeatLabel(item.key, rowByKey); label != "" {
			cloned["FeatLabel"] = label
		}
		expanded = append(expanded, cloned)
	}
	return expanded, nil
}

func rowReferencesMasterfeat(row map[string]any, masterfeatKey string, keyToID map[string]int) bool {
	masterID, ok := keyToID[masterfeatKey]
	if !ok {
		return false
	}
	value, ok := row["MASTERFEAT"]
	if !ok {
		return false
	}
	switch typed := value.(type) {
	case string:
		return typed == strconv.Itoa(masterID)
	case int:
		return typed == masterID
	case float64:
		return int(typed) == masterID
	case map[string]any:
		if id, ok := typed["id"].(string); ok {
			if id == masterfeatKey {
				return true
			}
			if refID, ok := keyToID[id]; ok {
				return refID == masterID
			}
			return id == "masterfeats:"+strconv.Itoa(masterID)
		}
		return false
	default:
		return false
	}
}

func lookupReferencedFeatLabel(featKey string, rowByKey map[string]map[string]any) string {
	row, ok := rowByKey[featKey]
	if !ok {
		return ""
	}
	label, _ := row["LABEL"].(string)
	return strings.TrimSpace(label)
}

func isNullLikeValue(value any) bool {
	switch typed := value.(type) {
	case nil:
		return true
	case string:
		trimmed := strings.TrimSpace(typed)
		return trimmed == "" || trimmed == nullValue
	default:
		return false
	}
}

func buildFeatSuccessorsIndex(rowByKey map[string]map[string]any, keyToID map[string]int) map[string]string {
	successors := make(map[string]string)
	for key, row := range rowByKey {
		if !strings.HasPrefix(key, "feat:") {
			continue
		}
		successorKey := resolveFeatSuccessor(row, keyToID)
		if successorKey != "" {
			successors[key] = successorKey
		}
	}
	return successors
}

func resolveFeatSuccessor(row map[string]any, keyToID map[string]int) string {
	value, ok := row["SUCCESSOR"]
	if !ok {
		return ""
	}
	switch typed := value.(type) {
	case map[string]any:
		if idValue, ok := typed["id"]; ok {
			switch idTyped := idValue.(type) {
			case string:
				trimmed := strings.TrimSpace(idTyped)
				if trimmed == "" || trimmed == nullValue || trimmed == "****" {
					return ""
				}
				if id, err := strconv.Atoi(trimmed); err == nil {
					for k, v := range keyToID {
						if v == id && strings.HasPrefix(k, "feat:") {
							return k
						}
					}
					return ""
				}
				if strings.HasPrefix(trimmed, "feat:") {
					return trimmed
				}
				return "feat:" + trimmed
			case int:
				for k, v := range keyToID {
					if v == idTyped && strings.HasPrefix(k, "feat:") {
						return k
					}
				}
				return ""
			case float64:
				targetID := int(idTyped)
				for k, v := range keyToID {
					if v == targetID && strings.HasPrefix(k, "feat:") {
						return k
					}
				}
				return ""
			}
		}
		return ""
	case string:
		trimmed := strings.TrimSpace(typed)
		if trimmed == "" || trimmed == nullValue || trimmed == "****" {
			return ""
		}
		if id, err := strconv.Atoi(trimmed); err == nil {
			for k, v := range keyToID {
				if v == id && strings.HasPrefix(k, "feat:") {
					return k
				}
			}
			return ""
		}
		if strings.HasPrefix(trimmed, "feat:") {
			return trimmed
		}
		return "feat:" + trimmed
	case int:
		for k, v := range keyToID {
			if v == typed && strings.HasPrefix(k, "feat:") {
				return k
			}
		}
		return ""
	case float64:
		targetID := int(typed)
		for k, v := range keyToID {
			if v == targetID && strings.HasPrefix(k, "feat:") {
				return k
			}
		}
		return ""
	default:
		return ""
	}
}

func buildClassSkillsIndex(tableRegistry resolvedTableRegistry, classKey string) map[string]bool {
	skillDatasetKey := classKey
	if !strings.HasPrefix(skillDatasetKey, "classes/skills:") {
		skillDatasetKey = "classes/skills:" + strings.TrimPrefix(classKey, "classes:")
	}

	table, ok := tableRegistry.resolveTableByKey(skillDatasetKey)
	if !ok {
		return nil
	}

	skills := make(map[string]bool)
	for _, rowMap := range table.Rows {
		classSkill, _ := rowMap["ClassSkill"].(string)
		if classSkill != "1" {
			continue
		}
		skillIndex, ok := rowMap["SkillIndex"].(map[string]any)
		if !ok {
			continue
		}
		skillKey, _ := skillIndex["id"].(string)
		if skillKey != "" {
			skills[skillKey] = true
		}
	}

	if len(skills) == 0 {
		return nil
	}
	return skills
}

func expandFeatSuccessors(rootFeatKey string, row map[string]any, featSuccessors map[string]string, rowByKey map[string]map[string]any) ([]map[string]any, error) {
	current := rootFeatKey
	visited := make(map[string]bool)
	var results []map[string]any
	for {
		if current == "" {
			break
		}
		if visited[current] {
			return nil, fmt.Errorf("successor chain cycles at %q", current)
		}
		visited[current] = true
		if rowByKey[current] == nil {
			break
		}
		cloned := deepCopyValue(row).(map[string]any)
		cloned["FeatIndex"] = map[string]any{"id": current}
		if label := lookupReferencedFeatLabel(current, rowByKey); label != "" {
			cloned["FeatLabel"] = label
		}
		results = append(results, cloned)
		current = featSuccessors[current]
	}
	if len(results) == 0 {
		return []map[string]any{row}, nil
	}
	return results, nil
}

func expandMasterfeatChildrenFiltered(masterfeatKey string, row map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, classSkills map[string]bool) ([]map[string]any, error) {
	if classSkills == nil || len(classSkills) == 0 {
		return []map[string]any{}, nil
	}
	children, err := expandMasterfeatChildren(masterfeatKey, row, keyToID, rowByKey)
	if err != nil {
		return nil, err
	}
	var filtered []map[string]any
	for _, child := range children {
		featRef, ok := child["FeatIndex"].(map[string]any)
		if !ok {
			continue
		}
		featKey, _ := featRef["id"].(string)
		featRow := rowByKey[featKey]
		if featRow == nil {
			continue
		}
		reqSkillKey := resolveReqSkill(featRow, keyToID)
		if reqSkillKey == "" {
			continue
		}
		if classSkills[reqSkillKey] {
			filtered = append(filtered, child)
		}
	}
	return filtered, nil
}

func extractClassKeyFromDatasetName(rowByKey map[string]map[string]any) string {
	for key := range rowByKey {
		if strings.HasPrefix(key, "classes:") {
			return key
		}
	}
	return ""
}

func resolveReqSkill(row map[string]any, keyToID map[string]int) string {
	value, ok := row["REQSKILL"]
	if !ok {
		return ""
	}
	switch typed := value.(type) {
	case string:
		trimmed := strings.TrimSpace(typed)
		if trimmed == "" || trimmed == nullValue || trimmed == "****" {
			return ""
		}
		if strings.HasPrefix(trimmed, "skills:") {
			return trimmed
		}
		if resolved := lookupKeyByIDPrefix(keyToID, "skills:", trimmed); resolved != "" {
			return resolved
		}
		return "skills:" + trimmed
	case map[string]any:
		if id, ok := typed["id"].(string); ok {
			if id == "" || id == nullValue || id == "****" {
				return ""
			}
			if strings.HasPrefix(id, "skills:") {
				return id
			}
			if resolved := lookupKeyByIDPrefix(keyToID, "skills:", id); resolved != "" {
				return resolved
			}
			return "skills:" + id
		}
		return ""
	default:
		return ""
	}
}

func lookupKeyByIDPrefix(keyToID map[string]int, prefix string, rawID string) string {
	idValue, err := strconv.Atoi(strings.TrimSpace(rawID))
	if err != nil {
		return ""
	}
	for key, existingID := range keyToID {
		if existingID == idValue && strings.HasPrefix(key, prefix) {
			return key
		}
	}
	return ""
}

type valueResolver struct {
	dataset       nativeDataset
	keyToID       map[string]int
	rowByKey      map[string]map[string]any
	tableRegistry resolvedTableRegistry
	compiler      *tlkCompiler
	cache         map[string]map[string]tlkCompiledValue
}

func (r *valueResolver) resolveTableByKey(key string) (resolvedTable, bool) {
	return r.tableRegistry.resolveTableByKey(key)
}

func (r *valueResolver) resolveRow(row map[string]any, stack []string) (map[string]any, error) {
	effectiveRow, err := r.resolveInheritedObject(row, rowIdentity(r.dataset.Name, row), row, stack)
	if err != nil {
		return nil, err
	}
	out := map[string]any{
		"id": effectiveRow["id"],
	}
	for _, column := range rowColumns(effectiveRow, r.dataset.Columns) {
		resolved, err := r.resolveField(effectiveRow, column, stack)
		if err != nil {
			return nil, err
		}
		out[column] = resolved.Value
	}
	return out, nil
}

func (r *valueResolver) resolveField(row map[string]any, field string, stack []string) (tlkCompiledValue, error) {
	rowKey := rowCacheIdentity(r.dataset.Name, row)
	if r.cache[rowKey] == nil {
		r.cache[rowKey] = map[string]tlkCompiledValue{}
	}
	if cached, ok := r.cache[rowKey][field]; ok {
		return cached, nil
	}

	stackKey := rowKey + "." + field
	for _, existing := range stack {
		if existing == stackKey {
			return tlkCompiledValue{}, fmt.Errorf("cyclic row/field reference detected at %s", stackKey)
		}
	}

	value, ok := lookupField(row, field)
	if !ok {
		resolved := tlkCompiledValue{Value: nullValue}
		r.cache[rowKey][field] = resolved
		return resolved, nil
	}
	resolved, err := r.resolveValue(row, field, value, append(stack, stackKey))
	if err != nil {
		return tlkCompiledValue{}, err
	}
	r.cache[rowKey][field] = resolved
	return resolved, nil
}

func (r *valueResolver) resolveValue(row map[string]any, field string, value any, stack []string) (tlkCompiledValue, error) {
	if typed, ok := value.(map[string]any); ok {
		merged, err := r.resolveInheritedObject(row, rowCacheIdentity(r.dataset.Name, row)+"."+field, typed, stack)
		if err != nil {
			return tlkCompiledValue{}, err
		}
		value = merged
	}
	if literal, ok := r.resolveLiteralFeatObject(row, field, value); ok {
		return tlkCompiledValue{Value: literal}, nil
	}
	allowBare := columnMatchesSpec(r.dataset.Spec, field)
	if payload, ok, err := parseTLKPayload(value, allowBare); err != nil {
		return tlkCompiledValue{}, err
	} else if ok {
		return r.resolveTLKPayload(row, field, payload, stack)
	}

	switch typed := value.(type) {
	case map[string]any:
		if refID, ok := typed["id"]; ok {
			if refKey, ok := refID.(string); ok && strings.Contains(refKey, ":") {
				id, ok := r.keyToID[refKey]
				if !ok {
					return tlkCompiledValue{}, fmt.Errorf("unknown key reference: %s (row=%s field=%s)", refKey, rowCacheIdentity(r.dataset.Name, row), field)
				}
				return tlkCompiledValue{Value: id}, nil
			}
		}
		if refKey, ok := typed["ref"]; ok {
			fieldValue, hasField := typed["field"]
			refKeyString, okKey := refKey.(string)
			fieldString, okField := fieldValue.(string)
			if hasField && okKey && okField {
				targetRow, ok := r.rowByKey[refKeyString]
				if !ok {
					return tlkCompiledValue{}, fmt.Errorf("unknown row reference: %s", refKeyString)
				}
				return r.resolveField(targetRow, fieldString, stack)
			}
		}
		if tableValue, ok := typed["table"]; ok {
			tableKey, ok := tableValue.(string)
			if ok {
				outputName, ok := r.tableRegistry.stemByKey[tableKey]
				if !ok {
					return tlkCompiledValue{}, fmt.Errorf("unknown table reference: %s", tableKey)
				}
				return tlkCompiledValue{Value: outputName}, nil
			}
		}

		resolved := map[string]any{}
		for _, key := range sortedKeys(typed) {
			item, err := r.resolveValue(row, key, typed[key], stack)
			if err != nil {
				return tlkCompiledValue{}, err
			}
			resolved[key] = item.Value
		}
		return tlkCompiledValue{Value: resolved}, nil
	case []any:
		resolved := make([]any, 0, len(typed))
		for _, item := range typed {
			value, err := r.resolveValue(row, field, item, stack)
			if err != nil {
				return tlkCompiledValue{}, err
			}
			resolved = append(resolved, value.Value)
		}
		return tlkCompiledValue{Value: resolved}, nil
	default:
		return tlkCompiledValue{Value: typed}, nil
	}
}

func (r *valueResolver) resolveLiteralFeatObject(row map[string]any, field string, value any) (string, bool) {
	if r.dataset.Name != "feat" {
		return "", false
	}
	typed, ok := value.(map[string]any)
	if !ok {
		return "", false
	}
	if refID, ok := typed["id"].(string); ok && len(typed) == 1 && strings.Contains(refID, ":") {
		if !shouldPreserveLiteralFeatIDObject(row, field, refID) {
			return "", false
		}
		if _, exists := r.keyToID[refID]; !exists {
			return "", false
		}
		return "{'id': '" + refID + "'}", true
	}
	if refKey, ok := typed["key"].(string); ok && len(typed) == 1 && strings.Contains(refKey, ":") {
		if !shouldPreserveLiteralFeatKeyObject(field) {
			return "", false
		}
		if _, exists := r.keyToID[refKey]; !exists && !isLegacyLiteralFeatKeyException(field, refKey) {
			return "", false
		}
		return "{'key': '" + refKey + "'}", true
	}
	return "", false
}

func shouldPreserveLiteralFeatKeyObject(field string) bool {
	switch field {
	case "SUCCESSOR", "PREREQFEAT1", "PREREQFEAT2":
		return true
	default:
		return false
	}
}

func shouldPreserveLiteralFeatIDObject(row map[string]any, field, refID string) bool {
	switch field {
	case "MinLevelClass":
		return shouldPreserveFeatMinLevelClass(row)
	case "SUCCESSOR":
		switch refID {
		case "feat:weaponproficiencysimple":
			return true
		default:
			return false
		}
	default:
		return false
	}
}

func shouldPreserveFeatMinLevelClass(row map[string]any) bool {
	rowKey, _ := row["key"].(string)
	switch {
	case strings.HasPrefix(rowKey, "feat:weaponfocus_"):
		return true
	case strings.HasPrefix(rowKey, "feat:weaponspecialization_"):
		return true
	case strings.HasPrefix(rowKey, "feat:greaterweaponfocus_"):
		return true
	case strings.HasPrefix(rowKey, "feat:greaterweaponspecialization_"):
		return true
	case strings.HasPrefix(rowKey, "feat:improvedcritical_"):
		return true
	case strings.HasPrefix(rowKey, "feat:overwhelmingcritical_"):
		return true
	case strings.HasPrefix(rowKey, "feat:epicweaponfocus_"):
		return true
	case strings.HasPrefix(rowKey, "feat:epicweaponspecialization_"):
		return true
	case rowKey == "masterfeats:weaponfocus":
		return true
	case rowKey == "masterfeats:weaponspecialization":
		return true
	case rowKey == "masterfeats:greaterweaponfocus":
		return true
	case rowKey == "masterfeats:greaterweaponspecialization":
		return true
	case rowKey == "masterfeats:improvedcritical":
		return true
	case rowKey == "masterfeats:overwhelmingcritical":
		return true
	default:
		return false
	}
}

func isLegacyLiteralFeatKeyException(field, refKey string) bool {
	if field != "SUCCESSOR" {
		return false
	}
	switch refKey {
	case "feat:improveddodge", "feat:improveduncannydodge":
		return true
	default:
		return false
	}
}

func (r *valueResolver) resolveTLKPayload(row map[string]any, field string, payload parsedTLKPayload, stack []string) (tlkCompiledValue, error) {
	if payload.Ref != "" {
		targetRow, ok := r.rowByKey[payload.Ref]
		if !ok {
			return tlkCompiledValue{}, fmt.Errorf("unknown TLK ref row %s", payload.Ref)
		}
		resolved, err := r.resolveField(targetRow, payload.RefField, stack)
		if err != nil {
			return tlkCompiledValue{}, err
		}
		if resolved.TLKKey == "" {
			switch resolved.Value.(type) {
			case string, int, float64:
				return resolved, nil
			default:
				return tlkCompiledValue{}, fmt.Errorf("TLK ref %s.%s does not resolve to TLK-backed text", payload.Ref, payload.RefField)
			}
		}
		return resolved, nil
	}
	if payload.Text != "" {
		key := payload.Key
		if key == "" {
			key = deriveAutoTLKKey(rowIdentity(r.dataset.Name, row), field)
		}
		return r.compiler.registerInline(key, tlkEntryData{
			Text:        payload.Text,
			SoundResRef: payload.SoundResRef,
			SoundLength: payload.SoundLength,
		})
	}
	return r.compiler.resolveLegacyKey(payload.Key)
}

func rowColumns(row map[string]any, ordered []string) []string {
	columns := make([]string, 0, len(row))
	seen := map[string]struct{}{}
	for _, key := range ordered {
		if _, ok := row[key]; ok {
			columns = append(columns, key)
			seen[key] = struct{}{}
		}
	}
	for key := range row {
		if key == "id" || key == "key" || key == "__columns" || key == "inherit" || isMetadataField(key) {
			continue
		}
		if _, ok := seen[key]; ok {
			continue
		}
		columns = append(columns, key)
	}
	return columns
}

func columnMatchesSpec(spec datasetTextSpec, field string) bool {
	return roleForField(spec, field) == textRoleName || roleForField(spec, field) == textRoleDescription
}

func rowIdentity(datasetName string, row map[string]any) string {
	if key, ok := row["key"].(string); ok && key != "" {
		return key
	}
	return fmt.Sprintf("%s:id:%d", datasetName, row["id"].(int))
}

func rowCacheIdentity(datasetName string, row map[string]any) string {
	return rowIdentity(datasetName, row)
}

func lookupField(row map[string]any, field string) (any, bool) {
	if value, ok := row[field]; ok {
		return value, true
	}
	for key, value := range row {
		if strings.EqualFold(key, field) {
			return value, true
		}
	}
	return nil, false
}

func parseColumns(baseData map[string]any, datasetName string) ([]string, error) {
	rawColumns, ok := baseData["columns"].([]any)
	if !ok {
		return nil, fmt.Errorf("dataset %s: columns must be an array", datasetName)
	}
	columns := make([]string, 0, len(rawColumns))
	for _, raw := range rawColumns {
		column, ok := raw.(string)
		if !ok {
			return nil, fmt.Errorf("dataset %s: columns must contain strings", datasetName)
		}
		if isDeprecatedAuthoringOnlyField(datasetName, column) || isMetadataField(column) {
			continue
		}
		columns = append(columns, column)
	}
	return columns, nil
}

func canonicalizeBaseRow(dataset nativeDataset, columns []string, raw map[string]any, index int) (map[string]any, error) {
	rowID := index
	if value, ok := raw["id"]; ok {
		parsed, err := asInt(value)
		if err != nil {
			return nil, fmt.Errorf("row id %v is not numeric", value)
		}
		rowID = parsed
	}

	row := map[string]any{"id": rowID}
	for _, column := range columns {
		row[column] = nullValue
	}
	for key, value := range raw {
		switch key {
		case "id", "_tlk":
		case "key":
			if value == nil {
				continue
			}
			text, ok := value.(string)
			if !ok {
				return nil, fmt.Errorf("row key must be a string")
			}
			row["key"] = text
		case "inherit":
			row["inherit"] = value
		default:
			if isMetadataField(key) {
				meta, err := parseRowMetadata(value)
				if err != nil {
					return nil, err
				}
				if meta != nil {
					row["meta"] = meta
				}
				continue
			}
			if isDeprecatedAuthoringOnlyField(dataset.Name, key) {
				if err := setLegacyWikiMetadata(row, key, value); err != nil {
					return nil, err
				}
				continue
			}
			columnName, ok := canonicalColumn(columns, key)
			if !ok {
				if isDeprecatedAuthoringOnlyField(dataset.Name, key) {
					continue
				}
				return nil, fmt.Errorf("unknown column %q", key)
			}
			row[columnName] = value
		}
	}

	return expandRowAuthoringSugar(dataset, columns, raw, row)
}

func canonicalizeEntry(dataset nativeDataset, columns []string, rowID int, key string, raw map[string]any) (map[string]any, error) {
	row := map[string]any{
		"id":  rowID,
		"key": key,
	}
	for _, column := range columns {
		row[column] = nullValue
	}
	for field, value := range raw {
		if field == "_tlk" {
			continue
		}
		if field == "inherit" {
			row["inherit"] = value
			continue
		}
		if isMetadataField(field) {
			meta, err := parseRowMetadata(value)
			if err != nil {
				return nil, err
			}
			if meta != nil {
				row["meta"] = meta
			}
			continue
		}
		if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
			if err := setLegacyWikiMetadata(row, field, value); err != nil {
				return nil, err
			}
			continue
		}
		columnName, ok := canonicalColumn(columns, field)
		if !ok {
			if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
				continue
			}
			return nil, fmt.Errorf("entry %q references unknown column %q", key, field)
		}
		row[columnName] = value
	}
	return expandRowAuthoringSugar(dataset, columns, raw, row)
}

func isAuthoringOnlyField(name string) bool {
	return strings.HasPrefix(strings.ToUpper(strings.TrimSpace(name)), "WIKI")
}

func preserveLegacyWikiColumn(datasetName, field string) bool {
	return filepath.ToSlash(strings.TrimSpace(datasetName)) == "classes/core" &&
		strings.EqualFold(strings.TrimSpace(field), "WIKIGENERATE")
}

func isDeprecatedAuthoringOnlyField(datasetName, field string) bool {
	return isAuthoringOnlyField(field) && !preserveLegacyWikiColumn(datasetName, field)
}

func isMetadataField(name string) bool {
	return strings.EqualFold(strings.TrimSpace(name), "meta")
}

func expandRowAuthoringSugar(dataset nativeDataset, columns []string, source map[string]any, row map[string]any) (map[string]any, error) {
	expanded, err := expandRowTLKSugar(dataset, columns, source, row)
	if err != nil {
		return nil, err
	}
	return expandRowInheritanceSugar(dataset, columns, source, expanded)
}

func expandRowTLKSugar(dataset nativeDataset, columns []string, source map[string]any, row map[string]any) (map[string]any, error) {
	rawTLK, ok := source["_tlk"]
	if !ok {
		return row, nil
	}
	tlkMap, ok := rawTLK.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("dataset %s: _tlk must be an object", dataset.Name)
	}

	out := map[string]any{}
	for key, value := range row {
		out[key] = value
	}
	for roleName, rawValue := range tlkMap {
		var role textRole
		switch strings.ToLower(roleName) {
		case "name":
			role = textRoleName
		case "description":
			role = textRoleDescription
		default:
			return nil, fmt.Errorf("dataset %s: unsupported _tlk role %q", dataset.Name, roleName)
		}
		column := columnForRole(columns, dataset.Spec, role)
		if column == "" {
			return nil, fmt.Errorf("dataset %s: _tlk.%s has no matching text column", dataset.Name, roleName)
		}
		if _, hasField := source[column]; hasField {
			return nil, fmt.Errorf("dataset %s: cannot use _tlk.%s and field %s together", dataset.Name, roleName, column)
		}

		switch typed := rawValue.(type) {
		case string:
			out[column] = map[string]any{"tlk": map[string]any{"text": typed}}
		case map[string]any:
			if _, ok := typed["tlk"]; ok {
				out[column] = typed
			} else {
				out[column] = map[string]any{"tlk": typed}
			}
		default:
			return nil, fmt.Errorf("dataset %s: _tlk.%s must be a string or object", dataset.Name, roleName)
		}
	}
	return out, nil
}

func expandRowInheritanceSugar(dataset nativeDataset, columns []string, source map[string]any, row map[string]any) (map[string]any, error) {
	spec, ok, err := parseRowInheritanceSpec(source)
	if err != nil {
		return nil, fmt.Errorf("dataset %s: %w", dataset.Name, err)
	}
	if !ok {
		return row, nil
	}

	parentKey, err := resolveInheritanceParentKey(row, spec.From)
	if err != nil {
		return nil, fmt.Errorf("dataset %s: %w", dataset.Name, err)
	}

	out := map[string]any{}
	for key, value := range row {
		out[key] = value
	}
	delete(out, "inherit")
	for _, field := range spec.Fields {
		columnName, ok := canonicalColumn(columns, field)
		if !ok {
			return nil, fmt.Errorf("inherit field %q is not a known column", field)
		}
		if rowFieldWasAuthored(source, columns, columnName, dataset.Spec) {
			continue
		}
		if current, ok := lookupField(out, columnName); ok && current != nullValue {
			continue
		}
		out[columnName] = map[string]any{
			"ref":   parentKey,
			"field": columnName,
		}
	}
	return out, nil
}

type rowInheritanceSpec struct {
	From   string
	Fields []string
}

type genericInheritanceSpec struct {
	Ref   string
	Field string
}

func parseRowInheritanceSpec(source map[string]any) (rowInheritanceSpec, bool, error) {
	raw, ok := source["inherit"]
	if !ok {
		return rowInheritanceSpec{}, false, nil
	}
	obj, ok := raw.(map[string]any)
	if !ok {
		return rowInheritanceSpec{}, false, fmt.Errorf("inherit must be an object")
	}
	if _, hasRef := obj["ref"]; hasRef {
		if _, hasFrom := obj["from"]; hasFrom {
			return rowInheritanceSpec{}, false, fmt.Errorf("inherit cannot mix row sugar (from/fields) with object inheritance (ref)")
		}
		return rowInheritanceSpec{}, false, nil
	}
	rawFrom, ok := obj["from"]
	if !ok {
		return rowInheritanceSpec{}, false, fmt.Errorf("inherit.from is required")
	}
	from, ok := rawFrom.(string)
	if !ok || strings.TrimSpace(from) == "" {
		return rowInheritanceSpec{}, false, fmt.Errorf("inherit.from must be a non-empty string")
	}
	rawFields, ok := obj["fields"]
	if !ok {
		return rowInheritanceSpec{}, false, fmt.Errorf("inherit.fields is required")
	}
	fieldList, ok := rawFields.([]any)
	if !ok || len(fieldList) == 0 {
		return rowInheritanceSpec{}, false, fmt.Errorf("inherit.fields must be a non-empty array")
	}
	fields := make([]string, 0, len(fieldList))
	seen := map[string]struct{}{}
	for _, rawField := range fieldList {
		field, ok := rawField.(string)
		if !ok || strings.TrimSpace(field) == "" {
			return rowInheritanceSpec{}, false, fmt.Errorf("inherit.fields must contain non-empty strings")
		}
		normalized := strings.ToLower(field)
		if _, exists := seen[normalized]; exists {
			continue
		}
		seen[normalized] = struct{}{}
		fields = append(fields, field)
	}
	return rowInheritanceSpec{From: from, Fields: fields}, true, nil
}

func parseGenericInheritanceSpec(source map[string]any) (genericInheritanceSpec, bool, error) {
	raw, ok := source["inherit"]
	if !ok {
		return genericInheritanceSpec{}, false, nil
	}
	obj, ok := raw.(map[string]any)
	if !ok {
		return genericInheritanceSpec{}, false, fmt.Errorf("inherit must be an object")
	}
	if _, hasFrom := obj["from"]; hasFrom {
		if _, hasRef := obj["ref"]; hasRef {
			return genericInheritanceSpec{}, false, fmt.Errorf("inherit cannot mix row sugar (from/fields) with object inheritance (ref)")
		}
		return genericInheritanceSpec{}, false, nil
	}
	rawRef, ok := obj["ref"]
	if !ok {
		return genericInheritanceSpec{}, false, fmt.Errorf("inherit.ref is required")
	}
	ref, ok := rawRef.(string)
	if !ok || strings.TrimSpace(ref) == "" {
		return genericInheritanceSpec{}, false, fmt.Errorf("inherit.ref must be a non-empty string")
	}
	spec := genericInheritanceSpec{Ref: strings.TrimSpace(ref)}
	if rawField, ok := obj["field"]; ok {
		field, ok := rawField.(string)
		if !ok || strings.TrimSpace(field) == "" {
			return genericInheritanceSpec{}, false, fmt.Errorf("inherit.field must be a non-empty string when present")
		}
		spec.Field = strings.TrimSpace(field)
	}
	return spec, true, nil
}

func resolveInheritanceParentKey(row map[string]any, from string) (string, error) {
	value, ok := lookupField(row, from)
	if !ok {
		return "", fmt.Errorf("inherit.from field %q was not found on row", from)
	}
	switch typed := value.(type) {
	case string:
		if strings.Contains(typed, ":") {
			return typed, nil
		}
	case map[string]any:
		if rawID, ok := typed["id"]; ok {
			if key, ok := rawID.(string); ok && strings.Contains(key, ":") {
				return key, nil
			}
		}
		if rawRef, ok := typed["ref"]; ok {
			if key, ok := rawRef.(string); ok && strings.Contains(key, ":") {
				if _, hasField := typed["field"]; !hasField {
					return key, nil
				}
			}
		}
	}
	return "", fmt.Errorf("inherit.from field %q must resolve to a keyed parent row reference", from)
}

func (r *valueResolver) resolveInheritedObject(ownerRow map[string]any, identity string, obj map[string]any, stack []string) (map[string]any, error) {
	spec, ok, err := parseGenericInheritanceSpec(obj)
	if err != nil {
		return nil, err
	}

	out := map[string]any{}
	for key, value := range obj {
		if key == "inherit" {
			continue
		}
		out[key] = cloneAuthoringValue(value)
	}
	if !ok {
		return out, nil
	}

	marker := "inherit:" + identity
	for _, existing := range stack {
		if existing == marker {
			return nil, fmt.Errorf("cyclic inheritance detected at %s", identity)
		}
	}

	parent, err := r.resolveGenericInheritanceTarget(ownerRow, spec, append(stack, marker))
	if err != nil {
		return nil, err
	}
	return mergeInheritedMaps(parent, out), nil
}

func (r *valueResolver) resolveGenericInheritanceTarget(ownerRow map[string]any, spec genericInheritanceSpec, stack []string) (map[string]any, error) {
	targetRow, ok := r.rowByKey[spec.Ref]
	if !ok {
		return nil, fmt.Errorf("inherit target %q was not found", spec.Ref)
	}
	if spec.Field == "" {
		return r.resolveInheritedObject(targetRow, spec.Ref, targetRow, stack)
	}
	value, ok := lookupField(targetRow, spec.Field)
	if !ok {
		return nil, fmt.Errorf("inherit target field %q was not found on %s", spec.Field, spec.Ref)
	}
	targetObj, ok := value.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("inherit target %s.%s must resolve to an object", spec.Ref, spec.Field)
	}
	return r.resolveInheritedObject(targetRow, spec.Ref+"."+spec.Field, targetObj, stack)
}

func mergeInheritedMaps(parent, local map[string]any) map[string]any {
	out := map[string]any{}
	for key, value := range parent {
		out[key] = cloneAuthoringValue(value)
	}
	for key, value := range local {
		if key == "inherit" {
			continue
		}
		parentValue, hasParent := out[key]
		if !hasParent {
			out[key] = cloneAuthoringValue(value)
			continue
		}
		out[key] = mergeInheritedValues(parentValue, value)
	}
	return out
}

func mergeInheritedValues(parent, local any) any {
	if isNullLike(local) {
		return cloneAuthoringValue(parent)
	}
	localMap, localIsMap := local.(map[string]any)
	parentMap, parentIsMap := parent.(map[string]any)
	if localIsMap && parentIsMap && !isAtomicAuthoringMap(localMap) && !isAtomicAuthoringMap(parentMap) {
		return mergeInheritedMaps(parentMap, localMap)
	}
	return cloneAuthoringValue(local)
}

func isAtomicAuthoringMap(value map[string]any) bool {
	for _, key := range []string{"tlk", "ref", "id", "table"} {
		if _, ok := value[key]; ok {
			return true
		}
	}
	return false
}

func cloneAuthoringValue(value any) any {
	switch typed := value.(type) {
	case map[string]any:
		out := map[string]any{}
		for key, item := range typed {
			out[key] = cloneAuthoringValue(item)
		}
		return out
	case []any:
		out := make([]any, 0, len(typed))
		for _, item := range typed {
			out = append(out, cloneAuthoringValue(item))
		}
		return out
	default:
		return typed
	}
}

func rowFieldWasAuthored(source map[string]any, columns []string, field string, spec datasetTextSpec) bool {
	if _, ok := source[field]; ok {
		return true
	}
	for key := range source {
		if canonical, ok := canonicalColumn(columns, key); ok && canonical == field {
			return true
		}
	}
	rawTLK, ok := source["_tlk"].(map[string]any)
	if !ok {
		return false
	}
	for roleName := range rawTLK {
		var role textRole
		switch strings.ToLower(roleName) {
		case "name":
			role = textRoleName
		case "description":
			role = textRoleDescription
		default:
			continue
		}
		column := columnForRole(columns, spec, role)
		if column == field {
			return true
		}
	}
	return false
}

func resolveOverrideTarget(datasetName string, override map[string]any, rowByID map[int]map[string]any, rowByKey map[string]map[string]any) (map[string]any, error) {
	if rawID, ok := override["id"]; ok {
		rowID, err := asInt(rawID)
		if err != nil {
			return nil, fmt.Errorf("dataset %s: override id %v is not numeric", datasetName, rawID)
		}
		row, ok := rowByID[rowID]
		if !ok {
			return nil, fmt.Errorf("dataset %s: override target id %d was not found", datasetName, rowID)
		}
		return row, nil
	}
	if rawKey, ok := override["key"]; ok {
		key, ok := rawKey.(string)
		if !ok || key == "" {
			return nil, fmt.Errorf("dataset %s: override key must be a string", datasetName)
		}
		if key != "" {
			row, ok := rowByKey[key]
			if ok {
				return row, nil
			}
			return nil, fmt.Errorf("dataset %s: override target key %q was not found", datasetName, key)
		}
	}
	return nil, fmt.Errorf("dataset %s: override must specify id or key", datasetName)
}

func updateOverrideRowKey(row map[string]any, expanded map[string]any, rowByKey map[string]map[string]any, lockData map[string]int) bool {
	oldKey, _ := row["key"].(string)
	newKeyValue, hasKey := expanded["key"]
	if !hasKey {
		return false
	}
	rowID, _ := row["id"].(int)
	changed := false
	if oldKey != "" {
		if mapped, ok := rowByKey[oldKey]; ok {
			mappedID, mappedHasID := mapped["id"].(int)
			rowHasID := rowID != 0 || row["id"] != nil
			if mappedHasID && rowHasID && mappedID == rowID {
				delete(rowByKey, oldKey)
			}
		}
	}
	if newKeyValue == nil {
		delete(row, "key")
		if oldKey != "" {
			delete(lockData, oldKey)
			changed = true
		}
		return changed
	}
	newKey, ok := newKeyValue.(string)
	if !ok || newKey == "" {
		delete(row, "key")
		if oldKey != "" {
			delete(lockData, oldKey)
			changed = true
		}
		return changed
	}
	if conflicting, ok := rowByKey[newKey]; ok && conflicting != nil {
		conflictingID, conflictingHasID := conflicting["id"].(int)
		rowHasID := rowID != 0 || row["id"] != nil
		if !conflictingHasID || !rowHasID || conflictingID != rowID {
			delete(conflicting, "key")
			changed = true
		}
	}
	row["key"] = newKey
	rowByKey[newKey] = row
	if oldKey != "" && oldKey != newKey {
		delete(lockData, oldKey)
		changed = true
	}
	if existingID, ok := lockData[newKey]; !ok || existingID != rowID {
		lockData[newKey] = rowID
		changed = true
	}
	return changed
}

func normalizeCollectedSpellImpactScripts(rows []map[string]any, baseRowKeys map[string]struct{}) {
	for _, row := range rows {
		rowKey, ok := row["key"].(string)
		if !ok || rowKey == "" {
			continue
		}
		if _, ok := baseRowKeys[rowKey]; ok {
			continue
		}
		rowID, ok := row["id"].(int)
		if !ok {
			continue
		}
		row["ImpactScript"] = fmt.Sprintf("ss_%d", rowID)
	}
}

func write2DA(data map[string]any, path string, denseRows bool) error {
	columns, ok := data["columns"].([]string)
	if !ok {
		return fmt.Errorf("2da columns must be []string")
	}
	rows, ok := data["rows"].([]map[string]any)
	if !ok {
		return fmt.Errorf("2da rows must be []map[string]any")
	}

	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return fmt.Errorf("create 2da output parent: %w", err)
	}

	var builder strings.Builder
	builder.WriteString("2DA V2.0\n\n")
	builder.WriteString(strings.Join(columns, "\t"))
	builder.WriteByte('\n')
	nextID := 0
	for _, row := range rows {
		rowID := row["id"].(int)
		if denseRows {
			for nextID < rowID {
				builder.WriteString(strconv.Itoa(nextID))
				for range columns {
					builder.WriteByte('\t')
					builder.WriteString(nullValue)
				}
				builder.WriteByte('\n')
				nextID++
			}
		}
		builder.WriteString(strconv.Itoa(rowID))
		for _, column := range columns {
			builder.WriteByte('\t')
			builder.WriteString(format2DAValue(row[column]))
		}
		builder.WriteByte('\n')
		if denseRows {
			nextID = rowID + 1
		}
	}
	return os.WriteFile(path, []byte(builder.String()), 0o644)
}

func format2DAValue(value any) string {
	switch typed := value.(type) {
	case nil:
		return nullValue
	case string:
		if typed == nullValue {
			return typed
		}
		if strings.ContainsAny(typed, " \t\r\n") {
			return strconv.Quote(typed)
		}
		return typed
	case int:
		return strconv.Itoa(typed)
	case float64:
		if typed == float64(int64(typed)) {
			return strconv.FormatInt(int64(typed), 10)
		}
		return strconv.FormatFloat(typed, 'f', -1, 64)
	case bool:
		if typed {
			return "1"
		}
		return "0"
	default:
		raw, err := json.Marshal(typed)
		if err != nil {
			return fmt.Sprint(typed)
		}
		text := string(raw)
		if strings.ContainsAny(text, " \t\r\n") {
			return strconv.Quote(text)
		}
		return text
	}
}

func outputStem(outputName string) string {
	return strings.TrimSuffix(outputName, filepath.Ext(outputName))
}

func loadJSONObject(path string) (map[string]any, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("read %s: %w", path, err)
	}
	var payload any
	if err := json.Unmarshal(raw, &payload); err != nil {
		return nil, fmt.Errorf("parse %s: %w", path, err)
	}
	obj, ok := payload.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("%s must contain a JSON object", path)
	}
	return obj, nil
}

func loadLockfile(path string) (map[string]int, error) {
	lockData := map[string]int{}
	if _, err := os.Stat(path); err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return lockData, nil
		}
		return nil, err
	}

	obj, err := loadJSONObject(path)
	if err != nil {
		return nil, err
	}
	for key, value := range obj {
		rowID, err := asInt(value)
		if err != nil {
			return nil, fmt.Errorf("lock entry %q is not numeric", key)
		}
		lockData[key] = rowID
	}
	return lockData, nil
}

func saveLockfile(path string, lockData map[string]int) error {
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return fmt.Errorf("create lockfile parent: %w", err)
	}
	keys, err := orderedLockfileKeys(path, lockData)
	if err != nil {
		return fmt.Errorf("read lockfile order: %w", err)
	}
	raw, err := marshalOrderedLockfile(keys, lockData)
	if err != nil {
		return fmt.Errorf("marshal lockfile: %w", err)
	}
	if err := os.WriteFile(path, raw, 0o644); err != nil {
		return fmt.Errorf("write lockfile %s: %w", path, err)
	}
	return nil
}

func orderedLockfileKeys(path string, lockData map[string]int) ([]string, error) {
	existingKeys, err := readLockfileKeyOrder(path)
	if err != nil && !errors.Is(err, os.ErrNotExist) {
		return nil, err
	}

	keys := make([]string, 0, len(lockData))
	seen := make(map[string]struct{}, len(lockData))
	for _, key := range existingKeys {
		if _, ok := lockData[key]; !ok {
			continue
		}
		keys = append(keys, key)
		seen[key] = struct{}{}
	}

	extraKeys := make([]string, 0, len(lockData)-len(keys))
	for key := range lockData {
		if _, ok := seen[key]; ok {
			continue
		}
		extraKeys = append(extraKeys, key)
	}
	slices.Sort(extraKeys)
	keys = append(keys, extraKeys...)
	return keys, nil
}

func readLockfileKeyOrder(path string) ([]string, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}

	decoder := json.NewDecoder(bytes.NewReader(raw))
	token, err := decoder.Token()
	if err != nil {
		return nil, err
	}
	delim, ok := token.(json.Delim)
	if !ok || delim != '{' {
		return nil, fmt.Errorf("%s must contain a JSON object", path)
	}

	keys := []string{}
	for decoder.More() {
		token, err := decoder.Token()
		if err != nil {
			return nil, err
		}
		key, ok := token.(string)
		if !ok {
			return nil, fmt.Errorf("%s contains a non-string object key", path)
		}
		var value json.RawMessage
		if err := decoder.Decode(&value); err != nil {
			return nil, err
		}
		keys = append(keys, key)
	}

	token, err = decoder.Token()
	if err != nil {
		return nil, err
	}
	delim, ok = token.(json.Delim)
	if !ok || delim != '}' {
		return nil, fmt.Errorf("%s must contain a JSON object", path)
	}
	return keys, nil
}

func marshalOrderedLockfile(keys []string, lockData map[string]int) ([]byte, error) {
	if len(keys) == 0 {
		return []byte("{}\n"), nil
	}

	var buffer bytes.Buffer
	buffer.WriteString("{\n")
	for index, key := range keys {
		keyRaw, err := json.Marshal(key)
		if err != nil {
			return nil, err
		}
		valueRaw, err := json.Marshal(lockData[key])
		if err != nil {
			return nil, err
		}
		buffer.WriteString("  ")
		buffer.Write(keyRaw)
		buffer.WriteString(": ")
		buffer.Write(valueRaw)
		if index < len(keys)-1 {
			buffer.WriteByte(',')
		}
		buffer.WriteByte('\n')
	}
	buffer.WriteString("}\n")
	return buffer.Bytes(), nil
}

func loadLegacyTLK(root string) (*legacyTLKData, error) {
	info, err := os.Stat(root)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	if !info.IsDir() {
		return nil, fmt.Errorf("legacy tlk root must be a directory: %s", root)
	}

	result := &legacyTLKData{
		Language: "en",
		Entries:  map[string]tlkEntryData{},
		Lock:     map[string]int{},
	}

	lockPath := filepath.Join(root, "lock.json")
	lockData, err := loadLockfile(lockPath)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	for key, id := range lockData {
		result.Lock[key] = id
	}

	basePath := filepath.Join(root, "base.json")
	if _, err := os.Stat(basePath); err == nil {
		base, err := loadJSONObject(basePath)
		if err != nil {
			return nil, err
		}
		if language, ok := base["language"].(string); ok && language != "" {
			result.Language = language
		}
		if entries, ok := base["entries"].(map[string]any); ok {
			normalized, err := normalizeLegacyTLKEntries(entries)
			if err != nil {
				return nil, err
			}
			for key, entry := range normalized {
				result.Entries[key] = entry
			}
		}
	}

	modulesDir := filepath.Join(root, "modules")
	modulePaths, err := collectModulePaths(modulesDir)
	if err != nil {
		if os.IsNotExist(err) {
			return result, nil
		}
		return nil, err
	}
	for _, path := range modulePaths {
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, err
		}
		entries, ok := obj["entries"].(map[string]any)
		if !ok {
			continue
		}
		normalized, err := normalizeLegacyTLKEntries(entries)
		if err != nil {
			return nil, fmt.Errorf("%s: %w", path, err)
		}
		for key, entry := range normalized {
			result.Entries[key] = entry
		}
	}
	return result, nil
}

func normalizeLegacyTLKEntries(entries map[string]any) (map[string]tlkEntryData, error) {
	normalized := map[string]tlkEntryData{}
	for _, key := range sortedKeys(entries) {
		value := entries[key]
		switch typed := value.(type) {
		case string:
			normalized[key] = tlkEntryData{Text: typed}
		case map[string]any:
			if shouldExpandLegacyKey(key, typed) {
				for _, subKey := range sortedKeys(typed) {
					subObj, ok := typed[subKey].(map[string]any)
					if !ok {
						continue
					}
					entry, err := parseLegacyTLKLeaf(subObj)
					if err != nil {
						return nil, err
					}
					normalized[key+"."+subKey] = entry
				}
				continue
			}
			entry, err := parseLegacyTLKLeaf(typed)
			if err != nil {
				return nil, err
			}
			normalized[key] = entry
		default:
			return nil, fmt.Errorf("legacy TLK entry %q must be an object or string", key)
		}
	}
	return normalized, nil
}

func shouldExpandLegacyKey(key string, obj map[string]any) bool {
	if strings.Contains(key[strings.LastIndex(key, ":")+1:], ".") {
		return false
	}
	for _, value := range obj {
		if _, ok := value.(map[string]any); ok {
			return true
		}
	}
	return false
}

func parseLegacyTLKLeaf(obj map[string]any) (tlkEntryData, error) {
	entry := tlkEntryData{}
	if rawText, ok := obj["text"]; ok {
		text, ok := rawText.(string)
		if !ok {
			return entry, fmt.Errorf("legacy TLK text must be a string")
		}
		entry.Text = text
	}
	if rawSound, ok := obj["sound_resref"]; ok {
		sound, ok := rawSound.(string)
		if !ok {
			return entry, fmt.Errorf("legacy TLK sound_resref must be a string")
		}
		entry.SoundResRef = sound
	}
	if rawLength, ok := obj["sound_length"]; ok {
		switch typed := rawLength.(type) {
		case float64:
			entry.SoundLength = float32(typed)
		case int:
			entry.SoundLength = float32(typed)
		default:
			return entry, fmt.Errorf("legacy TLK sound_length must be numeric")
		}
	}
	return entry, nil
}

func collectModulePaths(modulesDir string) ([]string, error) {
	if _, err := os.Stat(modulesDir); err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return nil, nil
		}
		return nil, err
	}
	var paths []string
	err := filepath.WalkDir(modulesDir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		if strings.HasSuffix(strings.ToLower(d.Name()), ".json") {
			paths = append(paths, path)
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	slices.Sort(paths)
	return paths, nil
}

func canonicalColumn(columns []string, name string) (string, bool) {
	for _, column := range columns {
		if strings.EqualFold(column, name) {
			return column, true
		}
	}
	return "", false
}

func optionalBool(value any, fallback bool) bool {
	switch typed := value.(type) {
	case bool:
		return typed
	case nil:
		return fallback
	default:
		return fallback
	}
}

func asInt(value any) (int, error) {
	switch typed := value.(type) {
	case int:
		return typed, nil
	case int64:
		return int(typed), nil
	case float64:
		return int(typed), nil
	case json.Number:
		n, err := typed.Int64()
		return int(n), err
	case string:
		return strconv.Atoi(typed)
	default:
		return 0, fmt.Errorf("not an int")
	}
}

func nextAvailableID(used map[int]struct{}) int {
	next := 0
	for {
		if _, ok := used[next]; !ok {
			return next
		}
		next++
	}
}

func hasJSONFiles(root string) (bool, error) {
	found := false
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		if strings.HasSuffix(strings.ToLower(d.Name()), ".json") {
			found = true
			return fs.SkipAll
		}
		return nil
	})
	if err != nil && !errors.Is(err, fs.SkipAll) {
		return false, err
	}
	return found, nil
}

func collectReferenceLockIDs(root string) (map[string]int, error) {
	result := map[string]int{}
	if _, err := os.Stat(root); err != nil {
		if os.IsNotExist(err) {
			return result, nil
		}
		return nil, err
	}
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() || d.Name() != "lock.json" {
			return nil
		}
		lockData, err := loadLockfile(path)
		if err != nil {
			return err
		}
		for key, rowID := range lockData {
			if _, exists := result[key]; !exists {
				result[key] = rowID
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return result, nil
}

func collectSupplementalLockIDs(sourceDir, referenceBuilderDir string) (map[string]int, error) {
	result := map[string]int{}
	snapshotIDs, err := collectReferenceLockIDs(filepath.Join(sourceDir, "migration_snapshot", "data"))
	if err != nil {
		return nil, err
	}
	for key, rowID := range snapshotIDs {
		result[key] = rowID
	}
	if strings.TrimSpace(referenceBuilderDir) == "" {
		return result, nil
	}
	referenceIDs, err := collectReferenceLockIDs(filepath.Join(referenceBuilderDir, "data"))
	if err != nil {
		return nil, err
	}
	for key, rowID := range referenceIDs {
		if _, ok := result[key]; ok {
			continue
		}
		result[key] = rowID
	}
	return result, nil
}