sow-tools/internal/topdata/native.go

package topdata

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"

	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)

const nullValue = "****"

type nativeDataset struct {
	Kind             nativeDatasetKind
	Name             string
	RootPath         string
	BasePath         string
	LockPath         string
	ModulesDir       string
	GeneratedDir     string
	OutputName       string
	Spec             datasetTextSpec
	Columns          []string
	CompareReference bool
}

type nativeDatasetKind string

const (
	nativeDatasetBase  nativeDatasetKind = "base"
	nativeDatasetPlain nativeDatasetKind = "plain"
)

type nativeCollectedDataset struct {
	Dataset      nativeDataset
	Columns      []string
	Rows         []map[string]any
	LockData     map[string]int
	RetiredKeys  map[string]struct{}
	TableKey     string
	LockAdded    int
	LockPruned   int
	LockModified bool
}

type resolvedTable struct {
	Key         string
	DatasetName string
	OutputName  string
	OutputStem  string
	Columns     []string
	Rows        []map[string]any
	LockData    map[string]int
	Kind        nativeDatasetKind
}

type resolvedTableRegistry struct {
	stemByKey  map[string]string
	tableByKey map[string]resolvedTable
}

func newResolvedTableRegistry(collected []nativeCollectedDataset) (resolvedTableRegistry, error) {
	registry := resolvedTableRegistry{
		stemByKey:  map[string]string{},
		tableByKey: map[string]resolvedTable{},
	}
	for _, collectedDataset := range collected {
		if collectedDataset.TableKey == "" {
			continue
		}
		key := collectedDataset.TableKey
		if existing, ok := registry.tableByKey[key]; ok {
			return resolvedTableRegistry{}, fmt.Errorf(
				"duplicate table key %q for %s (%s) conflicts with %s (%s)",
				key,
				collectedDataset.Dataset.Name,
				collectedDataset.Dataset.OutputName,
				existing.DatasetName,
				existing.OutputName,
			)
		}
		rows := make([]map[string]any, 0, len(collectedDataset.Rows))
		for _, row := range collectedDataset.Rows {
			rows = append(rows, cloneRowMap(row))
		}
		lockData := map[string]int{}
		for lockKey, rowID := range collectedDataset.LockData {
			lockData[lockKey] = rowID
		}
		registryTable := resolvedTable{
			Key:         key,
			DatasetName: collectedDataset.Dataset.Name,
			OutputName:  collectedDataset.Dataset.OutputName,
			OutputStem:  outputStem(collectedDataset.Dataset.OutputName),
			Columns:     append([]string(nil), collectedDataset.Columns...),
			Rows:        rows,
			LockData:    lockData,
			Kind:        collectedDataset.Dataset.Kind,
		}
		registry.stemByKey[key] = registryTable.OutputStem
		registry.tableByKey[key] = registryTable
	}
	return registry, nil
}

func (r resolvedTableRegistry) resolveTableByKey(key string) (resolvedTable, bool) {
	table, ok := r.tableByKey[key]
	return table, ok
}

func cloneRowMap(row map[string]any) map[string]any {
	if row == nil {
		return nil
	}
	cloned := make(map[string]any, len(row))
	for key, value := range row {
		cloned[key] = cloneAuthoringValue(value)
	}
	return cloned
}

type legacyTLKData struct {
	Language string
	Entries  map[string]tlkEntryData
	Lock     map[string]int
}

func Build(p *project.Project, progress func(string)) (BuildResult, error) {
	return BuildNative(p, progress)
}

type NativeBuildOptions struct {
	BuildWiki bool
	Force     bool
}

type BuildWikiOptions struct {
	Force bool
}

func BuildWikiWithOptions(p *project.Project, opts BuildWikiOptions, progress func(string)) (wikiResult, error) {
	if progress == nil {
		progress = func(string) {}
	}
	if !p.HasTopData() {
		return wikiResult{}, errors.New("topdata is not configured for this project")
	}

	output2DA := compiled2DAOutputDir(p)
	outputTLK := compiledTLKOutputDir(p)
	if _, err := os.Stat(output2DA); os.IsNotExist(err) {
		return wikiResult{}, fmt.Errorf("topdata build output missing: run build-topdata first")
	}
	if _, err := os.Stat(outputTLK); os.IsNotExist(err) {
		return wikiResult{}, fmt.Errorf("topdata tlk output missing: run build-topdata first")
	}

	nativeResult := BuildResult{
		Mode:         "native",
		Output2DADir: output2DA,
		OutputTLKDir: outputTLK,
	}

	wikiBuild, err := buildWiki(p, nativeResult, opts.Force, progress)
	if err != nil {
		return wikiResult{}, err
	}

	return wikiBuild, nil
}

func BuildNative(p *project.Project, progress func(string)) (BuildResult, error) {
	return BuildNativeWithOptions(p, NativeBuildOptions{BuildWiki: true}, progress)
}

func BuildNativeWithOptions(p *project.Project, opts NativeBuildOptions, progress func(string)) (BuildResult, error) {
	if progress == nil {
		progress = func(string) {}
	}
	if !p.HasTopData() {
		return BuildResult{}, errors.New("topdata is not configured for this project")
	}

	report := ValidateProject(p)
	if report.HasErrors() {
		summary := report.SummaryLines(3)
		if len(summary) > 0 {
			return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s): %s", report.ErrorCount(), summary[0])
		}
		return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount())
	}
	if warnings := report.WarningCount(); warnings > 0 {
		progress(fmt.Sprintf("topdata validation warnings: %d", warnings))
		for _, line := range report.SummaryLines(5) {
			if strings.HasPrefix(line, "warning:") {
				progress(line)
			}
		}
	}
	var prebuiltWiki *wikiResult
	if opts.BuildWiki {
		if newestWikiSource, _, err := newestRelevantWikiSource(p.TopDataSourceDir()); err == nil && !newestWikiSource.IsZero() {
			statePath := wikiOutputStatePath(p)
			outputDir := wikiOutputPagesDir(p)
			if stateInfo, err := os.Stat(statePath); err == nil && !newestWikiSource.After(stateInfo.ModTime()) {
				if outputInfo, err := os.Stat(outputDir); err == nil && outputInfo.IsDir() {
					if state, err := loadWikiState(statePath); err == nil {
						prebuiltWiki = &wikiResult{
							OutputDir: outputDir,
							PageCount: state.Pages,
							Status:    wikiSkippedStatus,
							Digest:    state.Digest,
						}
					}
				}
			}
		}
	}
	return buildNativeUnchecked(p, opts, progress, prebuiltWiki)
}

func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress func(string), prebuiltWiki *wikiResult) (BuildResult, error) {
	if progress == nil {
		progress = func(string) {}
	}
	sourceDir := p.TopDataSourceDir()
	dataDir := filepath.Join(sourceDir, "data")
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		return BuildResult{}, err
	}
	registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
	if err != nil {
		return BuildResult{}, err
	}
	if len(datasets) == 0 && len(registryDatasets) == 0 {
		return BuildResult{}, fmt.Errorf("no canonical topdata datasets were found in topdata/data")
	}

	compiler, err := newTLKCompiler(sourceDir, nil)
	if err != nil {
		return BuildResult{}, err
	}

	output2DA := compiled2DAOutputDir(p)
	outputTLK := compiledTLKOutputDir(p)
	progress("Preparing native topdata build outputs...")
	if err := os.RemoveAll(output2DA); err != nil {
		return BuildResult{}, fmt.Errorf("clean topdata 2da output dir: %w", err)
	}
	if err := os.MkdirAll(output2DA, 0o755); err != nil {
		return BuildResult{}, fmt.Errorf("create 2da output dir: %w", err)
	}
	if err := os.MkdirAll(outputTLK, 0o755); err != nil {
		return BuildResult{}, fmt.Errorf("create tlk output dir: %w", err)
	}

	progress("Building native topdata datasets...")
	collected := make([]nativeCollectedDataset, 0, len(datasets)+len(registryDatasets))
	globalKeyToID := map[string]int{}
	globalRowByKey := map[string]map[string]any{}
	for _, dataset := range datasets {
		collectedDataset, err := collectNativeDataset(dataset)
		if err != nil {
			return BuildResult{}, err
		}
		collected = append(collected, collectedDataset)
		for key, rowID := range collectedDataset.LockData {
			globalKeyToID[key] = rowID
		}
		for _, row := range collectedDataset.Rows {
			if key, ok := row["key"].(string); ok && key != "" {
				globalKeyToID[key] = row["id"].(int)
				globalRowByKey[key] = row
			}
		}
	}
	for _, collectedDataset := range registryDatasets {
		collected = append(collected, collectedDataset)
		for key, rowID := range collectedDataset.LockData {
			globalKeyToID[key] = rowID
		}
		for _, row := range collectedDataset.Rows {
			if key, ok := row["key"].(string); ok && key != "" {
				globalKeyToID[key] = row["id"].(int)
				globalRowByKey[key] = row
			}
		}
	}
	collected, err = applyAutogenConsumers(p, collected, progress)
	if err != nil {
		return BuildResult{}, err
	}
	collected, err = applyPartOverrides(sourceDir, collected)
	if err != nil {
		return BuildResult{}, err
	}

	tableRegistry, err := newResolvedTableRegistry(collected)
	if err != nil {
		return BuildResult{}, err
	}
	collected, err = mergeExpansionData(collected)
	if err != nil {
		return BuildResult{}, err
	}
	lockAdded, lockPruned, err := saveNativeLockfiles(collected)
	if err != nil {
		return BuildResult{}, err
	}
	if lockAdded > 0 || lockPruned > 0 {
		progress(fmt.Sprintf("Updated native lockfiles (%d new IDs, %d stale IDs pruned)", lockAdded, lockPruned))
	}
	globalKeyToID = map[string]int{}
	globalRowByKey = map[string]map[string]any{}
	for _, collectedDataset := range collected {
		for key, rowID := range collectedDataset.LockData {
			globalKeyToID[key] = rowID
		}
		for _, row := range collectedDataset.Rows {
			if key, ok := row["key"].(string); ok && key != "" {
				globalKeyToID[key] = row["id"].(int)
				globalRowByKey[key] = row
			}
		}
	}
	tableRegistry, err = newResolvedTableRegistry(collected)
	if err != nil {
		return BuildResult{}, err
	}
	supplementalKeyToID, err := collectSupplementalLockIDs(sourceDir, "")
	if err != nil {
		return BuildResult{}, err
	}
	for key, rowID := range supplementalKeyToID {
		if _, ok := globalKeyToID[key]; ok {
			continue
		}
		globalKeyToID[key] = rowID
	}

	groupStats := nativeCompileGroupStats(collected)
	currentGroup := ""
	for _, dataset := range collected {
		group := nativeCompileGroup(dataset.Dataset.Name)
		if group != currentGroup {
			currentGroup = group
			stats := groupStats[group]
			progress(fmt.Sprintf(
				"Compiling %s datasets (%d output tables from %d source fragments)...",
				group,
				stats.OutputTables,
				stats.SourceFragments,
			))
		}
		compiled, err := resolveNativeDataset(dataset, globalKeyToID, globalRowByKey, tableRegistry, compiler)
		if err != nil {
			return BuildResult{}, err
		}
		if err := write2DA(compiled, filepath.Join(output2DA, dataset.Dataset.OutputName), dataset.Dataset.Kind == nativeDatasetBase); err != nil {
			return BuildResult{}, err
		}
	}

	filesTLK, err := compiler.finish(outputTLK, filepath.Base(p.TopDataCompiledTLKPath()))
	if err != nil {
		return BuildResult{}, err
	}
	files2DA, err := countFiles(output2DA)
	if err != nil {
		return BuildResult{}, err
	}

	wikiBuild := wikiResult{Status: wikiSkippedStatus}
	if prebuiltWiki != nil {
		wikiBuild = *prebuiltWiki
	} else if opts.BuildWiki {
		wikiBuild, err = buildWiki(p, BuildResult{
			Mode:         "native",
			Output2DADir: output2DA,
			OutputTLKDir: outputTLK,
			Files2DA:     files2DA,
			FilesTLK:     filesTLK,
		}, opts.Force, progress)
		if err != nil {
			return BuildResult{}, err
		}
	}

	return BuildResult{
		Mode:          "native",
		Output2DADir:  output2DA,
		OutputTLKDir:  outputTLK,
		Files2DA:      files2DA,
		FilesTLK:      filesTLK,
		WikiOutputDir: wikiBuild.OutputDir,
		WikiPages:     wikiBuild.PageCount,
		WikiStatus:    wikiBuild.Status,
	}, nil
}

func nativeCompileGroup(datasetName string) string {
	head, _, ok := strings.Cut(datasetName, "/")
	if !ok {
		return datasetName
	}
	switch head {
	case "itemprops", "racialtypes", "damagetypes", "parts", "classes":
		return head
	default:
		return head
	}
}

func saveNativeLockfiles(collected []nativeCollectedDataset) (int, int, error) {
	added := 0
	pruned := 0
	for _, dataset := range collected {
		if dataset.Dataset.LockPath == "" {
			continue
		}
		current, err := loadLockfile(dataset.Dataset.LockPath)
		if err != nil {
			return 0, 0, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
		}
		added += countAddedLockEntries(current, dataset.LockData)
		pruned += countAddedLockEntries(dataset.LockData, current)
		if lockDataEqual(current, dataset.LockData) {
			continue
		}
		if err := saveLockfile(dataset.Dataset.LockPath, dataset.LockData); err != nil {
			return 0, 0, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
		}
	}
	return added, pruned, nil
}

func countAddedLockEntries(before, after map[string]int) int {
	count := 0
	for key, afterID := range after {
		if beforeID, ok := before[key]; !ok || beforeID != afterID {
			count++
		}
	}
	return count
}

func lockDataEqual(a, b map[string]int) bool {
	if len(a) != len(b) {
		return false
	}
	for key, aID := range a {
		if bID, ok := b[key]; !ok || bID != aID {
			return false
		}
	}
	return true
}

type nativeCompileGroupStat struct {
	OutputTables    int
	SourceFragments int
}

func nativeCompileGroupStats(collected []nativeCollectedDataset) map[string]nativeCompileGroupStat {
	stats := make(map[string]nativeCompileGroupStat)
	for _, dataset := range collected {
		group := nativeCompileGroup(dataset.Dataset.Name)
		groupStat := stats[group]
		groupStat.OutputTables++
		groupStat.SourceFragments += nativeDatasetSourceFragments(dataset.Dataset)
		stats[group] = groupStat
	}
	return stats
}

func nativeDatasetSourceFragments(dataset nativeDataset) int {
	fragments := 1
	if dataset.Kind == nativeDatasetBase {
		fragments += countModuleFiles(dataset.ModulesDir)
		fragments += countModuleFiles(dataset.GeneratedDir)
	}
	return fragments
}

func countModuleFiles(dir string) int {
	if strings.TrimSpace(dir) == "" {
		return 0
	}
	paths, err := collectModulePaths(dir)
	if err != nil {
		return 0
	}
	return len(paths)
}

func discoverNativeDatasets(dataDir string) ([]nativeDataset, error) {
	var datasets []nativeDataset
	appendPlainDataset := func(rootPath, filePath, relName string) error {
		tableData, err := loadJSONObject(filePath)
		if err != nil {
			return err
		}
		outputName, _ := tableData["output"].(string)
		if strings.TrimSpace(outputName) == "" {
			outputName = nativeDatasetDefaultOutputName(filePath, nativeDatasetPlain)
		}
		datasets = append(datasets, nativeDataset{
			Kind:             nativeDatasetPlain,
			Name:             filepath.ToSlash(relName),
			RootPath:         rootPath,
			BasePath:         filePath,
			LockPath:         filepath.Join(rootPath, "lock.json"),
			OutputName:       outputName,
			Spec:             specForDataset(filepath.ToSlash(relName)),
			CompareReference: optionalBool(tableData["compare_reference"], true),
		})
		return nil
	}
	err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if !d.IsDir() {
			return nil
		}
		if d.Name() == "modules" || d.Name() == "generated" {
			return filepath.SkipDir
		}

		rel, err := filepath.Rel(dataDir, path)
		if err != nil {
			return err
		}
		if rel == "." {
			entries, err := os.ReadDir(path)
			if err != nil {
				return err
			}
			for _, entry := range entries {
				if entry.IsDir() {
					continue
				}
				name := entry.Name()
				if !strings.HasSuffix(strings.ToLower(name), ".json") || name == "lock.json" || name == "base.json" {
					continue
				}
				filePath := filepath.Join(path, name)
				if err := appendPlainDataset(path, filePath, strings.TrimSuffix(name, filepath.Ext(name))); err != nil {
					return err
				}
			}
			return nil
		}
		if filepath.ToSlash(rel) == "parts/overrides" {
			return filepath.SkipDir
		}
		if filepath.ToSlash(rel) == itempropsRegistryDirName || filepath.ToSlash(rel) == damagetypesRegistryDirName || filepath.ToSlash(rel) == racialtypesRegistryDirName {
			return filepath.SkipDir
		}

		basePath := filepath.Join(path, "base.json")
		if _, err := os.Stat(basePath); err == nil {
			baseData, err := loadJSONObject(basePath)
			if err != nil {
				return err
			}
			outputName, _ := baseData["output"].(string)
			if strings.TrimSpace(outputName) == "" {
				outputName = nativeDatasetDefaultOutputName(path, nativeDatasetBase)
			}
			datasets = append(datasets, nativeDataset{
				Kind:             nativeDatasetBase,
				Name:             filepath.ToSlash(rel),
				RootPath:         path,
				BasePath:         basePath,
				LockPath:         filepath.Join(path, "lock.json"),
				ModulesDir:       filepath.Join(path, "modules"),
				GeneratedDir:     filepath.Join(path, "generated"),
				OutputName:       outputName,
				Spec:             specForDataset(filepath.ToSlash(rel)),
				CompareReference: optionalBool(baseData["compare_reference"], true),
			})
			return nil
		}

		entries, err := os.ReadDir(path)
		if err != nil {
			return err
		}
		for _, entry := range entries {
			if entry.IsDir() {
				continue
			}
			name := entry.Name()
			if strings.HasSuffix(strings.ToLower(name), ".json") && name != "lock.json" {
				filePath := filepath.Join(path, name)
				if err := appendPlainDataset(path, filePath, filepath.Join(rel, strings.TrimSuffix(name, filepath.Ext(name)))); err != nil {
					return err
				}
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	slices.SortFunc(datasets, func(a, b nativeDataset) int {
		return strings.Compare(a.Name, b.Name)
	})
	return datasets, nil
}

func discoverNativeOutputCatalog(dataDir string) (map[string]string, error) {
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		return nil, err
	}
	registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
	if err != nil {
		return nil, err
	}
	catalog := make(map[string]string, len(datasets)+len(registryDatasets))
	for _, dataset := range datasets {
		if !dataset.CompareReference {
			continue
		}
		catalog[dataset.OutputName] = dataset.Name
	}
	for _, dataset := range registryDatasets {
		catalog[dataset.Dataset.OutputName] = dataset.Dataset.Name
	}
	return catalog, nil
}

func missingNativeOutputs(projectOutputs, referenceOutputs map[string]string) []string {
	missing := make([]string, 0)
	for outputName, datasetName := range referenceOutputs {
		if _, ok := projectOutputs[outputName]; ok {
			continue
		}
		missing = append(missing, fmt.Sprintf("%s (%s)", outputName, datasetName))
	}
	slices.Sort(missing)
	return missing
}

func collectNativeDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
	switch dataset.Kind {
	case nativeDatasetBase:
		return collectBaseDataset(dataset)
	case nativeDatasetPlain:
		return collectPlainDataset(dataset)
	default:
		return nativeCollectedDataset{}, fmt.Errorf("unsupported native dataset kind %q", dataset.Kind)
	}
}

func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
	baseData, err := loadJSONObject(dataset.BasePath)
	if err != nil {
		return nativeCollectedDataset{}, err
	}

	columns, err := parseColumns(baseData, dataset.Name)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	modulePaths, err := collectModulePaths(dataset.ModulesDir)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	moduleData := make([]nativeGeneratedModule, 0, len(modulePaths))
	for _, path := range modulePaths {
		obj, err := loadJSONObject(path)
		if err != nil {
			return nativeCollectedDataset{}, err
		}
		moduleData = append(moduleData, nativeGeneratedModule{
			Name: path,
			Data: obj,
		})
	}
	for _, module := range moduleData {
		columns, err = extendColumns(columns, module.Data, dataset.Name, module.Name)
		if err != nil {
			return nativeCollectedDataset{}, err
		}
	}
	rawBaseRows, ok := baseData["rows"].([]any)
	if !ok {
		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: rows must be an array", dataset.Name)
	}
	baseBoundaryID := -1
	baseRowKeys := map[string]struct{}{}
	baseRowKeyCounts := map[string]int{}
	for index, raw := range rawBaseRows {
		rowObj, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		rowID := index
		if rawID, ok := rowObj["id"]; ok {
			parsed, err := asInt(rawID)
			if err == nil {
				rowID = parsed
			}
		}
		baseBoundaryID = rowID
		key, ok := rowObj["key"].(string)
		if !ok || key == "" {
			continue
		}
		baseRowKeys[key] = struct{}{}
		baseRowKeyCounts[key]++
	}

	lockData, err := loadLockfile(dataset.LockPath)
	if err != nil {
		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
	}

	explicitModuleIDs := map[string]int{}
	for _, module := range moduleData {
		explicitIDs, err := collectExplicitModuleIDs(dataset.Name, module.Name, module.Data)
		if err != nil {
			return nativeCollectedDataset{}, err
		}
		for key, rowID := range explicitIDs {
			explicitModuleIDs[key] = rowID
		}
	}

	rows := make([]map[string]any, 0, len(rawBaseRows))
	rowByID := map[int]map[string]any{}
	rowByKey := map[string]map[string]any{}
	usedIDs := map[int]struct{}{}
	lockModified := false
	lockAdded := 0
	lockPruned := 0
	retiredKeys := map[string]struct{}{}
	featFamilyAliases := generatedFamilyAliases{}
	if dataset.Name == "feat" {
		var err error
		featFamilyAliases, err = collectGeneratedFamilyAliases(dataset.GeneratedDir)
		if err != nil {
			return nativeCollectedDataset{}, err
		}
	}

	for key, rowID := range lockData {
		if rowID <= baseBoundaryID {
			if _, ok := baseRowKeys[key]; ok {
				continue
			}
			if explicitID, ok := explicitModuleIDs[key]; ok && explicitID == rowID {
				continue
			}
			delete(lockData, key)
			lockModified = true
			lockPruned++
		}
	}
	for _, rowID := range lockData {
		usedIDs[rowID] = struct{}{}
	}
	for index, raw := range rawBaseRows {
		rowObj, ok := raw.(map[string]any)
		if !ok {
			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d must be an object", dataset.Name, index)
		}
		row, err := canonicalizeBaseRow(dataset, columns, rowObj, index)
		if err != nil {
			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
		}
		rows = append(rows, row)
		rowID := row["id"].(int)
		if key, ok := row["key"].(string); ok && key != "" {
			if baseRowKeyCounts[key] > 1 {
				if lockedID, ok := lockData[key]; !ok || lockedID != rowID {
					lockData[key] = rowID
					lockModified = true
				}
			} else if _, ok := lockData[key]; !ok {
				lockData[key] = rowID
				lockModified = true
				lockAdded++
			}
		}
		rowByID[rowID] = row
		if key, ok := row["key"].(string); ok && key != "" {
			rowByKey[key] = row
		}
		usedIDs[rowID] = struct{}{}
	}
	for rowID := 0; rowID <= baseBoundaryID; rowID++ {
		usedIDs[rowID] = struct{}{}
	}
	for _, rowID := range explicitModuleIDs {
		usedIDs[rowID] = struct{}{}
	}

	nextID := nextAvailableID(usedIDs)
	allocateNextID := func() int {
		rowID := nextID
		usedIDs[rowID] = struct{}{}
		nextID = nextAvailableID(usedIDs)
		return rowID
	}
	lockedIDForKey := func(key string) (int, bool) {
		lockedID, ok := lockData[key]
		if !ok {
			return 0, false
		}
		return lockedID, true
	}
	stripMismatchedLockedOverrideID := func(override map[string]any) (map[string]any, error) {
		rawKey, hasKey := override["key"].(string)
		if !hasKey || rawKey == "" {
			return override, nil
		}
		lockedID, hasLockedID := lockData[rawKey]
		if !hasLockedID {
			return override, nil
		}
		rawID, hasID := override["id"]
		if !hasID {
			return override, nil
		}
		parsedID, err := asInt(rawID)
		if err != nil {
			return nil, fmt.Errorf("dataset %s: override id %v is not numeric", dataset.Name, rawID)
		}
		if parsedID == lockedID {
			return override, nil
		}
		if dataset.Name == "feat" && generatedCanonicalLockCanMove(rawKey, parsedID, lockData, featFamilyAliases) {
			delete(lockData, rawKey)
			lockModified = true
			return override, nil
		}
		return override, nil
	}
	validExplicitIDForNewKey := func(key string, rowID int) bool {
		if key == "" {
			return true
		}
		for lockedKey, lockedID := range lockData {
			if lockedKey != key && lockedID == rowID {
				if dataset.Name == "feat" && generatedAliasLockForKey(key, lockedKey, featFamilyAliases) {
					delete(lockData, lockedKey)
					lockModified = true
					continue
				}
				return false
			}
		}
		return true
	}
	applyModuleData := func(sourceLabel string, moduleData map[string]any) error {
		if rawEntries, ok := moduleData["entries"]; ok {
			entries, ok := rawEntries.(map[string]any)
			if !ok {
				return fmt.Errorf("dataset %s: entries in %s must be an object", dataset.Name, sourceLabel)
			}
			for _, key := range sortedKeys(entries) {
				rawEntry, ok := entries[key].(map[string]any)
				if !ok {
					return fmt.Errorf("dataset %s: entry %q in %s must be an object", dataset.Name, key, sourceLabel)
				}
				if existing, ok := rowByKey[key]; ok {
					candidate := map[string]any{}
					for field, value := range existing {
						candidate[field] = value
					}
					for field, value := range rawEntry {
						if field == "id" || field == "key" || field == "_tlk" {
							continue
						}
						if isMetadataField(field) {
							meta, err := parseRowMetadata(value)
							if err != nil {
								return fmt.Errorf("dataset %s: %w", dataset.Name, err)
							}
							if meta != nil {
								existingMeta, _ := parseExistingMetadata(candidate)
								candidate["meta"] = mergeMetadataMaps(existingMeta, meta)
							}
							continue
						}
						if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
							if err := setLegacyWikiMetadata(candidate, field, value); err != nil {
								return fmt.Errorf("dataset %s: %w", dataset.Name, err)
							}
							continue
						}
						columnName, ok := canonicalColumn(columns, field)
						if !ok {
							return fmt.Errorf("dataset %s: entry field %q is not a known column", dataset.Name, field)
						}
						candidate[columnName] = value
					}
					expanded, err := expandRowAuthoringSugar(dataset, columns, rawEntry, candidate)
					if err != nil {
						return err
					}
					keyChanged, retiredKey, err := updateOverrideRowKey(dataset.Name, existing, expanded, rowByKey, lockData)
					if err != nil {
						return err
					}
					if keyChanged {
						lockModified = true
					}
					if retiredKey != "" {
						retiredKeys[retiredKey] = struct{}{}
					}
					for field, value := range expanded {
						if field == "id" || field == "_tlk" {
							continue
						}
						existing[field] = value
					}
					continue
				}
				rowID, ok := lockedIDForKey(key)
				if !ok {
					rowID = allocateNextID()
					lockData[key] = rowID
					lockModified = true
					lockAdded++
				}
				row, err := canonicalizeEntry(dataset, columns, rowID, key, rawEntry)
				if err != nil {
					return fmt.Errorf("dataset %s: %w", dataset.Name, err)
				}
				rows = append(rows, row)
				rowByID[rowID] = row
				rowByKey[key] = row
			}
		}

		if rawOverrides, ok := moduleData["overrides"]; ok {
			overrideList, ok := rawOverrides.([]any)
			if !ok {
				return fmt.Errorf("dataset %s: overrides in %s must be an array", dataset.Name, sourceLabel)
			}
			for index, rawOverride := range overrideList {
				override, ok := rawOverride.(map[string]any)
				if !ok {
					return fmt.Errorf("dataset %s: override %d in %s must be an object", dataset.Name, index, sourceLabel)
				}
				override, err = stripMismatchedLockedOverrideID(override)
				if err != nil {
					return err
				}
				row, err := resolveOverrideTarget(dataset.Name, override, rowByID, rowByKey)
				if err != nil {
					rawKey, hasKey := override["key"].(string)
					rowID := 0
					hasRowID := false
					if rawID, hasID := override["id"]; hasID {
						parsedID, parseErr := asInt(rawID)
						if parseErr != nil {
							return fmt.Errorf("dataset %s: override id %v is not numeric", dataset.Name, rawID)
						}
						rawKey, _ := override["key"].(string)
						_, lockedIDExists := lockData[rawKey]
						if !lockedIDExists {
							if !validExplicitIDForNewKey(rawKey, parsedID) {
								return fmt.Errorf("dataset %s: override key %q id %d collides with an existing lockfile id", dataset.Name, rawKey, parsedID)
							}
							rowID = parsedID
							hasRowID = true
						}
					}
					if hasKey && rawKey != "" {
						if lockedID, hasLockedID := lockedIDForKey(rawKey); hasLockedID {
							rowID = lockedID
							hasRowID = true
						} else {
							if !hasRowID {
								rowID = allocateNextID()
								hasRowID = true
							}
							lockData[rawKey] = rowID
							lockModified = true
							lockAdded++
						}
					}
					if !hasRowID {
						return err
					}
					usedIDs[rowID] = struct{}{}
					row = map[string]any{"id": rowID}
					if hasKey && rawKey != "" {
						row["key"] = rawKey
					}
					rows = append(rows, row)
					rowByID[rowID] = row
					if hasKey && rawKey != "" {
						rowByKey[rawKey] = row
					}
				}
				if overrideRequestsNullRow(override) {
					retireRowIdentity(row, lockData, retiredKeys)
					nullifyOverrideRow(row, columns, rowByKey)
					continue
				}
				candidate := map[string]any{}
				for field, value := range row {
					candidate[field] = value
				}
				for field, value := range override {
					if field == "id" || field == "key" || field == "_tlk" {
						continue
					}
					if isMetadataField(field) {
						meta, err := parseRowMetadata(value)
						if err != nil {
							return fmt.Errorf("dataset %s: %w", dataset.Name, err)
						}
						if meta != nil {
							existingMeta, _ := parseExistingMetadata(candidate)
							candidate["meta"] = mergeMetadataMaps(existingMeta, meta)
						}
						continue
					}
					if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
						if err := setLegacyWikiMetadata(candidate, field, value); err != nil {
							return fmt.Errorf("dataset %s: %w", dataset.Name, err)
						}
						continue
					}
					columnName, ok := canonicalColumn(columns, field)
					if !ok {
						if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
							continue
						}
						return fmt.Errorf("dataset %s: override field %q is not a known column", dataset.Name, field)
					}
					candidate[columnName] = value
				}
				if rawKey, ok := override["key"]; ok {
					candidate["key"] = rawKey
				}
				expanded, err := expandRowAuthoringSugar(dataset, columns, override, candidate)
				if err != nil {
					return err
				}
				keyChanged, retiredKey, err := updateOverrideRowKey(dataset.Name, row, expanded, rowByKey, lockData)
				if err != nil {
					return err
				}
				if keyChanged {
					lockModified = true
				}
				if retiredKey != "" {
					retiredKeys[retiredKey] = struct{}{}
				}
				for field, value := range expanded {
					if field == "id" || field == "_tlk" {
						continue
					}
					row[field] = value
				}
			}
		}
		return nil
	}

	generatedModules, err := collectGeneratedModules(dataset, lockData)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	for _, module := range generatedModules {
		updatedColumns, err := extendColumns(columns, module.Data, dataset.Name, module.Name)
		if err != nil {
			return nativeCollectedDataset{}, err
		}
		if len(updatedColumns) != len(columns) {
			columns = updatedColumns
			ensureRowsExposeColumns(rows, columns)
		}
	}
	for _, module := range generatedModules {
		if module.ApplyAfterModules {
			continue
		}
		if err := applyModuleData(module.Name, module.Data); err != nil {
			return nativeCollectedDataset{}, err
		}
	}

	for _, module := range moduleData {
		if err := applyModuleData(module.Name, module.Data); err != nil {
			return nativeCollectedDataset{}, err
		}
	}
	for _, module := range generatedModules {
		if !module.ApplyAfterModules {
			continue
		}
		if err := applyModuleData(module.Name, module.Data); err != nil {
			return nativeCollectedDataset{}, err
		}
	}
	if dataset.Name == "spells" {
		normalizeCollectedSpellImpactScripts(rows, baseRowKeys)
	}
	referencedKeys, err := collectReferencedLockKeys(nativeDatasetSourceDir(dataset))
	if err != nil {
		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: collect referenced keys: %w", dataset.Name, err)
	}
	if pruned, updated := pruneLockDataToActiveRows(lockData, rows, referencedKeys, retiredKeys); pruned > 0 || updated > 0 {
		lockModified = true
		lockPruned += pruned
	}

	rows = dedupeCollectedRows(rows)
	slices.SortFunc(rows, func(a, b map[string]any) int {
		return a["id"].(int) - b["id"].(int)
	})
	return nativeCollectedDataset{
		Dataset:      dataset,
		Columns:      columns,
		Rows:         rows,
		LockData:     lockData,
		RetiredKeys:  retiredKeys,
		LockAdded:    lockAdded,
		LockPruned:   lockPruned,
		LockModified: lockModified,
	}, nil
}

func pruneLockDataToActiveRows(lockData map[string]int, rows []map[string]any, referencedKeys map[string]struct{}, retiredKeys map[string]struct{}) (int, int) {
	if len(lockData) == 0 {
		return 0, 0
	}
	activeKeys := make(map[string]int, len(rows))
	for _, row := range rows {
		if key, ok := row["key"].(string); ok && key != "" {
			rowID, _ := row["id"].(int)
			activeKeys[key] = rowID
		}
	}
	updated := 0
	for key, rowID := range activeKeys {
		if lockedID, ok := lockData[key]; !ok || lockedID != rowID {
			lockData[key] = rowID
			updated++
		}
	}
	pruned := 0
	for key := range lockData {
		if _, ok := activeKeys[key]; ok {
			continue
		}
		if _, retired := retiredKeys[key]; retired {
			delete(lockData, key)
			pruned++
			continue
		}
		if _, ok := referencedKeys[key]; ok {
			continue
		}
		delete(lockData, key)
		pruned++
	}
	return pruned, updated
}

func collectReferencedLockKeys(sourceDir string) (map[string]struct{}, error) {
	dataDir := filepath.Join(sourceDir, "data")
	referenced := map[string]struct{}{}
	if _, err := os.Stat(dataDir); err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return referenced, nil
		}
		return nil, err
	}
	err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		if filepath.Base(path) == "lock.json" || !strings.EqualFold(filepath.Ext(path), ".json") {
			return nil
		}
		obj, err := loadJSONObject(path)
		if err != nil {
			return err
		}
		collectReferencedLockKeysFromValue(obj, referenced)
		return nil
	})
	return referenced, err
}

func nativeDatasetSourceDir(dataset nativeDataset) string {
	dir := filepath.Clean(dataset.RootPath)
	for {
		if filepath.Base(dir) == "data" {
			return filepath.Dir(dir)
		}
		parent := filepath.Dir(dir)
		if parent == dir {
			return filepath.Dir(filepath.Dir(dataset.RootPath))
		}
		dir = parent
	}
}

func collectReferencedLockKeysFromValue(value any, referenced map[string]struct{}) {
	collectReferencedLockKeysFromValueWithExpansion(value, referenced, false)
}

func collectReferencedLockKeysFromValueWithExpansion(value any, referenced map[string]struct{}, inExpansionSubtree bool) {
	switch typed := value.(type) {
	case string:
		if strings.Contains(typed, ":") {
			referenced[typed] = struct{}{}
		}
	case []any:
		for _, item := range typed {
			collectReferencedLockKeysFromValueWithExpansion(item, referenced, inExpansionSubtree)
		}
	case map[string]any:
		_, hasValue := typed["value"]
		_, hasData := typed["data"]
		isExpansion := hasValue && hasData

		for key, item := range typed {
			if strings.Contains(key, ":") {
				referenced[key] = struct{}{}
			}
			if key == "key" {
				if inExpansionSubtree {
					if s, ok := item.(string); ok && strings.Contains(s, ":") {
						referenced[s] = struct{}{}
					}
				}
				continue
			}
			collectReferencedLockKeysFromValueWithExpansion(item, referenced, inExpansionSubtree || isExpansion)
		}
	}
}

func dedupeCollectedRows(rows []map[string]any) []map[string]any {
	if len(rows) <= 1 {
		return rows
	}
	seen := make(map[string]int, len(rows))
	out := make([]map[string]any, 0, len(rows))
	for _, row := range rows {
		rowID, _ := row["id"].(int)
		rowKey, _ := row["key"].(string)
		if rowKey == "" {
			out = append(out, row)
			continue
		}
		identity := strconv.Itoa(rowID) + "\x00" + rowKey
		if index, ok := seen[identity]; ok {
			out[index] = row
			continue
		}
		seen[identity] = len(out)
		out = append(out, row)
	}
	return out
}

type nativeGeneratedModule struct {
	Name              string
	Data              map[string]any
	ApplyAfterModules bool
}

func collectGeneratedModules(dataset nativeDataset, lockData map[string]int) ([]nativeGeneratedModule, error) {
	if dataset.Name != "feat" {
		return nil, nil
	}
	ctx, err := newFeatGeneratedContext(dataset, lockData)
	if err != nil {
		return nil, err
	}
	paths, err := collectModulePaths(dataset.GeneratedDir)
	if err != nil {
		return nil, err
	}
	type generatedSource struct {
		path string
		obj  map[string]any
	}
	sources := make([]generatedSource, 0, len(paths))
	for _, path := range paths {
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, err
		}
		if isFamilyExpansionObject(obj) {
			spec, err := parseFamilyExpansionSpec(path, obj)
			if err != nil {
				return nil, err
			}
			ctx.familySpecs[spec.FamilyKey] = spec
		}
		sources = append(sources, generatedSource{path: path, obj: obj})
	}
	modules := make([]nativeGeneratedModule, 0, len(paths))
	for _, source := range sources {
		applyAfterModules := false
		if isFamilyExpansionObject(source.obj) {
			spec, err := parseFamilyExpansionSpec(source.path, source.obj)
			if err != nil {
				return nil, err
			}
			applyAfterModules = spec.ApplyAfterModules
		}
		moduleData, err := buildFeatGeneratedModule(source.path, source.obj, ctx)
		if err != nil {
			return nil, err
		}
		modules = append(modules, nativeGeneratedModule{
			Name:              source.path,
			Data:              moduleData,
			ApplyAfterModules: applyAfterModules,
		})
	}
	racialtypesModule, err := buildRacialtypesSkillAffinityModule(ctx)
	if err != nil {
		return nil, err
	}
	if racialtypesModule != nil {
		modules = append(modules, nativeGeneratedModule{
			Name: filepath.Join(dataset.RootPath, "generated", "skill_affinity<racialtypes>"),
			Data: racialtypesModule,
		})
	}
	return modules, nil
}

type featGeneratedContext struct {
	sourceDir      string
	dataDir        string
	lockData       map[string]int
	supplementalID map[string]int
	baseDialog     *legacyTLKData
	tlkStateKeys   map[string]struct{}
	existingFeat   map[string]struct{}
	datasets       map[string]nativeDataset
	rowsByDataset  map[string]map[string]map[string]any
	familySpecs    map[string]familyExpansionSpec
}

type compactSourceOverride = map[string]map[string]any

type affinityGrant struct {
	featKey string
	races   []string
}

func newFeatGeneratedContext(dataset nativeDataset, lockData map[string]int) (*featGeneratedContext, error) {
	sourceDir := filepath.Dir(filepath.Dir(dataset.RootPath))
	dataDir := filepath.Join(sourceDir, "data")
	allDatasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		return nil, err
	}
	datasetMap := make(map[string]nativeDataset, len(allDatasets))
	for _, item := range allDatasets {
		datasetMap[item.Name] = item
	}
	supplementalID, err := collectSupplementalLockIDs(sourceDir, "")
	if err != nil {
		return nil, err
	}
	baseDialog, err := loadBaseDialogData(filepath.Join(sourceDir, "base_dialog.json"))
	if err != nil {
		return nil, err
	}
	tlkState, err := loadTLKState(filepath.Join(sourceDir, tlkStateFile))
	if err != nil {
		return nil, err
	}
	tlkStateKeys := make(map[string]struct{}, len(tlkState.Entries))
	for key := range tlkState.Entries {
		tlkStateKeys[key] = struct{}{}
	}
	lockCopy := make(map[string]int, len(lockData))
	for key, value := range lockData {
		lockCopy[key] = value
	}
	existingFeat, err := collectExistingFeatKeys(filepath.Join(dataDir, "feat"))
	if err != nil {
		return nil, err
	}
	retiredFeatKeys, err := collectRetiredFeatKeys(filepath.Join(dataDir, "feat"))
	if err != nil {
		return nil, err
	}
	for key, retiredID := range retiredFeatKeys {
		if lockedID, ok := lockCopy[key]; ok && lockedID == retiredID {
			delete(lockCopy, key)
		}
		delete(existingFeat, key)
	}
	return &featGeneratedContext{
		sourceDir:      sourceDir,
		dataDir:        dataDir,
		lockData:       lockCopy,
		supplementalID: supplementalID,
		baseDialog:     baseDialog,
		tlkStateKeys:   tlkStateKeys,
		existingFeat:   existingFeat,
		datasets:       datasetMap,
		rowsByDataset:  map[string]map[string]map[string]any{},
		familySpecs:    map[string]familyExpansionSpec{},
	}, nil
}

func collectExistingFeatKeys(featDir string) (map[string]struct{}, error) {
	keys := map[string]struct{}{}
	baseObj, err := loadJSONObject(filepath.Join(featDir, "base.json"))
	if err != nil {
		return nil, err
	}
	if rawRows, ok := baseObj["rows"].([]any); ok {
		for _, raw := range rawRows {
			row, ok := raw.(map[string]any)
			if !ok {
				continue
			}
			if key, ok := row["key"].(string); ok && key != "" {
				keys[key] = struct{}{}
			}
		}
	}
	modulePaths, err := collectModulePaths(filepath.Join(featDir, "modules"))
	if err != nil {
		return nil, err
	}
	for _, path := range modulePaths {
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, err
		}
		if rawEntries, ok := obj["entries"].(map[string]any); ok {
			for _, key := range sortedKeys(rawEntries) {
				if key != "" {
					keys[key] = struct{}{}
				}
			}
		}
		if rawOverrides, ok := obj["overrides"].([]any); ok {
			for _, raw := range rawOverrides {
				override, ok := raw.(map[string]any)
				if !ok {
					continue
				}
				if key, ok := override["key"].(string); ok && key != "" {
					keys[key] = struct{}{}
				}
			}
		}
	}
	return keys, nil
}

func (c *featGeneratedContext) featKeyExists(key string) bool {
	_, ok := c.existingFeat[key]
	return ok
}

func collectRetiredFeatKeys(featDir string) (map[string]int, error) {
	baseObj, err := loadJSONObject(filepath.Join(featDir, "base.json"))
	if err != nil {
		return nil, err
	}
	rowIDToKey := map[int]string{}
	rowKeyToID := map[string]int{}
	usedIDs := map[int]struct{}{}
	if rawRows, ok := baseObj["rows"].([]any); ok {
		for index, raw := range rawRows {
			row, ok := raw.(map[string]any)
			if !ok {
				continue
			}
			rowID := index
			if rawID, ok := row["id"]; ok {
				parsed, err := asInt(rawID)
				if err == nil {
					rowID = parsed
				}
			}
			usedIDs[rowID] = struct{}{}
			if key, ok := row["key"].(string); ok && key != "" {
				rowIDToKey[rowID] = key
				rowKeyToID[key] = rowID
			}
		}
	}
	nextID := nextAvailableID(usedIDs)
	allocateNextID := func() int {
		rowID := nextID
		usedIDs[rowID] = struct{}{}
		nextID = nextAvailableID(usedIDs)
		return rowID
	}
	retired := map[string]int{}
	modulePaths, err := collectModulePaths(filepath.Join(featDir, "modules"))
	if err != nil {
		return nil, err
	}
	for _, path := range modulePaths {
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, err
		}
		if rawEntries, ok := obj["entries"].(map[string]any); ok {
			for _, key := range sortedKeys(rawEntries) {
				rawEntry := rawEntries[key]
				entry, ok := rawEntry.(map[string]any)
				if !ok || key == "" {
					continue
				}
				if _, exists := rowKeyToID[key]; exists {
					continue
				}
				rowID := 0
				if rawID, ok := entry["id"]; ok {
					parsed, err := asInt(rawID)
					if err != nil {
						return nil, fmt.Errorf("entry %q in %s has non-numeric id %v", key, path, rawID)
					}
					rowID = parsed
					usedIDs[rowID] = struct{}{}
					nextID = nextAvailableID(usedIDs)
				} else {
					rowID = allocateNextID()
				}
				rowIDToKey[rowID] = key
				rowKeyToID[key] = rowID
			}
		}
		rawOverrides, ok := obj["overrides"].([]any)
		if !ok {
			continue
		}
		for index, rawOverride := range rawOverrides {
			override, ok := rawOverride.(map[string]any)
			if !ok {
				return nil, fmt.Errorf("override %d in %s must be an object", index, path)
			}
			rowID := 0
			hasRowID := false
			if rawID, ok := override["id"]; ok {
				parsed, err := asInt(rawID)
				if err != nil {
					return nil, fmt.Errorf("override %d in %s has non-numeric id %v", index, path, rawID)
				}
				rowID = parsed
				hasRowID = true
			} else if key, ok := override["key"].(string); ok && key != "" {
				if mappedID, exists := rowKeyToID[key]; exists {
					rowID = mappedID
					hasRowID = true
				}
			}
			if !hasRowID {
				continue
			}
			currentKey := rowIDToKey[rowID]
			if overrideRequestsNullRow(override) {
				if currentKey != "" {
					retired[currentKey] = rowID
					delete(rowKeyToID, currentKey)
					delete(rowIDToKey, rowID)
				}
				continue
			}
			if rawKey, present := override["key"]; present {
				if rawKey == nil {
					if currentKey != "" {
						retired[currentKey] = rowID
						delete(rowKeyToID, currentKey)
						delete(rowIDToKey, rowID)
					}
					continue
				}
				newKey, ok := rawKey.(string)
				if !ok || newKey == "" || strings.TrimSpace(newKey) == nullValue {
					if currentKey != "" {
						retired[currentKey] = rowID
						delete(rowKeyToID, currentKey)
						delete(rowIDToKey, rowID)
					}
					continue
				}
				if currentKey != "" && currentKey != newKey {
					retired[currentKey] = rowID
					delete(rowKeyToID, currentKey)
				}
				if previousID, exists := rowKeyToID[newKey]; exists && previousID != rowID {
					delete(rowIDToKey, previousID)
				}
				rowKeyToID[newKey] = rowID
				rowIDToKey[rowID] = newKey
			}
		}
	}
	return retired, nil
}

func (c *featGeneratedContext) familyHasExistingRows(familyKey string) bool {
	prefix := "feat:" + familyKey + "_"
	for key := range c.existingFeat {
		if strings.HasPrefix(key, prefix) {
			return true
		}
	}
	return false
}

func (c *featGeneratedContext) datasetRows(name string) (map[string]map[string]any, error) {
	if rows, ok := c.rowsByDataset[name]; ok {
		return rows, nil
	}
	dataset, ok := c.datasets[name]
	if !ok {
		return nil, fmt.Errorf("generated feat family requires canonical dataset %q", name)
	}
	collected, err := collectNativeDataset(dataset)
	if err != nil {
		return nil, err
	}
	rows := make(map[string]map[string]any, len(collected.Rows))
	for _, row := range collected.Rows {
		if key, ok := row["key"].(string); ok && key != "" {
			rows[key] = row
		}
	}
	c.rowsByDataset[name] = rows
	return rows, nil
}

func (c *featGeneratedContext) featID(key string, explicit any) (int, bool, error) {
	if explicit != nil && explicit != nullValue {
		parsed, err := featSourceID(explicit)
		if err != nil {
			return 0, false, err
		}
		if parsed > 0 {
			return parsed, true, nil
		}
	}
	if value, ok := c.lockData[key]; ok && value >= 0 {
		return value, true, nil
	}
	if value, ok := c.supplementalID[key]; ok && value > 0 {
		return value, true, nil
	}
	return 0, false, nil
}

func (c *featGeneratedContext) preferredGeneratedFeatKey(familyKey, sourceSlug string) string {
	exact := "feat:" + familyKey + "_" + sourceSlug
	target := normalizeKeyIdentity(sourceSlug)
	for _, candidates := range []map[string]int{c.lockData, c.supplementalID} {
		matches := make([]string, 0)
		for key, value := range candidates {
			if value < 0 {
				continue
			}
			if !strings.HasPrefix(key, "feat:"+familyKey+"_") {
				continue
			}
			suffix := strings.TrimPrefix(key, "feat:"+familyKey+"_")
			if normalizeKeyIdentity(suffix) == target {
				matches = append(matches, key)
			}
		}
		if best, ok := preferredExpandedKey(matches); ok {
			return best
		}
	}
	for _, candidates := range []map[string]int{c.lockData, c.supplementalID} {
		if value, ok := candidates[exact]; ok && value > 0 {
			return exact
		}
	}
	stateMatches := make([]string, 0)
	for key := range c.tlkStateKeys {
		if !strings.HasPrefix(key, "feat:"+familyKey+"_") {
			continue
		}
		suffix := strings.TrimPrefix(key, "feat:"+familyKey+"_")
		suffix = strings.TrimSuffix(strings.TrimSuffix(suffix, ".name"), ".description")
		if normalizeKeyIdentity(suffix) == target {
			stateMatches = append(stateMatches, "feat:"+familyKey+"_"+suffix)
		}
	}
	if best, ok := preferredExpandedKey(stateMatches); ok {
		return best
	}
	return exact
}

func preferredExpandedKey(matches []string) (string, bool) {
	if len(matches) == 0 {
		return "", false
	}
	best := matches[0]
	for _, candidate := range matches[1:] {
		best = betterExpandedKey(best, candidate)
	}
	return best, true
}

func betterExpandedKey(a, b string) string {
	aUnderscores := strings.Count(a, "_")
	bUnderscores := strings.Count(b, "_")
	if aUnderscores != bUnderscores {
		if aUnderscores < bUnderscores {
			return a
		}
		return b
	}
	if len(a) != len(b) {
		if len(a) < len(b) {
			return a
		}
		return b
	}
	if a <= b {
		return a
	}
	return b
}

func collectExplicitModuleIDs(datasetName, sourceLabel string, moduleData map[string]any) (map[string]int, error) {
	ids := map[string]int{}
	if rawEntries, ok := moduleData["entries"]; ok {
		entries, ok := rawEntries.(map[string]any)
		if !ok {
			return nil, fmt.Errorf("dataset %s: entries in %s must be an object", datasetName, sourceLabel)
		}
		for key, rawEntry := range entries {
			entry, ok := rawEntry.(map[string]any)
			if !ok {
				return nil, fmt.Errorf("dataset %s: entry %q in %s must be an object", datasetName, key, sourceLabel)
			}
			if rawID, ok := entry["id"]; ok {
				rowID, err := asInt(rawID)
				if err != nil {
					return nil, fmt.Errorf("dataset %s: entry %q id %v is not numeric", datasetName, key, rawID)
				}
				ids[key] = rowID
			}
		}
	}
	if rawOverrides, ok := moduleData["overrides"]; ok {
		switch overrides := rawOverrides.(type) {
		case []any:
			for index, rawOverride := range overrides {
				override, ok := rawOverride.(map[string]any)
				if !ok {
					return nil, fmt.Errorf("dataset %s: override %d in %s must be an object", datasetName, index, sourceLabel)
				}
				if rawID, ok := override["id"]; ok {
					rowID, err := asInt(rawID)
					if err != nil {
						return nil, fmt.Errorf("dataset %s: override id %v is not numeric", datasetName, rawID)
					}
					if key, ok := override["key"].(string); ok && key != "" {
						ids[key] = rowID
					}
				}
			}
		case map[string]any:
			for key, rawOverride := range overrides {
				override, ok := rawOverride.(map[string]any)
				if !ok {
					return nil, fmt.Errorf("dataset %s: override %q in %s must be an object", datasetName, key, sourceLabel)
				}
				if rawID, ok := override["id"]; ok {
					rowID, err := asInt(rawID)
					if err != nil {
						return nil, fmt.Errorf("dataset %s: override id %v is not numeric", datasetName, rawID)
					}
					ids[key] = rowID
				}
			}
		default:
			return nil, fmt.Errorf("dataset %s: overrides in %s must be an array or object", datasetName, sourceLabel)
		}
	}
	return ids, nil
}

func buildFeatGeneratedModule(path string, obj map[string]any, ctx *featGeneratedContext) (map[string]any, error) {
	family, _ := obj["family"].(string)
	if strings.TrimSpace(family) == "" {
		return nil, fmt.Errorf("generated feat file %s is missing family", path)
	}
	if isFamilyExpansionObject(obj) {
		return buildFamilyExpansionGeneratedModule(path, obj, ctx)
	}
	switch family {
	case "special_attacks":
		return buildLegacyFeatGeneratedModule(path, obj)
	default:
		return buildLegacyFeatGeneratedModule(path, obj)
	}
}

func buildLegacyFeatGeneratedModule(path string, obj map[string]any) (map[string]any, error) {
	module := map[string]any{}
	if rawEntries, ok := obj["entries"]; ok {
		entries, ok := rawEntries.(map[string]any)
		if !ok {
			return nil, fmt.Errorf("generated feat file %s entries must be an object", path)
		}
		module["entries"] = entries
	}
	if rawOverrides, ok := obj["overrides"]; ok {
		overrides, ok := rawOverrides.([]any)
		if !ok {
			return nil, fmt.Errorf("generated feat file %s overrides must be an array", path)
		}
		module["overrides"] = overrides
	}
	if len(module) == 0 {
		return nil, fmt.Errorf("generated feat file %s must contain entries and/or overrides", path)
	}
	return module, nil
}

func buildFamilyExpansionGeneratedModule(path string, obj map[string]any, ctx *featGeneratedContext) (map[string]any, error) {
	spec, err := parseFamilyExpansionSpec(path, obj)
	if err != nil {
		return nil, err
	}
	overrides, err := parseCompactSourceOverrides(obj["overrides"])
	if err != nil {
		return nil, fmt.Errorf("generated feat file %s: %w", path, err)
	}
	sourceRows, err := ctx.datasetRows(spec.ChildSource.Dataset)
	if err != nil {
		return nil, err
	}
	overrideList := make([]any, 0)
	familyAllowlist := spec.AllowExistingOnly && ctx.familyHasExistingRows(spec.FamilyKey)
	for _, sourceKey := range sortedStringMapKeys(sourceRows) {
		row := sourceRows[sourceKey]
		include, err := familyExpansionSourceEligible(spec, row)
		if err != nil {
			return nil, fmt.Errorf("generated feat file %s: %w", path, err)
		}
		if !include {
			continue
		}
		slug := strings.TrimPrefix(sourceKey, spec.ChildSource.Dataset+":")
		featKey, rowID, hasID, err := ctx.resolveGeneratedFeatIdentity(spec, slug, row)
		if err != nil {
			return nil, fmt.Errorf("generated feat file %s: %w", path, err)
		}
		childToken := childTokenFromExpandedKey(featKey, spec.FamilyKey)
		displayName := ctx.displayNameForGeneratedSource(spec.ChildSource.Dataset, row)
		titleChildName := applyFamilyExpansionTitleStyle(displayName, spec.TitleStyle)
		labelSuffix := upperSnake(displayName)
		override := map[string]any{
			"key":        featKey,
			"MASTERFEAT": map[string]any{"id": spec.Template},
			"meta":       familyMetadata(spec.FamilyKey, childToken, sourceKey, spec.Template),
		}
		if hasID {
			override["id"] = rowID
		}
		if familyAllowlist && !ctx.featKeyExists(featKey) {
			continue
		}
		if spec.ChildRefField != "" {
			override[spec.ChildRefField] = map[string]any{"id": sourceKey}
		}
		for field, prereqFamily := range spec.AutoPrereqFields {
			if prereqKey, ok := generatedFamilyPrereqKey(ctx, row, slug, prereqFamily); ok {
				override[field] = map[string]any{"id": prereqKey}
			}
		}
		if !ctx.featKeyExists(featKey) {
			if spec.LabelPrefix != "" {
				override["LABEL"] = spec.LabelPrefix + "_" + labelSuffix
			}
			if spec.NamePrefix != "" {
				override["FEAT"] = map[string]any{
					"tlk": map[string]any{
						"key":  featKey + ".name",
						"text": fmt.Sprintf("%s (%s)", spec.NamePrefix, titleChildName),
					},
				}
			}
			for _, field := range spec.TemplateFields {
				override[field] = map[string]any{"field": field, "ref": spec.Template}
			}
			if spec.ConstantPrefix != "" {
				override["Constant"] = spec.ConstantPrefix + "_" + labelSuffix
			}
		}
		for field, value := range spec.DefaultFields {
			override[field] = value
		}
		mergeGeneratedOverride(override, overrides[sourceKey])
		overrideList = append(overrideList, override)
	}
	return map[string]any{"overrides": overrideList}, nil
}

func buildRacialtypesSkillAffinityModule(ctx *featGeneratedContext) (map[string]any, error) {
	racesDir := filepath.Join(ctx.dataDir, "racialtypes", "registry", "races")
	entries, err := os.ReadDir(racesDir)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return nil, nil
		}
		return nil, err
	}
	grants := map[string]*affinityGrant{}
	for _, entry := range entries {
		if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
			continue
		}
		path := filepath.Join(racesDir, entry.Name())
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, err
		}
		raceName := displayNameForRaceFile(obj)
		rawFeats, ok := obj["feats"].([]any)
		if !ok {
			continue
		}
		for _, rawFeat := range rawFeats {
			featObj, ok := rawFeat.(map[string]any)
			if !ok {
				continue
			}
			featKey, ok := extractRaceFeatKey(featObj)
			if !ok || !isSkillAffinityFeatKey(featKey) {
				continue
			}
			item := grants[featKey]
			if item == nil {
				item = &affinityGrant{featKey: featKey}
				grants[featKey] = item
			}
			item.races = append(item.races, raceName)
		}
	}
	if len(grants) == 0 {
		return nil, nil
	}
	skillRows, err := ctx.datasetRows("skills")
	if err != nil {
		return nil, err
	}
	overrideList := make([]any, 0, len(grants))
	for _, featKey := range sortedGrantKeys(grants) {
		grant := grants[featKey]
		skillKey, bonus, err := parseSkillAffinityKey(featKey)
		if err != nil {
			return nil, err
		}
		skillKey = resolveAffinitySkillKey(skillRows, skillKey)
		skillRow, ok := skillRows[skillKey]
		skillName := affinitySkillDisplayName(featKey, skillKey)
		if ok {
			skillName = displayNameForSkill(skillRow)
		}
		rowID, hasID, err := ctx.featID(featKey, nil)
		if err != nil {
			return nil, err
		}
		races := dedupeAndSort(grant.races)
		title := affinityTitleFromKey(featKey, skillName)
		description := affinityDescription(skillName, bonus, races)
		label := affinityLabel(featKey)
		parent := "skillaffinity"
		if bonus == 1 {
			parent = "partialskillaffinity"
		}
		child := childTokenFromExpandedKey(featKey, parent)
		override := map[string]any{
			"key":   featKey,
			"LABEL": label,
			"FEAT": map[string]any{
				"tlk": map[string]any{
					"key":  featKey + ".name",
					"text": title,
				},
			},
			"DESCRIPTION": map[string]any{
				"tlk": map[string]any{
					"key":  featKey + ".description",
					"text": description,
				},
			},
			"ICON":             "ife_racial",
			"ALLCLASSESCANUSE": "0",
			"TOOLSCATEGORIES":  "6",
			"PreReqEpic":       "0",
			"ReqAction":        "1",
			"Constant":         label,
			"meta":             familyMetadata(parent, child, skillKey, ""),
		}
		if hasID {
			override["id"] = rowID
		}
		overrideList = append(overrideList, override)
	}
	return map[string]any{"overrides": overrideList}, nil
}

func affinitySkillDisplayName(featKey, skillKey string) string {
	slug := strings.TrimPrefix(skillKey, "skills:")
	if slug == "" {
		slug = strings.TrimPrefix(strings.TrimPrefix(featKey, "feat:skillaffinity"), "feat:partialskillaffinity")
	}
	parts := strings.Fields(strings.ReplaceAll(strings.ReplaceAll(slug, "_", " "), "-", " "))
	for i, part := range parts {
		if part == "" {
			continue
		}
		parts[i] = strings.ToUpper(part[:1]) + part[1:]
	}
	return strings.Join(parts, " ")
}

func requiredStringField(obj map[string]any, field string) (string, error) {
	raw, ok := obj[field]
	if !ok {
		return "", fmt.Errorf("%s is required", field)
	}
	text, ok := raw.(string)
	if !ok || strings.TrimSpace(text) == "" {
		return "", fmt.Errorf("%s must be a non-empty string", field)
	}
	return strings.TrimSpace(text), nil
}

func parseCompactSourceOverrides(raw any) (compactSourceOverride, error) {
	if raw == nil {
		return compactSourceOverride{}, nil
	}
	obj, ok := raw.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("overrides must be an object keyed by source dataset key")
	}
	result := make(compactSourceOverride, len(obj))
	for key, value := range obj {
		item, ok := value.(map[string]any)
		if !ok {
			return nil, fmt.Errorf("override %q must be an object", key)
		}
		result[key] = item
	}
	return result, nil
}

func sortedStringMapKeys[T any](items map[string]T) []string {
	keys := make([]string, 0, len(items))
	for key := range items {
		keys = append(keys, key)
	}
	slices.Sort(keys)
	return keys
}

func mergeGeneratedOverride(dst map[string]any, src map[string]any) {
	if len(src) == 0 {
		return
	}
	if name, ok := src["name"].(string); ok && strings.TrimSpace(name) != "" {
		key, _ := dst["key"].(string)
		dst["FEAT"] = map[string]any{
			"tlk": map[string]any{
				"key":  key + ".name",
				"text": strings.TrimSpace(name),
			},
		}
	}
	if description, ok := src["description_text"].(string); ok && strings.TrimSpace(description) != "" {
		key, _ := dst["key"].(string)
		dst["DESCRIPTION"] = map[string]any{
			"tlk": map[string]any{
				"key":  key + ".description",
				"text": strings.TrimSpace(description),
			},
		}
	}
	if icon, ok := src["icon"]; ok {
		dst["ICON"] = icon
	}
	for field, value := range src {
		switch field {
		case "name", "description_text", "icon":
			continue
		default:
			dst[field] = value
		}
	}
}

func isHiddenSkill(row map[string]any) bool {
	value, ok := lookupField(row, "HideFromLevelUp")
	if !ok {
		return false
	}
	switch typed := value.(type) {
	case string:
		return strings.TrimSpace(typed) == "1"
	case int:
		return typed == 1
	case float64:
		return typed == 1
	default:
		return false
	}
}

func displayNameForSkill(row map[string]any) string {
	if text, ok := displayTextFromValue(row["Name"]); ok {
		return text
	}
	if label, ok := row["Label"].(string); ok && strings.TrimSpace(label) != "" {
		return normalizeDisplayName(strings.TrimSpace(label))
	}
	if key, ok := row["key"].(string); ok {
		return displayNameFromSlug(strings.TrimPrefix(key, "skills:"))
	}
	return "Unknown Skill"
}

func displayNameForBaseitem(row map[string]any) string {
	if text, ok := displayTextFromValue(row["Name"]); ok {
		return text
	}
	if label, ok := row["label"].(string); ok && strings.TrimSpace(label) != "" {
		return normalizeDisplayName(strings.TrimSpace(label))
	}
	if key, ok := row["key"].(string); ok {
		return displayNameFromSlug(strings.TrimPrefix(key, "baseitems:"))
	}
	return "Unknown Weapon"
}

func (c *featGeneratedContext) displayNameForGeneratedSource(dataset string, row map[string]any) string {
	if text, ok := c.displayTextFromGeneratedValue(row["Name"]); ok {
		return text
	}
	switch dataset {
	case "skills":
		return displayNameForSkill(row)
	case "baseitems":
		return displayNameForBaseitem(row)
	default:
		if text, ok := displayTextFromValue(row["Name"]); ok {
			return text
		}
		if label, ok := row["Label"].(string); ok && strings.TrimSpace(label) != "" {
			return normalizeDisplayName(strings.TrimSpace(label))
		}
		if label, ok := row["label"].(string); ok && strings.TrimSpace(label) != "" {
			return normalizeDisplayName(strings.TrimSpace(label))
		}
		if key, ok := row["key"].(string); ok {
			return displayNameFromSlug(strings.TrimPrefix(key, dataset+":"))
		}
		return "Unknown Entry"
	}
}

func (c *featGeneratedContext) displayTextFromGeneratedValue(value any) (string, bool) {
	if text, ok := displayTextFromValue(value); ok {
		return text, true
	}
	if c == nil || c.baseDialog == nil {
		return "", false
	}
	key := ""
	switch typed := value.(type) {
	case string:
		if !isNumericText(typed) {
			return "", false
		}
		key = strings.TrimSpace(typed)
	case int:
		key = strconv.Itoa(typed)
	case float64:
		if typed != float64(int(typed)) {
			return "", false
		}
		key = strconv.Itoa(int(typed))
	default:
		return "", false
	}
	entry, ok := c.baseDialog.Entries[key]
	if !ok || strings.TrimSpace(entry.Text) == "" {
		return "", false
	}
	return normalizeDisplayName(strings.TrimSpace(entry.Text)), true
}

func applyFamilyExpansionTitleStyle(displayName string, style familyExpansionTitleStyle) string {
	if style.ChildParenthetical == "comma" {
		displayName = flattenGeneratedTitleChildParenthetical(displayName)
	}
	if style.ChildCase == "lower" {
		displayName = strings.ToLower(displayName)
	}
	return displayName
}

func flattenGeneratedTitleChildParenthetical(text string) string {
	text = strings.TrimSpace(text)
	if !strings.HasSuffix(text, ")") || strings.Count(text, "(") != 1 || strings.Count(text, ")") != 1 {
		return text
	}
	open := strings.LastIndex(text, "(")
	if open <= 0 || open >= len(text)-2 {
		return text
	}
	prefix := strings.TrimSpace(text[:open])
	suffix := strings.TrimSpace(strings.TrimSuffix(text[open+1:], ")"))
	if prefix == "" || suffix == "" {
		return text
	}
	return prefix + ", " + suffix
}

func displayNameForRaceFile(obj map[string]any) string {
	core, ok := obj["core"].(map[string]any)
	if ok {
		if text, ok := displayTextFromValue(core["Name"]); ok {
			return text
		}
		if label, ok := core["Label"].(string); ok && strings.TrimSpace(label) != "" {
			return normalizeDisplayName(strings.TrimSpace(label))
		}
	}
	if key, ok := obj["key"].(string); ok {
		return displayNameFromSlug(strings.TrimPrefix(key, "racialtypes:"))
	}
	return "Unknown Race"
}

func displayTextFromValue(value any) (string, bool) {
	switch typed := value.(type) {
	case string:
		if strings.TrimSpace(typed) == "" || isNumericText(typed) || typed == nullValue {
			return "", false
		}
		return normalizeDisplayName(strings.TrimSpace(typed)), true
	case map[string]any:
		if tlk, ok := typed["tlk"].(map[string]any); ok {
			if text, ok := tlk["text"].(string); ok && strings.TrimSpace(text) != "" {
				return strings.TrimSpace(text), true
			}
		}
	}
	return "", false
}

func isNumericText(text string) bool {
	if strings.TrimSpace(text) == "" {
		return false
	}
	for _, r := range text {
		if r < '0' || r > '9' {
			return false
		}
	}
	return true
}

func displayNameFromSlug(slug string) string {
	text := strings.TrimSpace(slug)
	text = strings.TrimPrefix(text, "knowledge")
	if strings.HasPrefix(slug, "knowledge") {
		remainder := strings.TrimPrefix(slug, "knowledge")
		if remainder == "" {
			return "Knowledge"
		}
		return "Knowledge (" + normalizeDisplayName(remainder) + ")"
	}
	if strings.HasPrefix(slug, "craft_") {
		remainder := strings.TrimPrefix(slug, "craft_")
		return "Craft (" + normalizeDisplayName(remainder) + ")"
	}
	return normalizeDisplayName(text)
}

func normalizeDisplayName(text string) string {
	text = strings.ReplaceAll(text, "_", " ")
	text = splitCamelText(text)
	parts := strings.Fields(text)
	for index, part := range parts {
		lower := strings.ToLower(part)
		switch lower {
		case "of", "and", "vs":
			parts[index] = lower
		default:
			runes := []rune(lower)
			if len(runes) == 0 {
				continue
			}
			runes[0] = []rune(strings.ToUpper(string(runes[0])))[0]
			parts[index] = string(runes)
		}
	}
	return strings.Join(parts, " ")
}

func splitCamelText(text string) string {
	var out []rune
	var prev rune
	for index, r := range text {
		if index > 0 && isCamelBoundary(prev, r) {
			out = append(out, ' ')
		}
		out = append(out, r)
		prev = r
	}
	return string(out)
}

func isCamelBoundary(prev, current rune) bool {
	return prev >= 'a' && prev <= 'z' && current >= 'A' && current <= 'Z'
}

func upperSnake(text string) string {
	normalized := normalizeDisplayName(text)
	normalized = strings.ReplaceAll(normalized, "(", " ")
	normalized = strings.ReplaceAll(normalized, ")", " ")
	normalized = strings.ReplaceAll(normalized, "-", " ")
	normalized = strings.Join(strings.Fields(normalized), "_")
	return strings.ToUpper(normalized)
}

func isNullishValue(value any) bool {
	switch typed := value.(type) {
	case nil:
		return true
	case string:
		return strings.TrimSpace(typed) == "" || typed == nullValue
	default:
		return false
	}
}

func featSourceID(value any) (int, error) {
	if isNullishValue(value) {
		return 0, nil
	}
	switch typed := value.(type) {
	case int:
		return typed, nil
	case float64:
		return int(typed), nil
	case string:
		return asInt(typed)
	default:
		return 0, fmt.Errorf("cannot derive feat id from %T", value)
	}
}

func (c *featGeneratedContext) resolveGeneratedFeatIdentityBySource(familyKey, slug string, rawSource any) (string, int, bool, error) {
	if ref, ok := rawSource.(string); ok && strings.HasPrefix(ref, "feat:") {
		return c.resolveGeneratedFeatIdentityFromRef(familyKey, ref)
	}
	if rawSource != nil && rawSource != nullValue {
		if parsedID, err := featSourceID(rawSource); err == nil && parsedID > 0 {
			if generatedKey, ok := c.generatedFeatKeyForID(familyKey, parsedID); ok {
				return generatedKey, parsedID, true, nil
			}
			return c.preferredGeneratedFeatKey(familyKey, slug), parsedID, true, nil
		}
	}
	if rawMap, ok := rawSource.(map[string]any); ok {
		if ref, ok := rawMap["id"].(string); ok && strings.HasPrefix(ref, "feat:") {
			return c.resolveGeneratedFeatIdentityFromRef(familyKey, ref)
		}
	}
	featKey := c.preferredGeneratedFeatKey(familyKey, slug)
	rowID, hasID, err := c.featID(featKey, rawSource)
	return featKey, rowID, hasID, err
}

func (c *featGeneratedContext) resolveGeneratedFeatIdentityFromRef(familyKey, ref string) (string, int, bool, error) {
	if generatedKey, rowID, ok := c.generatedFeatKeyFromCanonicalAlias(ref); ok {
		return generatedKey, rowID, true, nil
	}
	if rowID, hasID, err := c.featID(ref, nil); err != nil {
		return "", 0, false, err
	} else if hasID {
		if generatedKey, ok := c.generatedFeatKeyForID(familyKey, rowID); ok {
			return generatedKey, rowID, true, nil
		}
	}
	rowID, hasID, err := c.featID(ref, nil)
	return ref, rowID, hasID, err
}

func (c *featGeneratedContext) resolveGeneratedFeatIdentity(spec familyExpansionSpec, slug string, sourceRow map[string]any) (string, int, bool, error) {
	if spec.IdentitySource == "child_source_value" {
		rawSource, ok := lookupField(sourceRow, spec.ChildSource.Column)
		if !ok {
			rawSource = nil
		}
		return c.resolveGeneratedFeatIdentityBySource(spec.FamilyKey, slug, rawSource)
	}
	featKey := c.preferredGeneratedFeatKey(spec.FamilyKey, slug)
	if legacyKey, legacyID, ok := c.generatedFeatKeyFromLegacyAlias(spec.FamilyKey, slug); ok {
		return legacyKey, legacyID, true, nil
	}
	rowID, hasID, err := c.featID(featKey, nil)
	return featKey, rowID, hasID, err
}

func familyExpansionSourceEligible(spec familyExpansionSpec, row map[string]any) (bool, error) {
	if spec.ChildSource.Column != "" {
		rawValue, ok := lookupField(row, spec.ChildSource.Column)
		if !ok || isNullishValue(rawValue) {
			return false, nil
		}
	}
	switch spec.ChildSource.Predicate {
	case "":
		return true, nil
	case "accessible":
		return !isHiddenSkill(row), nil
	default:
		return false, fmt.Errorf("unsupported child_source.predicate %q", spec.ChildSource.Predicate)
	}
}

func generatedFamilyPrereqKey(ctx *featGeneratedContext, sourceRow map[string]any, slug, familyKey string) (string, bool) {
	if familyKey == "" {
		return "", false
	}
	spec, ok := ctx.familySpecs[familyKey]
	if !ok {
		return "", false
	}
	featKey, _, _, err := ctx.resolveGeneratedFeatIdentity(spec, slug, sourceRow)
	if err != nil {
		return "", false
	}
	return featKey, true
}

type generatedFamilyAliases map[string][]string

func collectGeneratedFamilyAliases(generatedDir string) (generatedFamilyAliases, error) {
	aliases := generatedFamilyAliases{}
	if generatedDir == "" {
		return aliases, nil
	}
	paths, err := collectModulePaths(generatedDir)
	if err != nil {
		return nil, err
	}
	for _, path := range paths {
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, err
		}
		if !isFamilyExpansionObject(obj) {
			continue
		}
		spec, err := parseFamilyExpansionSpec(path, obj)
		if err != nil {
			return nil, err
		}
		if len(spec.LegacyFamilyKeys) > 0 {
			aliases[spec.FamilyKey] = spec.LegacyFamilyKeys
		}
	}
	return aliases, nil
}

func configuredGeneratedFamilyLegacyAliases(familyKey string, aliases generatedFamilyAliases) []string {
	normalized := normalizeKeyIdentity(familyKey)
	for configuredFamilyKey, configured := range aliases {
		if normalizeKeyIdentity(configuredFamilyKey) == normalized && len(configured) > 0 {
			return configured
		}
	}
	return compatibilityGeneratedFamilyLegacyAliases(familyKey)
}

func compatibilityGeneratedFamilyLegacyAliases(familyKey string) []string {
	switch normalizeKeyIdentity(familyKey) {
	case "skillfocus":
		return []string{"skillfocus"}
	case "greaterskillfocus":
		return []string{"epicskillfocus"}
	case "greaterweaponfocus":
		return []string{"epicweaponfocus"}
	case "greaterweaponspecialization":
		return []string{"epicweaponspecialization"}
	case "overwhelmingcritical":
		return []string{"epicoverwhelmingcritical"}
	default:
		return nil
	}
}

func knownGeneratedFamilyKeys(aliases generatedFamilyAliases) []string {
	seen := map[string]struct{}{}
	keys := make([]string, 0, 32)
	add := func(key string) {
		key = strings.TrimSpace(key)
		if key == "" {
			return
		}
		if _, ok := seen[key]; ok {
			return
		}
		seen[key] = struct{}{}
		keys = append(keys, key)
	}
	for _, key := range []string{
		"greater_weapon_specialization",
		"epic_weapon_specialization",
		"greaterweaponspecialization",
		"epicweaponspecialization",
		"overwhelming_critical",
		"epic_overwhelming_critical",
		"overwhelmingcritical",
		"epicoverwhelmingcritical",
		"greater_weapon_focus",
		"epic_weapon_focus",
		"greaterweaponfocus",
		"epicweaponfocus",
		"greater_skill_focus",
		"epic_skill_focus",
		"greaterskillfocus",
		"epicskillfocus",
		"weapon_specialization",
		"weaponspecialization",
		"improved_critical",
		"improvedcritical",
		"weapon_of_choice",
		"weaponofchoice",
		"weapon_focus",
		"weaponfocus",
		"skill_focus",
		"skillfocus",
	} {
		add(key)
	}
	for familyKey, legacyKeys := range aliases {
		add(familyKey)
		for _, legacyKey := range legacyKeys {
			add(legacyKey)
		}
	}
	slices.SortFunc(keys, func(a, b string) int {
		if len(a) == len(b) {
			return strings.Compare(a, b)
		}
		return len(b) - len(a)
	})
	return keys
}

func splitKnownGeneratedFamilyIdentity(text string, aliases generatedFamilyAliases) familyIdentity {
	text = strings.TrimPrefix(strings.TrimSpace(text), "feat:")
	for _, parent := range knownGeneratedFamilyKeys(aliases) {
		if text == parent {
			return familyIdentity{Parent: parent}
		}
		prefix := parent + "_"
		if strings.HasPrefix(text, prefix) {
			return familyIdentity{
				Parent: parent,
				Child:  strings.TrimPrefix(text, prefix),
			}
		}
	}
	return splitFamilyExpansionIdentity(text)
}

func generatedAliasLockForKey(canonicalKey, lockedKey string, aliases generatedFamilyAliases) bool {
	canonicalKey = strings.TrimPrefix(strings.TrimSpace(canonicalKey), "feat:")
	lockedKey = strings.TrimPrefix(strings.TrimSpace(lockedKey), "feat:")
	if canonicalKey == "" || lockedKey == "" {
		return false
	}
	identity := splitKnownGeneratedFamilyIdentity(canonicalKey, aliases)
	if identity.Parent == "" || identity.Child == "" {
		return false
	}
	legacyChildren := legacyFamilyChildAliases(identity.Parent, identity.Child)
	for _, alias := range configuredGeneratedFamilyLegacyAliases(identity.Parent, aliases) {
		prefix := alias + "_"
		if !strings.HasPrefix(lockedKey, prefix) {
			continue
		}
		lockedChild := strings.TrimPrefix(lockedKey, prefix)
		for _, legacyChild := range legacyChildren {
			if normalizeKeyIdentity(lockedChild) == normalizeKeyIdentity(legacyChild) {
				return true
			}
		}
	}
	return false
}

func generatedCanonicalLockCanMove(canonicalKey string, rowID int, lockData map[string]int, aliases generatedFamilyAliases) bool {
	if rowID <= 0 {
		return false
	}
	for lockedKey, lockedID := range lockData {
		if lockedID == rowID && generatedAliasLockForKey(canonicalKey, lockedKey, aliases) {
			return true
		}
	}
	return false
}

func (c *featGeneratedContext) generatedFamilyAliases() generatedFamilyAliases {
	aliases := generatedFamilyAliases{}
	for familyKey, spec := range c.familySpecs {
		if len(spec.LegacyFamilyKeys) > 0 {
			aliases[familyKey] = spec.LegacyFamilyKeys
		}
	}
	return aliases
}

func (c *featGeneratedContext) generatedFamilyLegacyAliases(familyKey string) []string {
	normalized := normalizeKeyIdentity(familyKey)
	for specFamilyKey, spec := range c.familySpecs {
		if normalizeKeyIdentity(specFamilyKey) == normalized && len(spec.LegacyFamilyKeys) > 0 {
			return spec.LegacyFamilyKeys
		}
	}
	return compatibilityGeneratedFamilyLegacyAliases(familyKey)
}

func (c *featGeneratedContext) generatedFeatKeyForID(familyKey string, rowID int) (string, bool) {
	if rowID <= 0 {
		return "", false
	}
	for _, candidates := range []map[string]int{c.lockData, c.supplementalID} {
		matches := make([]string, 0)
		prefix := "feat:" + familyKey + "_"
		for key, candidateID := range candidates {
			if candidateID == rowID && strings.HasPrefix(key, prefix) {
				matches = append(matches, key)
			}
		}
		if best, ok := preferredExpandedKey(matches); ok {
			return best, true
		}
	}
	for _, alias := range c.generatedFamilyLegacyAliases(familyKey) {
		for _, candidates := range []map[string]int{c.lockData, c.supplementalID} {
			matches := make([]string, 0)
			prefix := "feat:" + alias + "_"
			for key, candidateID := range candidates {
				if candidateID != rowID || !strings.HasPrefix(key, prefix) {
					continue
				}
				child := strings.TrimPrefix(key, prefix)
				matches = append(matches, "feat:"+familyKey+"_"+child)
			}
			if best, ok := preferredExpandedKey(matches); ok {
				return best, true
			}
		}
	}
	return "", false
}

func (c *featGeneratedContext) generatedFeatKeyFromLegacyAlias(familyKey, sourceSlug string) (string, int, bool) {
	targets := legacyFamilyChildAliases(familyKey, sourceSlug)
	canonicalKey := c.preferredGeneratedFeatKey(familyKey, sourceSlug)
	for _, alias := range c.generatedFamilyLegacyAliases(familyKey) {
		for _, candidates := range []map[string]int{c.lockData, c.supplementalID} {
			prefix := "feat:" + alias + "_"
			for _, target := range targets {
				for key, rowID := range candidates {
					if rowID <= 0 || !strings.HasPrefix(key, prefix) {
						continue
					}
					child := strings.TrimPrefix(key, prefix)
					if normalizeKeyIdentity(child) == normalizeKeyIdentity(target) {
						return canonicalKey, rowID, true
					}
				}
			}
		}
	}
	return "", 0, false
}

func legacyFamilyChildAliases(familyKey, sourceSlug string) []string {
	aliases := []string{}
	normalizedFamilyKey := normalizeKeyIdentity(familyKey)
	if normalizedFamilyKey != "greaterskillfocus" && normalizedFamilyKey != "skillfocus" {
		return []string{sourceSlug}
	}
	switch normalizeKeyIdentity(sourceSlug) {
	case "acrobatics":
		aliases = append(aliases, "tumble")
	case "animalhandling":
		aliases = append(aliases, "animalempathy")
	case "craftarmorsmithing":
		aliases = append(aliases, "craftarmor")
	case "crafttinkering":
		aliases = append(aliases, "settrap")
	case "craftweaponsmithing":
		aliases = append(aliases, "craftweapon")
	case "craftwoodworking":
		aliases = append(aliases, "crafttrap")
	case "disabledevice":
		aliases = append(aliases, "disabletrap")
	case "influence":
		aliases = append(aliases, "persuade")
	case "sleightofhand":
		aliases = append(aliases, "pickpocket")
	}
	aliases = append(aliases, sourceSlug)
	return aliases
}

func (c *featGeneratedContext) generatedFeatKeyFromCanonicalAlias(canonicalKey string) (string, int, bool) {
	stripped := strings.TrimPrefix(strings.TrimSpace(canonicalKey), "feat:")
	identity := splitKnownGeneratedFamilyIdentity(stripped, c.generatedFamilyAliases())
	if identity.Parent == "" || identity.Child == "" {
		return "", 0, false
	}
	legacyChildren := legacyFamilyChildAliases(identity.Parent, identity.Child)
	for _, alias := range c.generatedFamilyLegacyAliases(identity.Parent) {
		for _, legacyChild := range legacyChildren {
			if _, rowID, ok := c.featKeyForFamilyChild(alias, legacyChild); ok {
				return "feat:" + identity.Parent + "_" + identity.Child, rowID, true
			}
		}
	}
	return "", 0, false
}

func (c *featGeneratedContext) featKeyForFamilyChild(familyKey, child string) (string, int, bool) {
	key := "feat:" + familyKey + "_" + child
	for _, candidates := range []map[string]int{c.lockData, c.supplementalID} {
		if rowID, ok := candidates[key]; ok && rowID > 0 {
			return key, rowID, true
		}
	}
	return "", 0, false
}

func (c *featGeneratedContext) preferredFeatKeyForID(familyKey string, rowID int) (string, bool) {
	prefix := "feat:" + familyKey
	type candidateBuckets struct {
		familyExisting []string
		otherExisting  []string
		familyLegacy   []string
		otherLegacy    []string
		familyOther    []string
		otherOther     []string
	}

	buckets := candidateBuckets{}
	appendMatches := func(candidates map[string]int, legacy bool) {
		for key, candidateID := range candidates {
			if candidateID != rowID {
				continue
			}
			inFamily := strings.HasPrefix(key, prefix)
			exists := c.featKeyExists(key)
			switch {
			case exists && inFamily:
				buckets.familyExisting = append(buckets.familyExisting, key)
			case exists:
				buckets.otherExisting = append(buckets.otherExisting, key)
			case legacy && inFamily:
				buckets.familyLegacy = append(buckets.familyLegacy, key)
			case legacy:
				buckets.otherLegacy = append(buckets.otherLegacy, key)
			case inFamily:
				buckets.familyOther = append(buckets.familyOther, key)
			default:
				buckets.otherOther = append(buckets.otherOther, key)
			}
		}
	}

	appendMatches(c.lockData, false)
	appendMatches(c.supplementalID, true)

	for _, group := range [][]string{
		buckets.familyExisting,
		buckets.otherExisting,
		buckets.otherLegacy,
		buckets.familyLegacy,
		buckets.familyOther,
		buckets.otherOther,
	} {
		if best, ok := preferredExpandedKey(group); ok {
			return best, true
		}
	}
	return "", false
}

func extractRaceFeatKey(obj map[string]any) (string, bool) {
	value, ok := obj["FeatIndex"].(map[string]any)
	if !ok {
		return "", false
	}
	key, ok := value["id"].(string)
	if !ok || !strings.HasPrefix(key, "feat:") {
		return "", false
	}
	return key, true
}

func isSkillAffinityFeatKey(key string) bool {
	return strings.HasPrefix(key, "feat:skillaffinity") || strings.HasPrefix(key, "feat:partialskillaffinity")
}

func parseSkillAffinityKey(featKey string) (string, int, error) {
	switch {
	case strings.HasPrefix(featKey, "feat:partialskillaffinity"):
		return "skills:" + strings.TrimPrefix(featKey, "feat:partialskillaffinity"), 1, nil
	case strings.HasPrefix(featKey, "feat:skillaffinity"):
		return "skills:" + strings.TrimPrefix(featKey, "feat:skillaffinity"), 2, nil
	default:
		return "", 0, fmt.Errorf("unsupported skill affinity key %q", featKey)
	}
}

func resolveAffinitySkillKey(skillRows map[string]map[string]any, skillKey string) string {
	if _, ok := skillRows[skillKey]; ok {
		return skillKey
	}
	target := normalizeKeyIdentity(strings.TrimPrefix(skillKey, "skills:"))
	for _, key := range sortedStringMapKeys(skillRows) {
		if normalizeKeyIdentity(strings.TrimPrefix(key, "skills:")) == target {
			return key
		}
	}
	return skillKey
}

func normalizeKeyIdentity(text string) string {
	text = strings.ToLower(strings.TrimSpace(text))
	text = strings.ReplaceAll(text, "_", "")
	text = strings.ReplaceAll(text, " ", "")
	return text
}

func resolveCanonicalDatasetKey(key string, keyToID map[string]int) string {
	key = strings.TrimSpace(key)
	if key == "" {
		return key
	}
	if _, ok := keyToID[key]; ok {
		return key
	}
	prefix, suffix, ok := strings.Cut(key, ":")
	if !ok || prefix == "" || suffix == "" {
		return key
	}
	normalizedSuffix := normalizeKeyIdentity(suffix)
	match := ""
	for candidate := range keyToID {
		candidatePrefix, candidateSuffix, ok := strings.Cut(candidate, ":")
		if !ok || candidatePrefix != prefix {
			continue
		}
		if normalizeKeyIdentity(candidateSuffix) != normalizedSuffix {
			continue
		}
		if match == "" || candidate < match {
			match = candidate
		}
	}
	if match != "" {
		return match
	}
	return key
}

func affinityTitleFromKey(featKey, skillName string) string {
	if strings.HasPrefix(featKey, "feat:partialskillaffinity") {
		return fmt.Sprintf("Partial Skill Affinity (%s)", skillName)
	}
	return fmt.Sprintf("Skill Affinity (%s)", skillName)
}

func affinityDescription(skillName string, bonus int, races []string) string {
	return fmt.Sprintf("Gain a +%d racial bonus to %s checks. Prerequisites: %s.", bonus, skillName, strings.Join(races, ", "))
}

func affinityLabel(featKey string) string {
	skillKey, _, err := parseSkillAffinityKey(featKey)
	if err != nil {
		return "FEAT_UNKNOWN_SKILL_AFFINITY"
	}
	skillSlug := strings.TrimPrefix(skillKey, "skills:")
	if strings.HasPrefix(featKey, "feat:partialskillaffinity") {
		return "FEAT_PARTIAL_SKILL_AFFINITY_" + upperSnake(displayNameFromSlug(skillSlug))
	}
	return "FEAT_SKILL_AFFINITY_" + upperSnake(displayNameFromSlug(skillSlug))
}

func dedupeAndSort(items []string) []string {
	seen := map[string]struct{}{}
	out := make([]string, 0, len(items))
	for _, item := range items {
		item = strings.TrimSpace(item)
		if item == "" {
			continue
		}
		if _, ok := seen[item]; ok {
			continue
		}
		seen[item] = struct{}{}
		out = append(out, item)
	}
	slices.Sort(out)
	return out
}

func sortedGrantKeys(items map[string]*affinityGrant) []string {
	keys := make([]string, 0, len(items))
	for key := range items {
		keys = append(keys, key)
	}
	slices.Sort(keys)
	return keys
}

func collectPlainDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
	tableData, err := loadJSONObject(dataset.BasePath)
	if err != nil {
		return nativeCollectedDataset{}, err
	}

	columns, err := parseColumns(tableData, dataset.Name)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	rawRows, ok := tableData["rows"].([]any)
	if !ok {
		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: rows must be an array", dataset.Name)
	}

	lockData, err := loadLockfile(dataset.LockPath)
	if err != nil {
		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
	}
	usedIDs := map[int]struct{}{}
	for _, rowID := range lockData {
		usedIDs[rowID] = struct{}{}
	}

	rows := make([]map[string]any, 0, len(rawRows))
	explicitID := make([]bool, 0, len(rawRows))
	for index, raw := range rawRows {
		rowObj, ok := raw.(map[string]any)
		if !ok {
			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d must be an object", dataset.Name, index)
		}
		row, err := canonicalizeBaseRow(dataset, columns, rowObj, index)
		if err != nil {
			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
		}
		rows = append(rows, row)
		_, hasExplicitID := rowObj["id"]
		explicitID = append(explicitID, hasExplicitID)
		if hasExplicitID {
			usedIDs[row["id"].(int)] = struct{}{}
		}
	}

	nextID := nextAvailableID(usedIDs)
	for index, row := range rows {
		key, hasKey := row["key"].(string)
		if hasKey && key != "" {
			if lockedID, ok := lockData[key]; ok {
				row["id"] = lockedID
				continue
			}
			if explicitID[index] {
				lockData[key] = row["id"].(int)
			} else {
				row["id"] = nextID
				lockData[key] = nextID
				usedIDs[nextID] = struct{}{}
				nextID = nextAvailableID(usedIDs)
			}
			continue
		}
		if !explicitID[index] {
			row["id"] = index
		}
	}

	slices.SortFunc(rows, func(a, b map[string]any) int {
		return a["id"].(int) - b["id"].(int)
	})
	tableKey, _ := tableData["key"].(string)
	return nativeCollectedDataset{
		Dataset:  dataset,
		Columns:  columns,
		Rows:     rows,
		LockData: lockData,
		TableKey: tableKey,
	}, nil
}

func resolveNativeDataset(dataset nativeCollectedDataset, keyToID map[string]int, globalRowByKey map[string]map[string]any, tableRegistry resolvedTableRegistry, compiler *tlkCompiler) (map[string]any, error) {
	rows := dataset.Rows
	if strings.HasPrefix(filepath.ToSlash(dataset.Dataset.Name), "classes/feats/") {
		classKey := "classes:" + dataset.Dataset.Name[strings.LastIndex(dataset.Dataset.Name, "/")+1:]
		featSuccessors := buildFeatSuccessorsIndex(globalRowByKey, keyToID)
		classSkills := buildClassSkillsIndex(tableRegistry, classKey)
		expanded, err := expandClassesFeatRows(rows, keyToID, globalRowByKey, featSuccessors, classSkills, globalRowByKey, classKey)
		if err != nil {
			return nil, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
		}
		rows = expanded
	}

	dataset.Dataset.Columns = dataset.Columns
	resolver := valueResolver{
		dataset:       dataset.Dataset,
		keyToID:       keyToID,
		rowByKey:      globalRowByKey,
		tableRegistry: tableRegistry,
		compiler:      compiler,
		cache:         map[string]map[string]tlkCompiledValue{},
	}

	resolvedRows := make([]map[string]any, 0, len(rows))
	for _, row := range rows {
		resolved, err := resolver.resolveRow(row, nil)
		if err != nil {
			return nil, fmt.Errorf("dataset %s: %w", dataset.Dataset.Name, err)
		}
		resolvedRows = append(resolvedRows, resolved)
	}
	return map[string]any{
		"columns": dataset.Columns,
		"rows":    resolvedRows,
	}, nil
}

var (
	classFeatGlobalRows = []map[string]any{
		{"FeatIndex": map[string]any{"id": "feat:combatexpertise"}, "List": "0", "GrantedOnLevel": "-1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:powerattack"}, "List": "0", "GrantedOnLevel": "-1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:specialattacks"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:throw"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:grapple"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:offensivefighting"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:defensivefighting"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
		{"FeatIndex": map[string]any{"id": "feat:horsemenu"}, "List": "3", "GrantedOnLevel": "1", "OnMenu": "1"},
	}
	classFeatClassSkillShorthandRows = []map[string]any{
		{"FeatIndex": map[string]any{"id": "masterfeats:skill_focus", "filter": "classskills"}, "List": "1", "GrantedOnLevel": "-1", "OnMenu": "0"},
		{"FeatIndex": map[string]any{"id": "masterfeats:greater_skill_focus", "filter": "classskills"}, "List": "1", "GrantedOnLevel": "12", "OnMenu": "0"},
	}
)

func expandClassesFeatRows(rows []map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, featSuccessors map[string]string, classSkills map[string]bool, allRowByKey map[string]map[string]any, classKey string) ([]map[string]any, error) {

	injected := make([]map[string]any, 0, len(classFeatGlobalRows)+len(classFeatClassSkillShorthandRows))
	existingRefIDs := make(map[string]struct{}, len(rows))
	for _, row := range rows {
		featRef, ok := row["FeatIndex"].(map[string]any)
		if !ok {
			continue
		}
		refID, _ := featRef["id"].(string)
		if refID == "" {
			continue
		}
		existingRefIDs[refID] = struct{}{}
	}

	for _, row := range classFeatGlobalRows {
		featKey := row["FeatIndex"].(map[string]any)["id"].(string)
		if _, exists := existingRefIDs[featKey]; exists {
			continue
		}
		if _, exists := allRowByKey[featKey]; exists {
			cloned := deepCopyValue(row).(map[string]any)
			if label := lookupReferencedFeatLabel(featKey, allRowByKey); label != "" {
				cloned["FeatLabel"] = label
			}
			injected = append(injected, cloned)
		}
	}

	hasClassSkills := classSkills != nil && len(classSkills) > 0
	for _, row := range classFeatClassSkillShorthandRows {
		if !hasClassSkills {
			continue
		}
		refID := row["FeatIndex"].(map[string]any)["id"].(string)
		if _, exists := existingRefIDs[refID]; exists {
			continue
		}
		rowItems, err := expandClassesFeatRow(row, keyToID, rowByKey, featSuccessors, classSkills, classKey)
		if err != nil {
			return nil, err
		}
		if len(rowItems) > 0 {
			injected = append(injected, rowItems...)
		}
	}

	expanded := make([]map[string]any, 0, len(rows))
	for _, row := range rows {
		rowItems, err := expandClassesFeatRow(row, keyToID, rowByKey, featSuccessors, classSkills, classKey)
		if err != nil {
			return nil, err
		}
		expanded = append(expanded, rowItems...)
	}

	combined := make([]map[string]any, 0, len(injected)+len(expanded))
	combined = append(combined, injected...)
	combined = append(combined, expanded...)

	for index, row := range combined {
		row["id"] = index
	}
	return combined, nil
}

func expandClassesFeatRow(row map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, featSuccessors map[string]string, classSkills map[string]bool, classKey string) ([]map[string]any, error) {
	featRef, ok := row["FeatIndex"].(map[string]any)
	if !ok {
		return []map[string]any{row}, nil
	}
	refKey, _ := featRef["id"].(string)
	filterMode, _ := featRef["filter"].(string)

	switch {
	case strings.HasPrefix(refKey, "feat:"):
		if filterMode == "successors" {
			return expandFeatSuccessors(refKey, row, featSuccessors, rowByKey)
		}
		cloned := deepCopyValue(row).(map[string]any)
		if label := lookupReferencedFeatLabel(refKey, rowByKey); label != "" && isNullLikeValue(cloned["FeatLabel"]) {
			cloned["FeatLabel"] = label
		}
		return []map[string]any{cloned}, nil
	case strings.HasPrefix(refKey, "masterfeats:"):
		masterfeatKey := resolveCanonicalDatasetKey(refKey, keyToID)
		children, err := expandMasterfeatChildren(masterfeatKey, row, keyToID, rowByKey, classKey)
		if err != nil {
			return nil, err
		}
		if filterMode == "classskills" {
			if classSkills == nil || len(classSkills) == 0 {
				return []map[string]any{}, nil
			}
			filtered := make([]map[string]any, 0, len(children))
			for _, child := range children {
				featRef, ok := child["FeatIndex"].(map[string]any)
				if !ok {
					continue
				}
				featKey, _ := featRef["id"].(string)
				featRow := rowByKey[featKey]
				if featRow == nil {
					continue
				}
				reqSkillKey := resolveReqSkill(featRow, keyToID)
				if reqSkillKey == "" {
					continue
				}
				if classSkills[reqSkillKey] {
					filtered = append(filtered, child)
				}
			}
			if len(filtered) == 0 {
				return []map[string]any{}, nil
			}
			return filtered, nil
		}
		if len(children) == 0 {
			return []map[string]any{row}, nil
		}
		return children, nil
	default:
		return []map[string]any{row}, nil
	}
}

func expandMasterfeatChildren(masterfeatKey string, row map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, classKey string) ([]map[string]any, error) {
	masterfeatKey = resolveCanonicalDatasetKey(masterfeatKey, keyToID)
	if _, ok := keyToID[masterfeatKey]; !ok {
		return nil, fmt.Errorf("unknown masterfeat reference %q", masterfeatKey)
	}
	type match struct {
		key string
		id  int
	}
	matches := make([]match, 0)
	for key, featRow := range rowByKey {
		if !strings.HasPrefix(key, "feat:") {
			continue
		}
		if !rowReferencesMasterfeat(featRow, masterfeatKey, keyToID) {
			continue
		}
		if !classFeatExpansionCanUseFeat(featRow, classKey, keyToID) {
			continue
		}
		featID, ok := keyToID[key]
		if !ok {
			continue
		}
		matches = append(matches, match{key: key, id: featID})
	}
	slices.SortFunc(matches, func(a, b match) int {
		return a.id - b.id
	})
	expanded := make([]map[string]any, 0, len(matches))
	for _, item := range matches {
		cloned := deepCopyValue(row).(map[string]any)
		cloned["FeatIndex"] = map[string]any{"id": item.key}
		if label := lookupReferencedFeatLabel(item.key, rowByKey); label != "" {
			cloned["FeatLabel"] = label
		}
		expanded = append(expanded, cloned)
	}
	return expanded, nil
}

func classFeatExpansionCanUseFeat(featRow map[string]any, classKey string, keyToID map[string]int) bool {
	value, ok := lookupField(featRow, "ALLCLASSESCANUSE")
	if !ok || isNullishValue(value) {
		return true
	}
	if strings.TrimSpace(fmt.Sprint(value)) != "0" {
		return true
	}
	if classKey == "" {
		return false
	}
	minLevelClass, ok := lookupField(featRow, "MinLevelClass")
	if !ok || isNullishValue(minLevelClass) {
		return false
	}
	return rowRefersToKey(minLevelClass, classKey, keyToID)
}

func rowRefersToKey(value any, targetKey string, keyToID map[string]int) bool {
	targetKey = resolveCanonicalDatasetKey(targetKey, keyToID)
	targetID, hasTargetID := keyToID[targetKey]
	switch typed := value.(type) {
	case string:
		text := resolveCanonicalDatasetKey(strings.TrimSpace(typed), keyToID)
		if text == "" || text == nullValue || text == "****" {
			return false
		}
		if text == targetKey {
			return true
		}
		if hasTargetID {
			if id, err := strconv.Atoi(text); err == nil {
				return id == targetID
			}
		}
		return false
	case int:
		return hasTargetID && typed == targetID
	case float64:
		return hasTargetID && int(typed) == targetID
	case map[string]any:
		if id, ok := typed["id"].(string); ok {
			return rowRefersToKey(id, targetKey, keyToID)
		}
		if key, ok := typed["key"].(string); ok {
			return rowRefersToKey(key, targetKey, keyToID)
		}
		return false
	default:
		return false
	}
}

func rowReferencesMasterfeat(row map[string]any, masterfeatKey string, keyToID map[string]int) bool {
	masterfeatKey = resolveCanonicalDatasetKey(masterfeatKey, keyToID)
	masterID, ok := keyToID[masterfeatKey]
	if !ok {
		return false
	}
	value, ok := row["MASTERFEAT"]
	if !ok {
		return false
	}
	switch typed := value.(type) {
	case string:
		return typed == strconv.Itoa(masterID)
	case int:
		return typed == masterID
	case float64:
		return int(typed) == masterID
	case map[string]any:
		if id, ok := typed["id"].(string); ok {
			id = resolveCanonicalDatasetKey(id, keyToID)
			if id == masterfeatKey {
				return true
			}
			if refID, ok := keyToID[id]; ok {
				return refID == masterID
			}
			return id == "masterfeats:"+strconv.Itoa(masterID)
		}
		return false
	default:
		return false
	}
}

func lookupReferencedFeatLabel(featKey string, rowByKey map[string]map[string]any) string {
	row, ok := rowByKey[featKey]
	if !ok {
		return ""
	}
	label, _ := row["LABEL"].(string)
	return strings.TrimSpace(label)
}

func isNullLikeValue(value any) bool {
	switch typed := value.(type) {
	case nil:
		return true
	case string:
		trimmed := strings.TrimSpace(typed)
		return trimmed == "" || trimmed == nullValue
	default:
		return false
	}
}

func buildFeatSuccessorsIndex(rowByKey map[string]map[string]any, keyToID map[string]int) map[string]string {
	successors := make(map[string]string)
	for key, row := range rowByKey {
		if !strings.HasPrefix(key, "feat:") {
			continue
		}
		successorKey := resolveFeatSuccessor(row, keyToID)
		if successorKey != "" {
			successors[key] = successorKey
		}
	}
	return successors
}

func resolveFeatSuccessor(row map[string]any, keyToID map[string]int) string {
	value, ok := row["SUCCESSOR"]
	if !ok {
		return ""
	}
	switch typed := value.(type) {
	case map[string]any:
		if idValue, ok := typed["id"]; ok {
			switch idTyped := idValue.(type) {
			case string:
				trimmed := strings.TrimSpace(idTyped)
				if trimmed == "" || trimmed == nullValue || trimmed == "****" {
					return ""
				}
				if id, err := strconv.Atoi(trimmed); err == nil {
					for k, v := range keyToID {
						if v == id && strings.HasPrefix(k, "feat:") {
							return k
						}
					}
					return ""
				}
				if strings.HasPrefix(trimmed, "feat:") {
					return trimmed
				}
				return "feat:" + trimmed
			case int:
				for k, v := range keyToID {
					if v == idTyped && strings.HasPrefix(k, "feat:") {
						return k
					}
				}
				return ""
			case float64:
				targetID := int(idTyped)
				for k, v := range keyToID {
					if v == targetID && strings.HasPrefix(k, "feat:") {
						return k
					}
				}
				return ""
			}
		}
		return ""
	case string:
		trimmed := strings.TrimSpace(typed)
		if trimmed == "" || trimmed == nullValue || trimmed == "****" {
			return ""
		}
		if id, err := strconv.Atoi(trimmed); err == nil {
			for k, v := range keyToID {
				if v == id && strings.HasPrefix(k, "feat:") {
					return k
				}
			}
			return ""
		}
		if strings.HasPrefix(trimmed, "feat:") {
			return trimmed
		}
		return "feat:" + trimmed
	case int:
		for k, v := range keyToID {
			if v == typed && strings.HasPrefix(k, "feat:") {
				return k
			}
		}
		return ""
	case float64:
		targetID := int(typed)
		for k, v := range keyToID {
			if v == targetID && strings.HasPrefix(k, "feat:") {
				return k
			}
		}
		return ""
	default:
		return ""
	}
}

func buildClassSkillsIndex(tableRegistry resolvedTableRegistry, classKey string) map[string]bool {
	skillDatasetKey := classKey
	if !strings.HasPrefix(skillDatasetKey, "classes/skills:") {
		skillDatasetKey = "classes/skills:" + strings.TrimPrefix(classKey, "classes:")
	}

	table, ok := tableRegistry.resolveTableByKey(skillDatasetKey)
	if !ok {
		return nil
	}

	skills := make(map[string]bool)
	for _, rowMap := range table.Rows {
		classSkill, _ := rowMap["ClassSkill"].(string)
		if classSkill != "1" {
			continue
		}
		skillIndex, ok := rowMap["SkillIndex"].(map[string]any)
		if !ok {
			continue
		}
		skillKey, _ := skillIndex["id"].(string)
		if skillKey != "" {
			skills[skillKey] = true
		}
	}

	if len(skills) == 0 {
		return nil
	}
	return skills
}

func expandFeatSuccessors(rootFeatKey string, row map[string]any, featSuccessors map[string]string, rowByKey map[string]map[string]any) ([]map[string]any, error) {
	current := rootFeatKey
	visited := make(map[string]bool)
	var results []map[string]any
	for {
		if current == "" {
			break
		}
		if visited[current] {
			return nil, fmt.Errorf("successor chain cycles at %q", current)
		}
		visited[current] = true
		if rowByKey[current] == nil {
			break
		}
		cloned := deepCopyValue(row).(map[string]any)
		cloned["FeatIndex"] = map[string]any{"id": current}
		if label := lookupReferencedFeatLabel(current, rowByKey); label != "" {
			cloned["FeatLabel"] = label
		}
		results = append(results, cloned)
		current = featSuccessors[current]
	}
	if len(results) == 0 {
		return []map[string]any{row}, nil
	}
	return results, nil
}

func expandMasterfeatChildrenFiltered(masterfeatKey string, row map[string]any, keyToID map[string]int, rowByKey map[string]map[string]any, classSkills map[string]bool) ([]map[string]any, error) {
	if classSkills == nil || len(classSkills) == 0 {
		return []map[string]any{}, nil
	}
	children, err := expandMasterfeatChildren(masterfeatKey, row, keyToID, rowByKey, "")
	if err != nil {
		return nil, err
	}
	var filtered []map[string]any
	for _, child := range children {
		featRef, ok := child["FeatIndex"].(map[string]any)
		if !ok {
			continue
		}
		featKey, _ := featRef["id"].(string)
		featRow := rowByKey[featKey]
		if featRow == nil {
			continue
		}
		reqSkillKey := resolveReqSkill(featRow, keyToID)
		if reqSkillKey == "" {
			continue
		}
		if classSkills[reqSkillKey] {
			filtered = append(filtered, child)
		}
	}
	return filtered, nil
}

func extractClassKeyFromDatasetName(rowByKey map[string]map[string]any) string {
	for key := range rowByKey {
		if strings.HasPrefix(key, "classes:") {
			return key
		}
	}
	return ""
}

func resolveReqSkill(row map[string]any, keyToID map[string]int) string {
	value, ok := row["REQSKILL"]
	if !ok {
		return ""
	}
	switch typed := value.(type) {
	case string:
		trimmed := strings.TrimSpace(typed)
		if trimmed == "" || trimmed == nullValue || trimmed == "****" {
			return ""
		}
		if strings.HasPrefix(trimmed, "skills:") {
			return trimmed
		}
		if resolved := lookupKeyByIDPrefix(keyToID, "skills:", trimmed); resolved != "" {
			return resolved
		}
		return "skills:" + trimmed
	case map[string]any:
		if id, ok := typed["id"].(string); ok {
			if id == "" || id == nullValue || id == "****" {
				return ""
			}
			if strings.HasPrefix(id, "skills:") {
				return id
			}
			if resolved := lookupKeyByIDPrefix(keyToID, "skills:", id); resolved != "" {
				return resolved
			}
			return "skills:" + id
		}
		return ""
	default:
		return ""
	}
}

func lookupKeyByIDPrefix(keyToID map[string]int, prefix string, rawID string) string {
	idValue, err := strconv.Atoi(strings.TrimSpace(rawID))
	if err != nil {
		return ""
	}
	for key, existingID := range keyToID {
		if existingID == idValue && strings.HasPrefix(key, prefix) {
			return key
		}
	}
	return ""
}

type valueResolver struct {
	dataset       nativeDataset
	keyToID       map[string]int
	rowByKey      map[string]map[string]any
	tableRegistry resolvedTableRegistry
	compiler      *tlkCompiler
	cache         map[string]map[string]tlkCompiledValue
}

func (r *valueResolver) resolveTableByKey(key string) (resolvedTable, bool) {
	return r.tableRegistry.resolveTableByKey(key)
}

func (r *valueResolver) resolveRow(row map[string]any, stack []string) (map[string]any, error) {
	effectiveRow, err := r.resolveInheritedObject(row, rowIdentity(r.dataset.Name, row), row, stack)
	if err != nil {
		return nil, err
	}
	out := map[string]any{
		"id": effectiveRow["id"],
	}
	for _, column := range rowColumns(effectiveRow, r.dataset.Columns) {
		resolved, err := r.resolveField(effectiveRow, column, stack)
		if err != nil {
			return nil, err
		}
		out[column] = resolved.Value
	}
	return out, nil
}

func (r *valueResolver) resolveField(row map[string]any, field string, stack []string) (tlkCompiledValue, error) {
	rowKey := rowCacheIdentity(r.dataset.Name, row)
	if r.cache[rowKey] == nil {
		r.cache[rowKey] = map[string]tlkCompiledValue{}
	}
	if cached, ok := r.cache[rowKey][field]; ok {
		return cached, nil
	}

	stackKey := rowKey + "." + field
	for _, existing := range stack {
		if existing == stackKey {
			return tlkCompiledValue{}, fmt.Errorf("cyclic row/field reference detected at %s", stackKey)
		}
	}

	value, ok := lookupField(row, field)
	if !ok {
		resolved := tlkCompiledValue{Value: nullValue}
		r.cache[rowKey][field] = resolved
		return resolved, nil
	}
	resolved, err := r.resolveValue(row, field, value, append(stack, stackKey))
	if err != nil {
		return tlkCompiledValue{}, err
	}
	r.cache[rowKey][field] = resolved
	return resolved, nil
}

func (r *valueResolver) resolveValue(row map[string]any, field string, value any, stack []string) (tlkCompiledValue, error) {
	if typed, ok := value.(map[string]any); ok {
		merged, err := r.resolveInheritedObject(row, rowCacheIdentity(r.dataset.Name, row)+"."+field, typed, stack)
		if err != nil {
			return tlkCompiledValue{}, err
		}
		value = merged
	}
	if literal, ok := r.resolveLiteralFeatObject(row, field, value); ok {
		return tlkCompiledValue{Value: literal}, nil
	}
	allowBare := columnMatchesSpec(r.dataset.Spec, field)
	if payload, ok, err := parseTLKPayload(value, allowBare); err != nil {
		return tlkCompiledValue{}, err
	} else if ok {
		return r.resolveTLKPayload(row, field, payload, stack)
	}

	switch typed := value.(type) {
	case map[string]any:
		if refID, ok := typed["id"]; ok {
			if refKey, ok := refID.(string); ok && strings.Contains(refKey, ":") {
				id, ok := r.keyToID[refKey]
				if !ok {
					return tlkCompiledValue{}, fmt.Errorf("unknown key reference: %s (row=%s field=%s)", refKey, rowCacheIdentity(r.dataset.Name, row), field)
				}
				return tlkCompiledValue{Value: id}, nil
			}
		}
		if refKey, ok := typed["ref"]; ok {
			fieldValue, hasField := typed["field"]
			refKeyString, okKey := refKey.(string)
			fieldString, okField := fieldValue.(string)
			if hasField && okKey && okField {
				targetRow, ok := r.rowByKey[refKeyString]
				if !ok {
					return tlkCompiledValue{}, fmt.Errorf("unknown row reference: %s", refKeyString)
				}
				return r.resolveField(targetRow, fieldString, stack)
			}
		}
		if tableValue, ok := typed["table"]; ok {
			tableKey, ok := tableValue.(string)
			if ok {
				outputName, ok := r.tableRegistry.stemByKey[tableKey]
				if !ok {
					return tlkCompiledValue{}, fmt.Errorf("unknown table reference: %s", tableKey)
				}
				return tlkCompiledValue{Value: outputName}, nil
			}
		}

		resolved := map[string]any{}
		for _, key := range sortedKeys(typed) {
			item, err := r.resolveValue(row, key, typed[key], stack)
			if err != nil {
				return tlkCompiledValue{}, err
			}
			resolved[key] = item.Value
		}
		return tlkCompiledValue{Value: resolved}, nil
	case []any:
		resolved := make([]any, 0, len(typed))
		for _, item := range typed {
			value, err := r.resolveValue(row, field, item, stack)
			if err != nil {
				return tlkCompiledValue{}, err
			}
			resolved = append(resolved, value.Value)
		}
		return tlkCompiledValue{Value: resolved}, nil
	default:
		return tlkCompiledValue{Value: typed}, nil
	}
}

func (r *valueResolver) resolveLiteralFeatObject(row map[string]any, field string, value any) (string, bool) {
	if r.dataset.Name != "feat" {
		return "", false
	}
	typed, ok := value.(map[string]any)
	if !ok {
		return "", false
	}
	if refID, ok := typed["id"].(string); ok && len(typed) == 1 && strings.Contains(refID, ":") {
		if !shouldPreserveLiteralFeatIDObject(row, field, refID) {
			return "", false
		}
		if _, exists := r.keyToID[refID]; !exists {
			return "", false
		}
		return "{'id': '" + refID + "'}", true
	}
	if refKey, ok := typed["key"].(string); ok && len(typed) == 1 && strings.Contains(refKey, ":") {
		if !shouldPreserveLiteralFeatKeyObject(field) {
			return "", false
		}
		if _, exists := r.keyToID[refKey]; !exists && !isLegacyLiteralFeatKeyException(field, refKey) {
			return "", false
		}
		return "{'key': '" + refKey + "'}", true
	}
	return "", false
}

func shouldPreserveLiteralFeatKeyObject(field string) bool {
	switch field {
	case "SUCCESSOR", "PREREQFEAT1", "PREREQFEAT2":
		return true
	default:
		return false
	}
}

func shouldPreserveLiteralFeatIDObject(row map[string]any, field, refID string) bool {
	switch field {
	case "MinLevelClass":
		return shouldPreserveFeatMinLevelClass(row)
	case "SUCCESSOR":
		switch refID {
		case "feat:weaponproficiencysimple":
			return true
		default:
			return false
		}
	default:
		return false
	}
}

func shouldPreserveFeatMinLevelClass(row map[string]any) bool {
	rowKey, _ := row["key"].(string)
	if prefix, suffix, ok := strings.Cut(strings.TrimSpace(rowKey), ":"); ok && prefix == "masterfeats" {
		switch normalizeKeyIdentity(suffix) {
		case "weaponfocus", "weaponspecialization", "greaterweaponfocus", "greaterweaponspecialization", "improvedcritical", "overwhelmingcritical":
			return true
		}
	}
	switch {
	case strings.HasPrefix(rowKey, "feat:weaponfocus_"):
		return true
	case strings.HasPrefix(rowKey, "feat:weaponspecialization_"):
		return true
	case strings.HasPrefix(rowKey, "feat:greaterweaponfocus_"):
		return true
	case strings.HasPrefix(rowKey, "feat:greaterweaponspecialization_"):
		return true
	case strings.HasPrefix(rowKey, "feat:improvedcritical_"):
		return true
	case strings.HasPrefix(rowKey, "feat:overwhelmingcritical_"):
		return true
	case strings.HasPrefix(rowKey, "feat:epicweaponfocus_"):
		return true
	case strings.HasPrefix(rowKey, "feat:epicweaponspecialization_"):
		return true
	default:
		return false
	}
}

func isLegacyLiteralFeatKeyException(field, refKey string) bool {
	if field != "SUCCESSOR" {
		return false
	}
	switch refKey {
	case "feat:improveddodge", "feat:improveduncannydodge":
		return true
	default:
		return false
	}
}

func (r *valueResolver) resolveTLKPayload(row map[string]any, field string, payload parsedTLKPayload, stack []string) (tlkCompiledValue, error) {
	if payload.Ref != "" {
		targetRow, ok := r.rowByKey[payload.Ref]
		if !ok {
			return tlkCompiledValue{}, fmt.Errorf("unknown TLK ref row %s", payload.Ref)
		}
		resolved, err := r.resolveField(targetRow, payload.RefField, stack)
		if err != nil {
			return tlkCompiledValue{}, err
		}
		if resolved.TLKKey == "" {
			switch resolved.Value.(type) {
			case string, int, float64:
				return resolved, nil
			default:
				return tlkCompiledValue{}, fmt.Errorf("TLK ref %s.%s does not resolve to TLK-backed text", payload.Ref, payload.RefField)
			}
		}
		return resolved, nil
	}
	if payload.Text != "" {
		key := payload.Key
		if key == "" {
			key = deriveAutoTLKKey(rowIdentity(r.dataset.Name, row), field)
		}
		return r.compiler.registerInline(key, tlkEntryData{
			Text:        payload.Text,
			SoundResRef: payload.SoundResRef,
			SoundLength: payload.SoundLength,
		})
	}
	return r.compiler.resolveLegacyKey(payload.Key)
}

func rowColumns(row map[string]any, ordered []string) []string {
	columns := make([]string, 0, len(row))
	seen := map[string]struct{}{}
	for _, key := range ordered {
		if _, ok := row[key]; ok {
			columns = append(columns, key)
			seen[key] = struct{}{}
		}
	}
	for key := range row {
		if key == "id" || key == "key" || key == "__columns" || key == "inherit" || isMetadataField(key) {
			continue
		}
		if _, ok := seen[key]; ok {
			continue
		}
		columns = append(columns, key)
	}
	return columns
}

func columnMatchesSpec(spec datasetTextSpec, field string) bool {
	return roleForField(spec, field) == textRoleName || roleForField(spec, field) == textRoleDescription
}

func rowIdentity(datasetName string, row map[string]any) string {
	if key, ok := row["key"].(string); ok && key != "" {
		return key
	}
	return fmt.Sprintf("%s:id:%d", datasetName, row["id"].(int))
}

func rowCacheIdentity(datasetName string, row map[string]any) string {
	return rowIdentity(datasetName, row)
}

func lookupField(row map[string]any, field string) (any, bool) {
	if value, ok := row[field]; ok {
		return value, true
	}
	for key, value := range row {
		if strings.EqualFold(key, field) {
			return value, true
		}
	}
	return nil, false
}

func parseColumns(baseData map[string]any, datasetName string) ([]string, error) {
	rawColumns, ok := baseData["columns"].([]any)
	if !ok {
		return nil, fmt.Errorf("dataset %s: columns must be an array", datasetName)
	}
	columns := make([]string, 0, len(rawColumns))
	for _, raw := range rawColumns {
		column, ok := raw.(string)
		if !ok {
			return nil, fmt.Errorf("dataset %s: columns must contain strings", datasetName)
		}
		if isDeprecatedAuthoringOnlyField(datasetName, column) || isMetadataField(column) {
			continue
		}
		columns = append(columns, column)
	}
	return columns, nil
}

func extendColumns(columns []string, obj map[string]any, datasetName, sourceLabel string) ([]string, error) {
	rawColumns, ok := obj["columns"]
	if !ok {
		return columns, nil
	}
	columnList, ok := rawColumns.([]any)
	if !ok {
		return nil, fmt.Errorf("dataset %s: columns in %s must be an array", datasetName, sourceLabel)
	}
	extended := append([]string(nil), columns...)
	for _, raw := range columnList {
		column, ok := raw.(string)
		if !ok {
			return nil, fmt.Errorf("dataset %s: columns in %s must contain strings", datasetName, sourceLabel)
		}
		if isDeprecatedAuthoringOnlyField(datasetName, column) || isMetadataField(column) {
			continue
		}
		if _, exists := canonicalColumn(extended, column); exists {
			continue
		}
		extended = append(extended, column)
	}
	return extended, nil
}

func ensureRowsExposeColumns(rows []map[string]any, columns []string) {
	for _, row := range rows {
		for _, column := range columns {
			if _, ok := row[column]; !ok {
				row[column] = nullValue
			}
		}
	}
}

func canonicalizeBaseRow(dataset nativeDataset, columns []string, raw map[string]any, index int) (map[string]any, error) {
	rowID := index
	if value, ok := raw["id"]; ok {
		parsed, err := asInt(value)
		if err != nil {
			return nil, fmt.Errorf("row id %v is not numeric", value)
		}
		rowID = parsed
	}

	row := map[string]any{"id": rowID}
	for _, column := range columns {
		row[column] = nullValue
	}
	for key, value := range raw {
		switch key {
		case "id", "_tlk":
		case "key":
			if value == nil {
				continue
			}
			text, ok := value.(string)
			if !ok {
				return nil, fmt.Errorf("row key must be a string")
			}
			row["key"] = text
		case "inherit":
			row["inherit"] = value
		default:
			if isMetadataField(key) {
				meta, err := parseRowMetadata(value)
				if err != nil {
					return nil, err
				}
				if meta != nil {
					row["meta"] = meta
				}
				continue
			}
			if isDeprecatedAuthoringOnlyField(dataset.Name, key) {
				if err := setLegacyWikiMetadata(row, key, value); err != nil {
					return nil, err
				}
				continue
			}
			columnName, ok := canonicalColumn(columns, key)
			if !ok {
				if isDeprecatedAuthoringOnlyField(dataset.Name, key) {
					continue
				}
				return nil, fmt.Errorf("unknown column %q", key)
			}
			row[columnName] = value
		}
	}

	return expandRowAuthoringSugar(dataset, columns, raw, row)
}

func canonicalizeEntry(dataset nativeDataset, columns []string, rowID int, key string, raw map[string]any) (map[string]any, error) {
	row := map[string]any{
		"id":  rowID,
		"key": key,
	}
	for _, column := range columns {
		row[column] = nullValue
	}
	for field, value := range raw {
		if field == "_tlk" {
			continue
		}
		if field == "inherit" {
			row["inherit"] = value
			continue
		}
		if isMetadataField(field) {
			meta, err := parseRowMetadata(value)
			if err != nil {
				return nil, err
			}
			if meta != nil {
				row["meta"] = meta
			}
			continue
		}
		if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
			if err := setLegacyWikiMetadata(row, field, value); err != nil {
				return nil, err
			}
			continue
		}
		columnName, ok := canonicalColumn(columns, field)
		if !ok {
			if isDeprecatedAuthoringOnlyField(dataset.Name, field) {
				continue
			}
			return nil, fmt.Errorf("entry %q references unknown column %q", key, field)
		}
		row[columnName] = value
	}
	return expandRowAuthoringSugar(dataset, columns, raw, row)
}

func isAuthoringOnlyField(name string) bool {
	return strings.HasPrefix(strings.ToUpper(strings.TrimSpace(name)), "WIKI")
}

func preserveLegacyWikiColumn(datasetName, field string) bool {
	return filepath.ToSlash(strings.TrimSpace(datasetName)) == "classes/core" &&
		strings.EqualFold(strings.TrimSpace(field), "WIKIGENERATE")
}

func isDeprecatedAuthoringOnlyField(datasetName, field string) bool {
	return isAuthoringOnlyField(field) && !preserveLegacyWikiColumn(datasetName, field)
}

func isMetadataField(name string) bool {
	return strings.EqualFold(strings.TrimSpace(name), "meta")
}

func expandRowAuthoringSugar(dataset nativeDataset, columns []string, source map[string]any, row map[string]any) (map[string]any, error) {
	expanded, err := expandRowTLKSugar(dataset, columns, source, row)
	if err != nil {
		return nil, err
	}
	return expandRowInheritanceSugar(dataset, columns, source, expanded)
}

func expandRowTLKSugar(dataset nativeDataset, columns []string, source map[string]any, row map[string]any) (map[string]any, error) {
	rawTLK, ok := source["_tlk"]
	if !ok {
		return row, nil
	}
	tlkMap, ok := rawTLK.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("dataset %s: _tlk must be an object", dataset.Name)
	}

	out := map[string]any{}
	for key, value := range row {
		out[key] = value
	}
	for roleName, rawValue := range tlkMap {
		var role textRole
		switch strings.ToLower(roleName) {
		case "name":
			role = textRoleName
		case "description":
			role = textRoleDescription
		default:
			return nil, fmt.Errorf("dataset %s: unsupported _tlk role %q", dataset.Name, roleName)
		}
		column := columnForRole(columns, dataset.Spec, role)
		if column == "" {
			return nil, fmt.Errorf("dataset %s: _tlk.%s has no matching text column", dataset.Name, roleName)
		}
		if _, hasField := source[column]; hasField {
			return nil, fmt.Errorf("dataset %s: cannot use _tlk.%s and field %s together", dataset.Name, roleName, column)
		}

		switch typed := rawValue.(type) {
		case string:
			out[column] = map[string]any{"tlk": map[string]any{"text": typed}}
		case map[string]any:
			if _, ok := typed["tlk"]; ok {
				out[column] = typed
			} else {
				out[column] = map[string]any{"tlk": typed}
			}
		default:
			return nil, fmt.Errorf("dataset %s: _tlk.%s must be a string or object", dataset.Name, roleName)
		}
	}
	return out, nil
}

func expandRowInheritanceSugar(dataset nativeDataset, columns []string, source map[string]any, row map[string]any) (map[string]any, error) {
	spec, ok, err := parseRowInheritanceSpec(source)
	if err != nil {
		return nil, fmt.Errorf("dataset %s: %w", dataset.Name, err)
	}
	if !ok {
		return row, nil
	}

	parentKey, err := resolveInheritanceParentKey(row, spec.From)
	if err != nil {
		return nil, fmt.Errorf("dataset %s: %w", dataset.Name, err)
	}

	out := map[string]any{}
	for key, value := range row {
		out[key] = value
	}
	delete(out, "inherit")
	for _, field := range spec.Fields {
		columnName, ok := canonicalColumn(columns, field)
		if !ok {
			return nil, fmt.Errorf("inherit field %q is not a known column", field)
		}
		if rowFieldWasAuthored(source, columns, columnName, dataset.Spec) {
			continue
		}
		if current, ok := lookupField(out, columnName); ok && current != nullValue {
			continue
		}
		out[columnName] = map[string]any{
			"ref":   parentKey,
			"field": columnName,
		}
	}
	return out, nil
}

type rowInheritanceSpec struct {
	From   string
	Fields []string
}

type genericInheritanceSpec struct {
	Ref   string
	Field string
}

func parseRowInheritanceSpec(source map[string]any) (rowInheritanceSpec, bool, error) {
	raw, ok := source["inherit"]
	if !ok {
		return rowInheritanceSpec{}, false, nil
	}
	obj, ok := raw.(map[string]any)
	if !ok {
		return rowInheritanceSpec{}, false, fmt.Errorf("inherit must be an object")
	}
	if _, hasRef := obj["ref"]; hasRef {
		if _, hasFrom := obj["from"]; hasFrom {
			return rowInheritanceSpec{}, false, fmt.Errorf("inherit cannot mix row sugar (from/fields) with object inheritance (ref)")
		}
		return rowInheritanceSpec{}, false, nil
	}
	rawFrom, ok := obj["from"]
	if !ok {
		return rowInheritanceSpec{}, false, fmt.Errorf("inherit.from is required")
	}
	from, ok := rawFrom.(string)
	if !ok || strings.TrimSpace(from) == "" {
		return rowInheritanceSpec{}, false, fmt.Errorf("inherit.from must be a non-empty string")
	}
	rawFields, ok := obj["fields"]
	if !ok {
		return rowInheritanceSpec{}, false, fmt.Errorf("inherit.fields is required")
	}
	fieldList, ok := rawFields.([]any)
	if !ok || len(fieldList) == 0 {
		return rowInheritanceSpec{}, false, fmt.Errorf("inherit.fields must be a non-empty array")
	}
	fields := make([]string, 0, len(fieldList))
	seen := map[string]struct{}{}
	for _, rawField := range fieldList {
		field, ok := rawField.(string)
		if !ok || strings.TrimSpace(field) == "" {
			return rowInheritanceSpec{}, false, fmt.Errorf("inherit.fields must contain non-empty strings")
		}
		normalized := strings.ToLower(field)
		if _, exists := seen[normalized]; exists {
			continue
		}
		seen[normalized] = struct{}{}
		fields = append(fields, field)
	}
	return rowInheritanceSpec{From: from, Fields: fields}, true, nil
}

func parseGenericInheritanceSpec(source map[string]any) (genericInheritanceSpec, bool, error) {
	raw, ok := source["inherit"]
	if !ok {
		return genericInheritanceSpec{}, false, nil
	}
	obj, ok := raw.(map[string]any)
	if !ok {
		return genericInheritanceSpec{}, false, fmt.Errorf("inherit must be an object")
	}
	if _, hasFrom := obj["from"]; hasFrom {
		if _, hasRef := obj["ref"]; hasRef {
			return genericInheritanceSpec{}, false, fmt.Errorf("inherit cannot mix row sugar (from/fields) with object inheritance (ref)")
		}
		return genericInheritanceSpec{}, false, nil
	}
	rawRef, ok := obj["ref"]
	if !ok {
		return genericInheritanceSpec{}, false, fmt.Errorf("inherit.ref is required")
	}
	ref, ok := rawRef.(string)
	if !ok || strings.TrimSpace(ref) == "" {
		return genericInheritanceSpec{}, false, fmt.Errorf("inherit.ref must be a non-empty string")
	}
	spec := genericInheritanceSpec{Ref: strings.TrimSpace(ref)}
	if rawField, ok := obj["field"]; ok {
		field, ok := rawField.(string)
		if !ok || strings.TrimSpace(field) == "" {
			return genericInheritanceSpec{}, false, fmt.Errorf("inherit.field must be a non-empty string when present")
		}
		spec.Field = strings.TrimSpace(field)
	}
	return spec, true, nil
}

func resolveInheritanceParentKey(row map[string]any, from string) (string, error) {
	value, ok := lookupField(row, from)
	if !ok {
		return "", fmt.Errorf("inherit.from field %q was not found on row", from)
	}
	switch typed := value.(type) {
	case string:
		if strings.Contains(typed, ":") {
			return typed, nil
		}
	case map[string]any:
		if rawID, ok := typed["id"]; ok {
			if key, ok := rawID.(string); ok && strings.Contains(key, ":") {
				return key, nil
			}
		}
		if rawRef, ok := typed["ref"]; ok {
			if key, ok := rawRef.(string); ok && strings.Contains(key, ":") {
				if _, hasField := typed["field"]; !hasField {
					return key, nil
				}
			}
		}
	}
	return "", fmt.Errorf("inherit.from field %q must resolve to a keyed parent row reference", from)
}

func (r *valueResolver) resolveInheritedObject(ownerRow map[string]any, identity string, obj map[string]any, stack []string) (map[string]any, error) {
	spec, ok, err := parseGenericInheritanceSpec(obj)
	if err != nil {
		return nil, err
	}

	out := map[string]any{}
	for key, value := range obj {
		if key == "inherit" {
			continue
		}
		out[key] = cloneAuthoringValue(value)
	}
	if !ok {
		return out, nil
	}

	marker := "inherit:" + identity
	for _, existing := range stack {
		if existing == marker {
			return nil, fmt.Errorf("cyclic inheritance detected at %s", identity)
		}
	}

	parent, err := r.resolveGenericInheritanceTarget(ownerRow, spec, append(stack, marker))
	if err != nil {
		return nil, err
	}
	return mergeInheritedMaps(parent, out), nil
}

func (r *valueResolver) resolveGenericInheritanceTarget(ownerRow map[string]any, spec genericInheritanceSpec, stack []string) (map[string]any, error) {
	targetRow, ok := r.rowByKey[spec.Ref]
	if !ok {
		return nil, fmt.Errorf("inherit target %q was not found", spec.Ref)
	}
	if spec.Field == "" {
		return r.resolveInheritedObject(targetRow, spec.Ref, targetRow, stack)
	}
	value, ok := lookupField(targetRow, spec.Field)
	if !ok {
		return nil, fmt.Errorf("inherit target field %q was not found on %s", spec.Field, spec.Ref)
	}
	targetObj, ok := value.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("inherit target %s.%s must resolve to an object", spec.Ref, spec.Field)
	}
	return r.resolveInheritedObject(targetRow, spec.Ref+"."+spec.Field, targetObj, stack)
}

func mergeInheritedMaps(parent, local map[string]any) map[string]any {
	out := map[string]any{}
	for key, value := range parent {
		out[key] = cloneAuthoringValue(value)
	}
	for key, value := range local {
		if key == "inherit" {
			continue
		}
		parentValue, hasParent := out[key]
		if !hasParent {
			out[key] = cloneAuthoringValue(value)
			continue
		}
		out[key] = mergeInheritedValues(parentValue, value)
	}
	return out
}

func mergeInheritedValues(parent, local any) any {
	if isNullLike(local) {
		return cloneAuthoringValue(parent)
	}
	localMap, localIsMap := local.(map[string]any)
	parentMap, parentIsMap := parent.(map[string]any)
	if localIsMap && parentIsMap && !isAtomicAuthoringMap(localMap) && !isAtomicAuthoringMap(parentMap) {
		return mergeInheritedMaps(parentMap, localMap)
	}
	return cloneAuthoringValue(local)
}

func isAtomicAuthoringMap(value map[string]any) bool {
	for _, key := range []string{"tlk", "ref", "id", "table"} {
		if _, ok := value[key]; ok {
			return true
		}
	}
	return false
}

func cloneAuthoringValue(value any) any {
	switch typed := value.(type) {
	case map[string]any:
		out := map[string]any{}
		for key, item := range typed {
			out[key] = cloneAuthoringValue(item)
		}
		return out
	case []any:
		out := make([]any, 0, len(typed))
		for _, item := range typed {
			out = append(out, cloneAuthoringValue(item))
		}
		return out
	default:
		return typed
	}
}

func rowFieldWasAuthored(source map[string]any, columns []string, field string, spec datasetTextSpec) bool {
	if _, ok := source[field]; ok {
		return true
	}
	for key := range source {
		if canonical, ok := canonicalColumn(columns, key); ok && canonical == field {
			return true
		}
	}
	rawTLK, ok := source["_tlk"].(map[string]any)
	if !ok {
		return false
	}
	for roleName := range rawTLK {
		var role textRole
		switch strings.ToLower(roleName) {
		case "name":
			role = textRoleName
		case "description":
			role = textRoleDescription
		default:
			continue
		}
		column := columnForRole(columns, spec, role)
		if column == field {
			return true
		}
	}
	return false
}

func resolveOverrideTarget(datasetName string, override map[string]any, rowByID map[int]map[string]any, rowByKey map[string]map[string]any) (map[string]any, error) {
	if rawID, ok := override["id"]; ok {
		rowID, err := asInt(rawID)
		if err != nil {
			return nil, fmt.Errorf("dataset %s: override id %v is not numeric", datasetName, rawID)
		}
		row, ok := rowByID[rowID]
		if !ok {
			return nil, fmt.Errorf("dataset %s: override target id %d was not found", datasetName, rowID)
		}
		return row, nil
	}
	if rawKey, ok := override["key"]; ok {
		key, ok := rawKey.(string)
		if !ok || key == "" {
			return nil, fmt.Errorf("dataset %s: override key must be a string", datasetName)
		}
		if key != "" {
			row, ok := rowByKey[key]
			if ok {
				return row, nil
			}
			return nil, fmt.Errorf("dataset %s: override target key %q was not found", datasetName, key)
		}
	}
	return nil, fmt.Errorf("dataset %s: override must specify id or key", datasetName)
}

func overrideRequestsNullRow(override map[string]any) bool {
	value, ok := override["null"]
	if !ok {
		return false
	}
	switch typed := value.(type) {
	case bool:
		return typed
	case string:
		trimmed := strings.TrimSpace(strings.ToLower(typed))
		return trimmed == "true" || trimmed == "1" || trimmed == "yes"
	case int:
		return typed != 0
	case float64:
		return typed != 0
	default:
		return false
	}
}

func nullifyOverrideRow(row map[string]any, columns []string, rowByKey map[string]map[string]any) {
	if oldKey, ok := row["key"].(string); ok && oldKey != "" {
		if mapped, exists := rowByKey[oldKey]; exists {
			mappedID, mappedHasID := mapped["id"].(int)
			rowID, rowHasID := row["id"].(int)
			if mappedHasID && rowHasID && mappedID == rowID {
				delete(rowByKey, oldKey)
			}
		}
	}
	for field := range row {
		if field != "id" {
			delete(row, field)
		}
	}
	for _, column := range columns {
		row[column] = nullValue
	}
}

func retireRowIdentity(row map[string]any, lockData map[string]int, retiredKeys map[string]struct{}) {
	if row == nil {
		return
	}
	oldKey, ok := row["key"].(string)
	if !ok || oldKey == "" {
		return
	}
	rowID, _ := row["id"].(int)
	if lockedID, ok := lockData[oldKey]; ok && lockedID == rowID {
		delete(lockData, oldKey)
	}
	if retiredKeys != nil {
		retiredKeys[oldKey] = struct{}{}
	}
}

func updateOverrideRowKey(datasetName string, row map[string]any, expanded map[string]any, rowByKey map[string]map[string]any, lockData map[string]int) (bool, string, error) {
	oldKey, _ := row["key"].(string)
	newKeyValue, hasKey := expanded["key"]
	if !hasKey {
		return false, "", nil
	}
	rowID, _ := row["id"].(int)
	changed := false
	retiredKey := ""
	if oldKey != "" {
		if mapped, ok := rowByKey[oldKey]; ok {
			mappedID, mappedHasID := mapped["id"].(int)
			rowHasID := rowID != 0 || row["id"] != nil
			if mappedHasID && rowHasID && mappedID == rowID {
				delete(rowByKey, oldKey)
			}
		}
	}
	if newKeyValue == nil {
		delete(row, "key")
		if oldKey != "" {
			retiredKey = oldKey
			if lockedID, ok := lockData[oldKey]; ok && lockedID == rowID {
				delete(lockData, oldKey)
			}
			changed = true
		}
		return changed, retiredKey, nil
	}
	newKey, ok := newKeyValue.(string)
	if !ok || newKey == "" || strings.TrimSpace(newKey) == nullValue {
		delete(row, "key")
		if oldKey != "" {
			retiredKey = oldKey
			if lockedID, ok := lockData[oldKey]; ok && lockedID == rowID {
				delete(lockData, oldKey)
			}
			changed = true
		}
		return changed, retiredKey, nil
	}
	if oldKey == newKey {
		row["key"] = newKey
		rowByKey[newKey] = row
		if _, ok := lockData[newKey]; !ok {
			lockData[newKey] = rowID
			changed = true
		}
		return changed, "", nil
	}
	conflictingID := -1
	if conflicting, ok := rowByKey[newKey]; ok && conflicting != nil {
		var conflictingHasID bool
		conflictingID, conflictingHasID = conflicting["id"].(int)
		rowHasID := rowID != 0 || row["id"] != nil
		if !conflictingHasID || !rowHasID || conflictingID != rowID {
			delete(conflicting, "key")
			changed = true
		}
	}
	row["key"] = newKey
	rowByKey[newKey] = row
	if oldKey != "" && oldKey != newKey {
		retiredKey = oldKey
		if lockedID, ok := lockData[oldKey]; ok && lockedID == rowID {
			delete(lockData, oldKey)
		}
		changed = true
	}
	if existingID, ok := lockData[newKey]; ok {
		if existingID != rowID {
			if conflictingID >= 0 && existingID == conflictingID {
				lockData[newKey] = rowID
				changed = true
			} else {
				return false, "", fmt.Errorf("dataset %s: key %q is locked to id %d and cannot be rewritten to id %d", datasetName, newKey, existingID, rowID)
			}
		}
	} else {
		lockData[newKey] = rowID
		changed = true
	}
	return changed, retiredKey, nil
}

func normalizeCollectedSpellImpactScripts(rows []map[string]any, baseRowKeys map[string]struct{}) {
	for _, row := range rows {
		rowKey, ok := row["key"].(string)
		if !ok || rowKey == "" {
			continue
		}
		if _, ok := baseRowKeys[rowKey]; ok {
			continue
		}
		rowID, ok := row["id"].(int)
		if !ok {
			continue
		}
		row["ImpactScript"] = fmt.Sprintf("ss_%d", rowID)
	}
}

func write2DA(data map[string]any, path string, denseRows bool) error {
	columns, ok := data["columns"].([]string)
	if !ok {
		return fmt.Errorf("2da columns must be []string")
	}
	rows, ok := data["rows"].([]map[string]any)
	if !ok {
		return fmt.Errorf("2da rows must be []map[string]any")
	}

	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return fmt.Errorf("create 2da output parent: %w", err)
	}

	var builder strings.Builder
	builder.WriteString("2DA V2.0\n\n")
	builder.WriteString(strings.Join(columns, "\t"))
	builder.WriteByte('\n')
	nextID := 0
	for _, row := range rows {
		rowID := row["id"].(int)
		if denseRows {
			for nextID < rowID {
				builder.WriteString(strconv.Itoa(nextID))
				for range columns {
					builder.WriteByte('\t')
					builder.WriteString(nullValue)
				}
				builder.WriteByte('\n')
				nextID++
			}
		}
		builder.WriteString(strconv.Itoa(rowID))
		for _, column := range columns {
			builder.WriteByte('\t')
			builder.WriteString(format2DAValue(row[column]))
		}
		builder.WriteByte('\n')
		if denseRows {
			nextID = rowID + 1
		}
	}
	return os.WriteFile(path, []byte(builder.String()), 0o644)
}

func format2DAValue(value any) string {
	switch typed := value.(type) {
	case nil:
		return nullValue
	case string:
		if typed == nullValue {
			return typed
		}
		if strings.ContainsAny(typed, " \t\r\n") {
			return strconv.Quote(typed)
		}
		return typed
	case int:
		return strconv.Itoa(typed)
	case float64:
		if typed == float64(int64(typed)) {
			return strconv.FormatInt(int64(typed), 10)
		}
		return strconv.FormatFloat(typed, 'f', -1, 64)
	case bool:
		if typed {
			return "1"
		}
		return "0"
	default:
		raw, err := json.Marshal(typed)
		if err != nil {
			return fmt.Sprint(typed)
		}
		text := string(raw)
		if strings.ContainsAny(text, " \t\r\n") {
			return strconv.Quote(text)
		}
		return text
	}
}

func outputStem(outputName string) string {
	return strings.TrimSuffix(outputName, filepath.Ext(outputName))
}

func nativeDatasetDefaultOutputName(path string, kind nativeDatasetKind) string {
	name := filepath.Base(path)
	if kind == nativeDatasetPlain {
		name = strings.TrimSuffix(name, filepath.Ext(name))
	}
	return name + ".2da"
}

func loadJSONObject(path string) (map[string]any, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("read %s: %w", path, err)
	}
	var payload any
	if err := json.Unmarshal(raw, &payload); err != nil {
		return nil, fmt.Errorf("parse %s: %w", path, err)
	}
	obj, ok := payload.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("%s must contain a JSON object", path)
	}
	return obj, nil
}

func loadLockfile(path string) (map[string]int, error) {
	lockData := map[string]int{}
	if _, err := os.Stat(path); err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return lockData, nil
		}
		return nil, err
	}

	obj, err := loadJSONObject(path)
	if err != nil {
		return nil, err
	}
	for key, value := range obj {
		rowID, err := asInt(value)
		if err != nil {
			return nil, fmt.Errorf("lock entry %q is not numeric", key)
		}
		lockData[key] = rowID
	}
	return lockData, nil
}

func saveLockfile(path string, lockData map[string]int) error {
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return fmt.Errorf("create lockfile parent: %w", err)
	}
	keys, err := orderedLockfileKeys(path, lockData)
	if err != nil {
		return fmt.Errorf("read lockfile order: %w", err)
	}
	raw, err := marshalOrderedLockfile(keys, lockData)
	if err != nil {
		return fmt.Errorf("marshal lockfile: %w", err)
	}
	current, err := os.ReadFile(path)
	if err == nil && bytes.Equal(current, raw) {
		return nil
	}
	if err != nil && !errors.Is(err, os.ErrNotExist) {
		return fmt.Errorf("read lockfile %s: %w", path, err)
	}
	if err := os.WriteFile(path, raw, 0o644); err != nil {
		return fmt.Errorf("write lockfile %s: %w", path, err)
	}
	return nil
}

func orderedLockfileKeys(path string, lockData map[string]int) ([]string, error) {
	existingKeys, err := readLockfileKeyOrder(path)
	if err != nil && !errors.Is(err, os.ErrNotExist) {
		return nil, err
	}

	keys := make([]string, 0, len(lockData))
	seen := make(map[string]struct{}, len(lockData))
	for _, key := range existingKeys {
		if _, ok := lockData[key]; !ok {
			continue
		}
		keys = append(keys, key)
		seen[key] = struct{}{}
	}

	extraKeys := make([]string, 0, len(lockData)-len(keys))
	for key := range lockData {
		if _, ok := seen[key]; ok {
			continue
		}
		extraKeys = append(extraKeys, key)
	}
	slices.Sort(extraKeys)
	keys = append(keys, extraKeys...)
	return keys, nil
}

func readLockfileKeyOrder(path string) ([]string, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}

	decoder := json.NewDecoder(bytes.NewReader(raw))
	token, err := decoder.Token()
	if err != nil {
		return nil, err
	}
	delim, ok := token.(json.Delim)
	if !ok || delim != '{' {
		return nil, fmt.Errorf("%s must contain a JSON object", path)
	}

	keys := []string{}
	for decoder.More() {
		token, err := decoder.Token()
		if err != nil {
			return nil, err
		}
		key, ok := token.(string)
		if !ok {
			return nil, fmt.Errorf("%s contains a non-string object key", path)
		}
		var value json.RawMessage
		if err := decoder.Decode(&value); err != nil {
			return nil, err
		}
		keys = append(keys, key)
	}

	token, err = decoder.Token()
	if err != nil {
		return nil, err
	}
	delim, ok = token.(json.Delim)
	if !ok || delim != '}' {
		return nil, fmt.Errorf("%s must contain a JSON object", path)
	}
	return keys, nil
}

func marshalOrderedLockfile(keys []string, lockData map[string]int) ([]byte, error) {
	if len(keys) == 0 {
		return []byte("{}\n"), nil
	}

	var buffer bytes.Buffer
	buffer.WriteString("{\n")
	for index, key := range keys {
		keyRaw, err := json.Marshal(key)
		if err != nil {
			return nil, err
		}
		valueRaw, err := json.Marshal(lockData[key])
		if err != nil {
			return nil, err
		}
		buffer.WriteString("  ")
		buffer.Write(keyRaw)
		buffer.WriteString(": ")
		buffer.Write(valueRaw)
		if index < len(keys)-1 {
			buffer.WriteByte(',')
		}
		buffer.WriteByte('\n')
	}
	buffer.WriteString("}\n")
	return buffer.Bytes(), nil
}

func loadLegacyTLK(root string) (*legacyTLKData, error) {
	info, err := os.Stat(root)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	if !info.IsDir() {
		return nil, fmt.Errorf("legacy tlk root must be a directory: %s", root)
	}

	result := &legacyTLKData{
		Language: "en",
		Entries:  map[string]tlkEntryData{},
		Lock:     map[string]int{},
	}

	lockPath := filepath.Join(root, "lock.json")
	lockData, err := loadLockfile(lockPath)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	for key, id := range lockData {
		result.Lock[key] = id
	}

	basePath := filepath.Join(root, "base.json")
	if _, err := os.Stat(basePath); err == nil {
		base, err := loadJSONObject(basePath)
		if err != nil {
			return nil, err
		}
		if language, ok := base["language"].(string); ok && language != "" {
			result.Language = language
		}
		if entries, ok := base["entries"].(map[string]any); ok {
			normalized, err := normalizeLegacyTLKEntries(entries)
			if err != nil {
				return nil, err
			}
			for key, entry := range normalized {
				result.Entries[key] = entry
			}
		}
	}

	modulesDir := filepath.Join(root, "modules")
	modulePaths, err := collectModulePaths(modulesDir)
	if err != nil {
		if os.IsNotExist(err) {
			return result, nil
		}
		return nil, err
	}
	for _, path := range modulePaths {
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, err
		}
		entries, ok := obj["entries"].(map[string]any)
		if !ok {
			continue
		}
		normalized, err := normalizeLegacyTLKEntries(entries)
		if err != nil {
			return nil, fmt.Errorf("%s: %w", path, err)
		}
		for key, entry := range normalized {
			result.Entries[key] = entry
		}
	}
	return result, nil
}

func normalizeLegacyTLKEntries(entries map[string]any) (map[string]tlkEntryData, error) {
	normalized := map[string]tlkEntryData{}
	for _, key := range sortedKeys(entries) {
		value := entries[key]
		switch typed := value.(type) {
		case string:
			normalized[key] = tlkEntryData{Text: typed}
		case map[string]any:
			if shouldExpandLegacyKey(key, typed) {
				for _, subKey := range sortedKeys(typed) {
					subObj, ok := typed[subKey].(map[string]any)
					if !ok {
						continue
					}
					entry, err := parseLegacyTLKLeaf(subObj)
					if err != nil {
						return nil, err
					}
					normalized[key+"."+subKey] = entry
				}
				continue
			}
			entry, err := parseLegacyTLKLeaf(typed)
			if err != nil {
				return nil, err
			}
			normalized[key] = entry
		default:
			return nil, fmt.Errorf("legacy TLK entry %q must be an object or string", key)
		}
	}
	return normalized, nil
}

func shouldExpandLegacyKey(key string, obj map[string]any) bool {
	if strings.Contains(key[strings.LastIndex(key, ":")+1:], ".") {
		return false
	}
	for _, value := range obj {
		if _, ok := value.(map[string]any); ok {
			return true
		}
	}
	return false
}

func parseLegacyTLKLeaf(obj map[string]any) (tlkEntryData, error) {
	entry := tlkEntryData{}
	if rawText, ok := obj["text"]; ok {
		text, ok := rawText.(string)
		if !ok {
			return entry, fmt.Errorf("legacy TLK text must be a string")
		}
		entry.Text = text
	}
	if rawSound, ok := obj["sound_resref"]; ok {
		sound, ok := rawSound.(string)
		if !ok {
			return entry, fmt.Errorf("legacy TLK sound_resref must be a string")
		}
		entry.SoundResRef = sound
	}
	if rawLength, ok := obj["sound_length"]; ok {
		switch typed := rawLength.(type) {
		case float64:
			entry.SoundLength = float32(typed)
		case int:
			entry.SoundLength = float32(typed)
		default:
			return entry, fmt.Errorf("legacy TLK sound_length must be numeric")
		}
	}
	return entry, nil
}

func collectModulePaths(modulesDir string) ([]string, error) {
	if _, err := os.Stat(modulesDir); err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return nil, nil
		}
		return nil, err
	}
	var paths []string
	err := filepath.WalkDir(modulesDir, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		if strings.HasSuffix(strings.ToLower(d.Name()), ".json") {
			paths = append(paths, path)
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	slices.Sort(paths)
	return paths, nil
}

func canonicalColumn(columns []string, name string) (string, bool) {
	for _, column := range columns {
		if strings.EqualFold(column, name) {
			return column, true
		}
	}
	return "", false
}

func optionalBool(value any, fallback bool) bool {
	switch typed := value.(type) {
	case bool:
		return typed
	case nil:
		return fallback
	default:
		return fallback
	}
}

func asInt(value any) (int, error) {
	switch typed := value.(type) {
	case int:
		return typed, nil
	case int64:
		return int(typed), nil
	case float64:
		return int(typed), nil
	case json.Number:
		n, err := typed.Int64()
		return int(n), err
	case string:
		return strconv.Atoi(typed)
	default:
		return 0, fmt.Errorf("not an int")
	}
}

func nextAvailableID(used map[int]struct{}) int {
	next := 0
	for {
		if _, ok := used[next]; !ok {
			return next
		}
		next++
	}
}

func hasJSONFiles(root string) (bool, error) {
	found := false
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		if strings.HasSuffix(strings.ToLower(d.Name()), ".json") {
			found = true
			return fs.SkipAll
		}
		return nil
	})
	if err != nil && !errors.Is(err, fs.SkipAll) {
		return false, err
	}
	return found, nil
}

func collectReferenceLockIDs(root string) (map[string]int, error) {
	result := map[string]int{}
	if _, err := os.Stat(root); err != nil {
		if os.IsNotExist(err) {
			return result, nil
		}
		return nil, err
	}
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() || d.Name() != "lock.json" {
			return nil
		}
		lockData, err := loadLockfile(path)
		if err != nil {
			return err
		}
		for key, rowID := range lockData {
			if _, exists := result[key]; !exists {
				result[key] = rowID
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return result, nil
}

func collectSupplementalLockIDs(sourceDir, referenceBuilderDir string) (map[string]int, error) {
	result := map[string]int{}
	snapshotIDs, err := collectReferenceLockIDs(filepath.Join(sourceDir, "migration_snapshot", "data"))
	if err != nil {
		return nil, err
	}
	for key, rowID := range snapshotIDs {
		result[key] = rowID
	}
	if strings.TrimSpace(referenceBuilderDir) == "" {
		return result, nil
	}
	referenceIDs, err := collectReferenceLockIDs(filepath.Join(referenceBuilderDir, "data"))
	if err != nil {
		return nil, err
	}
	for key, rowID := range referenceIDs {
		if _, ok := result[key]; ok {
			continue
		}
		result[key] = rowID
	}
	return result, nil
}