sow-tools/internal/topdata/topdata.go

package topdata

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"

	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)

type Severity string

const (
	SeverityError   Severity = "error"
	SeverityWarning Severity = "warning"
)

type Diagnostic struct {
	Severity Severity
	Path     string
	Message  string
}

type ValidationReport struct {
	Files       int
	DataFiles   int
	TLKFiles    int
	Diagnostics []Diagnostic
}

func (r ValidationReport) SummaryLines(maxPaths int) []string {
	type key struct {
		severity Severity
		message  string
	}
	groups := map[key][]string{}
	order := make([]key, 0)
	for _, diagnostic := range r.Diagnostics {
		k := key{severity: diagnostic.Severity, message: diagnostic.Message}
		if _, ok := groups[k]; !ok {
			order = append(order, k)
		}
		groups[k] = append(groups[k], diagnostic.Path)
	}
	slices.SortFunc(order, func(a, b key) int {
		if a.severity != b.severity {
			if a.severity == SeverityError {
				return -1
			}
			return 1
		}
		return strings.Compare(a.message, b.message)
	})
	lines := make([]string, 0, len(order))
	for _, k := range order {
		paths := append([]string(nil), groups[k]...)
		slices.Sort(paths)
		paths = slices.Compact(paths)
		lines = append(lines, formatDiagnosticSummaryLine(string(k.severity), k.message, paths, maxPaths))
	}
	return lines
}

func (r ValidationReport) HasErrors() bool {
	for _, diagnostic := range r.Diagnostics {
		if diagnostic.Severity == SeverityError {
			return true
		}
	}
	return false
}

func (r ValidationReport) ErrorCount() int {
	count := 0
	for _, diagnostic := range r.Diagnostics {
		if diagnostic.Severity == SeverityError {
			count++
		}
	}
	return count
}

func (r ValidationReport) WarningCount() int {
	count := 0
	for _, diagnostic := range r.Diagnostics {
		if diagnostic.Severity == SeverityWarning {
			count++
		}
	}
	return count
}

func formatDiagnosticSummaryLine(prefix, message string, paths []string, maxPaths int) string {
	if len(paths) == 0 {
		return fmt.Sprintf("%s: %s", prefix, message)
	}
	if maxPaths <= 0 {
		maxPaths = 3
	}
	display := paths
	if len(display) > maxPaths {
		display = display[:maxPaths]
	}
	location := strings.Join(display, ", ")
	if len(paths) > len(display) {
		location += fmt.Sprintf(", and %d more", len(paths)-len(display))
	}
	return fmt.Sprintf("%s: %s [%s]", prefix, message, location)
}

type BuildResult struct {
	Mode          string
	Output2DADir  string
	OutputTLKDir  string
	Files2DA      int
	FilesTLK      int
	WikiOutputDir string
	WikiPages     int
	WikiStatus    string
}

type PackageResult struct {
	Mode          string
	Output2DADir  string
	OutputTLKDir  string
	OutputHAKPath string
	OutputTLKPath string
	Files2DA      int
	FilesTLK      int
	HAKResources  int
	AssetFiles    int
	WikiOutputDir string
	WikiPages     int
	WikiStatus    string
}

type CompareResult struct {
	Mode            string
	Compared2DA     int
	ComparedTLK     int
	NativePass      int
	NotYetCanonical int
	NativeMismatch  int
}

func ValidateProject(p *project.Project) ValidationReport {
	report := ValidationReport{}
	if !p.HasTopData() {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityWarning,
			Path:     project.ConfigFile,
			Message:  "topdata is not configured for this project",
		})
		return report
	}

	sourceDir := p.TopDataSourceDir()
	info, err := os.Stat(sourceDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     sourceDir,
			Message:  err.Error(),
		})
		return report
	}
	if !info.IsDir() {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     sourceDir,
			Message:  "topdata source must be a directory",
		})
		return report
	}

	dataDir := filepath.Join(sourceDir, "data")
	// Canonical topdata no longer requires a dedicated source tlk directory.
	// When present, it is only validated for disallowed legacy module input.
	tlkDir := filepath.Join(sourceDir, "tlk")
	baseDialog := filepath.Join(sourceDir, "base_dialog.json")
	statePath := filepath.Join(sourceDir, tlkStateFile)

	for _, requiredDir := range []string{dataDir} {
		info, err := os.Stat(requiredDir)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     requiredDir,
				Message:  err.Error(),
			})
			continue
		}
		if !info.IsDir() {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     requiredDir,
				Message:  "must be a directory",
			})
		}
	}

	if info, err := os.Stat(tlkDir); err == nil && info.IsDir() {
		modulesDir := filepath.Join(tlkDir, "modules")
		legacyModuleFiles := 0
		if moduleInfo, moduleErr := os.Stat(modulesDir); moduleErr == nil && moduleInfo.IsDir() {
			_ = walkJSON(modulesDir, func(path string) {
				legacyModuleFiles++
				report.Files++
				report.TLKFiles++
				_ = validateJSONFile(path, "tlk", &report)
			})
		} else if moduleErr != nil && !errors.Is(moduleErr, os.ErrNotExist) {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     modulesDir,
				Message:  moduleErr.Error(),
			})
		}
		if legacyModuleFiles > 0 {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     modulesDir,
				Message:  "topdata/tlk/modules is no longer allowed for canonical authored input; inline TLK-in-data is required",
			})
		}
		if lockInfo, lockErr := os.Stat(filepath.Join(tlkDir, "lock.json")); lockErr == nil && !lockInfo.IsDir() {
			report.Files++
			report.TLKFiles++
			_ = validateJSONFile(filepath.Join(tlkDir, "lock.json"), "tlk", &report)
		} else if lockErr != nil && !errors.Is(lockErr, os.ErrNotExist) {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     filepath.Join(tlkDir, "lock.json"),
				Message:  lockErr.Error(),
			})
		}
	}

	if _, err := os.Stat(baseDialog); err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityWarning,
			Path:     baseDialog,
			Message:  "base_dialog.json is missing; dialog migration is not canonical yet",
		})
	} else {
		report.Files++
		if err := validateJSONFile(baseDialog, "base_dialog", &report); err == nil {
			// count only once for base dialog.
		}
	}

	_ = walkJSON(dataDir, func(path string) {
		report.Files++
		report.DataFiles++
		_ = validateJSONFile(path, "data", &report)
	})
	validateTopPackageAssets(sourceDir, dataDir, &report)
	validateNativeOutputCatalog(dataDir, &report)
	validateNativeEntryKeyUniqueness(dataDir, &report)
	validateNativeLockAllocation(dataDir, &report)
	validateGeneratedFeatFamilies(dataDir, &report)
	validateItempropsRegistryGraph(dataDir, &report)
	if _, err := os.Stat(statePath); err == nil {
		report.Files++
		_ = validateJSONFile(statePath, "state", &report)
	}
	if !report.HasErrors() {
		validateNativeAuthoringWarnings(dataDir, &report)
		validateNativeBuildability(p, &report)
	}

	return report
}

func validateJSONFile(path, domain string, report *ValidationReport) error {
	raw, err := os.ReadFile(path)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  err.Error(),
		})
		return err
	}

	var payload any
	validateDuplicateJSONKeys(path, raw, report)
	if err := json.Unmarshal(raw, &payload); err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("invalid JSON: %v", err),
		})
		return err
	}

	obj, ok := payload.(map[string]any)
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "topdata files must contain a JSON object at the root",
		})
		return errors.New("topdata file root must be object")
	}

	switch domain {
	case "data":
		validateDataObject(path, obj, report)
	case "tlk":
		validateTLKObject(path, obj, report)
	case "state":
		validateStateObject(path, obj, report)
	case "base_dialog":
		validateBaseDialogObject(path, obj, report)
	}
	return nil
}

func validateDuplicateJSONKeys(path string, raw []byte, report *ValidationReport) {
	decoder := json.NewDecoder(bytes.NewReader(raw))
	if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil {
		return
	}
}

func scanJSONValueForDuplicateKeys(decoder *json.Decoder, path string, report *ValidationReport) error {
	token, err := decoder.Token()
	if err != nil {
		return err
	}
	switch delim := token.(type) {
	case json.Delim:
		switch delim {
		case '{':
			seen := map[string]struct{}{}
			for decoder.More() {
				rawKey, err := decoder.Token()
				if err != nil {
					return err
				}
				key, ok := rawKey.(string)
				if !ok {
					return fmt.Errorf("expected object key")
				}
				if _, ok := seen[key]; ok {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("duplicate JSON object key %q", key),
					})
				}
				seen[key] = struct{}{}
				if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil {
					return err
				}
			}
			_, err := decoder.Token()
			return err
		case '[':
			for decoder.More() {
				if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil {
					return err
				}
			}
			_, err := decoder.Token()
			return err
		default:
			return nil
		}
	default:
		return nil
	}
}

func validateDataObject(path string, obj map[string]any, report *ValidationReport) {
	base := filepath.Base(path)
	slashed := filepath.ToSlash(path)
	inModules := strings.Contains(slashed, "/modules/")
	dataPath := parseTopdataDataPath(path)

	if invariant, ok := datasetInvariantForPath(dataPath); ok {
		if dataPath.IsRootBase {
			validateDatasetInvariantBaseFile(path, obj, invariant, report)
			return
		}
		if dataPath.IsRootLock {
			validateDatasetInvariantLockFile(path, obj, invariant, report)
			return
		}
	}
	if strings.Contains(slashed, "/data/feat/generated/") {
		validateFeatGeneratedFile(path, obj, report)
		return
	}
	if strings.Contains(slashed, "/data/parts/overrides/") {
		validateOverridesFile(path, obj, report)
		return
	}

	if strings.Contains(slashed, "/racialtypes/registry/races/") {
		validateRacialtypesRegistryRaceFile(path, obj, report)
		return
	}

	if base == "lock.json" {
		validateLockObject(path, obj, report)
		return
	}
	if base == "base.json" {
		validateRowsFile(path, obj, report, true)
		return
	}
	if inModules {
		if _, hasEntries := obj["entries"]; hasEntries {
			validateEntriesFile(path, obj, report)
			return
		}
		if _, hasOverrides := obj["overrides"]; hasOverrides {
			validateOverridesFile(path, obj, report)
			return
		}
		if _, hasRows := obj["rows"]; hasRows {
			validateRowsFile(path, obj, report, false)
			return
		}
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityWarning,
			Path:     path,
			Message:  "module file does not use a recognized canonical shape yet; expected one of entries, overrides, or rows",
		})
		return
	}

	if _, hasRows := obj["rows"]; hasRows {
		validateRowsFile(path, obj, report, false)
		return
	}
	report.Diagnostics = append(report.Diagnostics, Diagnostic{
		Severity: SeverityWarning,
		Path:     path,
		Message:  "unrecognized data file shape; parseable for compatibility, but not canonical yet",
	})
}

type topdataDataPath struct {
	Segments   []string
	IsRootBase bool
	IsRootLock bool
}

func parseTopdataDataPath(path string) topdataDataPath {
	slashed := filepath.ToSlash(path)
	marker := "/data/"
	index := strings.LastIndex(slashed, marker)
	if index == -1 {
		return topdataDataPath{}
	}
	rel := strings.Trim(slashed[index+len(marker):], "/")
	if rel == "" {
		return topdataDataPath{}
	}
	segments := strings.Split(rel, "/")
	return topdataDataPath{
		Segments:   segments,
		IsRootBase: len(segments) == 2 && segments[1] == "base.json",
		IsRootLock: len(segments) == 2 && segments[1] == "lock.json",
	}
}

type datasetValidationInvariant struct {
	Name            string
	KeyPrefix       string
	RequiredColumns []string
}

var datasetValidationInvariants = map[string]datasetValidationInvariant{
	"feat": {
		Name:      "feat",
		KeyPrefix: "feat:",
	},
	"masterfeats": {
		Name:            "masterfeats",
		KeyPrefix:       "masterfeats:",
		RequiredColumns: []string{"LABEL", "STRREF", "DESCRIPTION"},
	},
}

func datasetInvariantForPath(dataPath topdataDataPath) (datasetValidationInvariant, bool) {
	if len(dataPath.Segments) == 0 {
		return datasetValidationInvariant{}, false
	}
	invariant, ok := datasetValidationInvariants[dataPath.Segments[0]]
	return invariant, ok
}

func validateDatasetInvariantBaseFile(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) {
	validateRowsFile(path, obj, report, true)
	validateDerivedBaseOutput(path, obj, invariant.Name, report)
	validateRequiredColumns(path, obj, invariant, report)
	validateRowsKeyPrefix(path, obj, invariant, report)
}

func validateRequiredColumns(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) {
	if len(invariant.RequiredColumns) == 0 {
		return
	}
	columns := extractValidationColumns(obj)
	for _, required := range invariant.RequiredColumns {
		if _, ok := canonicalColumn(columns, required); ok {
			continue
		}
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("%s base.json must include %s in columns", invariant.Name, required),
		})
	}
}

func validateRowsKeyPrefix(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) {
	if invariant.KeyPrefix == "" {
		return
	}
	rows, ok := obj["rows"].([]any)
	if !ok {
		return
	}
	for index, raw := range rows {
		row, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		key, ok := row["key"].(string)
		if !ok || strings.TrimSpace(key) == "" {
			continue
		}
		if !strings.HasPrefix(key, invariant.KeyPrefix) {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s row %d key %q must start with %s", invariant.Name, index, key, invariant.KeyPrefix),
			})
		}
	}
}

func validateDatasetInvariantLockFile(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) {
	validateLockObject(path, obj, report)
	if invariant.KeyPrefix == "" {
		return
	}
	for key := range obj {
		if !strings.HasPrefix(key, invariant.KeyPrefix) {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s lock key %q must start with %s", invariant.Name, key, invariant.KeyPrefix),
			})
		}
	}
}

func validateDerivedBaseOutput(path string, obj map[string]any, datasetLabel string, report *ValidationReport) {
	if output, ok := obj["output"]; !ok {
		return
	} else if outputText, ok := output.(string); !ok || strings.TrimSpace(outputText) != nativeDatasetDefaultOutputName(filepath.Dir(path), nativeDatasetBase) {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("%s output must be %s", datasetLabel, nativeDatasetDefaultOutputName(filepath.Dir(path), nativeDatasetBase)),
		})
	}
}

func validateFeatGeneratedFile(path string, obj map[string]any, report *ValidationReport) {
	family, ok := obj["family"].(string)
	if !ok || strings.TrimSpace(family) == "" {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "feat generated file must contain a non-empty family string",
		})
		return
	}
	if isFamilyExpansionObject(obj) {
		validateFeatFamilyExpansionFile(path, obj, report)
		return
	}
	switch family {
	default:
		_, hasEntries := obj["entries"]
		_, hasOverrides := obj["overrides"]
		if !hasEntries && !hasOverrides {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  "feat generated file must contain entries and/or overrides",
			})
			return
		}
		if hasEntries {
			validateEntriesFile(path, obj, report)
		}
		if hasOverrides {
			validateOverridesFile(path, obj, report)
		}
	}
}

func validateFeatFamilyExpansionFile(path string, obj map[string]any, report *ValidationReport) {
	spec, err := parseFamilyExpansionSpec(path, obj)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  err.Error(),
		})
		return
	}
	if !strings.HasPrefix(spec.Template, "masterfeats:") {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "compact feat generated template must start with masterfeats:",
		})
	}
	if spec.NamePrefix == "" {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "family expansion file must contain a non-empty name_prefix",
		})
	}
	if spec.LabelPrefix == "" {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "family expansion file must contain a non-empty label_prefix",
		})
	}
	if spec.ConstantPrefix == "" {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "family expansion file must contain a non-empty constant_prefix",
		})
	}
	if raw, ok := obj["overrides"]; ok {
		if _, ok := raw.(map[string]any); !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  "family expansion overrides must be an object keyed by source dataset key",
			})
		}
	}
}

func validateRacialtypesRegistryRaceFile(path string, obj map[string]any, report *ValidationReport) {
	key, ok := obj["key"]
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "racialtypes registry race file must contain a key field",
		})
	} else if _, ok := key.(string); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "racialtypes registry race key must be a string",
		})
	}

	core, ok := obj["core"]
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "racialtypes registry race file must contain a core object",
		})
	} else if _, ok := core.(map[string]any); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "racialtypes registry race core must be a JSON object",
		})
	}

	if rawOutput, ok := obj["feat_output"]; ok {
		if _, ok := rawOutput.(string); !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  "racialtypes registry race feat_output must be a string",
			})
		}
		if rawFeats, ok := obj["feats"]; ok {
			if _, ok := rawFeats.([]any); !ok {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  "racialtypes registry race feats must be an array",
				})
			}
		} else {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  "racialtypes registry race feat_output requires a feats array",
			})
		}
		return
	}

	if _, ok := obj["feats"]; ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "racialtypes registry race feats requires feat_output",
		})
	}
}

func validateTLKObject(path string, obj map[string]any, report *ValidationReport) {
	if filepath.Base(path) == "lock.json" {
		validateLockObject(path, obj, report)
		return
	}
	validateLegacyDialogEntries(path, obj, report, "TLK")
}

func validateBaseDialogObject(path string, obj map[string]any, report *ValidationReport) {
	validateLegacyDialogEntries(path, obj, report, "base_dialog")
}

func validateLegacyDialogEntries(path string, obj map[string]any, report *ValidationReport, label string) {
	entries, ok := obj["entries"]
	if ok {
		switch entries.(type) {
		case map[string]any, []any:
			// accepted compatibility shapes
		default:
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s entries must be a JSON object or array", label),
			})
		}
	}
	if language, ok := obj["language"]; ok {
		switch language.(type) {
		case string, float64:
			// accepted compatibility shapes
		default:
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s language must be a string or numeric language id when present", label),
			})
		}
	}
	if label == "TLK" && !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "TLK file must contain an entries object",
		})
	}
}

func validateLockObject(path string, obj map[string]any, report *ValidationReport) {
	ids := map[int]string{}
	checkDuplicateIDs := !strings.Contains(filepath.ToSlash(path), "/data/itemprops/registry/lock.json")
	for key, value := range obj {
		var rowID int
		switch value.(type) {
		case float64:
			// JSON numbers land here.
			if parsed, err := asInt(value); err == nil {
				rowID = parsed
			}
		default:
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("lock entry %q must be numeric", key),
			})
		}
		if checkDuplicateIDs {
			if rowID <= 0 {
				continue
			}
			if other, ok := ids[rowID]; ok && other != key {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  fmt.Sprintf("lock id collision between %q and %q at id %d", other, key, rowID),
				})
			}
			ids[rowID] = key
		}
	}
}

func validateRowsFile(path string, obj map[string]any, report *ValidationReport, requireColumns bool) {
	rows, ok := obj["rows"]
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "rows file must contain a rows array",
		})
		return
	}
	if _, ok := rows.([]any); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "rows must be a JSON array",
		})
	}
	if rowsList, ok := rows.([]any); ok {
		validateRowCollection(path, obj, rowsList, report)
	}
	if columns, ok := obj["columns"]; ok {
		if list, ok := columns.([]any); !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  "columns must be a JSON array when present",
			})
		} else {
			for _, raw := range list {
				column, ok := raw.(string)
				if !ok {
					continue
				}
				if isAuthoringOnlyField(column) && !preserveLegacyWikiColumnForPath(path, column) {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("deprecated wiki column %q must be moved into meta.wiki", column),
					})
				}
			}
		}
	} else if requireColumns {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "base dataset file must contain columns",
		})
	}
}

func validateEntriesFile(path string, obj map[string]any, report *ValidationReport) {
	entries, ok := obj["entries"]
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "entries file must contain an entries object",
		})
		return
	}
	if _, ok := entries.(map[string]any); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "entries must be a JSON object",
		})
	}
	if entryMap, ok := entries.(map[string]any); ok {
		for key, raw := range entryMap {
			entry, ok := raw.(map[string]any)
			if !ok {
				continue
			}
			for field := range entry {
				if isAuthoringOnlyField(field) && !preserveLegacyWikiColumnForPath(path, field) {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("entry %q uses deprecated wiki field %q; move it into meta.wiki", key, field),
					})
				}
			}
			validateRowMetadata(path, fmt.Sprintf("entry %q", key), entry, report)
			validateInlineTextUsage(path, key, entry, report)
		}
	}
}

func validateOverridesFile(path string, obj map[string]any, report *ValidationReport) {
	overrides, ok := obj["overrides"]
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "override file must contain an overrides array",
		})
		return
	}
	if _, ok := overrides.([]any); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "overrides must be a JSON array",
		})
	}
	if overrideList, ok := overrides.([]any); ok {
		container := map[string]any{"rows": overrideList}
		validateRowCollection(path, container, overrideList, report)
	}
}

func validateStateObject(path string, obj map[string]any, report *ValidationReport) {
	entries, ok := obj["entries"].(map[string]any)
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "TLK state file must contain an entries object",
		})
		return
	}
	ids := map[int]string{}
	for key, raw := range entries {
		entry, ok := raw.(map[string]any)
		if !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("TLK state entry %q must be an object", key),
			})
			continue
		}
		idValue, ok := entry["id"]
		if !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("TLK state entry %q is missing id", key),
			})
			continue
		}
		id, err := asInt(idValue)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("TLK state entry %q id must be numeric", key),
			})
			continue
		}
		if other, ok := ids[id]; ok && other != key {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("TLK state id collision between %q and %q", other, key),
			})
		}
		ids[id] = key
	}
}

func validateNativeOutputCatalog(dataDir string, report *ValidationReport) {
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  err.Error(),
		})
		return
	}
	registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  err.Error(),
		})
		return
	}
	outputOwners := map[string]string{}
	for _, dataset := range datasets {
		recordNativeOutputOwner(dataset.OutputName, dataset.Name, outputOwners, report)
	}
	for _, dataset := range registryDatasets {
		recordNativeOutputOwner(dataset.Dataset.OutputName, dataset.Dataset.Name, outputOwners, report)
	}
}

func validateNativeEntryKeyUniqueness(dataDir string, report *ValidationReport) {
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  fmt.Sprintf("discover native datasets for entry key uniqueness validation: %v", err),
		})
		return
	}
	for _, dataset := range datasets {
		if dataset.Kind != nativeDatasetBase {
			continue
		}
		entryKeyPath := map[string]string{}
		for _, dir := range []string{dataset.ModulesDir, dataset.GeneratedDir} {
			paths, err := collectModulePaths(dir)
			if err != nil {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     dir,
					Message:  fmt.Sprintf("collect module paths for entry key uniqueness validation: %v", err),
				})
				continue
			}
			for _, path := range paths {
				obj, err := loadJSONObject(path)
				if err != nil {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("load module for entry key uniqueness validation: %v", err),
					})
					continue
				}
				entries, ok := obj["entries"].(map[string]any)
				if !ok {
					continue
				}
				for _, key := range sortedKeys(entries) {
					if key == "" {
						continue
					}
					if previousPath, exists := entryKeyPath[key]; exists {
						report.Diagnostics = append(report.Diagnostics, Diagnostic{
							Severity: SeverityError,
							Path:     path,
							Message:  fmt.Sprintf("duplicate entries key %q declared by %s and %s", key, previousPath, path),
						})
						continue
					}
					entryKeyPath[key] = path
				}
			}
		}
	}
}

func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  fmt.Sprintf("discover native datasets for lock allocation validation: %v", err),
		})
		return
	}
	for _, dataset := range datasets {
		if dataset.Kind != nativeDatasetBase {
			continue
		}
		baseObj, err := loadJSONObject(dataset.BasePath)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     dataset.BasePath,
				Message:  fmt.Sprintf("load base rows for lock allocation validation: %v", err),
			})
			continue
		}
		rawRows, ok := baseObj["rows"].([]any)
		if !ok || len(rawRows) == 0 {
			continue
		}
		baseBoundaryID := len(rawRows) - 1
		baseKeys := map[string]struct{}{}
		for index, raw := range rawRows {
			row, ok := raw.(map[string]any)
			if !ok {
				continue
			}
			rowID := index
			if rawID, ok := row["id"]; ok {
				if parsed, err := asInt(rawID); err == nil {
					rowID = parsed
				}
			}
			baseBoundaryID = rowID
			if key, ok := row["key"].(string); ok && key != "" {
				baseKeys[key] = struct{}{}
			}
		}
		explicitModuleIDs := map[string]int{}
		for _, dir := range []string{dataset.ModulesDir, dataset.GeneratedDir} {
			paths, err := collectModulePaths(dir)
			if err != nil {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     dir,
					Message:  fmt.Sprintf("collect module paths for lock allocation validation: %v", err),
				})
				continue
			}
			for _, path := range paths {
				obj, err := loadJSONObject(path)
				if err != nil {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("load module for lock allocation validation: %v", err),
					})
					continue
				}
				ids, err := collectExplicitModuleIDs(dataset.Name, path, obj)
				if err != nil {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  err.Error(),
					})
					continue
				}
				for key, rowID := range ids {
					explicitModuleIDs[key] = rowID
				}
			}
		}
		lockData, err := loadLockfile(dataset.LockPath)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     dataset.LockPath,
				Message:  fmt.Sprintf("load lockfile for allocation validation: %v", err),
			})
			continue
		}
		if dataset.Name == "feat" {
			generatedModules, err := collectGeneratedModules(dataset, lockData)
			if err != nil {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     dataset.GeneratedDir,
					Message:  fmt.Sprintf("collect generated feat modules for lock allocation validation: %v", err),
				})
				continue
			}
			for _, module := range generatedModules {
				ids, err := collectExplicitModuleIDs(dataset.Name, module.Name, module.Data)
				if err != nil {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     module.Name,
						Message:  err.Error(),
					})
					continue
				}
				for key, rowID := range ids {
					explicitModuleIDs[key] = rowID
				}
			}
		}
		invalid := 0
		for key, rowID := range lockData {
			if rowID > baseBoundaryID {
				continue
			}
			if _, ok := baseKeys[key]; ok {
				continue
			}
			if explicitID, ok := explicitModuleIDs[key]; ok && explicitID == rowID {
				continue
			}
			invalid++
		}
		if invalid > 0 {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityWarning,
				Path:     dataset.LockPath,
				Message:  fmt.Sprintf("%d lock entrie(s) are inside base id space and will be regenerated above row %d", invalid, baseBoundaryID),
			})
		}
	}
}

func validateNativeAuthoringWarnings(dataDir string, report *ValidationReport) {
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		return
	}
	for _, dataset := range datasets {
		if dataset.Kind != nativeDatasetBase {
			continue
		}
		validateDatasetOverrideTargetWarnings(dataset, report)
		validateDatasetNormalizedLockWarnings(dataset, report)
		validateDatasetGhostRowWarnings(dataset, report)
	}
}

func validateDatasetOverrideTargetWarnings(dataset nativeDataset, report *ValidationReport) {
	if dataset.BasePath == "" {
		return
	}
	if _, err := os.Stat(dataset.BasePath); err != nil {
		return
	}
	baseObj, err := loadJSONObject(dataset.BasePath)
	if err != nil {
		return
	}
	rawRows, ok := baseObj["rows"].([]any)
	if !ok {
		return
	}
	rowIDToKey := map[int]string{}
	keyToID := map[string]int{}
	usedIDs := map[int]struct{}{}
	for index, raw := range rawRows {
		row, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		rowID := index
		if rawID, ok := row["id"]; ok {
			if parsed, err := asInt(rawID); err == nil {
				rowID = parsed
			}
		}
		usedIDs[rowID] = struct{}{}
		if key, ok := row["key"].(string); ok && key != "" {
			rowIDToKey[rowID] = key
			keyToID[key] = rowID
		}
	}
	nextID := nextAvailableID(usedIDs)
	allocateNextID := func() int {
		rowID := nextID
		usedIDs[rowID] = struct{}{}
		nextID = nextAvailableID(usedIDs)
		return rowID
	}
	modulePaths, err := collectModulePaths(dataset.ModulesDir)
	if err != nil {
		return
	}
	for _, path := range modulePaths {
		obj, err := loadJSONObject(path)
		if err != nil {
			continue
		}
		if rawEntries, ok := obj["entries"].(map[string]any); ok {
			for key := range rawEntries {
				if key == "" {
					continue
				}
				if _, exists := keyToID[key]; exists {
					continue
				}
				rowID := allocateNextID()
				keyToID[key] = rowID
				rowIDToKey[rowID] = key
			}
		}
		rawOverrides, ok := obj["overrides"].([]any)
		if !ok {
			continue
		}
		for index, raw := range rawOverrides {
			override, ok := raw.(map[string]any)
			if !ok {
				continue
			}
			keyOnlyTarget := false
			keyText, hasKeyText := override["key"].(string)
			if _, hasID := override["id"]; !hasID && hasKeyText && keyText != "" {
				keyOnlyTarget = true
				if _, exists := keyToID[keyText]; !exists {
					canonical := resolveCanonicalDatasetKey(keyText, keyToID)
					if canonical != keyText {
						report.Diagnostics = append(report.Diagnostics, Diagnostic{
							Severity: SeverityWarning,
							Path:     path,
							Message:  fmt.Sprintf("override %d targets key %q, but the live canonical key at that point is %q; this key-only override will create or retarget a new row", index, keyText, canonical),
						})
					} else {
						report.Diagnostics = append(report.Diagnostics, Diagnostic{
							Severity: SeverityWarning,
							Path:     path,
							Message:  fmt.Sprintf("override %d targets key %q, but no live row owns that key at that point; this key-only override will create or retarget a new row", index, keyText),
						})
					}
				}
			}
			rowID := -1
			if rawID, ok := override["id"]; ok {
				parsed, err := asInt(rawID)
				if err == nil {
					rowID = parsed
				}
			} else if hasKeyText && keyText != "" {
				if mappedID, exists := keyToID[keyText]; exists {
					rowID = mappedID
				}
			}
			if rowID < 0 {
				if keyOnlyTarget {
					rowID = allocateNextID()
					keyToID[keyText] = rowID
					rowIDToKey[rowID] = keyText
				}
				continue
			}
			currentKey := rowIDToKey[rowID]
			if overrideRequestsNullRow(override) {
				if currentKey != "" {
					delete(keyToID, currentKey)
					delete(rowIDToKey, rowID)
				}
				continue
			}
			if rawKey, present := override["key"]; present {
				switch typed := rawKey.(type) {
				case nil:
					if currentKey != "" {
						delete(keyToID, currentKey)
						delete(rowIDToKey, rowID)
					}
				case string:
					if typed == "" || strings.TrimSpace(typed) == nullValue {
						if currentKey != "" {
							delete(keyToID, currentKey)
							delete(rowIDToKey, rowID)
						}
						continue
					}
					if currentKey != "" && currentKey != typed {
						delete(keyToID, currentKey)
					}
					if previousID, exists := keyToID[typed]; exists && previousID != rowID {
						delete(rowIDToKey, previousID)
					}
					keyToID[typed] = rowID
					rowIDToKey[rowID] = typed
				}
			}
		}
	}
}

func validateDatasetNormalizedLockWarnings(dataset nativeDataset, report *ValidationReport) {
	lockData, err := loadLockfile(dataset.LockPath)
	if err != nil {
		return
	}
	groups := map[string][]string{}
	for key := range lockData {
		prefix, suffix, ok := strings.Cut(key, ":")
		if !ok || prefix == "" || suffix == "" {
			continue
		}
		groups[prefix+":"+normalizeKeyIdentity(suffix)] = append(groups[prefix+":"+normalizeKeyIdentity(suffix)], key)
	}
	for _, keys := range groups {
		if len(keys) < 2 {
			continue
		}
		slices.Sort(keys)
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityWarning,
			Path:     dataset.LockPath,
			Message:  fmt.Sprintf("lockfile keeps multiple normalized-equivalent keys alive: %s", strings.Join(keys, ", ")),
		})
	}
}

func validateDatasetGhostRowWarnings(dataset nativeDataset, report *ValidationReport) {
	if dataset.BasePath == "" {
		return
	}
	if _, err := os.Stat(dataset.BasePath); err != nil {
		return
	}
	baseObj, err := loadJSONObject(dataset.BasePath)
	if err != nil {
		return
	}
	rawRows, ok := baseObj["rows"].([]any)
	if !ok {
		return
	}
	baseKeyedIDs := map[int]struct{}{}
	for index, raw := range rawRows {
		row, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		if key, ok := row["key"].(string); !ok || key == "" {
			continue
		}
		rowID := index
		if rawID, ok := row["id"]; ok {
			if parsed, err := asInt(rawID); err == nil {
				rowID = parsed
			}
		}
		baseKeyedIDs[rowID] = struct{}{}
	}
	collected, err := collectNativeDataset(dataset)
	if err != nil {
		return
	}
	for _, row := range collected.Rows {
		if key, ok := row["key"].(string); ok && key != "" {
			continue
		}
		rowID, _ := row["id"].(int)
		if _, hadBaseKey := baseKeyedIDs[rowID]; !hadBaseKey {
			continue
		}
		nonNullFields := make([]string, 0, 4)
		for field, value := range row {
			if field == "id" || field == "key" || field == "meta" {
				continue
			}
			if isEffectivelyNullValue(value) {
				continue
			}
			nonNullFields = append(nonNullFields, field)
		}
		if len(nonNullFields) == 0 || len(nonNullFields) > 3 {
			continue
		}
		slices.Sort(nonNullFields)
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityWarning,
			Path:     dataset.BasePath,
			Message:  fmt.Sprintf("row id %d has no canonical key but still carries fields %s; this usually indicates a key-only override hit a retired row or a row was partially de-identified", rowID, strings.Join(nonNullFields, ", ")),
		})
	}
}

func isEffectivelyNullValue(value any) bool {
	switch typed := value.(type) {
	case nil:
		return true
	case string:
		return strings.TrimSpace(typed) == "" || strings.TrimSpace(typed) == nullValue
	default:
		return false
	}
}

type requiredFeatFamily struct {
	FamilyKey string
	Dataset   string
	Column    string
	Predicate string
}

func validateGeneratedFeatFamilies(dataDir string, report *ValidationReport) {
	featDataset, ok, err := discoverNamedNativeDataset(dataDir, "feat")
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  fmt.Sprintf("discover feat dataset for generated family validation: %v", err),
		})
		return
	}
	if !ok {
		return
	}
	lockData, err := loadLockfile(featDataset.LockPath)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     featDataset.LockPath,
			Message:  fmt.Sprintf("load feat lockfile for generated family validation: %v", err),
		})
		return
	}
	ctx, err := newFeatGeneratedContext(featDataset, lockData)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     featDataset.GeneratedDir,
			Message:  fmt.Sprintf("prepare generated family validation: %v", err),
		})
		return
	}
	specs, specPaths, err := collectGeneratedFamilySpecs(featDataset.GeneratedDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     featDataset.GeneratedDir,
			Message:  fmt.Sprintf("load generated family specs: %v", err),
		})
		return
	}
	for familyKey, spec := range specs {
		ctx.familySpecs[familyKey] = spec
	}

	required := []requiredFeatFamily{
		{FamilyKey: "skillfocus", Dataset: "skills", Predicate: "accessible"},
		{FamilyKey: "greaterskillfocus", Dataset: "skills", Predicate: "accessible"},
		{FamilyKey: "weaponfocus", Dataset: "baseitems", Column: "WeaponFocusFeat"},
		{FamilyKey: "weaponspecialization", Dataset: "baseitems", Column: "WeaponSpecializationFeat"},
		{FamilyKey: "improvedcritical", Dataset: "baseitems", Column: "WeaponImprovedCriticalFeat"},
		{FamilyKey: "overwhelmingcritical", Dataset: "baseitems", Column: "EpicWeaponOverwhelmingCriticalFeat"},
		{FamilyKey: "greaterweaponfocus", Dataset: "baseitems", Column: "EpicWeaponFocusFeat"},
		{FamilyKey: "greaterweaponspecialization", Dataset: "baseitems", Column: "EpicWeaponSpecializationFeat"},
		{FamilyKey: "weaponofchoice", Dataset: "baseitems", Column: "WeaponOfChoiceFeat"},
	}
	for _, requirement := range required {
		spec, path, ok := generatedFamilySpecByNormalizedKey(specs, specPaths, requirement.FamilyKey)
		if !ok {
			continue
		}
		if spec.ChildSource.Dataset != requirement.Dataset ||
			spec.ChildSource.Column != requirement.Column ||
			spec.ChildSource.Predicate != requirement.Predicate {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("generated feat family %q must use child source dataset=%q column=%q predicate=%q", requirement.FamilyKey, requirement.Dataset, requirement.Column, requirement.Predicate),
			})
			continue
		}
		validateGeneratedFeatFamilyCompleteness(path, spec, ctx, report)
	}
	if hasGeneratedFamilySpecs(specs, coreWeaponGeneratedFamilyKeys) {
		validateBaseitemsWeaponFeatColumnCompleteness(featDataset.GeneratedDir, ctx, report)
	}
	validateRequiredProductionGeneratedFamilies(featDataset.GeneratedDir, specs, report)
}

var coreWeaponGeneratedFamilyKeys = []string{
	"weaponfocus",
	"weaponspecialization",
	"improvedcritical",
	"overwhelmingcritical",
	"greaterweaponfocus",
	"greaterweaponspecialization",
}

func hasGeneratedFamilySpecs(specs map[string]familyExpansionSpec, familyKeys []string) bool {
	for _, familyKey := range familyKeys {
		if _, _, ok := generatedFamilySpecByNormalizedKey(specs, nil, familyKey); !ok {
			return false
		}
	}
	return true
}

func validateBaseitemsWeaponFeatColumnCompleteness(path string, ctx *featGeneratedContext, report *ValidationReport) {
	rows, err := ctx.datasetRows("baseitems")
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("validate baseitems weapon feat coverage: %v", err),
		})
		return
	}
	coreColumns := []string{
		"WeaponFocusFeat",
		"WeaponSpecializationFeat",
		"WeaponImprovedCriticalFeat",
		"EpicWeaponFocusFeat",
		"EpicWeaponSpecializationFeat",
		"EpicWeaponOverwhelmingCriticalFeat",
	}
	for _, sourceKey := range sortedStringMapKeys(rows) {
		row := rows[sourceKey]
		hasAny := false
		missing := make([]string, 0)
		for _, column := range coreColumns {
			value, ok := lookupField(row, column)
			if !ok || isNullishValue(value) {
				missing = append(missing, column)
				continue
			}
			hasAny = true
		}
		if !hasAny || len(missing) == 0 {
			continue
		}
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("baseitems row %q declares partial core weapon feat coverage; missing %s", sourceKey, strings.Join(missing, ", ")),
		})
	}
}

func generatedFamilySpecByNormalizedKey(specs map[string]familyExpansionSpec, specPaths map[string]string, familyKey string) (familyExpansionSpec, string, bool) {
	normalized := normalizeKeyIdentity(familyKey)
	for candidateKey, spec := range specs {
		if normalizeKeyIdentity(candidateKey) == normalized {
			path := ""
			if specPaths != nil {
				path = specPaths[candidateKey]
			}
			return spec, path, true
		}
	}
	return familyExpansionSpec{}, "", false
}

func validateRequiredProductionGeneratedFamilies(path string, specs map[string]familyExpansionSpec, report *ValidationReport) {
	if !hasGeneratedFamilySpecs(specs, coreWeaponGeneratedFamilyKeys) {
		return
	}
	if _, _, ok := generatedFamilySpecByNormalizedKey(specs, nil, "weaponofchoice"); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "missing generated feat family \"weaponofchoice\"",
		})
	}
}

func discoverNamedNativeDataset(dataDir, name string) (nativeDataset, bool, error) {
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		return nativeDataset{}, false, err
	}
	for _, dataset := range datasets {
		if dataset.Name == name {
			return dataset, true, nil
		}
	}
	return nativeDataset{}, false, nil
}

func collectGeneratedFamilySpecs(generatedDir string) (map[string]familyExpansionSpec, map[string]string, error) {
	paths, err := collectModulePaths(generatedDir)
	if err != nil {
		return nil, nil, err
	}
	specs := map[string]familyExpansionSpec{}
	pathsByFamily := map[string]string{}
	for _, path := range paths {
		obj, err := loadJSONObject(path)
		if err != nil {
			return nil, nil, err
		}
		if !isFamilyExpansionObject(obj) {
			continue
		}
		spec, err := parseFamilyExpansionSpec(path, obj)
		if err != nil {
			return nil, nil, err
		}
		if previous, ok := pathsByFamily[spec.FamilyKey]; ok {
			return nil, nil, fmt.Errorf("generated family %q is declared by both %s and %s", spec.FamilyKey, previous, path)
		}
		specs[spec.FamilyKey] = spec
		pathsByFamily[spec.FamilyKey] = path
	}
	return specs, pathsByFamily, nil
}

func validateGeneratedFeatFamilyCompleteness(path string, spec familyExpansionSpec, ctx *featGeneratedContext, report *ValidationReport) {
	sourceRows, err := ctx.datasetRows(spec.ChildSource.Dataset)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  err.Error(),
		})
		return
	}
	moduleData, err := buildFamilyExpansionGeneratedModule(path, map[string]any{
		"family":              spec.Family,
		"family_key":          spec.FamilyKey,
		"template":            spec.Template,
		"name_prefix":         spec.NamePrefix,
		"label_prefix":        spec.LabelPrefix,
		"constant_prefix":     spec.ConstantPrefix,
		"template_fields":     validationStringSliceToAny(spec.TemplateFields),
		"default_fields":      spec.DefaultFields,
		"apply_after_modules": spec.ApplyAfterModules,
		"child_ref_field":     spec.ChildRefField,
		"identity_source":     spec.IdentitySource,
		"allow_existing_only": spec.AllowExistingOnly,
		"auto_prereq_fields":  stringMapToAny(spec.AutoPrereqFields),
		"legacy_family_keys":  validationStringSliceToAny(spec.LegacyFamilyKeys),
		"child_source": map[string]any{
			"dataset":   spec.ChildSource.Dataset,
			"column":    spec.ChildSource.Column,
			"predicate": spec.ChildSource.Predicate,
		},
		"overrides": map[string]any{},
	}, ctx)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  err.Error(),
		})
		return
	}
	expected := map[string]struct{}{}
	familyAllowlist := spec.AllowExistingOnly && ctx.familyHasExistingRows(spec.FamilyKey)
	for sourceKey, row := range sourceRows {
		include, err := familyExpansionSourceEligible(spec, row)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  err.Error(),
			})
			return
		}
		if include {
			if familyAllowlist {
				slug := strings.TrimPrefix(sourceKey, spec.ChildSource.Dataset+":")
				featKey, _, _, err := ctx.resolveGeneratedFeatIdentity(spec, slug, row)
				if err != nil {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  err.Error(),
					})
					return
				}
				if !ctx.featKeyExists(featKey) {
					continue
				}
			}
			expected[sourceKey] = struct{}{}
		}
	}
	actual := map[string]string{}
	overrides, _ := moduleData["overrides"].([]any)
	for index, raw := range overrides {
		override, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		source, ok := familySourceFromMeta(override["meta"])
		if !ok || source == "" {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("generated family %q row %d is missing family source metadata", spec.FamilyKey, index),
			})
			continue
		}
		if previousKey, ok := actual[source]; ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("generated family %q emits source %q more than once (%s and row %d)", spec.FamilyKey, source, previousKey, index),
			})
		}
		key, _ := override["key"].(string)
		actual[source] = key
	}
	for source := range expected {
		if _, ok := actual[source]; !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("generated family %q is missing source %q", spec.FamilyKey, source),
			})
		}
	}
	for source := range actual {
		if _, ok := expected[source]; !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("generated family %q includes unexpected source %q", spec.FamilyKey, source),
			})
		}
	}
}

func familySourceFromMeta(raw any) (string, bool) {
	meta, ok := raw.(map[string]any)
	if !ok {
		return "", false
	}
	family, ok := meta["family"].(map[string]any)
	if !ok {
		return "", false
	}
	source, ok := family["source"].(string)
	return source, ok
}

func validationStringSliceToAny(items []string) []any {
	if items == nil {
		return nil
	}
	out := make([]any, 0, len(items))
	for _, item := range items {
		out = append(out, item)
	}
	return out
}

func stringMapToAny(items map[string]string) map[string]any {
	if items == nil {
		return nil
	}
	out := make(map[string]any, len(items))
	for key, value := range items {
		out[key] = value
	}
	return out
}

func validateTopPackageAssets(sourceDir, dataDir string, report *ValidationReport) {
	assetsDir := filepath.Join(sourceDir, "assets")
	generated2DADir := filepath.Join(assetsDir, "2da")
	info, err := os.Stat(assetsDir)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return
		}
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     assetsDir,
			Message:  err.Error(),
		})
		return
	}
	if !info.IsDir() {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     assetsDir,
			Message:  "must be a directory",
		})
		return
	}

	projectOutputs, err := discoverNativeOutputCatalog(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     assetsDir,
			Message:  err.Error(),
		})
		return
	}

	seen := map[string]string{}
	err = filepath.WalkDir(assetsDir, func(path string, d fs.DirEntry, walkErr error) error {
		if walkErr != nil {
			return walkErr
		}
		if d.IsDir() && path == generated2DADir {
			return filepath.SkipDir
		}
		if d.IsDir() {
			return nil
		}
		report.Files++
		rel, err := filepath.Rel(assetsDir, path)
		if err != nil {
			return err
		}
		if strings.HasPrefix(filepath.Base(path), ".") {
			return nil
		}
		base := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)))
		ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
		if _, ok := erf.HAKResourceTypeForExtension(ext); !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("unsupported topdata asset HAK resource extension %q", filepath.Ext(path)),
			})
			return nil
		}
		key := base + "." + ext
		if previous, ok := seen[key]; ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("duplicate topdata asset resource %q also defined by %s", key, previous),
			})
			return nil
		}
		seen[key] = rel
		if owner, ok := projectOutputs[key]; ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("topdata asset resource %q collides with compiled topdata output from %s", key, owner),
			})
		}
		return nil
	})
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     assetsDir,
			Message:  err.Error(),
		})
	}
}

func recordNativeOutputOwner(outputName, owner string, outputOwners map[string]string, report *ValidationReport) {
	if existing, ok := outputOwners[outputName]; ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     owner,
			Message:  fmt.Sprintf("duplicate 2da output %q also produced by %s", outputName, existing),
		})
		return
	}
	outputOwners[outputName] = owner
}

func validateNativeBuildability(p *project.Project, report *ValidationReport) {
	dataDir := filepath.Join(p.TopDataSourceDir(), "data")
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  err.Error(),
		})
		return
	}
	registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  err.Error(),
		})
		return
	}
	if len(datasets) == 0 && len(registryDatasets) == 0 {
		return
	}
	tempBuild, err := os.MkdirTemp("", "sow-topdata-validate-*")
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     p.TopDataSourceDir(),
			Message:  fmt.Sprintf("create validation temp dir: %v", err),
		})
		return
	}
	defer os.RemoveAll(tempBuild)

	clone := *p
	clone.Config.TopData.Build = tempBuild
	lockSnapshot, err := snapshotLockfiles(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  fmt.Sprintf("snapshot lockfiles for validation build: %v", err),
		})
		return
	}
	defer restoreLockfiles(lockSnapshot)
	if _, err := buildNativeUnchecked(&clone, NativeBuildOptions{}, nil, nil); err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     p.TopDataSourceDir(),
			Message:  fmt.Sprintf("native topdata buildability check failed: %v", err),
		})
	}
}

type lockfileSnapshot struct {
	Root  string
	Files map[string][]byte
}

func snapshotLockfiles(dataDir string) (lockfileSnapshot, error) {
	snapshot := lockfileSnapshot{
		Root:  dataDir,
		Files: map[string][]byte{},
	}
	err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, walkErr error) error {
		if walkErr != nil {
			return walkErr
		}
		if d.IsDir() || filepath.Base(path) != "lock.json" {
			return nil
		}
		raw, err := os.ReadFile(path)
		if err != nil {
			return err
		}
		snapshot.Files[path] = raw
		return nil
	})
	return snapshot, err
}

func restoreLockfiles(snapshot lockfileSnapshot) {
	seen := map[string]struct{}{}
	for path, raw := range snapshot.Files {
		seen[path] = struct{}{}
		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
			continue
		}
		_ = os.WriteFile(path, raw, 0o644)
	}
	_ = filepath.WalkDir(snapshot.Root, func(path string, d fs.DirEntry, walkErr error) error {
		if walkErr != nil || d.IsDir() || filepath.Base(path) != "lock.json" {
			return nil
		}
		if _, ok := seen[path]; !ok {
			_ = os.Remove(path)
		}
		return nil
	})
}

func validateRowCollection(path string, obj map[string]any, rows []any, report *ValidationReport) {
	columns := extractValidationColumns(obj)
	declaredKeys := map[string]int{}
	checkDuplicateKeys := filepath.Base(path) != "base.json"
	rowHasIdentity := func(row map[string]any, fallbackID int) (string, bool) {
		if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" {
			return key, true
		}
		if _, ok := row["id"]; ok {
			return fmt.Sprintf("id:%d", fallbackID), true
		}
		return "", false
	}

	for index, raw := range rows {
		row, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		if checkDuplicateKeys {
			if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" {
				if previous, ok := declaredKeys[key]; ok {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("row %d repeats key %q already declared by row %d", index, key, previous),
					})
				}
				declaredKeys[key] = index
			}
		}
		for key := range row {
			if isAuthoringOnlyField(key) && !preserveLegacyWikiColumnForPath(path, key) {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  fmt.Sprintf("row %d uses deprecated wiki field %q; move it into meta.wiki", index, key),
				})
			}
		}
		if _, hasTLK := row["_tlk"]; hasTLK {
			if _, ok := rowHasIdentity(row, index); !ok {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  fmt.Sprintf("row %d uses _tlk but has no resolvable identity", index),
				})
			}
		}
		validateRowMetadata(path, fmt.Sprintf("row %d", index), row, report)
		validateInheritanceUsage(path, fmt.Sprintf("row %d", index), row, columns, report)
		validateGenericInheritanceObject(path, fmt.Sprintf("row %d", index), row, report)
		validateInlineTextUsage(path, fmt.Sprintf("row %d", index), row, report)
	}
	_ = obj
}

func validateRowMetadata(path, label string, row map[string]any, report *ValidationReport) {
	rawMeta, ok := lookupField(row, "meta")
	if !ok {
		return
	}
	if _, err := parseRowMetadata(rawMeta); err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("%s meta: %v", label, err),
		})
	}
}

func extractValidationColumns(obj map[string]any) []string {
	rawColumns, ok := obj["columns"].([]any)
	if !ok {
		return nil
	}
	columns := make([]string, 0, len(rawColumns))
	for _, raw := range rawColumns {
		column, ok := raw.(string)
		if !ok {
			continue
		}
		columns = append(columns, column)
	}
	return columns
}

func preserveLegacyWikiColumnForPath(path, field string) bool {
	return strings.HasSuffix(filepath.ToSlash(path), "/data/classes/core/base.json") &&
		strings.EqualFold(strings.TrimSpace(field), "WIKIGENERATE")
}

func validateInheritanceUsage(path, label string, row map[string]any, columns []string, report *ValidationReport) {
	spec, ok, err := parseRowInheritanceSpec(row)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("%s inherit: %v", label, err),
		})
		return
	}
	if !ok {
		return
	}
	if _, ok := row[spec.From]; !ok {
		found := false
		for key := range row {
			if strings.EqualFold(key, spec.From) {
				found = true
				break
			}
		}
		if !found {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s inherit.from field %q was not found on row", label, spec.From),
			})
		}
	}
	for _, field := range spec.Fields {
		if len(columns) == 0 {
			continue
		}
		if _, ok := canonicalColumn(columns, field); ok {
			continue
		}
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("%s inherit field %q is not a known column", label, field),
		})
	}
}

func validateGenericInheritanceObject(path, label string, value any, report *ValidationReport) {
	switch typed := value.(type) {
	case map[string]any:
		spec, ok, err := parseGenericInheritanceSpec(typed)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s inherit: %v", label, err),
			})
		} else if ok {
			if !strings.Contains(spec.Ref, ":") {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  fmt.Sprintf("%s inherit.ref %q must use dataset:key identity", label, spec.Ref),
				})
			}
		}
		for key, child := range typed {
			if key == "inherit" {
				continue
			}
			validateGenericInheritanceObject(path, label+"."+key, child, report)
		}
	case []any:
		for index, child := range typed {
			validateGenericInheritanceObject(path, fmt.Sprintf("%s[%d]", label, index), child, report)
		}
	}
}

func validateInlineTextUsage(path, label string, obj map[string]any, report *ValidationReport) {
	for key, value := range obj {
		if key == "_tlk" {
			typed, ok := value.(map[string]any)
			if !ok {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  fmt.Sprintf("%s _tlk must be an object", label),
				})
				continue
			}
			for role, roleValue := range typed {
				if strings.ToLower(role) != "name" && strings.ToLower(role) != "description" {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("%s _tlk role %q is not supported", label, role),
					})
					continue
				}
				if _, _, err := parseTLKPayload(roleValue, true); err != nil {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("%s _tlk.%s: %v", label, role, err),
					})
				}
			}
			continue
		}

		allowBare := true
		if typed, ok := value.(map[string]any); ok {
			_, hasWrappedTLK := typed["tlk"]
			if !hasWrappedTLK {
				if _, hasRef := typed["ref"]; hasRef {
					if _, hasText := typed["text"]; !hasText {
						if _, hasKey := typed["key"]; !hasKey {
							continue
						}
					}
				}
				allowBare = false
			}
		}
		payload, ok, err := parseTLKPayload(value, allowBare)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s %s: %v", label, key, err),
			})
			continue
		}
		if !ok {
			continue
		}
		if payload.Key == "" && payload.Text == "" && payload.Ref == "" {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s %s TLK payload is empty", label, key),
			})
		}
	}
}

func Compare(p *project.Project, progress func(string)) (CompareResult, error) {
	if progress == nil {
		progress = func(string) {}
	}
	output2DA := compiled2DAOutputDir(p)
	outputTLK := compiledTLKOutputDir(p)
	if _, err := os.Stat(output2DA); err != nil {
		return CompareResult{}, fmt.Errorf("topdata build output missing: run build-topdata first")
	}
	compared2DA, comparedTLK, nativePass, nativeMismatch, err := compareNativeSelfCheck(p, output2DA, outputTLK, progress)
	if err != nil {
		return CompareResult{}, err
	}

	return CompareResult{
		Mode:            "native",
		Compared2DA:     compared2DA,
		ComparedTLK:     comparedTLK,
		NativePass:      nativePass,
		NotYetCanonical: 0,
		NativeMismatch:  nativeMismatch,
	}, nil
}

func BuildReference(p *project.Project, progress func(string)) (BuildResult, error) {
	return BuildNative(p, progress)
}

func CompareReference(p *project.Project, progress func(string)) (CompareResult, error) {
	return Compare(p, progress)
}

func compareDirs(actualRoot, expectedRoot string) (int, error) {
	expectedFiles := make([]string, 0)
	err := filepath.WalkDir(expectedRoot, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		rel, err := filepath.Rel(expectedRoot, path)
		if err != nil {
			return err
		}
		expectedFiles = append(expectedFiles, filepath.ToSlash(rel))
		return nil
	})
	if err != nil {
		return 0, err
	}
	slices.Sort(expectedFiles)

	actualFiles := make([]string, 0)
	err = filepath.WalkDir(actualRoot, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		rel, err := filepath.Rel(actualRoot, path)
		if err != nil {
			return err
		}
		actualFiles = append(actualFiles, filepath.ToSlash(rel))
		return nil
	})
	if err != nil {
		return 0, err
	}
	slices.Sort(actualFiles)
	if !slices.Equal(actualFiles, expectedFiles) {
		return 0, fmt.Errorf("topdata output file list differs from reference")
	}

	for _, rel := range expectedFiles {
		actualBytes, err := os.ReadFile(filepath.Join(actualRoot, rel))
		if err != nil {
			return 0, err
		}
		expectedBytes, err := os.ReadFile(filepath.Join(expectedRoot, rel))
		if err != nil {
			return 0, err
		}
		if !bytes.Equal(actualBytes, expectedBytes) {
			return 0, fmt.Errorf("topdata output differs from reference: %s", rel)
		}
	}
	return len(expectedFiles), nil
}

func compareNativeSelfCheck(p *project.Project, actual2DA, actualTLK string, progress func(string)) (int, int, int, int, error) {
	if progress == nil {
		progress = func(string) {}
	}
	projectOutputs, err := discoverNativeOutputCatalog(filepath.Join(p.TopDataSourceDir(), "data"))
	if err != nil {
		return 0, 0, 0, 0, err
	}
	if len(projectOutputs) == 0 {
		return 0, 0, 0, 0, nil
	}

	tempBuild, err := os.MkdirTemp("", "sow-topdata-native-compare-*")
	if err != nil {
		return 0, 0, 0, 0, fmt.Errorf("create native compare temp dir: %w", err)
	}
	defer os.RemoveAll(tempBuild)

	nativeProject := *p
	nativeProject.Config.TopData.Build = tempBuild
	nativeResult, err := buildNativeUnchecked(&nativeProject, NativeBuildOptions{}, nil, nil)
	if err != nil {
		return 0, 0, 0, len(projectOutputs), nil
	}

	pass := 0
	mismatch := 0
	for outputName := range projectOutputs {
		actualBytes, err := os.ReadFile(filepath.Join(actual2DA, outputName))
		if err != nil {
			mismatch++
			continue
		}
		expectedBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, outputName))
		if err != nil {
			mismatch++
			continue
		}
		if bytes.Equal(normalize2DAForCompare(outputName, actualBytes), normalize2DAForCompare(outputName, expectedBytes)) {
			pass++
			continue
		}
		mismatch++
	}
	actual2DAFiles, err := listRelativeFiles(actual2DA)
	if err != nil {
		return 0, 0, 0, 0, err
	}
	expected2DAFiles, err := listRelativeFiles(nativeResult.Output2DADir)
	if err != nil {
		return 0, 0, 0, 0, err
	}
	if !slices.Equal(actual2DAFiles, expected2DAFiles) {
		mismatch += symmetricDiffCount(actual2DAFiles, expected2DAFiles)
	}
	comparedTLK, tlkMismatch, err := compareExactDirFilesByExtension(actualTLK, nativeResult.OutputTLKDir, ".tlk")
	if err != nil {
		return 0, 0, 0, 0, err
	}
	mismatch += tlkMismatch
	return len(projectOutputs), comparedTLK, pass, mismatch, nil
}

func compareNativeCoverage(p *project.Project, actual2DA string, progress func(string)) (int, int, int, error) {
	actualTLK := compiledTLKOutputDir(p)
	compared2DA, _, pass, mismatch, err := compareNativeSelfCheck(p, actual2DA, actualTLK, progress)
	if err != nil {
		return 0, 0, 0, err
	}
	if compared2DA == 0 {
		return 0, 0, mismatch, nil
	}
	return pass, 0, mismatch, nil
}

func normalize2DAForCompare(outputName string, raw []byte) []byte {
	text := strings.ReplaceAll(string(raw), "\r\n", "\n")
	lines := strings.Split(text, "\n")
	if len(lines) < 3 || !strings.HasPrefix(lines[0], "2DA V2.0") {
		return raw
	}
	headerIndex := 2
	columns := strings.Split(lines[headerIndex], "\t")
	keep := make([]int, 0, len(columns))
	filtered := make([]string, 0, len(columns))
	for index, column := range columns {
		if isAuthoringOnlyField(column) {
			continue
		}
		keep = append(keep, index)
		filtered = append(filtered, column)
	}
	var builder strings.Builder
	builder.WriteString(lines[0])
	builder.WriteByte('\n')
	builder.WriteByte('\n')
	builder.WriteString(strings.Join(filtered, "\t"))
	builder.WriteByte('\n')
	normalizedRows := make([][]string, 0, len(lines))
	for _, line := range lines[headerIndex+1:] {
		if line == "" {
			continue
		}
		fields := strings.Split(line, "\t")
		if len(fields) == 0 {
			continue
		}
		row := make([]string, 0, len(keep))
		for _, index := range keep {
			if index+1 < len(fields) {
				row = append(row, fields[index+1])
			} else {
				row = append(row, nullValue)
			}
		}
		normalizedRows = append(normalizedRows, row)
	}
	if shouldIgnore2DAPlacement(outputName) {
		slices.SortFunc(normalizedRows, func(a, b []string) int {
			return strings.Compare(strings.Join(a, "\x00"), strings.Join(b, "\x00"))
		})
	}
	for index, row := range normalizedRows {
		builder.WriteString(strconv.Itoa(index))
		for _, value := range row {
			builder.WriteByte('\t')
			builder.WriteString(value)
		}
		builder.WriteByte('\n')
	}
	return []byte(builder.String())
}

func shouldIgnore2DAPlacement(outputName string) bool {
	return strings.HasPrefix(outputName, "cls_feat_") && strings.HasSuffix(outputName, ".2da")
}

func walkJSON(root string, fn func(path string)) error {
	return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		if strings.HasSuffix(strings.ToLower(d.Name()), ".json") {
			fn(path)
		}
		return nil
	})
}

func countFiles(root string) (int, error) {
	count := 0
	err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if !d.IsDir() {
			count++
		}
		return nil
	})
	return count, err
}

type progressWriter struct {
	progress func(string)
}

func (w progressWriter) Write(p []byte) (int, error) {
	text := strings.TrimSpace(string(p))
	if text != "" {
		w.progress(text)
	}
	return len(p), nil
}

func ioDiscardProgress(progress func(string)) *progressWriter {
	return &progressWriter{progress: progress}
}

func listRelativeFiles(root string) ([]string, error) {
	return listRelativeFilesByExtension(root, "")
}

func listRelativeFilesByExtension(root, extension string) ([]string, error) {
	files := make([]string, 0)
	if _, err := os.Stat(root); err != nil {
		if os.IsNotExist(err) {
			return files, nil
		}
		return nil, err
	}
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		if extension != "" && !strings.EqualFold(filepath.Ext(path), extension) {
			return nil
		}
		rel, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		files = append(files, filepath.ToSlash(rel))
		return nil
	})
	if err != nil {
		return nil, err
	}
	slices.Sort(files)
	return files, nil
}

func symmetricDiffCount(left, right []string) int {
	leftSet := map[string]struct{}{}
	rightSet := map[string]struct{}{}
	for _, item := range left {
		leftSet[item] = struct{}{}
	}
	for _, item := range right {
		rightSet[item] = struct{}{}
	}
	count := 0
	for item := range leftSet {
		if _, ok := rightSet[item]; !ok {
			count++
		}
	}
	for item := range rightSet {
		if _, ok := leftSet[item]; !ok {
			count++
		}
	}
	return count
}

func compareExactDirFiles(actualRoot, expectedRoot string) (int, int, error) {
	return compareExactDirFilesByExtension(actualRoot, expectedRoot, "")
}

func compareExactDirFilesByExtension(actualRoot, expectedRoot, extension string) (int, int, error) {
	actualFiles, err := listRelativeFilesByExtension(actualRoot, extension)
	if err != nil {
		return 0, 0, err
	}
	expectedFiles, err := listRelativeFilesByExtension(expectedRoot, extension)
	if err != nil {
		return 0, 0, err
	}
	mismatch := 0
	if !slices.Equal(actualFiles, expectedFiles) {
		mismatch += symmetricDiffCount(actualFiles, expectedFiles)
	}
	shared := make([]string, 0, len(expectedFiles))
	for _, rel := range expectedFiles {
		if slices.Contains(actualFiles, rel) {
			shared = append(shared, rel)
		}
	}
	for _, rel := range shared {
		actualBytes, err := os.ReadFile(filepath.Join(actualRoot, rel))
		if err != nil {
			mismatch++
			continue
		}
		expectedBytes, err := os.ReadFile(filepath.Join(expectedRoot, rel))
		if err != nil {
			mismatch++
			continue
		}
		if !bytes.Equal(actualBytes, expectedBytes) {
			mismatch++
		}
	}
	return len(expectedFiles), mismatch, nil
}