sow-tools/internal/topdata/topdata.go

package topdata

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"

	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/erf"
	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)

type Severity string

const (
	SeverityError   Severity = "error"
	SeverityWarning Severity = "warning"
)

type Diagnostic struct {
	Severity Severity
	Path     string
	Message  string
}

type ValidationReport struct {
	Files       int
	DataFiles   int
	TLKFiles    int
	Diagnostics []Diagnostic
}

func (r ValidationReport) SummaryLines(maxPaths int) []string {
	type key struct {
		severity Severity
		message  string
	}
	groups := map[key][]string{}
	order := make([]key, 0)
	for _, diagnostic := range r.Diagnostics {
		k := key{severity: diagnostic.Severity, message: diagnostic.Message}
		if _, ok := groups[k]; !ok {
			order = append(order, k)
		}
		groups[k] = append(groups[k], diagnostic.Path)
	}
	slices.SortFunc(order, func(a, b key) int {
		if a.severity != b.severity {
			if a.severity == SeverityError {
				return -1
			}
			return 1
		}
		return strings.Compare(a.message, b.message)
	})
	lines := make([]string, 0, len(order))
	for _, k := range order {
		paths := append([]string(nil), groups[k]...)
		slices.Sort(paths)
		paths = slices.Compact(paths)
		lines = append(lines, formatDiagnosticSummaryLine(string(k.severity), k.message, paths, maxPaths))
	}
	return lines
}

func (r ValidationReport) HasErrors() bool {
	for _, diagnostic := range r.Diagnostics {
		if diagnostic.Severity == SeverityError {
			return true
		}
	}
	return false
}

func (r ValidationReport) ErrorCount() int {
	count := 0
	for _, diagnostic := range r.Diagnostics {
		if diagnostic.Severity == SeverityError {
			count++
		}
	}
	return count
}

func (r ValidationReport) WarningCount() int {
	count := 0
	for _, diagnostic := range r.Diagnostics {
		if diagnostic.Severity == SeverityWarning {
			count++
		}
	}
	return count
}

func formatDiagnosticSummaryLine(prefix, message string, paths []string, maxPaths int) string {
	if len(paths) == 0 {
		return fmt.Sprintf("%s: %s", prefix, message)
	}
	if maxPaths <= 0 {
		maxPaths = 3
	}
	display := paths
	if len(display) > maxPaths {
		display = display[:maxPaths]
	}
	location := strings.Join(display, ", ")
	if len(paths) > len(display) {
		location += fmt.Sprintf(", and %d more", len(paths)-len(display))
	}
	return fmt.Sprintf("%s: %s [%s]", prefix, message, location)
}

type BuildResult struct {
	Mode          string
	Output2DADir  string
	OutputTLKDir  string
	Files2DA      int
	FilesTLK      int
	WikiOutputDir string
	WikiPages     int
	WikiStatus    string
}

type PackageResult struct {
	Mode          string
	Output2DADir  string
	OutputTLKDir  string
	OutputHAKPath string
	OutputTLKPath string
	Files2DA      int
	FilesTLK      int
	HAKResources  int
	AssetFiles    int
	WikiOutputDir string
	WikiPages     int
	WikiStatus    string
}

type CompareResult struct {
	Mode            string
	Compared2DA     int
	ComparedTLK     int
	NativePass      int
	NotYetCanonical int
	NativeMismatch  int
}

func ValidateProject(p *project.Project) ValidationReport {
	report := ValidationReport{}
	if !p.HasTopData() {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityWarning,
			Path:     project.ConfigFile,
			Message:  "topdata is not configured for this project",
		})
		return report
	}

	sourceDir := p.TopDataSourceDir()
	info, err := os.Stat(sourceDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     sourceDir,
			Message:  err.Error(),
		})
		return report
	}
	if !info.IsDir() {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     sourceDir,
			Message:  "topdata source must be a directory",
		})
		return report
	}

	dataDir := filepath.Join(sourceDir, "data")
	// Canonical topdata no longer requires a dedicated source tlk directory.
	// When present, it is only validated for disallowed legacy module input.
	tlkDir := filepath.Join(sourceDir, "tlk")
	baseDialog := filepath.Join(sourceDir, "base_dialog.json")
	statePath := filepath.Join(sourceDir, tlkStateFile)

	for _, requiredDir := range []string{dataDir} {
		info, err := os.Stat(requiredDir)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     requiredDir,
				Message:  err.Error(),
			})
			continue
		}
		if !info.IsDir() {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     requiredDir,
				Message:  "must be a directory",
			})
		}
	}

	if info, err := os.Stat(tlkDir); err == nil && info.IsDir() {
		modulesDir := filepath.Join(tlkDir, "modules")
		legacyModuleFiles := 0
		if moduleInfo, moduleErr := os.Stat(modulesDir); moduleErr == nil && moduleInfo.IsDir() {
			_ = walkJSON(modulesDir, func(path string) {
				legacyModuleFiles++
				report.Files++
				report.TLKFiles++
				_ = validateJSONFile(path, "tlk", &report)
			})
		} else if moduleErr != nil && !errors.Is(moduleErr, os.ErrNotExist) {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     modulesDir,
				Message:  moduleErr.Error(),
			})
		}
		if legacyModuleFiles > 0 {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     modulesDir,
				Message:  "topdata/tlk/modules is no longer allowed for canonical authored input; inline TLK-in-data is required",
			})
		}
		if lockInfo, lockErr := os.Stat(filepath.Join(tlkDir, "lock.json")); lockErr == nil && !lockInfo.IsDir() {
			report.Files++
			report.TLKFiles++
			_ = validateJSONFile(filepath.Join(tlkDir, "lock.json"), "tlk", &report)
		} else if lockErr != nil && !errors.Is(lockErr, os.ErrNotExist) {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     filepath.Join(tlkDir, "lock.json"),
				Message:  lockErr.Error(),
			})
		}
	}

	if _, err := os.Stat(baseDialog); err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityWarning,
			Path:     baseDialog,
			Message:  "base_dialog.json is missing; dialog migration is not canonical yet",
		})
	} else {
		report.Files++
		if err := validateJSONFile(baseDialog, "base_dialog", &report); err == nil {
			// count only once for base dialog.
		}
	}

	_ = walkJSON(dataDir, func(path string) {
		report.Files++
		report.DataFiles++
		_ = validateJSONFile(path, "data", &report)
	})
	validateTopPackageAssets(sourceDir, dataDir, &report)
	validateNativeOutputCatalog(dataDir, &report)
	validateItempropsRegistryGraph(dataDir, &report)
	if _, err := os.Stat(statePath); err == nil {
		report.Files++
		_ = validateJSONFile(statePath, "state", &report)
	}
	if !report.HasErrors() {
		validateNativeBuildability(p, &report)
	}

	return report
}

func validateJSONFile(path, domain string, report *ValidationReport) error {
	raw, err := os.ReadFile(path)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  err.Error(),
		})
		return err
	}

	var payload any
	if err := json.Unmarshal(raw, &payload); err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("invalid JSON: %v", err),
		})
		return err
	}

	obj, ok := payload.(map[string]any)
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "topdata files must contain a JSON object at the root",
		})
		return errors.New("topdata file root must be object")
	}

	switch domain {
	case "data":
		validateDataObject(path, obj, report)
	case "tlk":
		validateTLKObject(path, obj, report)
	case "state":
		validateStateObject(path, obj, report)
	case "base_dialog":
		validateBaseDialogObject(path, obj, report)
	}
	return nil
}

func validateDataObject(path string, obj map[string]any, report *ValidationReport) {
	base := filepath.Base(path)
	slashed := filepath.ToSlash(path)
	inModules := strings.Contains(slashed, "/modules/")

	if strings.HasSuffix(slashed, "/data/masterfeats/base.json") {
		validateMasterfeatsBaseFile(path, obj, report)
		return
	}
	if strings.HasSuffix(slashed, "/data/masterfeats/lock.json") {
		validateMasterfeatsLockFile(path, obj, report)
		return
	}
	if strings.HasSuffix(slashed, "/data/feat/base.json") {
		validateFeatBaseFile(path, obj, report)
		return
	}
	if strings.HasSuffix(slashed, "/data/feat/lock.json") {
		validateFeatLockFile(path, obj, report)
		return
	}
	if strings.Contains(slashed, "/data/feat/generated/") {
		validateFeatGeneratedFile(path, obj, report)
		return
	}
	if strings.Contains(slashed, "/data/parts/overrides/") {
		validateOverridesFile(path, obj, report)
		return
	}

	if strings.Contains(slashed, "/racialtypes/registry/races/") {
		validateRacialtypesRegistryRaceFile(path, obj, report)
		return
	}

	if base == "lock.json" {
		validateLockObject(path, obj, report)
		return
	}
	if base == "base.json" {
		validateRowsFile(path, obj, report, true)
		return
	}
	if inModules {
		if _, hasEntries := obj["entries"]; hasEntries {
			validateEntriesFile(path, obj, report)
			return
		}
		if _, hasOverrides := obj["overrides"]; hasOverrides {
			validateOverridesFile(path, obj, report)
			return
		}
		if _, hasRows := obj["rows"]; hasRows {
			validateRowsFile(path, obj, report, false)
			return
		}
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityWarning,
			Path:     path,
			Message:  "module file does not use a recognized canonical shape yet; expected one of entries, overrides, or rows",
		})
		return
	}

	if _, hasRows := obj["rows"]; hasRows {
		validateRowsFile(path, obj, report, false)
		return
	}
	report.Diagnostics = append(report.Diagnostics, Diagnostic{
		Severity: SeverityWarning,
		Path:     path,
		Message:  "unrecognized data file shape; parseable for compatibility, but not canonical yet",
	})
}

func validateMasterfeatsBaseFile(path string, obj map[string]any, report *ValidationReport) {
	validateRowsFile(path, obj, report, true)

	if output, ok := obj["output"]; !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "masterfeats base.json must contain output set to masterfeats.2da",
		})
	} else if outputText, ok := output.(string); !ok || strings.TrimSpace(outputText) != "masterfeats.2da" {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "masterfeats output must be masterfeats.2da",
		})
	}

	columns := extractValidationColumns(obj)
	for _, required := range []string{"LABEL", "STRREF", "DESCRIPTION"} {
		if _, ok := canonicalColumn(columns, required); !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("masterfeats base.json must include %s in columns", required),
			})
		}
	}

	rows, ok := obj["rows"].([]any)
	if !ok {
		return
	}
	for index, raw := range rows {
		row, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		key, ok := row["key"].(string)
		if !ok || strings.TrimSpace(key) == "" {
			continue
		}
		if !strings.HasPrefix(key, "masterfeats:") {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("masterfeats row %d key %q must start with masterfeats:", index, key),
			})
		}
	}
}

func validateMasterfeatsLockFile(path string, obj map[string]any, report *ValidationReport) {
	validateLockObject(path, obj, report)
	for key := range obj {
		if !strings.HasPrefix(key, "masterfeats:") {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("masterfeats lock key %q must start with masterfeats:", key),
			})
		}
	}
}

func validateFeatBaseFile(path string, obj map[string]any, report *ValidationReport) {
	validateRowsFile(path, obj, report, true)

	if output, ok := obj["output"]; !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "feat base.json must contain output set to feat.2da",
		})
	} else if outputText, ok := output.(string); !ok || strings.TrimSpace(outputText) != "feat.2da" {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "feat output must be feat.2da",
		})
	}
}

func validateFeatLockFile(path string, obj map[string]any, report *ValidationReport) {
	validateLockObject(path, obj, report)
	for key := range obj {
		if !strings.HasPrefix(key, "feat:") {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("feat lock key %q must start with feat:", key),
			})
		}
	}
}

func validateFeatGeneratedFile(path string, obj map[string]any, report *ValidationReport) {
	family, ok := obj["family"].(string)
	if !ok || strings.TrimSpace(family) == "" {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "feat generated file must contain a non-empty family string",
		})
		return
	}
	if isFamilyExpansionObject(obj) {
		validateFeatFamilyExpansionFile(path, obj, report)
		return
	}
	switch family {
	default:
		_, hasEntries := obj["entries"]
		_, hasOverrides := obj["overrides"]
		if !hasEntries && !hasOverrides {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  "feat generated file must contain entries and/or overrides",
			})
			return
		}
		if hasEntries {
			validateEntriesFile(path, obj, report)
		}
		if hasOverrides {
			validateOverridesFile(path, obj, report)
		}
	}
}

func validateFeatFamilyExpansionFile(path string, obj map[string]any, report *ValidationReport) {
	spec, err := parseFamilyExpansionSpec(path, obj)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  err.Error(),
		})
		return
	}
	if !strings.HasPrefix(spec.Template, "masterfeats:") {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "compact feat generated template must start with masterfeats:",
		})
	}
	if spec.NamePrefix == "" {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "family expansion file must contain a non-empty name_prefix",
		})
	}
	if spec.LabelPrefix == "" {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "family expansion file must contain a non-empty label_prefix",
		})
	}
	if spec.ConstantPrefix == "" {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "family expansion file must contain a non-empty constant_prefix",
		})
	}
	if raw, ok := obj["overrides"]; ok {
		if _, ok := raw.(map[string]any); !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  "family expansion overrides must be an object keyed by source dataset key",
			})
		}
	}
}

func validateRacialtypesRegistryRaceFile(path string, obj map[string]any, report *ValidationReport) {
	key, ok := obj["key"]
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "racialtypes registry race file must contain a key field",
		})
	} else if _, ok := key.(string); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "racialtypes registry race key must be a string",
		})
	}

	core, ok := obj["core"]
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "racialtypes registry race file must contain a core object",
		})
	} else if _, ok := core.(map[string]any); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "racialtypes registry race core must be a JSON object",
		})
	}

	if rawOutput, ok := obj["feat_output"]; ok {
		if _, ok := rawOutput.(string); !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  "racialtypes registry race feat_output must be a string",
			})
		}
		if rawFeats, ok := obj["feats"]; ok {
			if _, ok := rawFeats.([]any); !ok {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  "racialtypes registry race feats must be an array",
				})
			}
		} else {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  "racialtypes registry race feat_output requires a feats array",
			})
		}
		return
	}

	if _, ok := obj["feats"]; ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "racialtypes registry race feats requires feat_output",
		})
	}
}

func validateTLKObject(path string, obj map[string]any, report *ValidationReport) {
	if filepath.Base(path) == "lock.json" {
		validateLockObject(path, obj, report)
		return
	}
	validateLegacyDialogEntries(path, obj, report, "TLK")
}

func validateBaseDialogObject(path string, obj map[string]any, report *ValidationReport) {
	validateLegacyDialogEntries(path, obj, report, "base_dialog")
}

func validateLegacyDialogEntries(path string, obj map[string]any, report *ValidationReport, label string) {
	entries, ok := obj["entries"]
	if ok {
		switch entries.(type) {
		case map[string]any, []any:
			// accepted compatibility shapes
		default:
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s entries must be a JSON object or array", label),
			})
		}
	}
	if language, ok := obj["language"]; ok {
		switch language.(type) {
		case string, float64:
			// accepted compatibility shapes
		default:
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s language must be a string or numeric language id when present", label),
			})
		}
	}
	if label == "TLK" && !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "TLK file must contain an entries object",
		})
	}
}

func validateLockObject(path string, obj map[string]any, report *ValidationReport) {
	for key, value := range obj {
		switch value.(type) {
		case float64:
			// JSON numbers land here.
		default:
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("lock entry %q must be numeric", key),
			})
		}
	}
}

func validateRowsFile(path string, obj map[string]any, report *ValidationReport, requireColumns bool) {
	rows, ok := obj["rows"]
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "rows file must contain a rows array",
		})
		return
	}
	if _, ok := rows.([]any); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "rows must be a JSON array",
		})
	}
	if rowsList, ok := rows.([]any); ok {
		validateRowCollection(path, obj, rowsList, report)
	}
	if columns, ok := obj["columns"]; ok {
		if list, ok := columns.([]any); !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  "columns must be a JSON array when present",
			})
		} else {
			for _, raw := range list {
				column, ok := raw.(string)
				if !ok {
					continue
				}
				if isAuthoringOnlyField(column) && !preserveLegacyWikiColumnForPath(path, column) {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("deprecated wiki column %q must be moved into meta.wiki", column),
					})
				}
			}
		}
	} else if requireColumns {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "base dataset file must contain columns",
		})
	}
}

func validateEntriesFile(path string, obj map[string]any, report *ValidationReport) {
	entries, ok := obj["entries"]
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "entries file must contain an entries object",
		})
		return
	}
	if _, ok := entries.(map[string]any); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "entries must be a JSON object",
		})
	}
	if entryMap, ok := entries.(map[string]any); ok {
		for key, raw := range entryMap {
			entry, ok := raw.(map[string]any)
			if !ok {
				continue
			}
			for field := range entry {
				if isAuthoringOnlyField(field) && !preserveLegacyWikiColumnForPath(path, field) {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("entry %q uses deprecated wiki field %q; move it into meta.wiki", key, field),
					})
				}
			}
			validateRowMetadata(path, fmt.Sprintf("entry %q", key), entry, report)
			validateInlineTextUsage(path, key, entry, report)
		}
	}
}

func validateOverridesFile(path string, obj map[string]any, report *ValidationReport) {
	overrides, ok := obj["overrides"]
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "override file must contain an overrides array",
		})
		return
	}
	if _, ok := overrides.([]any); !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "overrides must be a JSON array",
		})
	}
	if overrideList, ok := overrides.([]any); ok {
		container := map[string]any{"rows": overrideList}
		validateRowCollection(path, container, overrideList, report)
	}
}

func validateStateObject(path string, obj map[string]any, report *ValidationReport) {
	entries, ok := obj["entries"].(map[string]any)
	if !ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  "TLK state file must contain an entries object",
		})
		return
	}
	ids := map[int]string{}
	for key, raw := range entries {
		entry, ok := raw.(map[string]any)
		if !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("TLK state entry %q must be an object", key),
			})
			continue
		}
		idValue, ok := entry["id"]
		if !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("TLK state entry %q is missing id", key),
			})
			continue
		}
		id, err := asInt(idValue)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("TLK state entry %q id must be numeric", key),
			})
			continue
		}
		if other, ok := ids[id]; ok && other != key {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("TLK state id collision between %q and %q", other, key),
			})
		}
		ids[id] = key
	}
}

func validateNativeOutputCatalog(dataDir string, report *ValidationReport) {
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  err.Error(),
		})
		return
	}
	registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  err.Error(),
		})
		return
	}
	outputOwners := map[string]string{}
	for _, dataset := range datasets {
		recordNativeOutputOwner(dataset.OutputName, dataset.Name, outputOwners, report)
	}
	for _, dataset := range registryDatasets {
		recordNativeOutputOwner(dataset.Dataset.OutputName, dataset.Dataset.Name, outputOwners, report)
	}
}

func validateTopPackageAssets(sourceDir, dataDir string, report *ValidationReport) {
	assetsDir := filepath.Join(sourceDir, "assets")
	generated2DADir := filepath.Join(assetsDir, "2da")
	info, err := os.Stat(assetsDir)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return
		}
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     assetsDir,
			Message:  err.Error(),
		})
		return
	}
	if !info.IsDir() {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     assetsDir,
			Message:  "must be a directory",
		})
		return
	}

	projectOutputs, err := discoverNativeOutputCatalog(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     assetsDir,
			Message:  err.Error(),
		})
		return
	}

	seen := map[string]string{}
	err = filepath.WalkDir(assetsDir, func(path string, d fs.DirEntry, walkErr error) error {
		if walkErr != nil {
			return walkErr
		}
		if d.IsDir() && path == generated2DADir {
			return filepath.SkipDir
		}
		if d.IsDir() {
			return nil
		}
		report.Files++
		rel, err := filepath.Rel(assetsDir, path)
		if err != nil {
			return err
		}
		if strings.HasPrefix(filepath.Base(path), ".") {
			return nil
		}
		base := strings.ToLower(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)))
		ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
		if _, ok := erf.HAKResourceTypeForExtension(ext); !ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("unsupported topdata asset HAK resource extension %q", filepath.Ext(path)),
			})
			return nil
		}
		key := base + "." + ext
		if previous, ok := seen[key]; ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("duplicate topdata asset resource %q also defined by %s", key, previous),
			})
			return nil
		}
		seen[key] = rel
		if owner, ok := projectOutputs[key]; ok {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("topdata asset resource %q collides with compiled topdata output from %s", key, owner),
			})
		}
		return nil
	})
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     assetsDir,
			Message:  err.Error(),
		})
	}
}

func recordNativeOutputOwner(outputName, owner string, outputOwners map[string]string, report *ValidationReport) {
	if existing, ok := outputOwners[outputName]; ok {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     owner,
			Message:  fmt.Sprintf("duplicate 2da output %q also produced by %s", outputName, existing),
		})
		return
	}
	outputOwners[outputName] = owner
}

func validateNativeBuildability(p *project.Project, report *ValidationReport) {
	dataDir := filepath.Join(p.TopDataSourceDir(), "data")
	datasets, err := discoverNativeDatasets(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  err.Error(),
		})
		return
	}
	registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     dataDir,
			Message:  err.Error(),
		})
		return
	}
	if len(datasets) == 0 && len(registryDatasets) == 0 {
		return
	}
	tempBuild, err := os.MkdirTemp("", "sow-topdata-validate-*")
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     p.TopDataSourceDir(),
			Message:  fmt.Sprintf("create validation temp dir: %v", err),
		})
		return
	}
	defer os.RemoveAll(tempBuild)

	clone := *p
	clone.Config.TopData.Build = tempBuild
	if _, err := buildNativeUnchecked(&clone, nil, nil); err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     p.TopDataSourceDir(),
			Message:  fmt.Sprintf("native topdata buildability check failed: %v", err),
		})
	}
}

func validateRowCollection(path string, obj map[string]any, rows []any, report *ValidationReport) {
	columns := extractValidationColumns(obj)
	rowHasIdentity := func(row map[string]any, fallbackID int) (string, bool) {
		if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" {
			return key, true
		}
		if _, ok := row["id"]; ok {
			return fmt.Sprintf("id:%d", fallbackID), true
		}
		return "", false
	}

	for index, raw := range rows {
		row, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		for key := range row {
			if isAuthoringOnlyField(key) && !preserveLegacyWikiColumnForPath(path, key) {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  fmt.Sprintf("row %d uses deprecated wiki field %q; move it into meta.wiki", index, key),
				})
			}
		}
		if _, hasTLK := row["_tlk"]; hasTLK {
			if _, ok := rowHasIdentity(row, index); !ok {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  fmt.Sprintf("row %d uses _tlk but has no resolvable identity", index),
				})
			}
		}
		validateRowMetadata(path, fmt.Sprintf("row %d", index), row, report)
		validateInheritanceUsage(path, fmt.Sprintf("row %d", index), row, columns, report)
		validateGenericInheritanceObject(path, fmt.Sprintf("row %d", index), row, report)
		validateInlineTextUsage(path, fmt.Sprintf("row %d", index), row, report)
	}
	_ = obj
}

func validateRowMetadata(path, label string, row map[string]any, report *ValidationReport) {
	rawMeta, ok := lookupField(row, "meta")
	if !ok {
		return
	}
	if _, err := parseRowMetadata(rawMeta); err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("%s meta: %v", label, err),
		})
	}
}

func extractValidationColumns(obj map[string]any) []string {
	rawColumns, ok := obj["columns"].([]any)
	if !ok {
		return nil
	}
	columns := make([]string, 0, len(rawColumns))
	for _, raw := range rawColumns {
		column, ok := raw.(string)
		if !ok {
			continue
		}
		columns = append(columns, column)
	}
	return columns
}

func preserveLegacyWikiColumnForPath(path, field string) bool {
	return strings.HasSuffix(filepath.ToSlash(path), "/data/classes/core/base.json") &&
		strings.EqualFold(strings.TrimSpace(field), "WIKIGENERATE")
}

func validateInheritanceUsage(path, label string, row map[string]any, columns []string, report *ValidationReport) {
	spec, ok, err := parseRowInheritanceSpec(row)
	if err != nil {
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("%s inherit: %v", label, err),
		})
		return
	}
	if !ok {
		return
	}
	if _, ok := row[spec.From]; !ok {
		found := false
		for key := range row {
			if strings.EqualFold(key, spec.From) {
				found = true
				break
			}
		}
		if !found {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s inherit.from field %q was not found on row", label, spec.From),
			})
		}
	}
	for _, field := range spec.Fields {
		if len(columns) == 0 {
			continue
		}
		if _, ok := canonicalColumn(columns, field); ok {
			continue
		}
		report.Diagnostics = append(report.Diagnostics, Diagnostic{
			Severity: SeverityError,
			Path:     path,
			Message:  fmt.Sprintf("%s inherit field %q is not a known column", label, field),
		})
	}
}

func validateGenericInheritanceObject(path, label string, value any, report *ValidationReport) {
	switch typed := value.(type) {
	case map[string]any:
		spec, ok, err := parseGenericInheritanceSpec(typed)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s inherit: %v", label, err),
			})
		} else if ok {
			if !strings.Contains(spec.Ref, ":") {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  fmt.Sprintf("%s inherit.ref %q must use dataset:key identity", label, spec.Ref),
				})
			}
		}
		for key, child := range typed {
			if key == "inherit" {
				continue
			}
			validateGenericInheritanceObject(path, label+"."+key, child, report)
		}
	case []any:
		for index, child := range typed {
			validateGenericInheritanceObject(path, fmt.Sprintf("%s[%d]", label, index), child, report)
		}
	}
}

func validateInlineTextUsage(path, label string, obj map[string]any, report *ValidationReport) {
	for key, value := range obj {
		if key == "_tlk" {
			typed, ok := value.(map[string]any)
			if !ok {
				report.Diagnostics = append(report.Diagnostics, Diagnostic{
					Severity: SeverityError,
					Path:     path,
					Message:  fmt.Sprintf("%s _tlk must be an object", label),
				})
				continue
			}
			for role, roleValue := range typed {
				if strings.ToLower(role) != "name" && strings.ToLower(role) != "description" {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("%s _tlk role %q is not supported", label, role),
					})
					continue
				}
				if _, _, err := parseTLKPayload(roleValue, true); err != nil {
					report.Diagnostics = append(report.Diagnostics, Diagnostic{
						Severity: SeverityError,
						Path:     path,
						Message:  fmt.Sprintf("%s _tlk.%s: %v", label, role, err),
					})
				}
			}
			continue
		}

		allowBare := true
		if typed, ok := value.(map[string]any); ok {
			_, hasWrappedTLK := typed["tlk"]
			if !hasWrappedTLK {
				if _, hasRef := typed["ref"]; hasRef {
					if _, hasText := typed["text"]; !hasText {
						if _, hasKey := typed["key"]; !hasKey {
							continue
						}
					}
				}
				allowBare = false
			}
		}
		payload, ok, err := parseTLKPayload(value, allowBare)
		if err != nil {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s %s: %v", label, key, err),
			})
			continue
		}
		if !ok {
			continue
		}
		if payload.Key == "" && payload.Text == "" && payload.Ref == "" {
			report.Diagnostics = append(report.Diagnostics, Diagnostic{
				Severity: SeverityError,
				Path:     path,
				Message:  fmt.Sprintf("%s %s TLK payload is empty", label, key),
			})
		}
	}
}

func Compare(p *project.Project, progress func(string)) (CompareResult, error) {
	if progress == nil {
		progress = func(string) {}
	}
	output2DA := compiled2DAOutputDir(p)
	outputTLK := compiledTLKOutputDir(p)
	if _, err := os.Stat(output2DA); err != nil {
		return CompareResult{}, fmt.Errorf("topdata build output missing: run build-topdata first")
	}
	compared2DA, comparedTLK, nativePass, nativeMismatch, err := compareNativeSelfCheck(p, output2DA, outputTLK, progress)
	if err != nil {
		return CompareResult{}, err
	}

	return CompareResult{
		Mode:            "native",
		Compared2DA:     compared2DA,
		ComparedTLK:     comparedTLK,
		NativePass:      nativePass,
		NotYetCanonical: 0,
		NativeMismatch:  nativeMismatch,
	}, nil
}

func BuildReference(p *project.Project, progress func(string)) (BuildResult, error) {
	return BuildNative(p, progress)
}

func CompareReference(p *project.Project, progress func(string)) (CompareResult, error) {
	return Compare(p, progress)
}

func compareDirs(actualRoot, expectedRoot string) (int, error) {
	expectedFiles := make([]string, 0)
	err := filepath.WalkDir(expectedRoot, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		rel, err := filepath.Rel(expectedRoot, path)
		if err != nil {
			return err
		}
		expectedFiles = append(expectedFiles, filepath.ToSlash(rel))
		return nil
	})
	if err != nil {
		return 0, err
	}
	slices.Sort(expectedFiles)

	actualFiles := make([]string, 0)
	err = filepath.WalkDir(actualRoot, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		rel, err := filepath.Rel(actualRoot, path)
		if err != nil {
			return err
		}
		actualFiles = append(actualFiles, filepath.ToSlash(rel))
		return nil
	})
	if err != nil {
		return 0, err
	}
	slices.Sort(actualFiles)
	if !slices.Equal(actualFiles, expectedFiles) {
		return 0, fmt.Errorf("topdata output file list differs from reference")
	}

	for _, rel := range expectedFiles {
		actualBytes, err := os.ReadFile(filepath.Join(actualRoot, rel))
		if err != nil {
			return 0, err
		}
		expectedBytes, err := os.ReadFile(filepath.Join(expectedRoot, rel))
		if err != nil {
			return 0, err
		}
		if !bytes.Equal(actualBytes, expectedBytes) {
			return 0, fmt.Errorf("topdata output differs from reference: %s", rel)
		}
	}
	return len(expectedFiles), nil
}

func compareNativeSelfCheck(p *project.Project, actual2DA, actualTLK string, progress func(string)) (int, int, int, int, error) {
	if progress == nil {
		progress = func(string) {}
	}
	projectOutputs, err := discoverNativeOutputCatalog(filepath.Join(p.TopDataSourceDir(), "data"))
	if err != nil {
		return 0, 0, 0, 0, err
	}
	if len(projectOutputs) == 0 {
		return 0, 0, 0, 0, nil
	}

	tempBuild, err := os.MkdirTemp("", "sow-topdata-native-compare-*")
	if err != nil {
		return 0, 0, 0, 0, fmt.Errorf("create native compare temp dir: %w", err)
	}
	defer os.RemoveAll(tempBuild)

	nativeProject := *p
	nativeProject.Config.TopData.Build = tempBuild
	nativeResult, err := buildNativeUnchecked(&nativeProject, nil, nil)
	if err != nil {
		return 0, 0, 0, len(projectOutputs), nil
	}

	pass := 0
	mismatch := 0
	for outputName := range projectOutputs {
		actualBytes, err := os.ReadFile(filepath.Join(actual2DA, outputName))
		if err != nil {
			mismatch++
			continue
		}
		expectedBytes, err := os.ReadFile(filepath.Join(nativeResult.Output2DADir, outputName))
		if err != nil {
			mismatch++
			continue
		}
		if bytes.Equal(normalize2DAForCompare(outputName, actualBytes), normalize2DAForCompare(outputName, expectedBytes)) {
			pass++
			continue
		}
		mismatch++
	}
	actual2DAFiles, err := listRelativeFiles(actual2DA)
	if err != nil {
		return 0, 0, 0, 0, err
	}
	expected2DAFiles, err := listRelativeFiles(nativeResult.Output2DADir)
	if err != nil {
		return 0, 0, 0, 0, err
	}
	if !slices.Equal(actual2DAFiles, expected2DAFiles) {
		mismatch += symmetricDiffCount(actual2DAFiles, expected2DAFiles)
	}
	comparedTLK, tlkMismatch, err := compareExactDirFiles(actualTLK, nativeResult.OutputTLKDir)
	if err != nil {
		return 0, 0, 0, 0, err
	}
	mismatch += tlkMismatch
	return len(projectOutputs), comparedTLK, pass, mismatch, nil
}

func compareNativeCoverage(p *project.Project, actual2DA string, progress func(string)) (int, int, int, error) {
	actualTLK := compiledTLKOutputDir(p)
	compared2DA, _, pass, mismatch, err := compareNativeSelfCheck(p, actual2DA, actualTLK, progress)
	if err != nil {
		return 0, 0, 0, err
	}
	if compared2DA == 0 {
		return 0, 0, mismatch, nil
	}
	return pass, 0, mismatch, nil
}

func normalize2DAForCompare(outputName string, raw []byte) []byte {
	text := strings.ReplaceAll(string(raw), "\r\n", "\n")
	lines := strings.Split(text, "\n")
	if len(lines) < 3 || !strings.HasPrefix(lines[0], "2DA V2.0") {
		return raw
	}
	headerIndex := 2
	columns := strings.Split(lines[headerIndex], "\t")
	keep := make([]int, 0, len(columns))
	filtered := make([]string, 0, len(columns))
	for index, column := range columns {
		if isAuthoringOnlyField(column) {
			continue
		}
		keep = append(keep, index)
		filtered = append(filtered, column)
	}
	var builder strings.Builder
	builder.WriteString(lines[0])
	builder.WriteByte('\n')
	builder.WriteByte('\n')
	builder.WriteString(strings.Join(filtered, "\t"))
	builder.WriteByte('\n')
	normalizedRows := make([][]string, 0, len(lines))
	for _, line := range lines[headerIndex+1:] {
		if line == "" {
			continue
		}
		fields := strings.Split(line, "\t")
		if len(fields) == 0 {
			continue
		}
		row := make([]string, 0, len(keep))
		for _, index := range keep {
			if index+1 < len(fields) {
				row = append(row, fields[index+1])
			} else {
				row = append(row, nullValue)
			}
		}
		normalizedRows = append(normalizedRows, row)
	}
	if shouldIgnore2DAPlacement(outputName) {
		slices.SortFunc(normalizedRows, func(a, b []string) int {
			return strings.Compare(strings.Join(a, "\x00"), strings.Join(b, "\x00"))
		})
	}
	for index, row := range normalizedRows {
		builder.WriteString(strconv.Itoa(index))
		for _, value := range row {
			builder.WriteByte('\t')
			builder.WriteString(value)
		}
		builder.WriteByte('\n')
	}
	return []byte(builder.String())
}

func shouldIgnore2DAPlacement(outputName string) bool {
	return strings.HasPrefix(outputName, "cls_feat_") && strings.HasSuffix(outputName, ".2da")
}

func walkJSON(root string, fn func(path string)) error {
	return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		if strings.HasSuffix(strings.ToLower(d.Name()), ".json") {
			fn(path)
		}
		return nil
	})
}

func countFiles(root string) (int, error) {
	count := 0
	err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if !d.IsDir() {
			count++
		}
		return nil
	})
	return count, err
}

type progressWriter struct {
	progress func(string)
}

func (w progressWriter) Write(p []byte) (int, error) {
	text := strings.TrimSpace(string(p))
	if text != "" {
		w.progress(text)
	}
	return len(p), nil
}

func ioDiscardProgress(progress func(string)) *progressWriter {
	return &progressWriter{progress: progress}
}

func listRelativeFiles(root string) ([]string, error) {
	files := make([]string, 0)
	if _, err := os.Stat(root); err != nil {
		if os.IsNotExist(err) {
			return files, nil
		}
		return nil, err
	}
	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() {
			return nil
		}
		rel, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		files = append(files, filepath.ToSlash(rel))
		return nil
	})
	if err != nil {
		return nil, err
	}
	slices.Sort(files)
	return files, nil
}

func symmetricDiffCount(left, right []string) int {
	leftSet := map[string]struct{}{}
	rightSet := map[string]struct{}{}
	for _, item := range left {
		leftSet[item] = struct{}{}
	}
	for _, item := range right {
		rightSet[item] = struct{}{}
	}
	count := 0
	for item := range leftSet {
		if _, ok := rightSet[item]; !ok {
			count++
		}
	}
	for item := range rightSet {
		if _, ok := leftSet[item]; !ok {
			count++
		}
	}
	return count
}

func compareExactDirFiles(actualRoot, expectedRoot string) (int, int, error) {
	actualFiles, err := listRelativeFiles(actualRoot)
	if err != nil {
		return 0, 0, err
	}
	expectedFiles, err := listRelativeFiles(expectedRoot)
	if err != nil {
		return 0, 0, err
	}
	mismatch := 0
	if !slices.Equal(actualFiles, expectedFiles) {
		mismatch += symmetricDiffCount(actualFiles, expectedFiles)
	}
	shared := make([]string, 0, len(expectedFiles))
	for _, rel := range expectedFiles {
		if slices.Contains(actualFiles, rel) {
			shared = append(shared, rel)
		}
	}
	for _, rel := range shared {
		actualBytes, err := os.ReadFile(filepath.Join(actualRoot, rel))
		if err != nil {
			mismatch++
			continue
		}
		expectedBytes, err := os.ReadFile(filepath.Join(expectedRoot, rel))
		if err != nil {
			mismatch++
			continue
		}
		if !bytes.Equal(actualBytes, expectedBytes) {
			mismatch++
		}
	}
	return len(expectedFiles), mismatch, nil
}