Add topdata bridge commands and handoff

README.md

@@ -35,6 +35,10 @@ tools/             built development binary output
 
 - `sow-module` uses `sow-toolkit` for `build-module`, `extract`, `validate`, `compare`, and `apply-hak-manifest`
 - `sow-assets` uses `sow-toolkit` for `build-haks` and validation
+- `sow-module` now also uses `sow-toolkit` for the early topdata bridge:
+  - `validate-topdata`
+  - `build-topdata`
+  - `compare-topdata`
 
 During development, sibling repositories can point at `../sow-tools/tools/sow-toolkit`. For pinned releases, each consumer repo can instead carry its own copy at `tools/sow-toolkit`.
 
@@ -50,3 +54,16 @@ Consumer repos can fetch those binaries into their local `tools/` directory with
 
 - `install-tool.sh` on Linux/macOS
 - `install-tool.ps1` on Windows
+
+## TopData Migration
+
+The topdata rebuild is now staged in this repo.
+
+Current state:
+
+- the new command surface exists in `sow-toolkit`
+- `sow-module/topdata/` is the new long-term source home
+- `build-topdata` currently bridges through the Python builder in `sow-dev` for authoritative output generation
+- native Go merge/build parity is still a follow-up milestone
+
+See [TOPDATA_HANDOFF_2026-04-02.md](./TOPDATA_HANDOFF_2026-04-02.md) for the current handoff and migration notes.

TOPDATA_HANDOFF_2026-04-02.md

@@ -0,0 +1,110 @@
+# TopData Rebuild Handoff - 2026-04-02
+
+## Current State
+
+This handoff captures the first implementation milestone of the topdata rebuild.
+
+The migration direction is:
+
+- Python `2dabuilder` in `sow-dev` remains authoritative
+- `sow-toolkit` now owns the new topdata command surface
+- `sow-module` is the new long-term source home for migrated topdata content
+- wiki generation, wiki deploy, and tophak packaging are still deferred
+
+## Implemented In This Milestone
+
+### New `sow-toolkit` commands
+
+- `validate-topdata`
+- `build-topdata`
+- `compare-topdata`
+
+Current behavior:
+
+- `validate-topdata` validates the new `topdata/` source root in `sow-module`
+- `build-topdata` currently uses the Python builder as a reference backend and writes outputs into `build/topdata/`
+- `compare-topdata` rebuilds fresh reference output from Python and compares it against `build/topdata/`
+
+This is an intentional bridge: it preserves current production behavior while the deeper merge/build rules are audited and ported.
+
+### New `sow-module` layout
+
+Recommended migration root now exists:
+
+```text
+topdata/
+  data/
+  tlk/
+  base_dialog.json
+```
+
+Generated outputs are expected under:
+
+```text
+build/topdata/
+  2da/
+  tlk/
+```
+
+### Canonical compatibility validation
+
+The new validator already recognizes the current family of authoring shapes:
+
+- dataset `base.json` with `columns` + `rows`
+- lockfiles as object maps of key -> numeric id
+- module files using `entries`
+- module files using `overrides`
+- plain row files using `rows`
+- TLK files with `entries`
+
+It is intentionally permissive for now:
+
+- legacy parseable shapes can still load
+- warnings are used where a file is parseable but not canonical yet
+- this keeps migration practical while still nudging toward a formal schema
+
+## Important Rules Captured So Far
+
+These behaviors are considered core and must be preserved in the eventual native compiler:
+
+- base fallback from `base.json`
+- TLK fallback from `base_dialog.json` / TLK base data
+- snippet files and grouped files both being valid
+- grouped dataset families and plain tables both being valid
+- lockfile-backed IDs
+- ID allocation when a key has no existing lock entry
+- “only override what we intentionally touch”
+- deterministic output and deterministic compare behavior
+
+## What Is Still Deferred
+
+- native Go merge/build parity for current Python rules
+- shorthand expansion porting
+- dataset-specific rule porting
+- local wiki generation
+- wiki deploy
+- tophak packaging
+- canonical normalize/export command
+
+## Suggested Next Steps
+
+1. Audit the current Python rules dataset-by-dataset, especially:
+   - `classes`
+   - `feat`
+   - `itemprops`
+   - TLK merge behavior
+2. Capture representative parity fixtures from the Python builder.
+3. Replace the reference-backed `build-topdata` bridge with native Go build logic one dataset family at a time.
+4. Add a normalize/migrate command once the canonical JSON contract is nailed down.
+5. Only migrate wiki generation after compiler parity is stable.
+
+## Related UX State
+
+Separate from topdata, the content repos already have:
+
+- auto-installing `sow-toolkit` wrappers
+- Linux/macOS shell wrappers
+- Windows PowerShell and `.cmd` wrappers
+- simplified `START-HERE.md` docs in `sow-module` and `sow-assets`
+
+Windows still needs a real smoke test from an actual Windows machine.

internal/app/app.go

@@ -11,6 +11,7 @@ import (
 
 	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline"
 	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/topdata"
 	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/validator"
 )
 
@@ -24,6 +25,7 @@ type context struct {
 	stdout io.Writer
 	stderr io.Writer
 	cwd    string
+	args   []string
 }
 
 var commands = []command{
@@ -62,6 +64,21 @@ var commands = []command{
 		description: "Apply a generated HAK manifest to src/module/module.ifo.json.",
 		run:         runApplyHAKManifest,
 	},
+	{
+		name:        "validate-topdata",
+		description: "Validate topdata source layout and canonical JSON compatibility.",
+		run:         runValidateTopData,
+	},
+	{
+		name:        "build-topdata",
+		description: "Build topdata outputs into build/topdata using the reference builder bridge.",
+		run:         runBuildTopData,
+	},
+	{
+		name:        "compare-topdata",
+		description: "Compare built topdata outputs against fresh reference-builder output.",
+		run:         runCompareTopData,
+	},
 }
 
 func Run(args []string) (int, error) {
@@ -103,6 +120,7 @@ func newContext() (context, error) {
 		stdout: os.Stdout,
 		stderr: os.Stderr,
 		cwd:    cwd,
+		args:   os.Args[1:],
 	}, nil
 }
 
@@ -284,8 +302,8 @@ func runApplyHAKManifest(ctx context) error {
 	}
 
 	manifestPath := filepath.Join(p.BuildDir(), "haks.json")
-	if len(os.Args) > 2 {
+	if len(ctx.args) > 1 {
-		manifestPath = os.Args[2]
+		manifestPath = ctx.args[1]
 		if !filepath.IsAbs(manifestPath) {
 			manifestPath = filepath.Join(ctx.cwd, manifestPath)
 		}
@@ -303,6 +321,79 @@ func runApplyHAKManifest(ctx context) error {
 	return nil
 }
 
+func runValidateTopData(ctx context) error {
+	p, err := loadProject(ctx.cwd)
+	if err != nil {
+		return err
+	}
+
+	report := topdata.ValidateProject(p)
+	for _, diagnostic := range report.Diagnostics {
+		if diagnostic.Severity == topdata.SeverityWarning {
+			fmt.Fprintf(ctx.stderr, "warning: %s: %s\n", diagnostic.Path, diagnostic.Message)
+			continue
+		}
+		fmt.Fprintf(ctx.stderr, "error: %s: %s\n", diagnostic.Path, diagnostic.Message)
+	}
+	if report.HasErrors() {
+		return fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount())
+	}
+
+	fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
+	fmt.Fprintf(ctx.stdout, "topdata root: %s\n", p.TopDataSourceDir())
+	fmt.Fprintf(ctx.stdout, "topdata files: %d\n", report.Files)
+	fmt.Fprintf(ctx.stdout, "data files: %d\n", report.DataFiles)
+	fmt.Fprintf(ctx.stdout, "tlk files: %d\n", report.TLKFiles)
+	if warnings := report.WarningCount(); warnings > 0 {
+		fmt.Fprintf(ctx.stdout, "warnings: %d\n", warnings)
+	}
+	fmt.Fprintf(ctx.stdout, "topdata validation: ok\n")
+	return nil
+}
+
+func runBuildTopData(ctx context) error {
+	p, err := loadProject(ctx.cwd)
+	if err != nil {
+		return err
+	}
+
+	result, err := topdata.BuildReference(p, func(message string) {
+		fmt.Fprintf(ctx.stdout, "[build-topdata] %s\n", message)
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
+	fmt.Fprintf(ctx.stdout, "mode: %s\n", result.Mode)
+	fmt.Fprintf(ctx.stdout, "topdata 2da output: %s\n", result.Output2DADir)
+	fmt.Fprintf(ctx.stdout, "topdata tlk output: %s\n", result.OutputTLKDir)
+	fmt.Fprintf(ctx.stdout, "2da files: %d\n", result.Files2DA)
+	fmt.Fprintf(ctx.stdout, "tlk files: %d\n", result.FilesTLK)
+	return nil
+}
+
+func runCompareTopData(ctx context) error {
+	p, err := loadProject(ctx.cwd)
+	if err != nil {
+		return err
+	}
+
+	result, err := topdata.CompareReference(p, func(message string) {
+		fmt.Fprintf(ctx.stdout, "[compare-topdata] %s\n", message)
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
+	fmt.Fprintf(ctx.stdout, "mode: %s\n", result.Mode)
+	fmt.Fprintf(ctx.stdout, "checked 2da files: %d\n", result.Compared2DA)
+	fmt.Fprintf(ctx.stdout, "checked tlk files: %d\n", result.ComparedTLK)
+	fmt.Fprintf(ctx.stdout, "topdata compare: ok\n")
+	return nil
+}
+
 func loadProject(cwd string) (*project.Project, error) {
 	root, err := project.FindRoot(cwd)
 	if err != nil {

internal/project/project.go

@@ -29,9 +29,10 @@ type Project struct {
 }
 
 type Config struct {
-	Module ModuleConfig `json:"module"`
+	Module  ModuleConfig  `json:"module"`
-	Paths  PathConfig   `json:"paths"`
+	Paths   PathConfig    `json:"paths"`
-	HAKs   []HAKConfig  `json:"haks"`
+	HAKs    []HAKConfig   `json:"haks"`
+	TopData TopDataConfig `json:"topdata"`
 }
 
 type ModuleConfig struct {
@@ -55,6 +56,12 @@ type HAKConfig struct {
 	Include  []string `json:"include"`
 }
 
+type TopDataConfig struct {
+	Source           string `json:"source"`
+	Build            string `json:"build"`
+	ReferenceBuilder string `json:"reference_builder"`
+}
+
 type Inventory struct {
 	SourceFiles []string
 	ScriptFiles []string
@@ -211,6 +218,39 @@ func (p *Project) BuildDir() string {
 	return filepath.Join(p.Root, p.Config.Paths.Build)
 }
 
+func (p *Project) HasTopData() bool {
+	return strings.TrimSpace(p.Config.TopData.Source) != ""
+}
+
+func (p *Project) TopDataSourceDir() string {
+	if !p.HasTopData() {
+		return ""
+	}
+	return filepath.Join(p.Root, p.Config.TopData.Source)
+}
+
+func (p *Project) TopDataBuildDir() string {
+	buildPath := strings.TrimSpace(p.Config.TopData.Build)
+	if buildPath == "" {
+		buildPath = filepath.Join(p.Config.Paths.Build, "topdata")
+	}
+	if filepath.IsAbs(buildPath) {
+		return buildPath
+	}
+	return filepath.Join(p.Root, buildPath)
+}
+
+func (p *Project) TopDataReferenceBuilderDir() string {
+	ref := strings.TrimSpace(p.Config.TopData.ReferenceBuilder)
+	if ref == "" {
+		return ""
+	}
+	if filepath.IsAbs(ref) {
+		return ref
+	}
+	return filepath.Join(p.Root, ref)
+}
+
 func (i Inventory) Report() InventoryReport {
 	return InventoryReport{
 		SourceFiles: len(i.SourceFiles),
@@ -227,6 +267,9 @@ func defaultConfig() Config {
 			Assets: "assets",
 			Build:  "build",
 		},
+		TopData: TopDataConfig{
+			Build: "build/topdata",
+		},
 	}
 }
 

internal/topdata/topdata.go

@@ -0,0 +1,702 @@
+package topdata
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+type Severity string
+
+const (
+	SeverityError   Severity = "error"
+	SeverityWarning Severity = "warning"
+)
+
+type Diagnostic struct {
+	Severity Severity
+	Path     string
+	Message  string
+}
+
+type ValidationReport struct {
+	Files       int
+	DataFiles   int
+	TLKFiles    int
+	Diagnostics []Diagnostic
+}
+
+func (r ValidationReport) HasErrors() bool {
+	for _, diagnostic := range r.Diagnostics {
+		if diagnostic.Severity == SeverityError {
+			return true
+		}
+	}
+	return false
+}
+
+func (r ValidationReport) ErrorCount() int {
+	count := 0
+	for _, diagnostic := range r.Diagnostics {
+		if diagnostic.Severity == SeverityError {
+			count++
+		}
+	}
+	return count
+}
+
+func (r ValidationReport) WarningCount() int {
+	count := 0
+	for _, diagnostic := range r.Diagnostics {
+		if diagnostic.Severity == SeverityWarning {
+			count++
+		}
+	}
+	return count
+}
+
+type BuildResult struct {
+	Mode     string
+	Output2DADir string
+	OutputTLKDir string
+	Files2DA int
+	FilesTLK int
+}
+
+type CompareResult struct {
+	Mode         string
+	Compared2DA  int
+	ComparedTLK  int
+}
+
+func ValidateProject(p *project.Project) ValidationReport {
+	report := ValidationReport{}
+	if !p.HasTopData() {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityWarning,
+			Path:     project.ConfigFile,
+			Message:  "topdata is not configured for this project",
+		})
+		return report
+	}
+
+	sourceDir := p.TopDataSourceDir()
+	info, err := os.Stat(sourceDir)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     sourceDir,
+			Message:  err.Error(),
+		})
+		return report
+	}
+	if !info.IsDir() {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     sourceDir,
+			Message:  "topdata source must be a directory",
+		})
+		return report
+	}
+
+	dataDir := filepath.Join(sourceDir, "data")
+	tlkDir := filepath.Join(sourceDir, "tlk")
+	baseDialog := filepath.Join(sourceDir, "base_dialog.json")
+
+	for _, requiredDir := range []string{dataDir, tlkDir} {
+		info, err := os.Stat(requiredDir)
+		if err != nil {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     requiredDir,
+				Message:  err.Error(),
+			})
+			continue
+		}
+		if !info.IsDir() {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     requiredDir,
+				Message:  "must be a directory",
+			})
+		}
+	}
+
+	if _, err := os.Stat(baseDialog); err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityWarning,
+			Path:     baseDialog,
+			Message:  "base_dialog.json is missing; migrated datasets may rely on reference builder fallback for now",
+		})
+	} else {
+		report.Files++
+		if err := validateJSONFile(baseDialog, "base_dialog", &report); err == nil {
+			// count only once for base dialog.
+		}
+	}
+
+	_ = walkJSON(dataDir, func(path string) {
+		report.Files++
+		report.DataFiles++
+		_ = validateJSONFile(path, "data", &report)
+	})
+	_ = walkJSON(tlkDir, func(path string) {
+		report.Files++
+		report.TLKFiles++
+		_ = validateJSONFile(path, "tlk", &report)
+	})
+
+	refDir := p.TopDataReferenceBuilderDir()
+	if refDir == "" {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityWarning,
+			Path:     project.ConfigFile,
+			Message:  "topdata.reference_builder is not configured; build-topdata reference mode will be unavailable",
+		})
+	} else {
+		if _, err := os.Stat(filepath.Join(refDir, "build.py")); err != nil {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     refDir,
+				Message:  "reference builder does not contain build.py",
+			})
+		}
+	}
+
+	return report
+}
+
+func validateJSONFile(path, domain string, report *ValidationReport) error {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  err.Error(),
+		})
+		return err
+	}
+
+	var payload any
+	if err := json.Unmarshal(raw, &payload); err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  fmt.Sprintf("invalid JSON: %v", err),
+		})
+		return err
+	}
+
+	obj, ok := payload.(map[string]any)
+	if !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "topdata files must contain a JSON object at the root",
+		})
+		return errors.New("topdata file root must be object")
+	}
+
+	switch domain {
+	case "data":
+		validateDataObject(path, obj, report)
+	case "tlk":
+		validateTLKObject(path, obj, report)
+	case "base_dialog":
+		// Parseability is enough for now; compatibility is still reference-backed.
+	}
+	return nil
+}
+
+func validateDataObject(path string, obj map[string]any, report *ValidationReport) {
+	base := filepath.Base(path)
+	slashed := filepath.ToSlash(path)
+	inModules := strings.Contains(slashed, "/modules/")
+
+	if base == "lock.json" {
+		validateLockObject(path, obj, report)
+		return
+	}
+	if base == "base.json" {
+		validateRowsFile(path, obj, report, true)
+		return
+	}
+	if inModules {
+		if _, hasEntries := obj["entries"]; hasEntries {
+			validateEntriesFile(path, obj, report)
+			return
+		}
+		if _, hasOverrides := obj["overrides"]; hasOverrides {
+			validateOverridesFile(path, obj, report)
+			return
+		}
+		if _, hasRows := obj["rows"]; hasRows {
+			validateRowsFile(path, obj, report, false)
+			return
+		}
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityWarning,
+			Path:     path,
+			Message:  "module file does not use a recognized canonical shape yet; expected one of entries, overrides, or rows",
+		})
+		return
+	}
+
+	if _, hasRows := obj["rows"]; hasRows {
+		validateRowsFile(path, obj, report, false)
+		return
+	}
+	report.Diagnostics = append(report.Diagnostics, Diagnostic{
+		Severity: SeverityWarning,
+		Path:     path,
+		Message:  "unrecognized data file shape; parseable for compatibility, but not canonical yet",
+	})
+}
+
+func validateTLKObject(path string, obj map[string]any, report *ValidationReport) {
+	if filepath.Base(path) == "lock.json" {
+		validateLockObject(path, obj, report)
+		return
+	}
+	entries, ok := obj["entries"]
+	if !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "TLK file must contain an entries object",
+		})
+		return
+	}
+	if _, ok := entries.(map[string]any); !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "TLK entries must be a JSON object",
+		})
+	}
+	if language, ok := obj["language"]; ok {
+		if _, ok := language.(string); !ok {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  "TLK language must be a string when present",
+			})
+		}
+	}
+}
+
+func validateLockObject(path string, obj map[string]any, report *ValidationReport) {
+	for key, value := range obj {
+		switch value.(type) {
+		case float64:
+			// JSON numbers land here.
+		default:
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("lock entry %q must be numeric", key),
+			})
+		}
+	}
+}
+
+func validateRowsFile(path string, obj map[string]any, report *ValidationReport, requireColumns bool) {
+	rows, ok := obj["rows"]
+	if !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "rows file must contain a rows array",
+		})
+		return
+	}
+	if _, ok := rows.([]any); !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "rows must be a JSON array",
+		})
+	}
+	if columns, ok := obj["columns"]; ok {
+		if _, ok := columns.([]any); !ok {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  "columns must be a JSON array when present",
+			})
+		}
+	} else if requireColumns {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "base dataset file must contain columns",
+		})
+	}
+}
+
+func validateEntriesFile(path string, obj map[string]any, report *ValidationReport) {
+	entries, ok := obj["entries"]
+	if !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "entries file must contain an entries object",
+		})
+		return
+	}
+	if _, ok := entries.(map[string]any); !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "entries must be a JSON object",
+		})
+	}
+}
+
+func validateOverridesFile(path string, obj map[string]any, report *ValidationReport) {
+	overrides, ok := obj["overrides"]
+	if !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "override file must contain an overrides array",
+		})
+		return
+	}
+	if _, ok := overrides.([]any); !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "overrides must be a JSON array",
+		})
+	}
+}
+
+func BuildReference(p *project.Project, progress func(string)) (BuildResult, error) {
+	if progress == nil {
+		progress = func(string) {}
+	}
+	if !p.HasTopData() {
+		return BuildResult{}, errors.New("topdata is not configured for this project")
+	}
+
+	report := ValidateProject(p)
+	if report.HasErrors() {
+		return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount())
+	}
+
+	refDir := p.TopDataReferenceBuilderDir()
+	if refDir == "" {
+		return BuildResult{}, errors.New("topdata.reference_builder is not configured")
+	}
+
+	buildDir := p.TopDataBuildDir()
+	output2DA := filepath.Join(buildDir, "2da")
+	outputTLK := filepath.Join(buildDir, "tlk")
+	progress("Preparing topdata build directories...")
+	if err := os.RemoveAll(buildDir); err != nil {
+		return BuildResult{}, fmt.Errorf("clean topdata build dir: %w", err)
+	}
+	if err := os.MkdirAll(output2DA, 0o755); err != nil {
+		return BuildResult{}, fmt.Errorf("create 2da output dir: %w", err)
+	}
+	if err := os.MkdirAll(outputTLK, 0o755); err != nil {
+		return BuildResult{}, fmt.Errorf("create tlk output dir: %w", err)
+	}
+
+	progress("Building topdata outputs through the Python reference builder...")
+	bridgeDir, cleanup, err := prepareReferenceBridge(refDir)
+	if err != nil {
+		return BuildResult{}, err
+	}
+	defer cleanup()
+
+	cmd := exec.Command("python3", "build.py", "--out", output2DA, "--tlk-dir", outputTLK)
+	cmd.Dir = bridgeDir
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	cmd.Stdout = ioDiscardProgress(progress)
+	if err := cmd.Run(); err != nil {
+		message := strings.TrimSpace(stderr.String())
+		if message == "" {
+			message = err.Error()
+		}
+		return BuildResult{}, fmt.Errorf("reference builder failed: %s", message)
+	}
+
+	files2DA, err := countFiles(output2DA)
+	if err != nil {
+		return BuildResult{}, err
+	}
+	filesTLK, err := countFiles(outputTLK)
+	if err != nil {
+		return BuildResult{}, err
+	}
+
+	return BuildResult{
+		Mode:       "reference",
+		Output2DADir: output2DA,
+		OutputTLKDir: outputTLK,
+		Files2DA:   files2DA,
+		FilesTLK:   filesTLK,
+	}, nil
+}
+
+func CompareReference(p *project.Project, progress func(string)) (CompareResult, error) {
+	if progress == nil {
+		progress = func(string) {}
+	}
+	buildDir := p.TopDataBuildDir()
+	output2DA := filepath.Join(buildDir, "2da")
+	outputTLK := filepath.Join(buildDir, "tlk")
+	if _, err := os.Stat(output2DA); err != nil {
+		return CompareResult{}, fmt.Errorf("topdata build output missing: run build-topdata first")
+	}
+
+	tempDir, err := os.MkdirTemp("", "sow-topdata-compare-*")
+	if err != nil {
+		return CompareResult{}, fmt.Errorf("create compare temp dir: %w", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	ref2DA := filepath.Join(tempDir, "2da")
+	refTLK := filepath.Join(tempDir, "tlk")
+	if err := os.MkdirAll(ref2DA, 0o755); err != nil {
+		return CompareResult{}, err
+	}
+	if err := os.MkdirAll(refTLK, 0o755); err != nil {
+		return CompareResult{}, err
+	}
+
+	progress("Building fresh reference topdata outputs for comparison...")
+	bridgeDir, cleanup, err := prepareReferenceBridge(p.TopDataReferenceBuilderDir())
+	if err != nil {
+		return CompareResult{}, err
+	}
+	defer cleanup()
+
+	cmd := exec.Command("python3", "build.py", "--out", ref2DA, "--tlk-dir", refTLK)
+	cmd.Dir = bridgeDir
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	cmd.Stdout = ioDiscardProgress(progress)
+	if err := cmd.Run(); err != nil {
+		message := strings.TrimSpace(stderr.String())
+		if message == "" {
+			message = err.Error()
+		}
+		return CompareResult{}, fmt.Errorf("reference builder failed: %s", message)
+	}
+
+	compared2DA, err := compareDirs(output2DA, ref2DA)
+	if err != nil {
+		return CompareResult{}, err
+	}
+	comparedTLK, err := compareDirs(outputTLK, refTLK)
+	if err != nil {
+		return CompareResult{}, err
+	}
+
+	return CompareResult{
+		Mode:        "reference",
+		Compared2DA: compared2DA,
+		ComparedTLK: comparedTLK,
+	}, nil
+}
+
+func compareDirs(actualRoot, expectedRoot string) (int, error) {
+	expectedFiles := make([]string, 0)
+	err := filepath.WalkDir(expectedRoot, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		rel, err := filepath.Rel(expectedRoot, path)
+		if err != nil {
+			return err
+		}
+		expectedFiles = append(expectedFiles, filepath.ToSlash(rel))
+		return nil
+	})
+	if err != nil {
+		return 0, err
+	}
+	slices.Sort(expectedFiles)
+
+	actualFiles := make([]string, 0)
+	err = filepath.WalkDir(actualRoot, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		rel, err := filepath.Rel(actualRoot, path)
+		if err != nil {
+			return err
+		}
+		actualFiles = append(actualFiles, filepath.ToSlash(rel))
+		return nil
+	})
+	if err != nil {
+		return 0, err
+	}
+	slices.Sort(actualFiles)
+	if !slices.Equal(actualFiles, expectedFiles) {
+		return 0, fmt.Errorf("topdata output file list differs from reference")
+	}
+
+	for _, rel := range expectedFiles {
+		actualBytes, err := os.ReadFile(filepath.Join(actualRoot, rel))
+		if err != nil {
+			return 0, err
+		}
+		expectedBytes, err := os.ReadFile(filepath.Join(expectedRoot, rel))
+		if err != nil {
+			return 0, err
+		}
+		if !bytes.Equal(actualBytes, expectedBytes) {
+			return 0, fmt.Errorf("topdata output differs from reference: %s", rel)
+		}
+	}
+	return len(expectedFiles), nil
+}
+
+func walkJSON(root string, fn func(path string)) error {
+	return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		if strings.HasSuffix(strings.ToLower(d.Name()), ".json") {
+			fn(path)
+		}
+		return nil
+	})
+}
+
+func countFiles(root string) (int, error) {
+	count := 0
+	err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() {
+			count++
+		}
+		return nil
+	})
+	return count, err
+}
+
+type progressWriter struct {
+	progress func(string)
+}
+
+func (w progressWriter) Write(p []byte) (int, error) {
+	text := strings.TrimSpace(string(p))
+	if text != "" {
+		w.progress(text)
+	}
+	return len(p), nil
+}
+
+func ioDiscardProgress(progress func(string)) *progressWriter {
+	return &progressWriter{progress: progress}
+}
+
+func prepareReferenceBridge(referenceBuilderDir string) (string, func(), error) {
+	tempRoot, err := os.MkdirTemp("", "sow-topdata-ref-*")
+	if err != nil {
+		return "", nil, fmt.Errorf("create reference bridge dir: %w", err)
+	}
+
+	cleanup := func() {
+		_ = os.RemoveAll(tempRoot)
+	}
+
+	builderLink := filepath.Join(tempRoot, "tools", "2dabuilder")
+	if err := os.MkdirAll(filepath.Dir(builderLink), 0o755); err != nil {
+		cleanup()
+		return "", nil, fmt.Errorf("create reference bridge tools dir: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Join(tempRoot, "assets"), 0o755); err != nil {
+		cleanup()
+		return "", nil, fmt.Errorf("create reference bridge assets dir: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Join(tempRoot, "staging"), 0o755); err != nil {
+		cleanup()
+		return "", nil, fmt.Errorf("create reference bridge staging dir: %w", err)
+	}
+	if err := copyReferenceBuilder(referenceBuilderDir, builderLink); err != nil {
+		cleanup()
+		return "", nil, err
+	}
+	if err := rewriteBridgeDefaults(filepath.Join(builderLink, "build.py"), filepath.Join(tempRoot, "assets"), filepath.Join(tempRoot, "staging")); err != nil {
+		cleanup()
+		return "", nil, err
+	}
+	return builderLink, cleanup, nil
+}
+
+func copyReferenceBuilder(src, dest string) error {
+	return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		if rel == "." {
+			return os.MkdirAll(dest, 0o755)
+		}
+		name := d.Name()
+		if d.IsDir() {
+			if name == ".venv" || name == ".pytest_cache" || name == "__pycache__" {
+				return filepath.SkipDir
+			}
+			return os.MkdirAll(filepath.Join(dest, rel), 0o755)
+		}
+		if strings.HasSuffix(name, ".pyc") || name == "2dabuilder.log" {
+			return nil
+		}
+		target := filepath.Join(dest, rel)
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		return os.WriteFile(target, data, info.Mode())
+	})
+}
+
+func rewriteBridgeDefaults(buildScriptPath, assetsDir, stagingDir string) error {
+	raw, err := os.ReadFile(buildScriptPath)
+	if err != nil {
+		return fmt.Errorf("read bridge build.py: %w", err)
+	}
+	text := string(raw)
+	text = strings.Replace(text, `DEFAULT_ASSETS_DIR = "../../assets"`, fmt.Sprintf("DEFAULT_ASSETS_DIR = %q", assetsDir), 1)
+	text = strings.Replace(text, `DEFAULT_STAGING_DIR = "../../staging"`, fmt.Sprintf("DEFAULT_STAGING_DIR = %q", stagingDir), 1)
+	if err := os.WriteFile(buildScriptPath, []byte(text), 0o644); err != nil {
+		return fmt.Errorf("rewrite bridge build.py defaults: %w", err)
+	}
+	return nil
+}

internal/topdata/topdata_test.go

@@ -0,0 +1,115 @@
+package topdata
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+func TestValidateProjectAcceptsCanonicalTopDataLayout(t *testing.T) {
+	root := t.TempDir()
+	mkdirAll(t, filepath.Join(root, "src"))
+	mkdirAll(t, filepath.Join(root, "assets"))
+	mkdirAll(t, filepath.Join(root, "build"))
+	mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules"))
+	mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules"))
+	writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{"output":"feat.2da","columns":["LABEL"],"rows":[{"id":0,"LABEL":"Feat","key":"feat:test"}]}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "entry.json"), `{"overrides":[{"id":0,"LABEL":"Feat"}]}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "tlk", "base.json"), `{"language":"en","entries":{"feat:test.name":"Feat"}}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "tlk", "lock.json"), `{"feat:test.name":0}`+"\n")
+	mkdirAll(t, filepath.Join(root, "reference"))
+	writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n")
+
+	p := &project.Project{
+		Root: root,
+		Config: project.Config{
+			Module: project.ModuleConfig{Name: "Test", ResRef: "test"},
+			Paths:  project.PathConfig{Source: "src", Assets: "assets", Build: "build"},
+			TopData: project.TopDataConfig{
+				Source:           "topdata",
+				Build:            "build/topdata",
+				ReferenceBuilder: "reference",
+			},
+		},
+	}
+
+	report := ValidateProject(p)
+	if report.HasErrors() {
+		t.Fatalf("expected no topdata validation errors, got %#v", report.Diagnostics)
+	}
+}
+
+func TestBuildReferenceAndCompareReference(t *testing.T) {
+	root := t.TempDir()
+	mkdirAll(t, filepath.Join(root, "src"))
+	mkdirAll(t, filepath.Join(root, "assets"))
+	mkdirAll(t, filepath.Join(root, "build"))
+	mkdirAll(t, filepath.Join(root, "topdata", "data"))
+	mkdirAll(t, filepath.Join(root, "topdata", "tlk"))
+	writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
+	mkdirAll(t, filepath.Join(root, "reference"))
+	writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3
+import os, sys
+out = None
+tlk = None
+args = sys.argv[1:]
+for i, arg in enumerate(args):
+    if arg == "--out":
+        out = args[i + 1]
+    if arg == "--tlk-dir":
+        tlk = args[i + 1]
+os.makedirs(out, exist_ok=True)
+os.makedirs(tlk, exist_ok=True)
+with open(os.path.join(out, "feat.2da"), "w", encoding="utf-8") as f:
+    f.write("2DA V2.0\n\n")
+with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f:
+    f.write(b"TLK")
+`)
+
+	p := &project.Project{
+		Root: root,
+		Config: project.Config{
+			Module: project.ModuleConfig{Name: "Test", ResRef: "test"},
+			Paths:  project.PathConfig{Source: "src", Assets: "assets", Build: "build"},
+			TopData: project.TopDataConfig{
+				Source:           "topdata",
+				Build:            "build/topdata",
+				ReferenceBuilder: "reference",
+			},
+		},
+	}
+
+	result, err := BuildReference(p, nil)
+	if err != nil {
+		t.Fatalf("BuildReference failed: %v", err)
+	}
+	if result.Files2DA != 1 || result.FilesTLK != 1 {
+		t.Fatalf("unexpected build result: %#v", result)
+	}
+
+	compare, err := CompareReference(p, nil)
+	if err != nil {
+		t.Fatalf("CompareReference failed: %v", err)
+	}
+	if compare.Compared2DA != 1 || compare.ComparedTLK != 1 {
+		t.Fatalf("unexpected compare result: %#v", compare)
+	}
+}
+
+func mkdirAll(t *testing.T, path string) {
+	t.Helper()
+	if err := os.MkdirAll(path, 0o755); err != nil {
+		t.Fatalf("mkdir %s: %v", path, err)
+	}
+}
+
+func writeFile(t *testing.T, path, content string) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(content), 0o755); err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}