From d67b6e3f1269011ed693e08dc890743d434b3598 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Thu, 2 Apr 2026 21:40:59 +0200 Subject: [PATCH] Add topdata bridge commands and handoff --- README.md | 17 + TOPDATA_HANDOFF_2026-04-02.md | 110 +++++ internal/app/app.go | 95 ++++- internal/project/project.go | 49 ++- internal/topdata/topdata.go | 702 +++++++++++++++++++++++++++++++ internal/topdata/topdata_test.go | 115 +++++ 6 files changed, 1083 insertions(+), 5 deletions(-) create mode 100644 TOPDATA_HANDOFF_2026-04-02.md create mode 100644 internal/topdata/topdata.go create mode 100644 internal/topdata/topdata_test.go diff --git a/README.md b/README.md index 70e9423..15efe5a 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,10 @@ tools/ built development binary output - `sow-module` uses `sow-toolkit` for `build-module`, `extract`, `validate`, `compare`, and `apply-hak-manifest` - `sow-assets` uses `sow-toolkit` for `build-haks` and validation +- `sow-module` now also uses `sow-toolkit` for the early topdata bridge: + - `validate-topdata` + - `build-topdata` + - `compare-topdata` During development, sibling repositories can point at `../sow-tools/tools/sow-toolkit`. For pinned releases, each consumer repo can instead carry its own copy at `tools/sow-toolkit`. @@ -50,3 +54,16 @@ Consumer repos can fetch those binaries into their local `tools/` directory with - `install-tool.sh` on Linux/macOS - `install-tool.ps1` on Windows + +## TopData Migration + +The topdata rebuild is now staged in this repo. + +Current state: + +- the new command surface exists in `sow-toolkit` +- `sow-module/topdata/` is the new long-term source home +- `build-topdata` currently bridges through the Python builder in `sow-dev` for authoritative output generation +- native Go merge/build parity is still a follow-up milestone + +See [TOPDATA_HANDOFF_2026-04-02.md](./TOPDATA_HANDOFF_2026-04-02.md) for the current handoff and migration notes. diff --git a/TOPDATA_HANDOFF_2026-04-02.md b/TOPDATA_HANDOFF_2026-04-02.md new file mode 100644 index 0000000..af0add0 --- /dev/null +++ b/TOPDATA_HANDOFF_2026-04-02.md @@ -0,0 +1,110 @@ +# TopData Rebuild Handoff - 2026-04-02 + +## Current State + +This handoff captures the first implementation milestone of the topdata rebuild. + +The migration direction is: + +- Python `2dabuilder` in `sow-dev` remains authoritative +- `sow-toolkit` now owns the new topdata command surface +- `sow-module` is the new long-term source home for migrated topdata content +- wiki generation, wiki deploy, and tophak packaging are still deferred + +## Implemented In This Milestone + +### New `sow-toolkit` commands + +- `validate-topdata` +- `build-topdata` +- `compare-topdata` + +Current behavior: + +- `validate-topdata` validates the new `topdata/` source root in `sow-module` +- `build-topdata` currently uses the Python builder as a reference backend and writes outputs into `build/topdata/` +- `compare-topdata` rebuilds fresh reference output from Python and compares it against `build/topdata/` + +This is an intentional bridge: it preserves current production behavior while the deeper merge/build rules are audited and ported. + +### New `sow-module` layout + +Recommended migration root now exists: + +```text +topdata/ + data/ + tlk/ + base_dialog.json +``` + +Generated outputs are expected under: + +```text +build/topdata/ + 2da/ + tlk/ +``` + +### Canonical compatibility validation + +The new validator already recognizes the current family of authoring shapes: + +- dataset `base.json` with `columns` + `rows` +- lockfiles as object maps of key -> numeric id +- module files using `entries` +- module files using `overrides` +- plain row files using `rows` +- TLK files with `entries` + +It is intentionally permissive for now: + +- legacy parseable shapes can still load +- warnings are used where a file is parseable but not canonical yet +- this keeps migration practical while still nudging toward a formal schema + +## Important Rules Captured So Far + +These behaviors are considered core and must be preserved in the eventual native compiler: + +- base fallback from `base.json` +- TLK fallback from `base_dialog.json` / TLK base data +- snippet files and grouped files both being valid +- grouped dataset families and plain tables both being valid +- lockfile-backed IDs +- ID allocation when a key has no existing lock entry +- “only override what we intentionally touch” +- deterministic output and deterministic compare behavior + +## What Is Still Deferred + +- native Go merge/build parity for current Python rules +- shorthand expansion porting +- dataset-specific rule porting +- local wiki generation +- wiki deploy +- tophak packaging +- canonical normalize/export command + +## Suggested Next Steps + +1. Audit the current Python rules dataset-by-dataset, especially: + - `classes` + - `feat` + - `itemprops` + - TLK merge behavior +2. Capture representative parity fixtures from the Python builder. +3. Replace the reference-backed `build-topdata` bridge with native Go build logic one dataset family at a time. +4. Add a normalize/migrate command once the canonical JSON contract is nailed down. +5. Only migrate wiki generation after compiler parity is stable. + +## Related UX State + +Separate from topdata, the content repos already have: + +- auto-installing `sow-toolkit` wrappers +- Linux/macOS shell wrappers +- Windows PowerShell and `.cmd` wrappers +- simplified `START-HERE.md` docs in `sow-module` and `sow-assets` + +Windows still needs a real smoke test from an actual Windows machine. diff --git a/internal/app/app.go b/internal/app/app.go index 7da076a..417a1fa 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -11,6 +11,7 @@ import ( "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/pipeline" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/topdata" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/validator" ) @@ -24,6 +25,7 @@ type context struct { stdout io.Writer stderr io.Writer cwd string + args []string } var commands = []command{ @@ -62,6 +64,21 @@ var commands = []command{ description: "Apply a generated HAK manifest to src/module/module.ifo.json.", run: runApplyHAKManifest, }, + { + name: "validate-topdata", + description: "Validate topdata source layout and canonical JSON compatibility.", + run: runValidateTopData, + }, + { + name: "build-topdata", + description: "Build topdata outputs into build/topdata using the reference builder bridge.", + run: runBuildTopData, + }, + { + name: "compare-topdata", + description: "Compare built topdata outputs against fresh reference-builder output.", + run: runCompareTopData, + }, } func Run(args []string) (int, error) { @@ -103,6 +120,7 @@ func newContext() (context, error) { stdout: os.Stdout, stderr: os.Stderr, cwd: cwd, + args: os.Args[1:], }, nil } @@ -284,8 +302,8 @@ func runApplyHAKManifest(ctx context) error { } manifestPath := filepath.Join(p.BuildDir(), "haks.json") - if len(os.Args) > 2 { - manifestPath = os.Args[2] + if len(ctx.args) > 1 { + manifestPath = ctx.args[1] if !filepath.IsAbs(manifestPath) { manifestPath = filepath.Join(ctx.cwd, manifestPath) } @@ -303,6 +321,79 @@ func runApplyHAKManifest(ctx context) error { return nil } +func runValidateTopData(ctx context) error { + p, err := loadProject(ctx.cwd) + if err != nil { + return err + } + + report := topdata.ValidateProject(p) + for _, diagnostic := range report.Diagnostics { + if diagnostic.Severity == topdata.SeverityWarning { + fmt.Fprintf(ctx.stderr, "warning: %s: %s\n", diagnostic.Path, diagnostic.Message) + continue + } + fmt.Fprintf(ctx.stderr, "error: %s: %s\n", diagnostic.Path, diagnostic.Message) + } + if report.HasErrors() { + return fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount()) + } + + fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name) + fmt.Fprintf(ctx.stdout, "topdata root: %s\n", p.TopDataSourceDir()) + fmt.Fprintf(ctx.stdout, "topdata files: %d\n", report.Files) + fmt.Fprintf(ctx.stdout, "data files: %d\n", report.DataFiles) + fmt.Fprintf(ctx.stdout, "tlk files: %d\n", report.TLKFiles) + if warnings := report.WarningCount(); warnings > 0 { + fmt.Fprintf(ctx.stdout, "warnings: %d\n", warnings) + } + fmt.Fprintf(ctx.stdout, "topdata validation: ok\n") + return nil +} + +func runBuildTopData(ctx context) error { + p, err := loadProject(ctx.cwd) + if err != nil { + return err + } + + result, err := topdata.BuildReference(p, func(message string) { + fmt.Fprintf(ctx.stdout, "[build-topdata] %s\n", message) + }) + if err != nil { + return err + } + + fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name) + fmt.Fprintf(ctx.stdout, "mode: %s\n", result.Mode) + fmt.Fprintf(ctx.stdout, "topdata 2da output: %s\n", result.Output2DADir) + fmt.Fprintf(ctx.stdout, "topdata tlk output: %s\n", result.OutputTLKDir) + fmt.Fprintf(ctx.stdout, "2da files: %d\n", result.Files2DA) + fmt.Fprintf(ctx.stdout, "tlk files: %d\n", result.FilesTLK) + return nil +} + +func runCompareTopData(ctx context) error { + p, err := loadProject(ctx.cwd) + if err != nil { + return err + } + + result, err := topdata.CompareReference(p, func(message string) { + fmt.Fprintf(ctx.stdout, "[compare-topdata] %s\n", message) + }) + if err != nil { + return err + } + + fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name) + fmt.Fprintf(ctx.stdout, "mode: %s\n", result.Mode) + fmt.Fprintf(ctx.stdout, "checked 2da files: %d\n", result.Compared2DA) + fmt.Fprintf(ctx.stdout, "checked tlk files: %d\n", result.ComparedTLK) + fmt.Fprintf(ctx.stdout, "topdata compare: ok\n") + return nil +} + func loadProject(cwd string) (*project.Project, error) { root, err := project.FindRoot(cwd) if err != nil { diff --git a/internal/project/project.go b/internal/project/project.go index 71c6824..af5258c 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -29,9 +29,10 @@ type Project struct { } type Config struct { - Module ModuleConfig `json:"module"` - Paths PathConfig `json:"paths"` - HAKs []HAKConfig `json:"haks"` + Module ModuleConfig `json:"module"` + Paths PathConfig `json:"paths"` + HAKs []HAKConfig `json:"haks"` + TopData TopDataConfig `json:"topdata"` } type ModuleConfig struct { @@ -55,6 +56,12 @@ type HAKConfig struct { Include []string `json:"include"` } +type TopDataConfig struct { + Source string `json:"source"` + Build string `json:"build"` + ReferenceBuilder string `json:"reference_builder"` +} + type Inventory struct { SourceFiles []string ScriptFiles []string @@ -211,6 +218,39 @@ func (p *Project) BuildDir() string { return filepath.Join(p.Root, p.Config.Paths.Build) } +func (p *Project) HasTopData() bool { + return strings.TrimSpace(p.Config.TopData.Source) != "" +} + +func (p *Project) TopDataSourceDir() string { + if !p.HasTopData() { + return "" + } + return filepath.Join(p.Root, p.Config.TopData.Source) +} + +func (p *Project) TopDataBuildDir() string { + buildPath := strings.TrimSpace(p.Config.TopData.Build) + if buildPath == "" { + buildPath = filepath.Join(p.Config.Paths.Build, "topdata") + } + if filepath.IsAbs(buildPath) { + return buildPath + } + return filepath.Join(p.Root, buildPath) +} + +func (p *Project) TopDataReferenceBuilderDir() string { + ref := strings.TrimSpace(p.Config.TopData.ReferenceBuilder) + if ref == "" { + return "" + } + if filepath.IsAbs(ref) { + return ref + } + return filepath.Join(p.Root, ref) +} + func (i Inventory) Report() InventoryReport { return InventoryReport{ SourceFiles: len(i.SourceFiles), @@ -227,6 +267,9 @@ func defaultConfig() Config { Assets: "assets", Build: "build", }, + TopData: TopDataConfig{ + Build: "build/topdata", + }, } } diff --git a/internal/topdata/topdata.go b/internal/topdata/topdata.go new file mode 100644 index 0000000..1eb58d6 --- /dev/null +++ b/internal/topdata/topdata.go @@ -0,0 +1,702 @@ +package topdata + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +type Severity string + +const ( + SeverityError Severity = "error" + SeverityWarning Severity = "warning" +) + +type Diagnostic struct { + Severity Severity + Path string + Message string +} + +type ValidationReport struct { + Files int + DataFiles int + TLKFiles int + Diagnostics []Diagnostic +} + +func (r ValidationReport) HasErrors() bool { + for _, diagnostic := range r.Diagnostics { + if diagnostic.Severity == SeverityError { + return true + } + } + return false +} + +func (r ValidationReport) ErrorCount() int { + count := 0 + for _, diagnostic := range r.Diagnostics { + if diagnostic.Severity == SeverityError { + count++ + } + } + return count +} + +func (r ValidationReport) WarningCount() int { + count := 0 + for _, diagnostic := range r.Diagnostics { + if diagnostic.Severity == SeverityWarning { + count++ + } + } + return count +} + +type BuildResult struct { + Mode string + Output2DADir string + OutputTLKDir string + Files2DA int + FilesTLK int +} + +type CompareResult struct { + Mode string + Compared2DA int + ComparedTLK int +} + +func ValidateProject(p *project.Project) ValidationReport { + report := ValidationReport{} + if !p.HasTopData() { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: project.ConfigFile, + Message: "topdata is not configured for this project", + }) + return report + } + + sourceDir := p.TopDataSourceDir() + info, err := os.Stat(sourceDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: sourceDir, + Message: err.Error(), + }) + return report + } + if !info.IsDir() { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: sourceDir, + Message: "topdata source must be a directory", + }) + return report + } + + dataDir := filepath.Join(sourceDir, "data") + tlkDir := filepath.Join(sourceDir, "tlk") + baseDialog := filepath.Join(sourceDir, "base_dialog.json") + + for _, requiredDir := range []string{dataDir, tlkDir} { + info, err := os.Stat(requiredDir) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: requiredDir, + Message: err.Error(), + }) + continue + } + if !info.IsDir() { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: requiredDir, + Message: "must be a directory", + }) + } + } + + if _, err := os.Stat(baseDialog); err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: baseDialog, + Message: "base_dialog.json is missing; migrated datasets may rely on reference builder fallback for now", + }) + } else { + report.Files++ + if err := validateJSONFile(baseDialog, "base_dialog", &report); err == nil { + // count only once for base dialog. + } + } + + _ = walkJSON(dataDir, func(path string) { + report.Files++ + report.DataFiles++ + _ = validateJSONFile(path, "data", &report) + }) + _ = walkJSON(tlkDir, func(path string) { + report.Files++ + report.TLKFiles++ + _ = validateJSONFile(path, "tlk", &report) + }) + + refDir := p.TopDataReferenceBuilderDir() + if refDir == "" { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: project.ConfigFile, + Message: "topdata.reference_builder is not configured; build-topdata reference mode will be unavailable", + }) + } else { + if _, err := os.Stat(filepath.Join(refDir, "build.py")); err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: refDir, + Message: "reference builder does not contain build.py", + }) + } + } + + return report +} + +func validateJSONFile(path, domain string, report *ValidationReport) error { + raw, err := os.ReadFile(path) + if err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: err.Error(), + }) + return err + } + + var payload any + if err := json.Unmarshal(raw, &payload); err != nil { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("invalid JSON: %v", err), + }) + return err + } + + obj, ok := payload.(map[string]any) + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "topdata files must contain a JSON object at the root", + }) + return errors.New("topdata file root must be object") + } + + switch domain { + case "data": + validateDataObject(path, obj, report) + case "tlk": + validateTLKObject(path, obj, report) + case "base_dialog": + // Parseability is enough for now; compatibility is still reference-backed. + } + return nil +} + +func validateDataObject(path string, obj map[string]any, report *ValidationReport) { + base := filepath.Base(path) + slashed := filepath.ToSlash(path) + inModules := strings.Contains(slashed, "/modules/") + + if base == "lock.json" { + validateLockObject(path, obj, report) + return + } + if base == "base.json" { + validateRowsFile(path, obj, report, true) + return + } + if inModules { + if _, hasEntries := obj["entries"]; hasEntries { + validateEntriesFile(path, obj, report) + return + } + if _, hasOverrides := obj["overrides"]; hasOverrides { + validateOverridesFile(path, obj, report) + return + } + if _, hasRows := obj["rows"]; hasRows { + validateRowsFile(path, obj, report, false) + return + } + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: path, + Message: "module file does not use a recognized canonical shape yet; expected one of entries, overrides, or rows", + }) + return + } + + if _, hasRows := obj["rows"]; hasRows { + validateRowsFile(path, obj, report, false) + return + } + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: path, + Message: "unrecognized data file shape; parseable for compatibility, but not canonical yet", + }) +} + +func validateTLKObject(path string, obj map[string]any, report *ValidationReport) { + if filepath.Base(path) == "lock.json" { + validateLockObject(path, obj, report) + return + } + entries, ok := obj["entries"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "TLK file must contain an entries object", + }) + return + } + if _, ok := entries.(map[string]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "TLK entries must be a JSON object", + }) + } + if language, ok := obj["language"]; ok { + if _, ok := language.(string); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "TLK language must be a string when present", + }) + } + } +} + +func validateLockObject(path string, obj map[string]any, report *ValidationReport) { + for key, value := range obj { + switch value.(type) { + case float64: + // JSON numbers land here. + default: + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: fmt.Sprintf("lock entry %q must be numeric", key), + }) + } + } +} + +func validateRowsFile(path string, obj map[string]any, report *ValidationReport, requireColumns bool) { + rows, ok := obj["rows"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "rows file must contain a rows array", + }) + return + } + if _, ok := rows.([]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "rows must be a JSON array", + }) + } + if columns, ok := obj["columns"]; ok { + if _, ok := columns.([]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "columns must be a JSON array when present", + }) + } + } else if requireColumns { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "base dataset file must contain columns", + }) + } +} + +func validateEntriesFile(path string, obj map[string]any, report *ValidationReport) { + entries, ok := obj["entries"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "entries file must contain an entries object", + }) + return + } + if _, ok := entries.(map[string]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "entries must be a JSON object", + }) + } +} + +func validateOverridesFile(path string, obj map[string]any, report *ValidationReport) { + overrides, ok := obj["overrides"] + if !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "override file must contain an overrides array", + }) + return + } + if _, ok := overrides.([]any); !ok { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityError, + Path: path, + Message: "overrides must be a JSON array", + }) + } +} + +func BuildReference(p *project.Project, progress func(string)) (BuildResult, error) { + if progress == nil { + progress = func(string) {} + } + if !p.HasTopData() { + return BuildResult{}, errors.New("topdata is not configured for this project") + } + + report := ValidateProject(p) + if report.HasErrors() { + return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount()) + } + + refDir := p.TopDataReferenceBuilderDir() + if refDir == "" { + return BuildResult{}, errors.New("topdata.reference_builder is not configured") + } + + buildDir := p.TopDataBuildDir() + output2DA := filepath.Join(buildDir, "2da") + outputTLK := filepath.Join(buildDir, "tlk") + progress("Preparing topdata build directories...") + if err := os.RemoveAll(buildDir); err != nil { + return BuildResult{}, fmt.Errorf("clean topdata build dir: %w", err) + } + if err := os.MkdirAll(output2DA, 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create 2da output dir: %w", err) + } + if err := os.MkdirAll(outputTLK, 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create tlk output dir: %w", err) + } + + progress("Building topdata outputs through the Python reference builder...") + bridgeDir, cleanup, err := prepareReferenceBridge(refDir) + if err != nil { + return BuildResult{}, err + } + defer cleanup() + + cmd := exec.Command("python3", "build.py", "--out", output2DA, "--tlk-dir", outputTLK) + cmd.Dir = bridgeDir + var stderr bytes.Buffer + cmd.Stderr = &stderr + cmd.Stdout = ioDiscardProgress(progress) + if err := cmd.Run(); err != nil { + message := strings.TrimSpace(stderr.String()) + if message == "" { + message = err.Error() + } + return BuildResult{}, fmt.Errorf("reference builder failed: %s", message) + } + + files2DA, err := countFiles(output2DA) + if err != nil { + return BuildResult{}, err + } + filesTLK, err := countFiles(outputTLK) + if err != nil { + return BuildResult{}, err + } + + return BuildResult{ + Mode: "reference", + Output2DADir: output2DA, + OutputTLKDir: outputTLK, + Files2DA: files2DA, + FilesTLK: filesTLK, + }, nil +} + +func CompareReference(p *project.Project, progress func(string)) (CompareResult, error) { + if progress == nil { + progress = func(string) {} + } + buildDir := p.TopDataBuildDir() + output2DA := filepath.Join(buildDir, "2da") + outputTLK := filepath.Join(buildDir, "tlk") + if _, err := os.Stat(output2DA); err != nil { + return CompareResult{}, fmt.Errorf("topdata build output missing: run build-topdata first") + } + + tempDir, err := os.MkdirTemp("", "sow-topdata-compare-*") + if err != nil { + return CompareResult{}, fmt.Errorf("create compare temp dir: %w", err) + } + defer os.RemoveAll(tempDir) + + ref2DA := filepath.Join(tempDir, "2da") + refTLK := filepath.Join(tempDir, "tlk") + if err := os.MkdirAll(ref2DA, 0o755); err != nil { + return CompareResult{}, err + } + if err := os.MkdirAll(refTLK, 0o755); err != nil { + return CompareResult{}, err + } + + progress("Building fresh reference topdata outputs for comparison...") + bridgeDir, cleanup, err := prepareReferenceBridge(p.TopDataReferenceBuilderDir()) + if err != nil { + return CompareResult{}, err + } + defer cleanup() + + cmd := exec.Command("python3", "build.py", "--out", ref2DA, "--tlk-dir", refTLK) + cmd.Dir = bridgeDir + var stderr bytes.Buffer + cmd.Stderr = &stderr + cmd.Stdout = ioDiscardProgress(progress) + if err := cmd.Run(); err != nil { + message := strings.TrimSpace(stderr.String()) + if message == "" { + message = err.Error() + } + return CompareResult{}, fmt.Errorf("reference builder failed: %s", message) + } + + compared2DA, err := compareDirs(output2DA, ref2DA) + if err != nil { + return CompareResult{}, err + } + comparedTLK, err := compareDirs(outputTLK, refTLK) + if err != nil { + return CompareResult{}, err + } + + return CompareResult{ + Mode: "reference", + Compared2DA: compared2DA, + ComparedTLK: comparedTLK, + }, nil +} + +func compareDirs(actualRoot, expectedRoot string) (int, error) { + expectedFiles := make([]string, 0) + err := filepath.WalkDir(expectedRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(expectedRoot, path) + if err != nil { + return err + } + expectedFiles = append(expectedFiles, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return 0, err + } + slices.Sort(expectedFiles) + + actualFiles := make([]string, 0) + err = filepath.WalkDir(actualRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(actualRoot, path) + if err != nil { + return err + } + actualFiles = append(actualFiles, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + return 0, err + } + slices.Sort(actualFiles) + if !slices.Equal(actualFiles, expectedFiles) { + return 0, fmt.Errorf("topdata output file list differs from reference") + } + + for _, rel := range expectedFiles { + actualBytes, err := os.ReadFile(filepath.Join(actualRoot, rel)) + if err != nil { + return 0, err + } + expectedBytes, err := os.ReadFile(filepath.Join(expectedRoot, rel)) + if err != nil { + return 0, err + } + if !bytes.Equal(actualBytes, expectedBytes) { + return 0, fmt.Errorf("topdata output differs from reference: %s", rel) + } + } + return len(expectedFiles), nil +} + +func walkJSON(root string, fn func(path string)) error { + return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.HasSuffix(strings.ToLower(d.Name()), ".json") { + fn(path) + } + return nil + }) +} + +func countFiles(root string) (int, error) { + count := 0 + err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + count++ + } + return nil + }) + return count, err +} + +type progressWriter struct { + progress func(string) +} + +func (w progressWriter) Write(p []byte) (int, error) { + text := strings.TrimSpace(string(p)) + if text != "" { + w.progress(text) + } + return len(p), nil +} + +func ioDiscardProgress(progress func(string)) *progressWriter { + return &progressWriter{progress: progress} +} + +func prepareReferenceBridge(referenceBuilderDir string) (string, func(), error) { + tempRoot, err := os.MkdirTemp("", "sow-topdata-ref-*") + if err != nil { + return "", nil, fmt.Errorf("create reference bridge dir: %w", err) + } + + cleanup := func() { + _ = os.RemoveAll(tempRoot) + } + + builderLink := filepath.Join(tempRoot, "tools", "2dabuilder") + if err := os.MkdirAll(filepath.Dir(builderLink), 0o755); err != nil { + cleanup() + return "", nil, fmt.Errorf("create reference bridge tools dir: %w", err) + } + if err := os.MkdirAll(filepath.Join(tempRoot, "assets"), 0o755); err != nil { + cleanup() + return "", nil, fmt.Errorf("create reference bridge assets dir: %w", err) + } + if err := os.MkdirAll(filepath.Join(tempRoot, "staging"), 0o755); err != nil { + cleanup() + return "", nil, fmt.Errorf("create reference bridge staging dir: %w", err) + } + if err := copyReferenceBuilder(referenceBuilderDir, builderLink); err != nil { + cleanup() + return "", nil, err + } + if err := rewriteBridgeDefaults(filepath.Join(builderLink, "build.py"), filepath.Join(tempRoot, "assets"), filepath.Join(tempRoot, "staging")); err != nil { + cleanup() + return "", nil, err + } + return builderLink, cleanup, nil +} + +func copyReferenceBuilder(src, dest string) error { + return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + if rel == "." { + return os.MkdirAll(dest, 0o755) + } + name := d.Name() + if d.IsDir() { + if name == ".venv" || name == ".pytest_cache" || name == "__pycache__" { + return filepath.SkipDir + } + return os.MkdirAll(filepath.Join(dest, rel), 0o755) + } + if strings.HasSuffix(name, ".pyc") || name == "2dabuilder.log" { + return nil + } + target := filepath.Join(dest, rel) + info, err := d.Info() + if err != nil { + return err + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + return os.WriteFile(target, data, info.Mode()) + }) +} + +func rewriteBridgeDefaults(buildScriptPath, assetsDir, stagingDir string) error { + raw, err := os.ReadFile(buildScriptPath) + if err != nil { + return fmt.Errorf("read bridge build.py: %w", err) + } + text := string(raw) + text = strings.Replace(text, `DEFAULT_ASSETS_DIR = "../../assets"`, fmt.Sprintf("DEFAULT_ASSETS_DIR = %q", assetsDir), 1) + text = strings.Replace(text, `DEFAULT_STAGING_DIR = "../../staging"`, fmt.Sprintf("DEFAULT_STAGING_DIR = %q", stagingDir), 1) + if err := os.WriteFile(buildScriptPath, []byte(text), 0o644); err != nil { + return fmt.Errorf("rewrite bridge build.py defaults: %w", err) + } + return nil +} diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go new file mode 100644 index 0000000..09423bd --- /dev/null +++ b/internal/topdata/topdata_test.go @@ -0,0 +1,115 @@ +package topdata + +import ( + "os" + "path/filepath" + "testing" + + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" +) + +func TestValidateProjectAcceptsCanonicalTopDataLayout(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{"output":"feat.2da","columns":["LABEL"],"rows":[{"id":0,"LABEL":"Feat","key":"feat:test"}]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{"feat:test":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "modules", "entry.json"), `{"overrides":[{"id":0,"LABEL":"Feat"}]}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "base.json"), `{"language":"en","entries":{"feat:test.name":"Feat"}}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "tlk", "lock.json"), `{"feat:test.name":0}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + p := &project.Project{ + Root: root, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: project.TopDataConfig{ + Source: "topdata", + Build: "build/topdata", + ReferenceBuilder: "reference", + }, + }, + } + + report := ValidateProject(p) + if report.HasErrors() { + t.Fatalf("expected no topdata validation errors, got %#v", report.Diagnostics) + } +} + +func TestBuildReferenceAndCompareReference(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "assets")) + mkdirAll(t, filepath.Join(root, "build")) + mkdirAll(t, filepath.Join(root, "topdata", "data")) + mkdirAll(t, filepath.Join(root, "topdata", "tlk")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), `#!/usr/bin/env python3 +import os, sys +out = None +tlk = None +args = sys.argv[1:] +for i, arg in enumerate(args): + if arg == "--out": + out = args[i + 1] + if arg == "--tlk-dir": + tlk = args[i + 1] +os.makedirs(out, exist_ok=True) +os.makedirs(tlk, exist_ok=True) +with open(os.path.join(out, "feat.2da"), "w", encoding="utf-8") as f: + f.write("2DA V2.0\n\n") +with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: + f.write(b"TLK") +`) + + p := &project.Project{ + Root: root, + Config: project.Config{ + Module: project.ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: project.PathConfig{Source: "src", Assets: "assets", Build: "build"}, + TopData: project.TopDataConfig{ + Source: "topdata", + Build: "build/topdata", + ReferenceBuilder: "reference", + }, + }, + } + + result, err := BuildReference(p, nil) + if err != nil { + t.Fatalf("BuildReference failed: %v", err) + } + if result.Files2DA != 1 || result.FilesTLK != 1 { + t.Fatalf("unexpected build result: %#v", result) + } + + compare, err := CompareReference(p, nil) + if err != nil { + t.Fatalf("CompareReference failed: %v", err) + } + if compare.Compared2DA != 1 || compare.ComparedTLK != 1 { + t.Fatalf("unexpected compare result: %#v", compare) + } +} + +func mkdirAll(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatalf("write %s: %v", path, err) + } +}