package topdata import ( "bytes" "encoding/json" "errors" "fmt" "io/fs" "os" "os/exec" "path/filepath" "slices" "strings" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) type Severity string const ( SeverityError Severity = "error" SeverityWarning Severity = "warning" ) type Diagnostic struct { Severity Severity Path string Message string } type ValidationReport struct { Files int DataFiles int TLKFiles int Diagnostics []Diagnostic } func (r ValidationReport) HasErrors() bool { for _, diagnostic := range r.Diagnostics { if diagnostic.Severity == SeverityError { return true } } return false } func (r ValidationReport) ErrorCount() int { count := 0 for _, diagnostic := range r.Diagnostics { if diagnostic.Severity == SeverityError { count++ } } return count } func (r ValidationReport) WarningCount() int { count := 0 for _, diagnostic := range r.Diagnostics { if diagnostic.Severity == SeverityWarning { count++ } } return count } type BuildResult struct { Mode string Output2DADir string OutputTLKDir string Files2DA int FilesTLK int } type CompareResult struct { Mode string Compared2DA int ComparedTLK int } func ValidateProject(p *project.Project) ValidationReport { report := ValidationReport{} if !p.HasTopData() { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityWarning, Path: project.ConfigFile, Message: "topdata is not configured for this project", }) return report } sourceDir := p.TopDataSourceDir() info, err := os.Stat(sourceDir) if err != nil { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: sourceDir, Message: err.Error(), }) return report } if !info.IsDir() { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: sourceDir, Message: "topdata source must be a directory", }) return report } dataDir := filepath.Join(sourceDir, "data") tlkDir := filepath.Join(sourceDir, "tlk") baseDialog := filepath.Join(sourceDir, "base_dialog.json") for _, requiredDir := range []string{dataDir, tlkDir} { info, err := os.Stat(requiredDir) if err != nil { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: requiredDir, Message: err.Error(), }) continue } if !info.IsDir() { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: requiredDir, Message: "must be a directory", }) } } if _, err := os.Stat(baseDialog); err != nil { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityWarning, Path: baseDialog, Message: "base_dialog.json is missing; migrated datasets may rely on reference builder fallback for now", }) } else { report.Files++ if err := validateJSONFile(baseDialog, "base_dialog", &report); err == nil { // count only once for base dialog. } } _ = walkJSON(dataDir, func(path string) { report.Files++ report.DataFiles++ _ = validateJSONFile(path, "data", &report) }) _ = walkJSON(tlkDir, func(path string) { report.Files++ report.TLKFiles++ _ = validateJSONFile(path, "tlk", &report) }) refDir := p.TopDataReferenceBuilderDir() if refDir == "" { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityWarning, Path: project.ConfigFile, Message: "topdata.reference_builder is not configured; build-topdata reference mode will be unavailable", }) } else { if _, err := os.Stat(filepath.Join(refDir, "build.py")); err != nil { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: refDir, Message: "reference builder does not contain build.py", }) } } return report } func validateJSONFile(path, domain string, report *ValidationReport) error { raw, err := os.ReadFile(path) if err != nil { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: err.Error(), }) return err } var payload any if err := json.Unmarshal(raw, &payload); err != nil { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: fmt.Sprintf("invalid JSON: %v", err), }) return err } obj, ok := payload.(map[string]any) if !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "topdata files must contain a JSON object at the root", }) return errors.New("topdata file root must be object") } switch domain { case "data": validateDataObject(path, obj, report) case "tlk": validateTLKObject(path, obj, report) case "base_dialog": // Parseability is enough for now; compatibility is still reference-backed. } return nil } func validateDataObject(path string, obj map[string]any, report *ValidationReport) { base := filepath.Base(path) slashed := filepath.ToSlash(path) inModules := strings.Contains(slashed, "/modules/") if base == "lock.json" { validateLockObject(path, obj, report) return } if base == "base.json" { validateRowsFile(path, obj, report, true) return } if inModules { if _, hasEntries := obj["entries"]; hasEntries { validateEntriesFile(path, obj, report) return } if _, hasOverrides := obj["overrides"]; hasOverrides { validateOverridesFile(path, obj, report) return } if _, hasRows := obj["rows"]; hasRows { validateRowsFile(path, obj, report, false) return } report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityWarning, Path: path, Message: "module file does not use a recognized canonical shape yet; expected one of entries, overrides, or rows", }) return } if _, hasRows := obj["rows"]; hasRows { validateRowsFile(path, obj, report, false) return } report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityWarning, Path: path, Message: "unrecognized data file shape; parseable for compatibility, but not canonical yet", }) } func validateTLKObject(path string, obj map[string]any, report *ValidationReport) { if filepath.Base(path) == "lock.json" { validateLockObject(path, obj, report) return } entries, ok := obj["entries"] if !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "TLK file must contain an entries object", }) return } if _, ok := entries.(map[string]any); !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "TLK entries must be a JSON object", }) } if language, ok := obj["language"]; ok { if _, ok := language.(string); !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "TLK language must be a string when present", }) } } } func validateLockObject(path string, obj map[string]any, report *ValidationReport) { for key, value := range obj { switch value.(type) { case float64: // JSON numbers land here. default: report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: fmt.Sprintf("lock entry %q must be numeric", key), }) } } } func validateRowsFile(path string, obj map[string]any, report *ValidationReport, requireColumns bool) { rows, ok := obj["rows"] if !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "rows file must contain a rows array", }) return } if _, ok := rows.([]any); !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "rows must be a JSON array", }) } if columns, ok := obj["columns"]; ok { if _, ok := columns.([]any); !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "columns must be a JSON array when present", }) } } else if requireColumns { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "base dataset file must contain columns", }) } } func validateEntriesFile(path string, obj map[string]any, report *ValidationReport) { entries, ok := obj["entries"] if !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "entries file must contain an entries object", }) return } if _, ok := entries.(map[string]any); !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "entries must be a JSON object", }) } } func validateOverridesFile(path string, obj map[string]any, report *ValidationReport) { overrides, ok := obj["overrides"] if !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "override file must contain an overrides array", }) return } if _, ok := overrides.([]any); !ok { report.Diagnostics = append(report.Diagnostics, Diagnostic{ Severity: SeverityError, Path: path, Message: "overrides must be a JSON array", }) } } func BuildReference(p *project.Project, progress func(string)) (BuildResult, error) { if progress == nil { progress = func(string) {} } if !p.HasTopData() { return BuildResult{}, errors.New("topdata is not configured for this project") } report := ValidateProject(p) if report.HasErrors() { return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount()) } refDir := p.TopDataReferenceBuilderDir() if refDir == "" { return BuildResult{}, errors.New("topdata.reference_builder is not configured") } buildDir := p.TopDataBuildDir() output2DA := filepath.Join(buildDir, "2da") outputTLK := filepath.Join(buildDir, "tlk") progress("Preparing topdata build directories...") if err := os.RemoveAll(buildDir); err != nil { return BuildResult{}, fmt.Errorf("clean topdata build dir: %w", err) } if err := os.MkdirAll(output2DA, 0o755); err != nil { return BuildResult{}, fmt.Errorf("create 2da output dir: %w", err) } if err := os.MkdirAll(outputTLK, 0o755); err != nil { return BuildResult{}, fmt.Errorf("create tlk output dir: %w", err) } progress("Building topdata outputs through the Python reference builder...") bridgeDir, cleanup, err := prepareReferenceBridge(refDir) if err != nil { return BuildResult{}, err } defer cleanup() cmd := exec.Command("python3", "build.py", "--out", output2DA, "--tlk-dir", outputTLK) cmd.Dir = bridgeDir var stderr bytes.Buffer cmd.Stderr = &stderr cmd.Stdout = ioDiscardProgress(progress) if err := cmd.Run(); err != nil { message := strings.TrimSpace(stderr.String()) if message == "" { message = err.Error() } return BuildResult{}, fmt.Errorf("reference builder failed: %s", message) } files2DA, err := countFiles(output2DA) if err != nil { return BuildResult{}, err } filesTLK, err := countFiles(outputTLK) if err != nil { return BuildResult{}, err } return BuildResult{ Mode: "reference", Output2DADir: output2DA, OutputTLKDir: outputTLK, Files2DA: files2DA, FilesTLK: filesTLK, }, nil } func CompareReference(p *project.Project, progress func(string)) (CompareResult, error) { if progress == nil { progress = func(string) {} } buildDir := p.TopDataBuildDir() output2DA := filepath.Join(buildDir, "2da") outputTLK := filepath.Join(buildDir, "tlk") if _, err := os.Stat(output2DA); err != nil { return CompareResult{}, fmt.Errorf("topdata build output missing: run build-topdata first") } tempDir, err := os.MkdirTemp("", "sow-topdata-compare-*") if err != nil { return CompareResult{}, fmt.Errorf("create compare temp dir: %w", err) } defer os.RemoveAll(tempDir) ref2DA := filepath.Join(tempDir, "2da") refTLK := filepath.Join(tempDir, "tlk") if err := os.MkdirAll(ref2DA, 0o755); err != nil { return CompareResult{}, err } if err := os.MkdirAll(refTLK, 0o755); err != nil { return CompareResult{}, err } progress("Building fresh reference topdata outputs for comparison...") bridgeDir, cleanup, err := prepareReferenceBridge(p.TopDataReferenceBuilderDir()) if err != nil { return CompareResult{}, err } defer cleanup() cmd := exec.Command("python3", "build.py", "--out", ref2DA, "--tlk-dir", refTLK) cmd.Dir = bridgeDir var stderr bytes.Buffer cmd.Stderr = &stderr cmd.Stdout = ioDiscardProgress(progress) if err := cmd.Run(); err != nil { message := strings.TrimSpace(stderr.String()) if message == "" { message = err.Error() } return CompareResult{}, fmt.Errorf("reference builder failed: %s", message) } compared2DA, err := compareDirs(output2DA, ref2DA) if err != nil { return CompareResult{}, err } comparedTLK, err := compareDirs(outputTLK, refTLK) if err != nil { return CompareResult{}, err } return CompareResult{ Mode: "reference", Compared2DA: compared2DA, ComparedTLK: comparedTLK, }, nil } func compareDirs(actualRoot, expectedRoot string) (int, error) { expectedFiles := make([]string, 0) err := filepath.WalkDir(expectedRoot, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } rel, err := filepath.Rel(expectedRoot, path) if err != nil { return err } expectedFiles = append(expectedFiles, filepath.ToSlash(rel)) return nil }) if err != nil { return 0, err } slices.Sort(expectedFiles) actualFiles := make([]string, 0) err = filepath.WalkDir(actualRoot, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } rel, err := filepath.Rel(actualRoot, path) if err != nil { return err } actualFiles = append(actualFiles, filepath.ToSlash(rel)) return nil }) if err != nil { return 0, err } slices.Sort(actualFiles) if !slices.Equal(actualFiles, expectedFiles) { return 0, fmt.Errorf("topdata output file list differs from reference") } for _, rel := range expectedFiles { actualBytes, err := os.ReadFile(filepath.Join(actualRoot, rel)) if err != nil { return 0, err } expectedBytes, err := os.ReadFile(filepath.Join(expectedRoot, rel)) if err != nil { return 0, err } if !bytes.Equal(actualBytes, expectedBytes) { return 0, fmt.Errorf("topdata output differs from reference: %s", rel) } } return len(expectedFiles), nil } func walkJSON(root string, fn func(path string)) error { return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } if strings.HasSuffix(strings.ToLower(d.Name()), ".json") { fn(path) } return nil }) } func countFiles(root string) (int, error) { count := 0 err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error { if err != nil { return err } if !d.IsDir() { count++ } return nil }) return count, err } type progressWriter struct { progress func(string) } func (w progressWriter) Write(p []byte) (int, error) { text := strings.TrimSpace(string(p)) if text != "" { w.progress(text) } return len(p), nil } func ioDiscardProgress(progress func(string)) *progressWriter { return &progressWriter{progress: progress} } func prepareReferenceBridge(referenceBuilderDir string) (string, func(), error) { tempRoot, err := os.MkdirTemp("", "sow-topdata-ref-*") if err != nil { return "", nil, fmt.Errorf("create reference bridge dir: %w", err) } cleanup := func() { _ = os.RemoveAll(tempRoot) } builderLink := filepath.Join(tempRoot, "tools", "2dabuilder") if err := os.MkdirAll(filepath.Dir(builderLink), 0o755); err != nil { cleanup() return "", nil, fmt.Errorf("create reference bridge tools dir: %w", err) } if err := os.MkdirAll(filepath.Join(tempRoot, "assets"), 0o755); err != nil { cleanup() return "", nil, fmt.Errorf("create reference bridge assets dir: %w", err) } if err := os.MkdirAll(filepath.Join(tempRoot, "staging"), 0o755); err != nil { cleanup() return "", nil, fmt.Errorf("create reference bridge staging dir: %w", err) } if err := copyReferenceBuilder(referenceBuilderDir, builderLink); err != nil { cleanup() return "", nil, err } if err := rewriteBridgeDefaults(filepath.Join(builderLink, "build.py"), filepath.Join(tempRoot, "assets"), filepath.Join(tempRoot, "staging")); err != nil { cleanup() return "", nil, err } return builderLink, cleanup, nil } func copyReferenceBuilder(src, dest string) error { return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } rel, err := filepath.Rel(src, path) if err != nil { return err } if rel == "." { return os.MkdirAll(dest, 0o755) } name := d.Name() if d.IsDir() { if name == ".venv" || name == ".pytest_cache" || name == "__pycache__" { return filepath.SkipDir } return os.MkdirAll(filepath.Join(dest, rel), 0o755) } if strings.HasSuffix(name, ".pyc") || name == "2dabuilder.log" { return nil } target := filepath.Join(dest, rel) info, err := d.Info() if err != nil { return err } data, err := os.ReadFile(path) if err != nil { return err } return os.WriteFile(target, data, info.Mode()) }) } func rewriteBridgeDefaults(buildScriptPath, assetsDir, stagingDir string) error { raw, err := os.ReadFile(buildScriptPath) if err != nil { return fmt.Errorf("read bridge build.py: %w", err) } text := string(raw) text = strings.Replace(text, `DEFAULT_ASSETS_DIR = "../../assets"`, fmt.Sprintf("DEFAULT_ASSETS_DIR = %q", assetsDir), 1) text = strings.Replace(text, `DEFAULT_STAGING_DIR = "../../staging"`, fmt.Sprintf("DEFAULT_STAGING_DIR = %q", stagingDir), 1) if err := os.WriteFile(buildScriptPath, []byte(text), 0o644); err != nil { return fmt.Errorf("rewrite bridge build.py defaults: %w", err) } return nil }