Add topdata bridge commands and handoff

2026-04-02 21:40:59 +02:00
parent 7c8d58586a
commit d67b6e3f12
6 changed files with 1083 additions and 5 deletions
@@ -0,0 +1,702 @@
+package topdata
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
+)
+
+type Severity string
+
+const (
+	SeverityError   Severity = "error"
+	SeverityWarning Severity = "warning"
+)
+
+type Diagnostic struct {
+	Severity Severity
+	Path     string
+	Message  string
+}
+
+type ValidationReport struct {
+	Files       int
+	DataFiles   int
+	TLKFiles    int
+	Diagnostics []Diagnostic
+}
+
+func (r ValidationReport) HasErrors() bool {
+	for _, diagnostic := range r.Diagnostics {
+		if diagnostic.Severity == SeverityError {
+			return true
+		}
+	}
+	return false
+}
+
+func (r ValidationReport) ErrorCount() int {
+	count := 0
+	for _, diagnostic := range r.Diagnostics {
+		if diagnostic.Severity == SeverityError {
+			count++
+		}
+	}
+	return count
+}
+
+func (r ValidationReport) WarningCount() int {
+	count := 0
+	for _, diagnostic := range r.Diagnostics {
+		if diagnostic.Severity == SeverityWarning {
+			count++
+		}
+	}
+	return count
+}
+
+type BuildResult struct {
+	Mode     string
+	Output2DADir string
+	OutputTLKDir string
+	Files2DA int
+	FilesTLK int
+}
+
+type CompareResult struct {
+	Mode         string
+	Compared2DA  int
+	ComparedTLK  int
+}
+
+func ValidateProject(p *project.Project) ValidationReport {
+	report := ValidationReport{}
+	if !p.HasTopData() {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityWarning,
+			Path:     project.ConfigFile,
+			Message:  "topdata is not configured for this project",
+		})
+		return report
+	}
+
+	sourceDir := p.TopDataSourceDir()
+	info, err := os.Stat(sourceDir)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     sourceDir,
+			Message:  err.Error(),
+		})
+		return report
+	}
+	if !info.IsDir() {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     sourceDir,
+			Message:  "topdata source must be a directory",
+		})
+		return report
+	}
+
+	dataDir := filepath.Join(sourceDir, "data")
+	tlkDir := filepath.Join(sourceDir, "tlk")
+	baseDialog := filepath.Join(sourceDir, "base_dialog.json")
+
+	for _, requiredDir := range []string{dataDir, tlkDir} {
+		info, err := os.Stat(requiredDir)
+		if err != nil {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     requiredDir,
+				Message:  err.Error(),
+			})
+			continue
+		}
+		if !info.IsDir() {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     requiredDir,
+				Message:  "must be a directory",
+			})
+		}
+	}
+
+	if _, err := os.Stat(baseDialog); err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityWarning,
+			Path:     baseDialog,
+			Message:  "base_dialog.json is missing; migrated datasets may rely on reference builder fallback for now",
+		})
+	} else {
+		report.Files++
+		if err := validateJSONFile(baseDialog, "base_dialog", &report); err == nil {
+			// count only once for base dialog.
+		}
+	}
+
+	_ = walkJSON(dataDir, func(path string) {
+		report.Files++
+		report.DataFiles++
+		_ = validateJSONFile(path, "data", &report)
+	})
+	_ = walkJSON(tlkDir, func(path string) {
+		report.Files++
+		report.TLKFiles++
+		_ = validateJSONFile(path, "tlk", &report)
+	})
+
+	refDir := p.TopDataReferenceBuilderDir()
+	if refDir == "" {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityWarning,
+			Path:     project.ConfigFile,
+			Message:  "topdata.reference_builder is not configured; build-topdata reference mode will be unavailable",
+		})
+	} else {
+		if _, err := os.Stat(filepath.Join(refDir, "build.py")); err != nil {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     refDir,
+				Message:  "reference builder does not contain build.py",
+			})
+		}
+	}
+
+	return report
+}
+
+func validateJSONFile(path, domain string, report *ValidationReport) error {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  err.Error(),
+		})
+		return err
+	}
+
+	var payload any
+	if err := json.Unmarshal(raw, &payload); err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  fmt.Sprintf("invalid JSON: %v", err),
+		})
+		return err
+	}
+
+	obj, ok := payload.(map[string]any)
+	if !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "topdata files must contain a JSON object at the root",
+		})
+		return errors.New("topdata file root must be object")
+	}
+
+	switch domain {
+	case "data":
+		validateDataObject(path, obj, report)
+	case "tlk":
+		validateTLKObject(path, obj, report)
+	case "base_dialog":
+		// Parseability is enough for now; compatibility is still reference-backed.
+	}
+	return nil
+}
+
+func validateDataObject(path string, obj map[string]any, report *ValidationReport) {
+	base := filepath.Base(path)
+	slashed := filepath.ToSlash(path)
+	inModules := strings.Contains(slashed, "/modules/")
+
+	if base == "lock.json" {
+		validateLockObject(path, obj, report)
+		return
+	}
+	if base == "base.json" {
+		validateRowsFile(path, obj, report, true)
+		return
+	}
+	if inModules {
+		if _, hasEntries := obj["entries"]; hasEntries {
+			validateEntriesFile(path, obj, report)
+			return
+		}
+		if _, hasOverrides := obj["overrides"]; hasOverrides {
+			validateOverridesFile(path, obj, report)
+			return
+		}
+		if _, hasRows := obj["rows"]; hasRows {
+			validateRowsFile(path, obj, report, false)
+			return
+		}
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityWarning,
+			Path:     path,
+			Message:  "module file does not use a recognized canonical shape yet; expected one of entries, overrides, or rows",
+		})
+		return
+	}
+
+	if _, hasRows := obj["rows"]; hasRows {
+		validateRowsFile(path, obj, report, false)
+		return
+	}
+	report.Diagnostics = append(report.Diagnostics, Diagnostic{
+		Severity: SeverityWarning,
+		Path:     path,
+		Message:  "unrecognized data file shape; parseable for compatibility, but not canonical yet",
+	})
+}
+
+func validateTLKObject(path string, obj map[string]any, report *ValidationReport) {
+	if filepath.Base(path) == "lock.json" {
+		validateLockObject(path, obj, report)
+		return
+	}
+	entries, ok := obj["entries"]
+	if !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "TLK file must contain an entries object",
+		})
+		return
+	}
+	if _, ok := entries.(map[string]any); !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "TLK entries must be a JSON object",
+		})
+	}
+	if language, ok := obj["language"]; ok {
+		if _, ok := language.(string); !ok {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  "TLK language must be a string when present",
+			})
+		}
+	}
+}
+
+func validateLockObject(path string, obj map[string]any, report *ValidationReport) {
+	for key, value := range obj {
+		switch value.(type) {
+		case float64:
+			// JSON numbers land here.
+		default:
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("lock entry %q must be numeric", key),
+			})
+		}
+	}
+}
+
+func validateRowsFile(path string, obj map[string]any, report *ValidationReport, requireColumns bool) {
+	rows, ok := obj["rows"]
+	if !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "rows file must contain a rows array",
+		})
+		return
+	}
+	if _, ok := rows.([]any); !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "rows must be a JSON array",
+		})
+	}
+	if columns, ok := obj["columns"]; ok {
+		if _, ok := columns.([]any); !ok {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  "columns must be a JSON array when present",
+			})
+		}
+	} else if requireColumns {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "base dataset file must contain columns",
+		})
+	}
+}
+
+func validateEntriesFile(path string, obj map[string]any, report *ValidationReport) {
+	entries, ok := obj["entries"]
+	if !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "entries file must contain an entries object",
+		})
+		return
+	}
+	if _, ok := entries.(map[string]any); !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "entries must be a JSON object",
+		})
+	}
+}
+
+func validateOverridesFile(path string, obj map[string]any, report *ValidationReport) {
+	overrides, ok := obj["overrides"]
+	if !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "override file must contain an overrides array",
+		})
+		return
+	}
+	if _, ok := overrides.([]any); !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "overrides must be a JSON array",
+		})
+	}
+}
+
+func BuildReference(p *project.Project, progress func(string)) (BuildResult, error) {
+	if progress == nil {
+		progress = func(string) {}
+	}
+	if !p.HasTopData() {
+		return BuildResult{}, errors.New("topdata is not configured for this project")
+	}
+
+	report := ValidateProject(p)
+	if report.HasErrors() {
+		return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount())
+	}
+
+	refDir := p.TopDataReferenceBuilderDir()
+	if refDir == "" {
+		return BuildResult{}, errors.New("topdata.reference_builder is not configured")
+	}
+
+	buildDir := p.TopDataBuildDir()
+	output2DA := filepath.Join(buildDir, "2da")
+	outputTLK := filepath.Join(buildDir, "tlk")
+	progress("Preparing topdata build directories...")
+	if err := os.RemoveAll(buildDir); err != nil {
+		return BuildResult{}, fmt.Errorf("clean topdata build dir: %w", err)
+	}
+	if err := os.MkdirAll(output2DA, 0o755); err != nil {
+		return BuildResult{}, fmt.Errorf("create 2da output dir: %w", err)
+	}
+	if err := os.MkdirAll(outputTLK, 0o755); err != nil {
+		return BuildResult{}, fmt.Errorf("create tlk output dir: %w", err)
+	}
+
+	progress("Building topdata outputs through the Python reference builder...")
+	bridgeDir, cleanup, err := prepareReferenceBridge(refDir)
+	if err != nil {
+		return BuildResult{}, err
+	}
+	defer cleanup()
+
+	cmd := exec.Command("python3", "build.py", "--out", output2DA, "--tlk-dir", outputTLK)
+	cmd.Dir = bridgeDir
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	cmd.Stdout = ioDiscardProgress(progress)
+	if err := cmd.Run(); err != nil {
+		message := strings.TrimSpace(stderr.String())
+		if message == "" {
+			message = err.Error()
+		}
+		return BuildResult{}, fmt.Errorf("reference builder failed: %s", message)
+	}
+
+	files2DA, err := countFiles(output2DA)
+	if err != nil {
+		return BuildResult{}, err
+	}
+	filesTLK, err := countFiles(outputTLK)
+	if err != nil {
+		return BuildResult{}, err
+	}
+
+	return BuildResult{
+		Mode:       "reference",
+		Output2DADir: output2DA,
+		OutputTLKDir: outputTLK,
+		Files2DA:   files2DA,
+		FilesTLK:   filesTLK,
+	}, nil
+}
+
+func CompareReference(p *project.Project, progress func(string)) (CompareResult, error) {
+	if progress == nil {
+		progress = func(string) {}
+	}
+	buildDir := p.TopDataBuildDir()
+	output2DA := filepath.Join(buildDir, "2da")
+	outputTLK := filepath.Join(buildDir, "tlk")
+	if _, err := os.Stat(output2DA); err != nil {
+		return CompareResult{}, fmt.Errorf("topdata build output missing: run build-topdata first")
+	}
+
+	tempDir, err := os.MkdirTemp("", "sow-topdata-compare-*")
+	if err != nil {
+		return CompareResult{}, fmt.Errorf("create compare temp dir: %w", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	ref2DA := filepath.Join(tempDir, "2da")
+	refTLK := filepath.Join(tempDir, "tlk")
+	if err := os.MkdirAll(ref2DA, 0o755); err != nil {
+		return CompareResult{}, err
+	}
+	if err := os.MkdirAll(refTLK, 0o755); err != nil {
+		return CompareResult{}, err
+	}
+
+	progress("Building fresh reference topdata outputs for comparison...")
+	bridgeDir, cleanup, err := prepareReferenceBridge(p.TopDataReferenceBuilderDir())
+	if err != nil {
+		return CompareResult{}, err
+	}
+	defer cleanup()
+
+	cmd := exec.Command("python3", "build.py", "--out", ref2DA, "--tlk-dir", refTLK)
+	cmd.Dir = bridgeDir
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	cmd.Stdout = ioDiscardProgress(progress)
+	if err := cmd.Run(); err != nil {
+		message := strings.TrimSpace(stderr.String())
+		if message == "" {
+			message = err.Error()
+		}
+		return CompareResult{}, fmt.Errorf("reference builder failed: %s", message)
+	}
+
+	compared2DA, err := compareDirs(output2DA, ref2DA)
+	if err != nil {
+		return CompareResult{}, err
+	}
+	comparedTLK, err := compareDirs(outputTLK, refTLK)
+	if err != nil {
+		return CompareResult{}, err
+	}
+
+	return CompareResult{
+		Mode:        "reference",
+		Compared2DA: compared2DA,
+		ComparedTLK: comparedTLK,
+	}, nil
+}
+
+func compareDirs(actualRoot, expectedRoot string) (int, error) {
+	expectedFiles := make([]string, 0)
+	err := filepath.WalkDir(expectedRoot, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		rel, err := filepath.Rel(expectedRoot, path)
+		if err != nil {
+			return err
+		}
+		expectedFiles = append(expectedFiles, filepath.ToSlash(rel))
+		return nil
+	})
+	if err != nil {
+		return 0, err
+	}
+	slices.Sort(expectedFiles)
+
+	actualFiles := make([]string, 0)
+	err = filepath.WalkDir(actualRoot, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		rel, err := filepath.Rel(actualRoot, path)
+		if err != nil {
+			return err
+		}
+		actualFiles = append(actualFiles, filepath.ToSlash(rel))
+		return nil
+	})
+	if err != nil {
+		return 0, err
+	}
+	slices.Sort(actualFiles)
+	if !slices.Equal(actualFiles, expectedFiles) {
+		return 0, fmt.Errorf("topdata output file list differs from reference")
+	}
+
+	for _, rel := range expectedFiles {
+		actualBytes, err := os.ReadFile(filepath.Join(actualRoot, rel))
+		if err != nil {
+			return 0, err
+		}
+		expectedBytes, err := os.ReadFile(filepath.Join(expectedRoot, rel))
+		if err != nil {
+			return 0, err
+		}
+		if !bytes.Equal(actualBytes, expectedBytes) {
+			return 0, fmt.Errorf("topdata output differs from reference: %s", rel)
+		}
+	}
+	return len(expectedFiles), nil
+}
+
+func walkJSON(root string, fn func(path string)) error {
+	return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		if strings.HasSuffix(strings.ToLower(d.Name()), ".json") {
+			fn(path)
+		}
+		return nil
+	})
+}
+
+func countFiles(root string) (int, error) {
+	count := 0
+	err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() {
+			count++
+		}
+		return nil
+	})
+	return count, err
+}
+
+type progressWriter struct {
+	progress func(string)
+}
+
+func (w progressWriter) Write(p []byte) (int, error) {
+	text := strings.TrimSpace(string(p))
+	if text != "" {
+		w.progress(text)
+	}
+	return len(p), nil
+}
+
+func ioDiscardProgress(progress func(string)) *progressWriter {
+	return &progressWriter{progress: progress}
+}
+
+func prepareReferenceBridge(referenceBuilderDir string) (string, func(), error) {
+	tempRoot, err := os.MkdirTemp("", "sow-topdata-ref-*")
+	if err != nil {
+		return "", nil, fmt.Errorf("create reference bridge dir: %w", err)
+	}
+
+	cleanup := func() {
+		_ = os.RemoveAll(tempRoot)
+	}
+
+	builderLink := filepath.Join(tempRoot, "tools", "2dabuilder")
+	if err := os.MkdirAll(filepath.Dir(builderLink), 0o755); err != nil {
+		cleanup()
+		return "", nil, fmt.Errorf("create reference bridge tools dir: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Join(tempRoot, "assets"), 0o755); err != nil {
+		cleanup()
+		return "", nil, fmt.Errorf("create reference bridge assets dir: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Join(tempRoot, "staging"), 0o755); err != nil {
+		cleanup()
+		return "", nil, fmt.Errorf("create reference bridge staging dir: %w", err)
+	}
+	if err := copyReferenceBuilder(referenceBuilderDir, builderLink); err != nil {
+		cleanup()
+		return "", nil, err
+	}
+	if err := rewriteBridgeDefaults(filepath.Join(builderLink, "build.py"), filepath.Join(tempRoot, "assets"), filepath.Join(tempRoot, "staging")); err != nil {
+		cleanup()
+		return "", nil, err
+	}
+	return builderLink, cleanup, nil
+}
+
+func copyReferenceBuilder(src, dest string) error {
+	return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		if rel == "." {
+			return os.MkdirAll(dest, 0o755)
+		}
+		name := d.Name()
+		if d.IsDir() {
+			if name == ".venv" || name == ".pytest_cache" || name == "__pycache__" {
+				return filepath.SkipDir
+			}
+			return os.MkdirAll(filepath.Join(dest, rel), 0o755)
+		}
+		if strings.HasSuffix(name, ".pyc") || name == "2dabuilder.log" {
+			return nil
+		}
+		target := filepath.Join(dest, rel)
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		return os.WriteFile(target, data, info.Mode())
+	})
+}
+
+func rewriteBridgeDefaults(buildScriptPath, assetsDir, stagingDir string) error {
+	raw, err := os.ReadFile(buildScriptPath)
+	if err != nil {
+		return fmt.Errorf("read bridge build.py: %w", err)
+	}
+	text := string(raw)
+	text = strings.Replace(text, `DEFAULT_ASSETS_DIR = "../../assets"`, fmt.Sprintf("DEFAULT_ASSETS_DIR = %q", assetsDir), 1)
+	text = strings.Replace(text, `DEFAULT_STAGING_DIR = "../../staging"`, fmt.Sprintf("DEFAULT_STAGING_DIR = %q", stagingDir), 1)
+	if err := os.WriteFile(buildScriptPath, []byte(text), 0o644); err != nil {
+		return fmt.Errorf("rewrite bridge build.py defaults: %w", err)
+	}
+	return nil
+}