Topdata Migration (#2)

Native topdata migration summary

- Replaced the legacy 2dabuilder runtime path with the native topdata builder.
- Native build, validate, compare, and convert flows now cover the canonical topdata pipeline.
- compare-topdata is now a native self-check; legacy reference-builder runtime usage was removed.
- Parts generation follows the native asset-scan contract and uses sow-assets via project asset resolution.
- Generated feat families, class-feat injects, masterfeat/successor expansion, and dataset-derived feat generation were brought to parity and regression-covered.
- Remaining mirrored datasets were migrated into native-owned canonical data while preserving lock IDs.
- normalize-topdata bridge behavior and migration-era ownership markers were removed.
- Documentation was rewritten around the native workflow and active contracts were cleaned up.
- Final hardening pass removed stale validator assumptions and cleaned ignored/generated artifacts for PR readiness.

Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/2
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>

internal/topdata/legacy_dataset_migrate.go

@@ -0,0 +1,176 @@
+package topdata
+
+import (
+	"encoding/json"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+type legacyDatasetTransform func(relativePath string, obj map[string]any)
+
+func importLegacyDatasetMirror(referenceBuilderDir, dataDir, datasetName, outputName string, transform legacyDatasetTransform) (int, error) {
+	legacyDir := filepath.Join(referenceBuilderDir, "data", datasetName)
+	if _, err := os.Stat(legacyDir); err != nil {
+		if os.IsNotExist(err) {
+			return 0, nil
+		}
+		return 0, err
+	}
+
+	targetDir := filepath.Join(dataDir, datasetName)
+	baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json"))
+	if err != nil {
+		return 0, err
+	}
+	if outputName != "" {
+		baseObj["output"] = outputName
+	}
+	if transform != nil {
+		transform("base.json", baseObj)
+	}
+
+	lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json"))
+	if err != nil {
+		return 0, err
+	}
+	if transform != nil {
+		transform("lock.json", lockObj)
+	}
+
+	moduleRelPaths, err := collectJSONRelativePaths(filepath.Join(legacyDir, "modules"))
+	if err != nil {
+		return 0, err
+	}
+
+	moduleObjs := make(map[string]map[string]any, len(moduleRelPaths))
+	for _, relPath := range moduleRelPaths {
+		obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", relPath))
+		if err != nil {
+			return 0, err
+		}
+		normalizedRel := filepath.ToSlash(relPath)
+		if transform != nil {
+			transform(filepath.ToSlash(filepath.Join("modules", normalizedRel)), obj)
+		}
+		moduleObjs[normalizedRel] = obj
+	}
+
+	if err := os.MkdirAll(targetDir, 0o755); err != nil {
+		return 0, err
+	}
+
+	updated := 0
+	for _, write := range []struct {
+		path string
+		obj  map[string]any
+	}{
+		{path: filepath.Join(targetDir, "base.json"), obj: baseObj},
+		{path: filepath.Join(targetDir, "lock.json"), obj: lockObj},
+	} {
+		changed, err := writeJSONObjectIfChanged(write.path, write.obj)
+		if err != nil {
+			return 0, err
+		}
+		if changed {
+			updated++
+		}
+	}
+
+	targetModulesDir := filepath.Join(targetDir, "modules")
+	if len(moduleObjs) > 0 {
+		if err := os.MkdirAll(targetModulesDir, 0o755); err != nil {
+			return 0, err
+		}
+	}
+	for relPath, obj := range moduleObjs {
+		targetPath := filepath.Join(targetModulesDir, filepath.FromSlash(relPath))
+		if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil {
+			return 0, err
+		}
+		changed, err := writeJSONObjectIfChanged(targetPath, obj)
+		if err != nil {
+			return 0, err
+		}
+		if changed {
+			updated++
+		}
+	}
+
+	staleModules, err := collectJSONRelativePaths(targetModulesDir)
+	if err != nil {
+		return 0, err
+	}
+	for _, relPath := range staleModules {
+		normalizedRel := filepath.ToSlash(relPath)
+		if _, ok := moduleObjs[normalizedRel]; ok {
+			continue
+		}
+		if err := os.Remove(filepath.Join(targetModulesDir, filepath.FromSlash(normalizedRel))); err != nil && !os.IsNotExist(err) {
+			return 0, err
+		}
+		updated++
+	}
+
+	return updated, nil
+}
+
+func collectJSONRelativePaths(root string) ([]string, error) {
+	if _, err := os.Stat(root); err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	var relPaths []string
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		if !strings.EqualFold(filepath.Ext(d.Name()), ".json") {
+			return nil
+		}
+		relPath, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		relPaths = append(relPaths, filepath.ToSlash(relPath))
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(relPaths)
+	return relPaths, nil
+}
+
+func writeJSONObjectIfChanged(path string, obj map[string]any) (bool, error) {
+	raw, err := json.MarshalIndent(obj, "", "  ")
+	if err != nil {
+		return false, err
+	}
+	raw = append(raw, '\n')
+
+	current, err := os.ReadFile(path)
+	if err == nil && string(current) == string(raw) {
+		return false, nil
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return false, err
+	}
+	if err := os.WriteFile(path, raw, 0o644); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func fileExists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}