claude: fold in old sow-tools

2026-06-13 09:49:29 +02:00
parent d4a435883b
commit 56c67132f3
112 changed files with 70812 additions and 74 deletions
@@ -0,0 +1,237 @@
+package topdata
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+func importLegacyMasterfeats(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
+	legacyDir := filepath.Join(referenceBuilderDir, "data", "feat", "masterfeats")
+	if _, err := os.Stat(legacyDir); err != nil {
+		if os.IsNotExist(err) {
+			return 0, nil
+		}
+		return 0, err
+	}
+
+	targetDir := filepath.Join(dataDir, "masterfeats")
+	targetPath := filepath.Join(targetDir, "base.json")
+	legacyPlainPath := filepath.Join(targetDir, "masterfeats.json")
+	if _, err := os.Stat(targetPath); err == nil {
+		obj, err := loadJSONObject(targetPath)
+		if err == nil && countLegacyTLKRefsInValue(obj) == 0 {
+			return 0, nil
+		}
+	}
+
+	collected, err := importLegacyMasterfeatsDataset(legacyDir)
+	if err != nil {
+		return 0, err
+	}
+	referenceIDs, err := collectReferenceLockIDs(filepath.Join(referenceBuilderDir, "data"))
+	if err != nil {
+		return 0, err
+	}
+
+	rows := make([]any, 0, len(collected.Rows))
+	for _, row := range collected.Rows {
+		copyRow, ok := deepCopyValue(row).(map[string]any)
+		if !ok {
+			continue
+		}
+		if legacyTLK != nil {
+			if _, _, err := inlineLegacyTLKValue(copyRow, legacyTLK); err != nil {
+				return 0, err
+			}
+		}
+		copyRow = rewriteImportedIDRefs(copyRow, referenceIDs)
+		rows = append(rows, copyRow)
+	}
+
+	if err := os.MkdirAll(targetDir, 0o755); err != nil {
+		return 0, err
+	}
+	writes := []struct {
+		path string
+		obj  map[string]any
+	}{
+		{
+			path: targetPath,
+			obj: map[string]any{
+				"output":  collected.Dataset.OutputName,
+				"columns": stringSliceToAny(collected.Columns),
+				"rows":    rows,
+			},
+		},
+		{
+			path: filepath.Join(targetDir, "lock.json"),
+			obj:  anyMapInt(collected.LockData),
+		},
+	}
+	for _, write := range writes {
+		raw, err := json.MarshalIndent(write.obj, "", "  ")
+		if err != nil {
+			return 0, err
+		}
+		raw = append(raw, '\n')
+		if err := os.WriteFile(write.path, raw, 0o644); err != nil {
+			return 0, err
+		}
+	}
+	updatedFiles := len(writes)
+	if err := os.Remove(legacyPlainPath); err == nil {
+		updatedFiles++
+	} else if err != nil && !os.IsNotExist(err) {
+		return 0, err
+	}
+	return updatedFiles, nil
+}
+
+func importLegacyMasterfeatsDataset(legacyDir string) (nativeCollectedDataset, error) {
+	dataset := nativeDataset{
+		Name:     "masterfeats",
+		BasePath: filepath.Join(legacyDir, "masterfeats.json"),
+		LockPath: filepath.Join(legacyDir, "lock.json"),
+		Spec:     specForDataset("masterfeats"),
+	}
+	tableData, err := loadJSONObject(dataset.BasePath)
+	if err != nil {
+		return nativeCollectedDataset{}, err
+	}
+	columns, err := parseColumns(tableData, dataset.Name)
+	if err != nil {
+		return nativeCollectedDataset{}, err
+	}
+	rawRows, ok := tableData["rows"].([]any)
+	if !ok {
+		return nativeCollectedDataset{}, nil
+	}
+	lockData, err := loadLockfile(dataset.LockPath)
+	if err != nil {
+		return nativeCollectedDataset{}, err
+	}
+	usedIDs := map[int]struct{}{}
+	for _, rowID := range lockData {
+		usedIDs[rowID] = struct{}{}
+	}
+
+	rows := make([]map[string]any, 0, len(rawRows))
+	explicitID := make([]bool, 0, len(rawRows))
+	for index, raw := range rawRows {
+		rowObj, ok := raw.(map[string]any)
+		if !ok {
+			continue
+		}
+		rowID := index
+		if value, ok := rowObj["id"]; ok {
+			parsed, err := asInt(value)
+			if err != nil {
+				return nativeCollectedDataset{}, err
+			}
+			rowID = parsed
+		}
+		row := map[string]any{"id": rowID}
+		for _, column := range columns {
+			row[column] = nullValue
+		}
+		for key, value := range rowObj {
+			switch key {
+			case "id":
+			case "key":
+				if keyText, ok := value.(string); ok && keyText != "" {
+					row["key"] = keyText
+				}
+			default:
+				columnName, ok := canonicalColumn(columns, key)
+				if !ok {
+					continue
+				}
+				row[columnName] = value
+			}
+		}
+		rows = append(rows, row)
+		_, hasExplicitID := rowObj["id"]
+		explicitID = append(explicitID, hasExplicitID)
+		if hasExplicitID {
+			usedIDs[rowID] = struct{}{}
+		}
+	}
+
+	nextID := nextAvailableID(usedIDs)
+	lockModified := false
+	for index, row := range rows {
+		key, hasKey := row["key"].(string)
+		if hasKey && key != "" {
+			if lockedID, ok := lockData[key]; ok {
+				row["id"] = lockedID
+				continue
+			}
+			if explicitID[index] {
+				lockData[key] = row["id"].(int)
+			} else {
+				row["id"] = nextID
+				lockData[key] = nextID
+				usedIDs[nextID] = struct{}{}
+				nextID = nextAvailableID(usedIDs)
+			}
+			lockModified = true
+			continue
+		}
+		if !explicitID[index] {
+			row["id"] = index
+		}
+	}
+	if lockModified {
+		_ = lockModified
+	}
+
+	return nativeCollectedDataset{
+		Dataset: nativeDataset{
+			Name:       dataset.Name,
+			OutputName: "masterfeats.2da",
+			Columns:    columns,
+			Spec:       dataset.Spec,
+		},
+		Columns:  columns,
+		Rows:     rows,
+		LockData: lockData,
+	}, nil
+}
+
+func rewriteImportedIDRefs(value any, ids map[string]int) map[string]any {
+	row, ok := rewriteImportedIDRefsValue(value, ids).(map[string]any)
+	if !ok {
+		return map[string]any{}
+	}
+	return row
+}
+
+func rewriteImportedIDRefsValue(value any, ids map[string]int) any {
+	switch typed := value.(type) {
+	case map[string]any:
+		if len(typed) == 1 {
+			if rawID, ok := typed["id"]; ok {
+				if key, ok := rawID.(string); ok {
+					if resolved, ok := ids[key]; ok {
+						return strconv.Itoa(resolved)
+					}
+				}
+			}
+		}
+		out := make(map[string]any, len(typed))
+		for key, child := range typed {
+			out[key] = rewriteImportedIDRefsValue(child, ids)
+		}
+		return out
+	case []any:
+		out := make([]any, 0, len(typed))
+		for _, child := range typed {
+			out = append(out, rewriteImportedIDRefsValue(child, ids))
+		}
+		return out
+	default:
+		return value
+	}
+}