sow-tools/internal/topdata/masterfeats_migrate.go

package topdata

import (
	"encoding/json"
	"os"
	"path/filepath"
	"strconv"
)

func importLegacyMasterfeats(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) {
	legacyDir := filepath.Join(referenceBuilderDir, "data", "feat", "masterfeats")
	if _, err := os.Stat(legacyDir); err != nil {
		if os.IsNotExist(err) {
			return 0, nil
		}
		return 0, err
	}

	targetDir := filepath.Join(dataDir, "masterfeats")
	targetPath := filepath.Join(targetDir, "base.json")
	legacyPlainPath := filepath.Join(targetDir, "masterfeats.json")
	if _, err := os.Stat(targetPath); err == nil {
		obj, err := loadJSONObject(targetPath)
		if err == nil && countLegacyTLKRefsInValue(obj) == 0 {
			return 0, nil
		}
	}

	collected, err := importLegacyMasterfeatsDataset(legacyDir)
	if err != nil {
		return 0, err
	}
	referenceIDs, err := collectReferenceLockIDs(filepath.Join(referenceBuilderDir, "data"))
	if err != nil {
		return 0, err
	}

	rows := make([]any, 0, len(collected.Rows))
	for _, row := range collected.Rows {
		copyRow, ok := deepCopyValue(row).(map[string]any)
		if !ok {
			continue
		}
		if legacyTLK != nil {
			if _, _, err := inlineLegacyTLKValue(copyRow, legacyTLK); err != nil {
				return 0, err
			}
		}
		copyRow = rewriteImportedIDRefs(copyRow, referenceIDs)
		rows = append(rows, copyRow)
	}

	if err := os.MkdirAll(targetDir, 0o755); err != nil {
		return 0, err
	}
	writes := []struct {
		path string
		obj  map[string]any
	}{
		{
			path: targetPath,
			obj: map[string]any{
				"output":  collected.Dataset.OutputName,
				"columns": stringSliceToAny(collected.Columns),
				"rows":    rows,
			},
		},
		{
			path: filepath.Join(targetDir, "lock.json"),
			obj:  anyMapInt(collected.LockData),
		},
	}
	for _, write := range writes {
		raw, err := json.MarshalIndent(write.obj, "", "  ")
		if err != nil {
			return 0, err
		}
		raw = append(raw, '\n')
		if err := os.WriteFile(write.path, raw, 0o644); err != nil {
			return 0, err
		}
	}
	updatedFiles := len(writes)
	if err := os.Remove(legacyPlainPath); err == nil {
		updatedFiles++
	} else if err != nil && !os.IsNotExist(err) {
		return 0, err
	}
	return updatedFiles, nil
}

func importLegacyMasterfeatsDataset(legacyDir string) (nativeCollectedDataset, error) {
	dataset := nativeDataset{
		Name:     "masterfeats",
		BasePath: filepath.Join(legacyDir, "masterfeats.json"),
		LockPath: filepath.Join(legacyDir, "lock.json"),
		Spec:     specForDataset("masterfeats"),
	}
	tableData, err := loadJSONObject(dataset.BasePath)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	columns, err := parseColumns(tableData, dataset.Name)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	rawRows, ok := tableData["rows"].([]any)
	if !ok {
		return nativeCollectedDataset{}, nil
	}
	lockData, err := loadLockfile(dataset.LockPath)
	if err != nil {
		return nativeCollectedDataset{}, err
	}
	usedIDs := map[int]struct{}{}
	for _, rowID := range lockData {
		usedIDs[rowID] = struct{}{}
	}

	rows := make([]map[string]any, 0, len(rawRows))
	explicitID := make([]bool, 0, len(rawRows))
	for index, raw := range rawRows {
		rowObj, ok := raw.(map[string]any)
		if !ok {
			continue
		}
		rowID := index
		if value, ok := rowObj["id"]; ok {
			parsed, err := asInt(value)
			if err != nil {
				return nativeCollectedDataset{}, err
			}
			rowID = parsed
		}
		row := map[string]any{"id": rowID}
		for _, column := range columns {
			row[column] = nullValue
		}
		for key, value := range rowObj {
			switch key {
			case "id":
			case "key":
				if keyText, ok := value.(string); ok && keyText != "" {
					row["key"] = keyText
				}
			default:
				columnName, ok := canonicalColumn(columns, key)
				if !ok {
					continue
				}
				row[columnName] = value
			}
		}
		rows = append(rows, row)
		_, hasExplicitID := rowObj["id"]
		explicitID = append(explicitID, hasExplicitID)
		if hasExplicitID {
			usedIDs[rowID] = struct{}{}
		}
	}

	nextID := nextAvailableID(usedIDs)
	lockModified := false
	for index, row := range rows {
		key, hasKey := row["key"].(string)
		if hasKey && key != "" {
			if lockedID, ok := lockData[key]; ok {
				row["id"] = lockedID
				continue
			}
			if explicitID[index] {
				lockData[key] = row["id"].(int)
			} else {
				row["id"] = nextID
				lockData[key] = nextID
				usedIDs[nextID] = struct{}{}
				nextID = nextAvailableID(usedIDs)
			}
			lockModified = true
			continue
		}
		if !explicitID[index] {
			row["id"] = index
		}
	}
	if lockModified {
		_ = lockModified
	}

	return nativeCollectedDataset{
		Dataset: nativeDataset{
			Name:       dataset.Name,
			OutputName: "masterfeats.2da",
			Columns:    columns,
			Spec:       dataset.Spec,
		},
		Columns:  columns,
		Rows:     rows,
		LockData: lockData,
	}, nil
}

func rewriteImportedIDRefs(value any, ids map[string]int) map[string]any {
	row, ok := rewriteImportedIDRefsValue(value, ids).(map[string]any)
	if !ok {
		return map[string]any{}
	}
	return row
}

func rewriteImportedIDRefsValue(value any, ids map[string]int) any {
	switch typed := value.(type) {
	case map[string]any:
		if len(typed) == 1 {
			if rawID, ok := typed["id"]; ok {
				if key, ok := rawID.(string); ok {
					if resolved, ok := ids[key]; ok {
						return strconv.Itoa(resolved)
					}
				}
			}
		}
		out := make(map[string]any, len(typed))
		for key, child := range typed {
			out[key] = rewriteImportedIDRefsValue(child, ids)
		}
		return out
	case []any:
		out := make([]any, 0, len(typed))
		for _, child := range typed {
			out = append(out, rewriteImportedIDRefsValue(child, ids))
		}
		return out
	default:
		return value
	}
}