sow-tools/internal/topdata/expansion_native.go

package topdata

import (
	"fmt"
	"slices"
	"strings"
)

func normalizeMetadataKey(key string) string {
	replacer := strings.NewReplacer("-", "_", " ", "_")
	return strings.ToLower(replacer.Replace(strings.TrimSpace(key)))
}

func parseRowMetadata(raw any) (map[string]any, error) {
	if raw == nil {
		return nil, nil
	}
	obj, ok := raw.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("meta must be an object")
	}
	if len(obj) == 0 {
		return map[string]any{}, nil
	}

	meta := make(map[string]any, len(obj))
	for key, value := range obj {
		switch normalizeMetadataKey(key) {
		case "family":
			family, err := parseFamilyMetadata(value)
			if err != nil {
				return nil, fmt.Errorf("meta.family: %w", err)
			}
			meta["family"] = family
		case "wiki":
			wiki, err := parseWikiMetadata(value)
			if err != nil {
				return nil, fmt.Errorf("meta.wiki: %w", err)
			}
			if len(wiki) > 0 {
				meta["wiki"] = wiki
			}
		default:
			return nil, fmt.Errorf("unknown metadata key %q", key)
		}
	}
	return meta, nil
}

func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) {
	datasetIndexMap := make(map[string]int)
	for i, ds := range collected {
		datasetIndexMap[ds.Dataset.Name] = i
		datasetIndexMap[ds.Dataset.OutputName] = i
	}

	rowsWithExpansion := []struct {
		datasetName string
		row         map[string]any
	}{}

	for _, ds := range collected {
		for _, row := range ds.Rows {
			if hasExpansionData(row) {
				rowsWithExpansion = append(rowsWithExpansion, struct {
					datasetName string
					row         map[string]any
				}{ds.Dataset.Name, row})
			}
		}
	}

	if len(rowsWithExpansion) == 0 {
		return collected, nil
	}

	modifiedTargets := map[int]struct{}{}
	for _, src := range rowsWithExpansion {
		expansion, _ := extractExpansionData(src.row)
		for targetDatasetName, targetRows := range expansion.Data {
			targetIndex, ok := datasetIndexMap[targetDatasetName]
			if !ok {
				outputName := targetDatasetName
				if !strings.HasSuffix(outputName, ".2da") {
					outputName = outputName + ".2da"
				}
				targetIndex, ok = datasetIndexMap[outputName]
			}
			if !ok {
				return nil, fmt.Errorf("expansion targets unknown dataset %q", targetDatasetName)
			}
			targetDS := collected[targetIndex]
			originalLockData, _ := loadLockfile(targetDS.Dataset.LockPath)
			if originalLockData == nil {
				originalLockData = map[string]int{}
			}
			lockModified := false
			usedIDs := map[int]struct{}{}
			usedKeys := map[string]struct{}{}
			for _, rowID := range targetDS.LockData {
				usedIDs[rowID] = struct{}{}
			}
			for _, row := range targetDS.Rows {
				if id, ok := row["id"].(int); ok {
					usedIDs[id] = struct{}{}
				}
				if key, ok := row["key"].(string); ok {
					usedKeys[key] = struct{}{}
				}
			}
			nextID := nextAvailableIDAtLeast(usedIDs, targetDS.Dataset.RowGenerationMinRow)

			for _, targetRow := range targetRows {
				var rowID int
				var hasID bool
				if rawID, ok := targetRow["id"]; ok {
					switch typed := rawID.(type) {
					case int:
						rowID = typed
						hasID = true
					case float64:
						rowID = int(typed)
						hasID = true
					case string:
						parsed, err := asInt(typed)
						if err == nil {
							rowID = parsed
							hasID = true
						}
					}
				}

				key, _ := targetRow["key"].(string)
				if strings.TrimSpace(key) == "" {
					return nil, fmt.Errorf("expansion into %s: injected rows must specify key", targetDatasetName)
				}

				if key != "" {
					if existingID, exists := targetDS.LockData[key]; exists {
						rowID = existingID
						hasID = true
					} else if existingID, exists := originalLockData[key]; exists && targetDS.Dataset.RowGeneration != "first_null_row" {
						rowID = existingID
						hasID = true
						targetDS.LockData[key] = existingID
						lockModified = true
						targetDS.LockModified = true
					}
					if _, seen := usedKeys[key]; seen {
						continue
					}
					usedKeys[key] = struct{}{}
				}

				if !hasID {
					rowID = nextID
					usedIDs[rowID] = struct{}{}
					nextID = nextAvailableIDAtLeast(usedIDs, targetDS.Dataset.RowGenerationMinRow)
				} else {
					if _, exists := usedIDs[rowID]; exists && key == "" {
						return nil, fmt.Errorf("expansion into %s: row id %d already exists", targetDatasetName, rowID)
					}
					usedIDs[rowID] = struct{}{}
				}

				if key != "" {
					if _, exists := targetDS.LockData[key]; !exists {
						targetDS.LockData[key] = rowID
						lockModified = true
						targetDS.LockAdded++
						targetDS.LockModified = true
					}
				}

				newRow := map[string]any{
					"id": rowID,
				}
				for k, v := range targetRow {
					if k != "id" {
						newRow[k] = v
					}
				}
				if key != "" {
					newRow["key"] = key
				}
				targetDS.Rows = append(targetDS.Rows, newRow)
			}

			if lockModified {
				modifiedTargets[targetIndex] = struct{}{}
			}

			slices.SortFunc(targetDS.Rows, func(a, b map[string]any) int {
				return a["id"].(int) - b["id"].(int)
			})
			collected[targetIndex] = targetDS
		}

		if err := applyExpansionValueToRow(src.row, expansion.Value); err != nil {
			return nil, err
		}
	}

	for targetIndex := range modifiedTargets {
		targetDS := collected[targetIndex]
		referencedKeys, err := collectReferencedLockKeys(nativeDatasetSourceDir(targetDS.Dataset))
		if err != nil {
			return nil, fmt.Errorf("dataset %s: collect referenced keys: %w", targetDS.Dataset.Name, err)
		}
		if pruned, updated := pruneLockDataToActiveRows(targetDS.LockData, targetDS.Rows, referencedKeys, targetDS.RetiredKeys); pruned > 0 || updated > 0 {
			targetDS.LockPruned += pruned
			targetDS.LockModified = true
			collected[targetIndex] = targetDS
		}
	}

	return collected, nil
}

type expansionSpec struct {
	Value string
	Data  map[string][]map[string]any
}

func hasExpansionData(row map[string]any) bool {
	for _, value := range row {
		if isExpansionValue(value) {
			return true
		}
	}
	return false
}

func isExpansionValue(value any) bool {
	obj, ok := value.(map[string]any)
	if !ok {
		return false
	}
	_, hasValue := obj["value"]
	_, hasData := obj["data"]
	return hasValue && hasData
}

func extractExpansionData(row map[string]any) (expansionSpec, bool) {
	for _, value := range row {
		if isExpansionValue(value) {
			obj := value.(map[string]any)
			valueStr, _ := obj["value"].(string)
			dataMap, _ := obj["data"].(map[string]any)

			result := expansionSpec{
				Value: valueStr,
				Data:  make(map[string][]map[string]any),
			}

			for datasetName, rawRows := range dataMap {
				switch typed := rawRows.(type) {
				case []any:
					rows := make([]map[string]any, 0, len(typed))
					for _, r := range typed {
						if rowObj, ok := r.(map[string]any); ok {
							rows = append(rows, rowObj)
						}
					}
					result.Data[datasetName] = rows
				case map[string]any:
					result.Data[datasetName] = []map[string]any{typed}
				}
			}

			return result, true
		}
	}
	return expansionSpec{}, false
}

func applyExpansionValueToRow(row map[string]any, value string) error {
	for field, oldValue := range row {
		if isExpansionValue(oldValue) {
			row[field] = value
		}
	}
	return nil
}