Global json adjustments

internal/topdata/native.go

@@ -58,6 +58,19 @@ type nativeCollectedDataset struct {
 	LockModified bool
 }
 
+type nativeRowSource string
+
+const (
+	nativeRowSourceBase     nativeRowSource = "base"
+	nativeRowSourceEntries  nativeRowSource = "entries"
+	nativeRowSourceOverride nativeRowSource = "overrides"
+)
+
+type nativeRowIdentity struct {
+	ID  int
+	Key string
+}
+
 type resolvedTable struct {
 	Key         string
 	DatasetName string
@@ -1198,6 +1211,87 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 	lockAdded := 0
 	lockPruned := 0
 	retiredKeys := map[string]struct{}{}
+	rowIdentity := func(row map[string]any) (nativeRowIdentity, bool) {
+		rowID, ok := row["id"].(int)
+		if !ok {
+			return nativeRowIdentity{}, false
+		}
+		key, _ := row["key"].(string)
+		return nativeRowIdentity{ID: rowID, Key: key}, true
+	}
+	rowSourceByIdentity := map[nativeRowIdentity]nativeRowSource{}
+	sourceForIdentity := func(identity nativeRowIdentity, ok bool) nativeRowSource {
+		if !ok {
+			return nativeRowSourceBase
+		}
+		if source, ok := rowSourceByIdentity[identity]; ok {
+			return source
+		}
+		return nativeRowSourceBase
+	}
+	setRowSource := func(row map[string]any, source nativeRowSource) {
+		identity, ok := rowIdentity(row)
+		if !ok {
+			return
+		}
+		rowSourceByIdentity[identity] = source
+	}
+	moveRowSource := func(before nativeRowIdentity, beforeOK bool, row map[string]any, source nativeRowSource) {
+		after, afterOK := rowIdentity(row)
+		if beforeOK && (!afterOK || before != after) {
+			delete(rowSourceByIdentity, before)
+		}
+		if afterOK {
+			rowSourceByIdentity[after] = source
+		}
+	}
+	rowSource := func(row map[string]any) nativeRowSource {
+		identity, ok := rowIdentity(row)
+		return sourceForIdentity(identity, ok)
+	}
+	type pendingRowSourceMove struct {
+		Row      map[string]any
+		Before   nativeRowIdentity
+		BeforeOK bool
+		Source   nativeRowSource
+	}
+	captureDisplacedRowSource := func(row map[string]any, expanded map[string]any) pendingRowSourceMove {
+		newKeyValue, hasKey := expanded["key"]
+		if !hasKey {
+			return pendingRowSourceMove{}
+		}
+		newKey, ok := newKeyValue.(string)
+		if !ok || newKey == "" || strings.TrimSpace(newKey) == nullValue {
+			return pendingRowSourceMove{}
+		}
+		oldKey, _ := row["key"].(string)
+		if oldKey == newKey {
+			return pendingRowSourceMove{}
+		}
+		conflicting, ok := rowByKey[newKey]
+		if !ok || conflicting == nil {
+			return pendingRowSourceMove{}
+		}
+		conflictingID, conflictingHasID := conflicting["id"].(int)
+		rowID, _ := row["id"].(int)
+		rowHasID := rowID != 0 || row["id"] != nil
+		if conflictingHasID && rowHasID && conflictingID == rowID {
+			return pendingRowSourceMove{}
+		}
+		identity, identityOK := rowIdentity(conflicting)
+		return pendingRowSourceMove{
+			Row:      conflicting,
+			Before:   identity,
+			BeforeOK: identityOK,
+			Source:   rowSource(conflicting),
+		}
+	}
+	moveCapturedRowSource := func(move pendingRowSourceMove) {
+		if move.Row == nil {
+			return
+		}
+		moveRowSource(move.Before, move.BeforeOK, move.Row, move.Source)
+	}
 	featFamilyAliases := generatedFamilyAliases{}
 	if dataset.Name == "feat" {
 		var err error
@@ -1257,6 +1351,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 			continue
 		}
 		rows = append(rows, row)
+		setRowSource(row, nativeRowSourceBase)
 		if key, ok := row["key"].(string); ok && key != "" {
 			if baseRowKeyCounts[key] > 1 {
 				if lockedID, ok := lockData[key]; !ok || lockedID != rowID {
@@ -1342,6 +1437,8 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 		return true
 	}
 	applyOverrideFields := func(override map[string]any, row map[string]any) (bool, string, error) {
+		sourceBefore := rowSource(row)
+		identityBefore, identityBeforeOK := rowIdentity(row)
 		candidate := map[string]any{}
 		for field, value := range row {
 			candidate[field] = value
@@ -1383,6 +1480,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 		if err != nil {
 			return false, "", err
 		}
+		displacedSource := captureDisplacedRowSource(row, expanded)
 		keyChanged, retiredKey, err := updateOverrideRowKey(dataset.Name, row, expanded, rowByKey, lockData)
 		if err != nil {
 			return false, "", err
@@ -1393,6 +1491,8 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 			}
 			row[field] = value
 		}
+		moveRowSource(identityBefore, identityBeforeOK, row, sourceBefore)
+		moveCapturedRowSource(displacedSource)
 		return keyChanged, retiredKey, nil
 	}
 	applyModuleData := func(sourceLabel string, moduleData map[string]any, globalSource bool) error {
@@ -1442,6 +1542,8 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 					if err != nil {
 						return err
 					}
+					identityBefore, identityBeforeOK := rowIdentity(existing)
+					displacedSource := captureDisplacedRowSource(existing, expanded)
 					keyChanged, retiredKey, err := updateOverrideRowKey(dataset.Name, existing, expanded, rowByKey, lockData)
 					if err != nil {
 						return err
@@ -1458,6 +1560,8 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 						}
 						existing[field] = value
 					}
+					moveRowSource(identityBefore, identityBeforeOK, existing, nativeRowSourceEntries)
+					moveCapturedRowSource(displacedSource)
 					continue
 				}
 				rowID, ok := lockedIDForKey(key)
@@ -1472,6 +1576,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 					return fmt.Errorf("dataset %s: %w", dataset.Name, err)
 				}
 				rows = append(rows, row)
+				setRowSource(row, nativeRowSourceEntries)
 				rowByID[rowID] = row
 				rowByKey[key] = row
 			}
@@ -1556,14 +1661,18 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 						row["key"] = rawKey
 					}
 					rows = append(rows, row)
+					setRowSource(row, nativeRowSourceOverride)
 					rowByID[rowID] = row
 					if hasKey && rawKey != "" {
 						rowByKey[rawKey] = row
 					}
 				}
 				if overrideRequestsNullRow(override) {
+					sourceBefore := rowSource(row)
+					identityBefore, identityBeforeOK := rowIdentity(row)
 					retireRowIdentity(row, lockData, retiredKeys)
 					nullifyOverrideRow(row, columns, rowByKey)
+					moveRowSource(identityBefore, identityBeforeOK, row, sourceBefore)
 					continue
 				}
 				keyChanged, retiredKey, err := applyOverrideFields(override, row)
@@ -1580,6 +1689,66 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 		}
 		return nil
 	}
+	applyGlobalDefaults := func(sourceLabel string, moduleData map[string]any) error {
+		rawDefaults, ok := moduleData["defaults"]
+		if !ok {
+			return nil
+		}
+		defaultList, ok := rawDefaults.([]any)
+		if !ok {
+			return fmt.Errorf("dataset %s: defaults in %s must be an array", dataset.Name, sourceLabel)
+		}
+		for index, rawDefault := range defaultList {
+			defaultRule, ok := rawDefault.(map[string]any)
+			if !ok {
+				return fmt.Errorf("dataset %s: default %d in %s must be an object", dataset.Name, index, sourceLabel)
+			}
+			rawValues, ok := defaultRule["values"]
+			if !ok {
+				return fmt.Errorf("dataset %s: default %d in %s must contain values", dataset.Name, index, sourceLabel)
+			}
+			values, ok := rawValues.(map[string]any)
+			if !ok {
+				return fmt.Errorf("dataset %s: default %d values in %s must be an object", dataset.Name, index, sourceLabel)
+			}
+			parsedValues := make([]parsedGlobalDefaultField, 0, len(values))
+			for _, field := range sortedKeys(values) {
+				columnName, ok := canonicalColumn(columns, field)
+				if !ok {
+					return fmt.Errorf("dataset %s: default %d field %q in %s is not a known column", dataset.Name, index, field, sourceLabel)
+				}
+				value, err := parseGlobalDefaultValue(values[field])
+				if err != nil {
+					return fmt.Errorf("dataset %s: default %d field %q in %s: %w", dataset.Name, index, field, sourceLabel, err)
+				}
+				parsedValues = append(parsedValues, parsedGlobalDefaultField{
+					Field:  field,
+					Column: columnName,
+					Value:  value,
+				})
+			}
+			for _, row := range rows {
+				matches, err := globalDefaultMatchesRow(defaultRule["match"], rowSource(row))
+				if err != nil {
+					return fmt.Errorf("dataset %s: default %d in %s: %w", dataset.Name, index, sourceLabel, err)
+				}
+				if !matches {
+					continue
+				}
+				for _, parsedValue := range parsedValues {
+					if !isTopDataNullLike(row[parsedValue.Column]) {
+						continue
+					}
+					value, err := evaluateGlobalDefaultValue(parsedValue.Value, row)
+					if err != nil {
+						return fmt.Errorf("dataset %s: default %d field %q in %s: %w", dataset.Name, index, parsedValue.Field, sourceLabel, err)
+					}
+					row[parsedValue.Column] = value
+				}
+			}
+		}
+		return nil
+	}
 
 	generatedModules, err := collectGeneratedModules(dataset, lockData)
 	if err != nil {
@@ -1621,9 +1790,9 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 		if err := applyModuleData(module.Name, module.Data, true); err != nil {
 			return nativeCollectedDataset{}, err
 		}
-	}
-	if dataset.Name == "spells" {
-		normalizeCollectedSpellImpactScripts(rows, baseRowKeys)
+		if err := applyGlobalDefaults(module.Name, module.Data); err != nil {
+			return nativeCollectedDataset{}, err
+		}
 	}
 	referencedKeys, err := collectReferencedLockKeys(nativeDatasetSourceDir(dataset))
 	if err != nil {
@@ -3513,6 +3682,11 @@ func collectPlainDataset(dataset nativeDataset) (nativeCollectedDataset, error)
 	if err != nil {
 		return nativeCollectedDataset{}, err
 	}
+	for _, module := range globalData {
+		if _, ok := module.Data["defaults"]; ok {
+			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: defaults in %s are only supported for base datasets", dataset.Name, module.Name)
+		}
+	}
 
 	columns, err := parseColumns(tableData, dataset.Name)
 	if err != nil {
@@ -6013,21 +6187,163 @@ func updateOverrideRowKey(datasetName string, row map[string]any, expanded map[s
 	return changed, retiredKey, nil
 }
 
-func normalizeCollectedSpellImpactScripts(rows []map[string]any, baseRowKeys map[string]struct{}) {
-	for _, row := range rows {
-		rowKey, ok := row["key"].(string)
-		if !ok || rowKey == "" {
-			continue
-		}
-		if _, ok := baseRowKeys[rowKey]; ok {
-			continue
-		}
-		rowID, ok := row["id"].(int)
-		if !ok {
-			continue
-		}
-		row["ImpactScript"] = fmt.Sprintf("ss_%d", rowID)
+func isTopDataNullLike(value any) bool {
+	if value == nil {
+		return true
 	}
+	text, ok := value.(string)
+	if !ok {
+		return false
+	}
+	trimmed := strings.TrimSpace(text)
+	return trimmed == "" || trimmed == nullValue
+}
+
+type parsedGlobalDefaultField struct {
+	Field  string
+	Column string
+	Value  parsedGlobalDefaultValue
+}
+
+type parsedGlobalDefaultValue struct {
+	Literal  any
+	Format   string
+	IsFormat bool
+}
+
+func globalDefaultMatchesRow(rawMatch any, source nativeRowSource) (bool, error) {
+	if rawMatch == nil {
+		return true, nil
+	}
+	if text, ok := rawMatch.(string); ok {
+		if strings.EqualFold(strings.TrimSpace(text), "all") {
+			return true, nil
+		}
+		return false, fmt.Errorf("unsupported defaults match value %q", text)
+	}
+	obj, ok := rawMatch.(map[string]any)
+	if !ok {
+		return false, fmt.Errorf("defaults match must be \"all\" or an object")
+	}
+	rawSource, ok := obj["source"]
+	if !ok {
+		return false, fmt.Errorf("defaults match object must contain source")
+	}
+	sourceText, ok := rawSource.(string)
+	if !ok || strings.TrimSpace(sourceText) == "" {
+		return false, fmt.Errorf("defaults match source must be a non-empty string")
+	}
+	for key := range obj {
+		if key != "source" {
+			return false, fmt.Errorf("defaults match field %q is not supported", key)
+		}
+	}
+	switch strings.ToLower(strings.TrimSpace(sourceText)) {
+	case string(nativeRowSourceBase):
+		return source == nativeRowSourceBase, nil
+	case string(nativeRowSourceEntries):
+		return source == nativeRowSourceEntries, nil
+	case string(nativeRowSourceOverride):
+		return source == nativeRowSourceOverride, nil
+	default:
+		return false, fmt.Errorf("defaults match source %q is not supported", sourceText)
+	}
+}
+
+func parseGlobalDefaultValue(value any) (parsedGlobalDefaultValue, error) {
+	obj, ok := value.(map[string]any)
+	if !ok {
+		return parsedGlobalDefaultValue{Literal: value}, nil
+	}
+	rawFormat, hasFormat := obj["format"]
+	if !hasFormat {
+		return parsedGlobalDefaultValue{Literal: value}, nil
+	}
+	if len(obj) != 1 {
+		return parsedGlobalDefaultValue{}, fmt.Errorf("default format object must contain only format")
+	}
+	format, ok := rawFormat.(string)
+	if !ok {
+		return parsedGlobalDefaultValue{}, fmt.Errorf("default format must be a string")
+	}
+	if err := validateTopDataRowFormat(format); err != nil {
+		return parsedGlobalDefaultValue{}, err
+	}
+	return parsedGlobalDefaultValue{Format: format, IsFormat: true}, nil
+}
+
+func evaluateGlobalDefaultValue(value parsedGlobalDefaultValue, row map[string]any) (any, error) {
+	if !value.IsFormat {
+		return value.Literal, nil
+	}
+	return formatTopDataRowValue(value.Format, row)
+}
+
+func validateTopDataRowFormat(format string) error {
+	return walkTopDataRowFormat(format, nil, nil)
+}
+
+func formatTopDataRowValue(format string, row map[string]any) (string, error) {
+	var builder strings.Builder
+	err := walkTopDataRowFormat(format, func(text string) {
+		builder.WriteString(text)
+	}, func(token string) error {
+		switch token {
+		case "id":
+			rowID, ok := row["id"].(int)
+			if !ok {
+				return fmt.Errorf("default format token {id} requires numeric row id")
+			}
+			builder.WriteString(strconv.Itoa(rowID))
+		case "key":
+			key, ok := row["key"].(string)
+			if !ok {
+				return fmt.Errorf("default format token {key} requires row key")
+			}
+			builder.WriteString(key)
+		}
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+	return builder.String(), nil
+}
+
+func walkTopDataRowFormat(format string, literal func(string), tokenValue func(string) error) error {
+	for index := 0; index < len(format); {
+		switch format[index] {
+		case '{':
+			end := strings.IndexByte(format[index+1:], '}')
+			if end < 0 {
+				return fmt.Errorf("unterminated default format token in %q", format)
+			}
+			token := format[index+1 : index+1+end]
+			switch token {
+			case "id", "key":
+			default:
+				return fmt.Errorf("default format token {%s} is not supported", token)
+			}
+			if tokenValue != nil {
+				if err := tokenValue(token); err != nil {
+					return err
+				}
+			}
+			index += end + 2
+		case '}':
+			return fmt.Errorf("unexpected default format token terminator in %q", format)
+		default:
+			start := index
+			for index < len(format) && format[index] != '{' && format[index] != '}' {
+				index++
+			}
+			if literal != nil {
+				literal(format[start:index])
+			}
+			continue
+		}
+	}
+	return nil
 }
 
 func write2DA(data map[string]any, path string, denseRows bool, denseRowMax int) error {