Fix: replaced keys become stale

internal/topdata/expansion_native.go

@@ -207,7 +207,7 @@ func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDa
 		if err != nil {
 			return nil, fmt.Errorf("dataset %s: collect referenced keys: %w", targetDS.Dataset.Name, err)
 		}
-		if pruned, updated := pruneLockDataToActiveRows(targetDS.LockData, targetDS.Rows, referencedKeys); pruned > 0 || updated > 0 {
+		if pruned, updated := pruneLockDataToActiveRows(targetDS.LockData, targetDS.Rows, referencedKeys, targetDS.RetiredKeys); pruned > 0 || updated > 0 {
 			targetDS.LockPruned += pruned
 			targetDS.LockModified = true
 			collected[targetIndex] = targetDS

internal/topdata/native.go

@@ -43,6 +43,7 @@ type nativeCollectedDataset struct {
 	Columns      []string
 	Rows         []map[string]any
 	LockData     map[string]int
+	RetiredKeys  map[string]struct{}
 	TableKey     string
 	LockAdded    int
 	LockPruned   int
@@ -714,6 +715,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 	lockModified := false
 	lockAdded := 0
 	lockPruned := 0
+	retiredKeys := map[string]struct{}{}
 
 	for key, rowID := range lockData {
 		if rowID <= baseBoundaryID {
@@ -871,13 +873,16 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 					if err != nil {
 						return err
 					}
-					keyChanged, err := updateOverrideRowKey(dataset.Name, existing, expanded, rowByKey, lockData)
+					keyChanged, retiredKey, err := updateOverrideRowKey(dataset.Name, existing, expanded, rowByKey, lockData)
 					if err != nil {
 						return err
 					}
 					if keyChanged {
 						lockModified = true
 					}
+					if retiredKey != "" {
+						retiredKeys[retiredKey] = struct{}{}
+					}
 					for field, value := range expanded {
 						if field == "id" || field == "_tlk" {
 							continue
@@ -1010,13 +1015,16 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 				if err != nil {
 					return err
 				}
-				keyChanged, err := updateOverrideRowKey(dataset.Name, row, expanded, rowByKey, lockData)
+				keyChanged, retiredKey, err := updateOverrideRowKey(dataset.Name, row, expanded, rowByKey, lockData)
 				if err != nil {
 					return err
 				}
 				if keyChanged {
 					lockModified = true
 				}
+				if retiredKey != "" {
+					retiredKeys[retiredKey] = struct{}{}
+				}
 				for field, value := range expanded {
 					if field == "id" || field == "_tlk" {
 						continue
@@ -1068,7 +1076,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 	if err != nil {
 		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: collect referenced keys: %w", dataset.Name, err)
 	}
-	if pruned, updated := pruneLockDataToActiveRows(lockData, rows, referencedKeys); pruned > 0 || updated > 0 {
+	if pruned, updated := pruneLockDataToActiveRows(lockData, rows, referencedKeys, retiredKeys); pruned > 0 || updated > 0 {
 		lockModified = true
 		lockPruned += pruned
 	}
@@ -1082,13 +1090,14 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 		Columns:      columns,
 		Rows:         rows,
 		LockData:     lockData,
+		RetiredKeys:  retiredKeys,
 		LockAdded:    lockAdded,
 		LockPruned:   lockPruned,
 		LockModified: lockModified,
 	}, nil
 }
 
-func pruneLockDataToActiveRows(lockData map[string]int, rows []map[string]any, referencedKeys map[string]struct{}) (int, int) {
+func pruneLockDataToActiveRows(lockData map[string]int, rows []map[string]any, referencedKeys map[string]struct{}, retiredKeys map[string]struct{}) (int, int) {
 	if len(lockData) == 0 {
 		return 0, 0
 	}
@@ -1108,6 +1117,11 @@ func pruneLockDataToActiveRows(lockData map[string]int, rows []map[string]any, r
 	}
 	pruned := 0
 	for key := range lockData {
+		if _, retired := retiredKeys[key]; retired {
+			delete(lockData, key)
+			pruned++
+			continue
+		}
 		if _, ok := activeKeys[key]; ok {
 			continue
 		}
@@ -4123,14 +4137,15 @@ func nullifyOverrideRow(row map[string]any, columns []string, rowByKey map[strin
 	}
 }
 
-func updateOverrideRowKey(datasetName string, row map[string]any, expanded map[string]any, rowByKey map[string]map[string]any, lockData map[string]int) (bool, error) {
+func updateOverrideRowKey(datasetName string, row map[string]any, expanded map[string]any, rowByKey map[string]map[string]any, lockData map[string]int) (bool, string, error) {
 	oldKey, _ := row["key"].(string)
 	newKeyValue, hasKey := expanded["key"]
 	if !hasKey {
-		return false, nil
+		return false, "", nil
 	}
 	rowID, _ := row["id"].(int)
 	changed := false
+	retiredKey := ""
 	if oldKey != "" {
 		if mapped, ok := rowByKey[oldKey]; ok {
 			mappedID, mappedHasID := mapped["id"].(int)
@@ -4142,12 +4157,22 @@ func updateOverrideRowKey(datasetName string, row map[string]any, expanded map[s
 	}
 	if newKeyValue == nil {
 		delete(row, "key")
-		return changed, nil
+		if oldKey != "" {
+			retiredKey = oldKey
+			delete(lockData, oldKey)
+			changed = true
+		}
+		return changed, retiredKey, nil
 	}
 	newKey, ok := newKeyValue.(string)
 	if !ok || newKey == "" || strings.TrimSpace(newKey) == nullValue {
 		delete(row, "key")
-		return changed, nil
+		if oldKey != "" {
+			retiredKey = oldKey
+			delete(lockData, oldKey)
+			changed = true
+		}
+		return changed, retiredKey, nil
 	}
 	conflictingID := -1
 	if conflicting, ok := rowByKey[newKey]; ok && conflicting != nil {
@@ -4161,20 +4186,25 @@ func updateOverrideRowKey(datasetName string, row map[string]any, expanded map[s
 	}
 	row["key"] = newKey
 	rowByKey[newKey] = row
+	if oldKey != "" && oldKey != newKey {
+		retiredKey = oldKey
+		delete(lockData, oldKey)
+		changed = true
+	}
 	if existingID, ok := lockData[newKey]; ok {
 		if existingID != rowID {
 			if conflictingID >= 0 && existingID == conflictingID {
 				lockData[newKey] = rowID
 				changed = true
 			} else {
-				return false, fmt.Errorf("dataset %s: key %q is locked to id %d and cannot be rewritten to id %d", datasetName, newKey, existingID, rowID)
+				return false, "", fmt.Errorf("dataset %s: key %q is locked to id %d and cannot be rewritten to id %d", datasetName, newKey, existingID, rowID)
 			}
 		}
 	} else {
 		lockData[newKey] = rowID
 		changed = true
 	}
-	return changed, nil
+	return changed, retiredKey, nil
 }
 
 func normalizeCollectedSpellImpactScripts(rows []map[string]any, baseRowKeys map[string]struct{}) {

internal/topdata/topdata_test.go

@@ -6433,6 +6433,57 @@ func TestResolveOverrideTargetPrefersIDOverKey(t *testing.T) {
 	}
 }
 
+func TestUpdateOverrideRowKeyRetiresOldKeyEvenIfReferenced(t *testing.T) {
+	row := map[string]any{"id": 443, "key": "spells:etherealness"}
+	rowByKey := map[string]map[string]any{
+		"spells:etherealness": row,
+	}
+	lockData := map[string]int{
+		"spells:etherealness": 443,
+	}
+	expanded := map[string]any{
+		"key": "spells:greater_sanctuary",
+	}
+
+	changed, retiredKey, err := updateOverrideRowKey("spells", row, expanded, rowByKey, lockData)
+	if err != nil {
+		t.Fatalf("updateOverrideRowKey failed: %v", err)
+	}
+	if !changed {
+		t.Fatal("expected key reassignment to mark the row changed")
+	}
+	if retiredKey != "spells:etherealness" {
+		t.Fatalf("expected retired key spells:etherealness, got %q", retiredKey)
+	}
+	if row["key"] != "spells:greater_sanctuary" {
+		t.Fatalf("expected row key to be updated, got %v", row["key"])
+	}
+	if _, ok := lockData["spells:etherealness"]; ok {
+		t.Fatalf("expected old key to be removed from lock data, got %#v", lockData)
+	}
+	if lockData["spells:greater_sanctuary"] != 443 {
+		t.Fatalf("expected new key to be locked to row 443, got %#v", lockData)
+	}
+
+	referencedKeys := map[string]struct{}{
+		"spells:etherealness": {},
+	}
+	retiredKeys := map[string]struct{}{
+		retiredKey: {},
+	}
+	rows := []map[string]any{row}
+	pruned, updated := pruneLockDataToActiveRows(lockData, rows, referencedKeys, retiredKeys)
+	if updated != 0 {
+		t.Fatalf("expected no lock id updates after prune, got %d", updated)
+	}
+	if pruned != 0 {
+		t.Fatalf("expected no additional prune once old key was already retired, got %d", pruned)
+	}
+	if _, ok := lockData["spells:etherealness"]; ok {
+		t.Fatalf("expected retired key to stay removed even when referenced, got %#v", lockData)
+	}
+}
+
 func TestBuildPrunesLockedIDThatTargetsManualBaseSpace(t *testing.T) {
 	root := testProjectRoot(t)
 	mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules"))