Fix lockfile generation, Key authoring hardening, and generic table injection

internal/topdata/native.go

@@ -39,11 +39,13 @@ const (
 )
 
 type nativeCollectedDataset struct {
-	Dataset  nativeDataset
-	Columns  []string
-	Rows     []map[string]any
-	LockData map[string]int
-	TableKey string
+	Dataset    nativeDataset
+	Columns    []string
+	Rows       []map[string]any
+	LockData   map[string]int
+	TableKey   string
+	LockAdded  int
+	LockPruned int
 }
 
 type resolvedTable struct {
@@ -236,7 +238,6 @@ func buildNativeUnchecked(p *project.Project, progress func(string), prebuiltWik
 			}
 		}
 	}
-
 	if hasCollectedPartsDatasets(collected) {
 		progress("Resolving released parts manifest...")
 		autogenerated, err := resolveAutogeneratedPartsInventory(p, progress)
@@ -261,6 +262,10 @@ func buildNativeUnchecked(p *project.Project, progress func(string), prebuiltWik
 	if err != nil {
 		return BuildResult{}, err
 	}
+	lockAdded, lockPruned := nativeLockChangeStats(collected)
+	if lockAdded > 0 || lockPruned > 0 {
+		progress(fmt.Sprintf("Updated native lockfiles (%d new IDs, %d stale IDs pruned)", lockAdded, lockPruned))
+	}
 	globalKeyToID = map[string]int{}
 	globalRowByKey = map[string]map[string]any{}
 	for _, collectedDataset := range collected {
@@ -365,6 +370,16 @@ func nativeCompileGroup(datasetName string) string {
 	}
 }
 
+func nativeLockChangeStats(collected []nativeCollectedDataset) (int, int) {
+	added := 0
+	pruned := 0
+	for _, dataset := range collected {
+		added += dataset.LockAdded
+		pruned += dataset.LockPruned
+	}
+	return added, pruned
+}
+
 type nativeCompileGroupStat struct {
 	OutputTables    int
 	SourceFragments int
@@ -577,17 +592,28 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 	if !ok {
 		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: rows must be an array", dataset.Name)
 	}
+	baseBoundaryID := -1
 	baseRowKeys := map[string]struct{}{}
-	for _, raw := range rawBaseRows {
+	baseRowKeyCounts := map[string]int{}
+	for index, raw := range rawBaseRows {
 		rowObj, ok := raw.(map[string]any)
 		if !ok {
 			continue
 		}
+		rowID := index
+		if rawID, ok := rowObj["id"]; ok {
+			parsed, err := asInt(rawID)
+			if err == nil {
+				rowID = parsed
+			}
+		}
+		baseBoundaryID = rowID
 		key, ok := rowObj["key"].(string)
 		if !ok || key == "" {
 			continue
 		}
 		baseRowKeys[key] = struct{}{}
+		baseRowKeyCounts[key]++
 	}
 
 	lockData, err := loadLockfile(dataset.LockPath)
@@ -595,12 +621,38 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
 	}
 
+	explicitModuleIDs := map[string]int{}
+	for _, module := range moduleData {
+		explicitIDs, err := collectExplicitModuleIDs(dataset.Name, module.Name, module.Data)
+		if err != nil {
+			return nativeCollectedDataset{}, err
+		}
+		for key, rowID := range explicitIDs {
+			explicitModuleIDs[key] = rowID
+		}
+	}
+
 	rows := make([]map[string]any, 0, len(rawBaseRows))
 	rowByID := map[int]map[string]any{}
 	rowByKey := map[string]map[string]any{}
 	usedIDs := map[int]struct{}{}
 	lockModified := false
+	lockAdded := 0
+	lockPruned := 0
 
+	for key, rowID := range lockData {
+		if rowID <= baseBoundaryID {
+			if _, ok := baseRowKeys[key]; ok {
+				continue
+			}
+			if explicitID, ok := explicitModuleIDs[key]; ok && explicitID == rowID {
+				continue
+			}
+			delete(lockData, key)
+			lockModified = true
+			lockPruned++
+		}
+	}
 	for _, rowID := range lockData {
 		usedIDs[rowID] = struct{}{}
 	}
@@ -615,18 +667,87 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 		}
 		rows = append(rows, row)
 		rowID := row["id"].(int)
+		if key, ok := row["key"].(string); ok && key != "" {
+			if baseRowKeyCounts[key] > 1 {
+				if lockedID, ok := lockData[key]; !ok || lockedID != rowID {
+					lockData[key] = rowID
+					lockModified = true
+				}
+			} else if lockedID, ok := lockData[key]; ok {
+				row["id"] = lockedID
+				rowID = lockedID
+			} else {
+				lockData[key] = rowID
+				lockModified = true
+				lockAdded++
+			}
+		}
 		rowByID[rowID] = row
 		if key, ok := row["key"].(string); ok && key != "" {
 			rowByKey[key] = row
-			if lockData[key] != rowID {
-				lockData[key] = rowID
-				lockModified = true
-			}
 		}
 		usedIDs[rowID] = struct{}{}
 	}
+	for rowID := 0; rowID <= baseBoundaryID; rowID++ {
+		usedIDs[rowID] = struct{}{}
+	}
+	for _, rowID := range explicitModuleIDs {
+		usedIDs[rowID] = struct{}{}
+	}
 
 	nextID := nextAvailableID(usedIDs)
+	allocateNextID := func() int {
+		rowID := nextID
+		usedIDs[rowID] = struct{}{}
+		nextID = nextAvailableID(usedIDs)
+		return rowID
+	}
+	lockedIDForKey := func(key string) (int, bool) {
+		lockedID, ok := lockData[key]
+		if !ok {
+			return 0, false
+		}
+		return lockedID, true
+	}
+	stripMismatchedLockedOverrideID := func(override map[string]any) (map[string]any, error) {
+		rawKey, hasKey := override["key"].(string)
+		if !hasKey || rawKey == "" {
+			return override, nil
+		}
+		lockedID, hasLockedID := lockData[rawKey]
+		if !hasLockedID {
+			return override, nil
+		}
+		rawID, hasID := override["id"]
+		if !hasID {
+			return override, nil
+		}
+		parsedID, err := asInt(rawID)
+		if err != nil {
+			return nil, fmt.Errorf("dataset %s: override id %v is not numeric", dataset.Name, rawID)
+		}
+		if parsedID == lockedID {
+			return override, nil
+		}
+		cloned := make(map[string]any, len(override)-1)
+		for field, value := range override {
+			if field != "id" {
+				cloned[field] = value
+			}
+		}
+		return cloned, nil
+	}
+	validExplicitIDForNewKey := func(key string, rowID int) bool {
+		if key == "" {
+			return true
+		}
+		for lockedKey, lockedID := range lockData {
+			if lockedKey != key && lockedID == rowID {
+				return false
+			}
+		}
+		return true
+	}
 	applyModuleData := func(sourceLabel string, moduleData map[string]any) error {
 		if rawEntries, ok := moduleData["entries"]; ok {
 			entries, ok := rawEntries.(map[string]any)
@@ -674,7 +795,10 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 					if err != nil {
 						return err
 					}
-					keyChanged := updateOverrideRowKey(existing, expanded, rowByKey, lockData)
+					keyChanged, err := updateOverrideRowKey(dataset.Name, existing, expanded, rowByKey, lockData)
+					if err != nil {
+						return err
+					}
 					if keyChanged {
 						lockModified = true
 					}
@@ -686,13 +810,12 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 					}
 					continue
 				}
-				rowID, ok := lockData[key]
+				rowID, ok := lockedIDForKey(key)
 				if !ok {
-					rowID = nextID
+					rowID = allocateNextID()
 					lockData[key] = rowID
 					lockModified = true
-					usedIDs[rowID] = struct{}{}
-					nextID = nextAvailableID(usedIDs)
+					lockAdded++
 				}
 				row, err := canonicalizeEntry(dataset, columns, rowID, key, rawEntry)
 				if err != nil {
@@ -714,6 +837,10 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 				if !ok {
 					return fmt.Errorf("dataset %s: override %d in %s must be an object", dataset.Name, index, sourceLabel)
 				}
+				override, err = stripMismatchedLockedOverrideID(override)
+				if err != nil {
+					return err
+				}
 				row, err := resolveOverrideTarget(dataset.Name, override, rowByID, rowByKey)
 				if err != nil {
 					rawKey, hasKey := override["key"].(string)
@@ -724,21 +851,28 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 						if parseErr != nil {
 							return fmt.Errorf("dataset %s: override id %v is not numeric", dataset.Name, rawID)
 						}
-						rowID = parsedID
-						hasRowID = true
+						rawKey, _ := override["key"].(string)
+						_, lockedIDExists := lockData[rawKey]
+						if !lockedIDExists {
+							if !validExplicitIDForNewKey(rawKey, parsedID) {
+								return fmt.Errorf("dataset %s: override key %q id %d collides with an existing lockfile id", dataset.Name, rawKey, parsedID)
+							}
+							rowID = parsedID
+							hasRowID = true
+						}
 					}
 					if hasKey && rawKey != "" {
-						if lockedID, hasLockedID := lockData[rawKey]; hasLockedID {
+						if lockedID, hasLockedID := lockedIDForKey(rawKey); hasLockedID {
 							rowID = lockedID
 							hasRowID = true
 						} else {
 							if !hasRowID {
-								rowID = nextID
+								rowID = allocateNextID()
 								hasRowID = true
-								nextID = nextAvailableID(usedIDs)
 							}
 							lockData[rawKey] = rowID
 							lockModified = true
+							lockAdded++
 						}
 					}
 					if !hasRowID {
@@ -796,7 +930,10 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 				if err != nil {
 					return err
 				}
-				keyChanged := updateOverrideRowKey(row, expanded, rowByKey, lockData)
+				keyChanged, err := updateOverrideRowKey(dataset.Name, row, expanded, rowByKey, lockData)
+				if err != nil {
+					return err
+				}
 				if keyChanged {
 					lockModified = true
 				}
@@ -847,6 +984,14 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 	if dataset.Name == "spells" {
 		normalizeCollectedSpellImpactScripts(rows, baseRowKeys)
 	}
+	referencedKeys, err := collectReferencedLockKeys(nativeDatasetSourceDir(dataset))
+	if err != nil {
+		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: collect referenced keys: %w", dataset.Name, err)
+	}
+	if pruned, updated := pruneLockDataToActiveRows(lockData, rows, referencedKeys); pruned > 0 || updated > 0 {
+		lockModified = true
+		lockPruned += pruned
+	}
 
 	if lockModified {
 		if err := saveLockfile(dataset.LockPath, lockData); err != nil {
@@ -859,13 +1004,113 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
 		return a["id"].(int) - b["id"].(int)
 	})
 	return nativeCollectedDataset{
-		Dataset:  dataset,
-		Columns:  columns,
-		Rows:     rows,
-		LockData: lockData,
+		Dataset:    dataset,
+		Columns:    columns,
+		Rows:       rows,
+		LockData:   lockData,
+		LockAdded:  lockAdded,
+		LockPruned: lockPruned,
 	}, nil
 }
 
+func pruneLockDataToActiveRows(lockData map[string]int, rows []map[string]any, referencedKeys map[string]struct{}) (int, int) {
+	if len(lockData) == 0 {
+		return 0, 0
+	}
+	activeKeys := make(map[string]int, len(rows))
+	for _, row := range rows {
+		if key, ok := row["key"].(string); ok && key != "" {
+			rowID, _ := row["id"].(int)
+			activeKeys[key] = rowID
+		}
+	}
+	updated := 0
+	for key, rowID := range activeKeys {
+		if lockedID, ok := lockData[key]; ok && lockedID != rowID {
+			lockData[key] = rowID
+			updated++
+		}
+	}
+	pruned := 0
+	for key := range lockData {
+		if _, ok := activeKeys[key]; ok {
+			continue
+		}
+		if _, ok := referencedKeys[key]; ok {
+			continue
+		}
+		delete(lockData, key)
+		pruned++
+	}
+	return pruned, updated
+}
+
+func collectReferencedLockKeys(sourceDir string) (map[string]struct{}, error) {
+	dataDir := filepath.Join(sourceDir, "data")
+	referenced := map[string]struct{}{}
+	if _, err := os.Stat(dataDir); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return referenced, nil
+		}
+		return nil, err
+	}
+	err := filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		if filepath.Base(path) == "lock.json" || !strings.EqualFold(filepath.Ext(path), ".json") {
+			return nil
+		}
+		obj, err := loadJSONObject(path)
+		if err != nil {
+			return err
+		}
+		collectReferencedLockKeysFromValue(obj, referenced)
+		return nil
+	})
+	return referenced, err
+}
+
+func nativeDatasetSourceDir(dataset nativeDataset) string {
+	dir := filepath.Clean(dataset.RootPath)
+	for {
+		if filepath.Base(dir) == "data" {
+			return filepath.Dir(dir)
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			return filepath.Dir(filepath.Dir(dataset.RootPath))
+		}
+		dir = parent
+	}
+}
+
+func collectReferencedLockKeysFromValue(value any, referenced map[string]struct{}) {
+	switch typed := value.(type) {
+	case string:
+		if strings.Contains(typed, ":") {
+			referenced[typed] = struct{}{}
+		}
+	case []any:
+		for _, item := range typed {
+			collectReferencedLockKeysFromValue(item, referenced)
+		}
+	case map[string]any:
+		for key, item := range typed {
+			if strings.Contains(key, ":") {
+				referenced[key] = struct{}{}
+			}
+			if key == "key" {
+				continue
+			}
+			collectReferencedLockKeysFromValue(item, referenced)
+		}
+	}
+}
+
 func dedupeCollectedRows(rows []map[string]any) []map[string]any {
 	if len(rows) <= 1 {
 		return rows
@@ -1118,7 +1363,7 @@ func (c *featGeneratedContext) featID(key string, explicit any) (int, bool, erro
 			return parsed, true, nil
 		}
 	}
-	if value, ok := c.lockData[key]; ok && value > 0 {
+	if value, ok := c.lockData[key]; ok && value >= 0 {
 		return value, true, nil
 	}
 	if value, ok := c.supplementalID[key]; ok && value > 0 {
@@ -1133,7 +1378,7 @@ func (c *featGeneratedContext) preferredGeneratedFeatKey(familyKey, sourceSlug s
 	for _, candidates := range []map[string]int{c.lockData, c.supplementalID} {
 		matches := make([]string, 0)
 		for key, value := range candidates {
-			if value <= 0 {
+			if value < 0 {
 				continue
 			}
 			if !strings.HasPrefix(key, "feat:"+familyKey+"_") {
@@ -1202,6 +1447,66 @@ func betterExpandedKey(a, b string) string {
 	return b
 }
 
+func collectExplicitModuleIDs(datasetName, sourceLabel string, moduleData map[string]any) (map[string]int, error) {
+	ids := map[string]int{}
+	if rawEntries, ok := moduleData["entries"]; ok {
+		entries, ok := rawEntries.(map[string]any)
+		if !ok {
+			return nil, fmt.Errorf("dataset %s: entries in %s must be an object", datasetName, sourceLabel)
+		}
+		for key, rawEntry := range entries {
+			entry, ok := rawEntry.(map[string]any)
+			if !ok {
+				return nil, fmt.Errorf("dataset %s: entry %q in %s must be an object", datasetName, key, sourceLabel)
+			}
+			if rawID, ok := entry["id"]; ok {
+				rowID, err := asInt(rawID)
+				if err != nil {
+					return nil, fmt.Errorf("dataset %s: entry %q id %v is not numeric", datasetName, key, rawID)
+				}
+				ids[key] = rowID
+			}
+		}
+	}
+	if rawOverrides, ok := moduleData["overrides"]; ok {
+		switch overrides := rawOverrides.(type) {
+		case []any:
+			for index, rawOverride := range overrides {
+				override, ok := rawOverride.(map[string]any)
+				if !ok {
+					return nil, fmt.Errorf("dataset %s: override %d in %s must be an object", datasetName, index, sourceLabel)
+				}
+				if rawID, ok := override["id"]; ok {
+					rowID, err := asInt(rawID)
+					if err != nil {
+						return nil, fmt.Errorf("dataset %s: override id %v is not numeric", datasetName, rawID)
+					}
+					if key, ok := override["key"].(string); ok && key != "" {
+						ids[key] = rowID
+					}
+				}
+			}
+		case map[string]any:
+			for key, rawOverride := range overrides {
+				override, ok := rawOverride.(map[string]any)
+				if !ok {
+					return nil, fmt.Errorf("dataset %s: override %q in %s must be an object", datasetName, key, sourceLabel)
+				}
+				if rawID, ok := override["id"]; ok {
+					rowID, err := asInt(rawID)
+					if err != nil {
+						return nil, fmt.Errorf("dataset %s: override id %v is not numeric", datasetName, rawID)
+					}
+					ids[key] = rowID
+				}
+			}
+		default:
+			return nil, fmt.Errorf("dataset %s: overrides in %s must be an array or object", datasetName, sourceLabel)
+		}
+	}
+	return ids, nil
+}
+
 func buildFeatGeneratedModule(path string, obj map[string]any, ctx *featGeneratedContext) (map[string]any, error) {
 	family, _ := obj["family"].(string)
 	if strings.TrimSpace(family) == "" {
@@ -3413,11 +3718,11 @@ func resolveOverrideTarget(datasetName string, override map[string]any, rowByID
 	return nil, fmt.Errorf("dataset %s: override must specify id or key", datasetName)
 }
 
-func updateOverrideRowKey(row map[string]any, expanded map[string]any, rowByKey map[string]map[string]any, lockData map[string]int) bool {
+func updateOverrideRowKey(datasetName string, row map[string]any, expanded map[string]any, rowByKey map[string]map[string]any, lockData map[string]int) (bool, error) {
 	oldKey, _ := row["key"].(string)
 	newKeyValue, hasKey := expanded["key"]
 	if !hasKey {
-		return false
+		return false, nil
 	}
 	rowID, _ := row["id"].(int)
 	changed := false
@@ -3432,20 +3737,12 @@ func updateOverrideRowKey(row map[string]any, expanded map[string]any, rowByKey
 	}
 	if newKeyValue == nil {
 		delete(row, "key")
-		if oldKey != "" {
-			delete(lockData, oldKey)
-			changed = true
-		}
-		return changed
+		return changed, nil
 	}
 	newKey, ok := newKeyValue.(string)
 	if !ok || newKey == "" {
 		delete(row, "key")
-		if oldKey != "" {
-			delete(lockData, oldKey)
-			changed = true
-		}
-		return changed
+		return changed, nil
 	}
 	if conflicting, ok := rowByKey[newKey]; ok && conflicting != nil {
 		conflictingID, conflictingHasID := conflicting["id"].(int)
@@ -3457,15 +3754,15 @@ func updateOverrideRowKey(row map[string]any, expanded map[string]any, rowByKey
 	}
 	row["key"] = newKey
 	rowByKey[newKey] = row
-	if oldKey != "" && oldKey != newKey {
-		delete(lockData, oldKey)
-		changed = true
-	}
-	if existingID, ok := lockData[newKey]; !ok || existingID != rowID {
+	if existingID, ok := lockData[newKey]; ok {
+		if existingID != rowID {
+			return false, fmt.Errorf("dataset %s: key %q is locked to id %d and cannot be rewritten to id %d", datasetName, newKey, existingID, rowID)
+		}
+	} else {
 		lockData[newKey] = rowID
 		changed = true
 	}
-	return changed
+	return changed, nil
 }
 
 func normalizeCollectedSpellImpactScripts(rows []map[string]any, baseRowKeys map[string]struct{}) {