package topdata import ( "fmt" "slices" "strings" ) func normalizeMetadataKey(key string) string { replacer := strings.NewReplacer("-", "_", " ", "_") return strings.ToLower(replacer.Replace(strings.TrimSpace(key))) } func parseRowMetadata(raw any) (map[string]any, error) { if raw == nil { return nil, nil } obj, ok := raw.(map[string]any) if !ok { return nil, fmt.Errorf("meta must be an object") } if len(obj) == 0 { return map[string]any{}, nil } meta := make(map[string]any, len(obj)) for key, value := range obj { switch normalizeMetadataKey(key) { case "family": family, err := parseFamilyMetadata(value) if err != nil { return nil, fmt.Errorf("meta.family: %w", err) } meta["family"] = family case "wiki": wiki, err := parseWikiMetadata(value) if err != nil { return nil, fmt.Errorf("meta.wiki: %w", err) } if len(wiki) > 0 { meta["wiki"] = wiki } default: return nil, fmt.Errorf("unknown metadata key %q", key) } } return meta, nil } func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) { datasetIndexMap := make(map[string]int) for i, ds := range collected { datasetIndexMap[ds.Dataset.Name] = i datasetIndexMap[ds.Dataset.OutputName] = i } rowsWithExpansion := []struct { datasetName string row map[string]any }{} for _, ds := range collected { for _, row := range ds.Rows { if hasExpansionData(row) { rowsWithExpansion = append(rowsWithExpansion, struct { datasetName string row map[string]any }{ds.Dataset.Name, row}) } } } if len(rowsWithExpansion) == 0 { return collected, nil } modifiedTargets := map[int]struct{}{} for _, src := range rowsWithExpansion { expansion, _ := extractExpansionData(src.row) for targetDatasetName, targetRows := range expansion.Data { targetIndex, ok := datasetIndexMap[targetDatasetName] if !ok { outputName := targetDatasetName if !strings.HasSuffix(outputName, ".2da") { outputName = outputName + ".2da" } targetIndex, ok = datasetIndexMap[outputName] } if !ok { return nil, fmt.Errorf("expansion targets unknown dataset %q", targetDatasetName) } targetDS := collected[targetIndex] originalLockData, _ := loadLockfile(targetDS.Dataset.LockPath) if originalLockData == nil { originalLockData = map[string]int{} } lockModified := false usedIDs := map[int]struct{}{} usedKeys := map[string]struct{}{} for _, rowID := range targetDS.LockData { usedIDs[rowID] = struct{}{} } for _, row := range targetDS.Rows { if id, ok := row["id"].(int); ok { usedIDs[id] = struct{}{} } if key, ok := row["key"].(string); ok { usedKeys[key] = struct{}{} } } nextID := nextAvailableIDAtLeast(usedIDs, targetDS.Dataset.RowGenerationMinRow) for _, targetRow := range targetRows { var rowID int var hasID bool if rawID, ok := targetRow["id"]; ok { switch typed := rawID.(type) { case int: rowID = typed hasID = true case float64: rowID = int(typed) hasID = true case string: parsed, err := asInt(typed) if err == nil { rowID = parsed hasID = true } } } key, _ := targetRow["key"].(string) if strings.TrimSpace(key) == "" { return nil, fmt.Errorf("expansion into %s: injected rows must specify key", targetDatasetName) } if key != "" { if existingID, exists := targetDS.LockData[key]; exists { rowID = existingID hasID = true } else if existingID, exists := originalLockData[key]; exists && targetDS.Dataset.RowGeneration != "first_null_row" { rowID = existingID hasID = true targetDS.LockData[key] = existingID lockModified = true targetDS.LockModified = true } if _, seen := usedKeys[key]; seen { continue } usedKeys[key] = struct{}{} } if !hasID { rowID = nextID usedIDs[rowID] = struct{}{} nextID = nextAvailableIDAtLeast(usedIDs, targetDS.Dataset.RowGenerationMinRow) } else { if _, exists := usedIDs[rowID]; exists && key == "" { return nil, fmt.Errorf("expansion into %s: row id %d already exists", targetDatasetName, rowID) } usedIDs[rowID] = struct{}{} } if key != "" { if _, exists := targetDS.LockData[key]; !exists { targetDS.LockData[key] = rowID lockModified = true targetDS.LockAdded++ targetDS.LockModified = true } } newRow := map[string]any{ "id": rowID, } for k, v := range targetRow { if k != "id" { newRow[k] = v } } if key != "" { newRow["key"] = key } targetDS.Rows = append(targetDS.Rows, newRow) } if lockModified { modifiedTargets[targetIndex] = struct{}{} } slices.SortFunc(targetDS.Rows, func(a, b map[string]any) int { return a["id"].(int) - b["id"].(int) }) collected[targetIndex] = targetDS } if err := applyExpansionValueToRow(src.row, expansion.Value); err != nil { return nil, err } } for targetIndex := range modifiedTargets { targetDS := collected[targetIndex] referencedKeys, err := collectReferencedLockKeys(nativeDatasetSourceDir(targetDS.Dataset)) if err != nil { return nil, fmt.Errorf("dataset %s: collect referenced keys: %w", targetDS.Dataset.Name, err) } if pruned, updated := pruneLockDataToActiveRows(targetDS.LockData, targetDS.Rows, referencedKeys, targetDS.RetiredKeys); pruned > 0 || updated > 0 { targetDS.LockPruned += pruned targetDS.LockModified = true collected[targetIndex] = targetDS } } return collected, nil } type expansionSpec struct { Value string Data map[string][]map[string]any } func hasExpansionData(row map[string]any) bool { for _, value := range row { if isExpansionValue(value) { return true } } return false } func isExpansionValue(value any) bool { obj, ok := value.(map[string]any) if !ok { return false } _, hasValue := obj["value"] _, hasData := obj["data"] return hasValue && hasData } func extractExpansionData(row map[string]any) (expansionSpec, bool) { for _, value := range row { if isExpansionValue(value) { obj := value.(map[string]any) valueStr, _ := obj["value"].(string) dataMap, _ := obj["data"].(map[string]any) result := expansionSpec{ Value: valueStr, Data: make(map[string][]map[string]any), } for datasetName, rawRows := range dataMap { switch typed := rawRows.(type) { case []any: rows := make([]map[string]any, 0, len(typed)) for _, r := range typed { if rowObj, ok := r.(map[string]any); ok { rows = append(rows, rowObj) } } result.Data[datasetName] = rows case map[string]any: result.Data[datasetName] = []map[string]any{typed} } } return result, true } } return expansionSpec{}, false } func applyExpansionValueToRow(row map[string]any, value string) error { for field, oldValue := range row { if isExpansionValue(oldValue) { row[field] = value } } return nil }