package topdata import ( "encoding/json" "os" "path/filepath" "strconv" ) func importLegacyMasterfeats(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { legacyDir := filepath.Join(referenceBuilderDir, "data", "feat", "masterfeats") if _, err := os.Stat(legacyDir); err != nil { if os.IsNotExist(err) { return 0, nil } return 0, err } targetDir := filepath.Join(dataDir, "masterfeats") targetPath := filepath.Join(targetDir, "base.json") legacyPlainPath := filepath.Join(targetDir, "masterfeats.json") if _, err := os.Stat(targetPath); err == nil { obj, err := loadJSONObject(targetPath) if err == nil && countLegacyTLKRefsInValue(obj) == 0 { return 0, nil } } collected, err := importLegacyMasterfeatsDataset(legacyDir) if err != nil { return 0, err } referenceIDs, err := collectReferenceLockIDs(filepath.Join(referenceBuilderDir, "data")) if err != nil { return 0, err } rows := make([]any, 0, len(collected.Rows)) for _, row := range collected.Rows { copyRow, ok := deepCopyValue(row).(map[string]any) if !ok { continue } if legacyTLK != nil { if _, _, err := inlineLegacyTLKValue(copyRow, legacyTLK); err != nil { return 0, err } } copyRow = rewriteImportedIDRefs(copyRow, referenceIDs) rows = append(rows, copyRow) } if err := os.MkdirAll(targetDir, 0o755); err != nil { return 0, err } writes := []struct { path string obj map[string]any }{ { path: targetPath, obj: map[string]any{ "output": collected.Dataset.OutputName, "columns": stringSliceToAny(collected.Columns), "rows": rows, }, }, { path: filepath.Join(targetDir, "lock.json"), obj: anyMapInt(collected.LockData), }, } for _, write := range writes { raw, err := json.MarshalIndent(write.obj, "", " ") if err != nil { return 0, err } raw = append(raw, '\n') if err := os.WriteFile(write.path, raw, 0o644); err != nil { return 0, err } } updatedFiles := len(writes) if err := os.Remove(legacyPlainPath); err == nil { updatedFiles++ } else if err != nil && !os.IsNotExist(err) { return 0, err } return updatedFiles, nil } func importLegacyMasterfeatsDataset(legacyDir string) (nativeCollectedDataset, error) { dataset := nativeDataset{ Name: "masterfeats", BasePath: filepath.Join(legacyDir, "masterfeats.json"), LockPath: filepath.Join(legacyDir, "lock.json"), Spec: specForDataset("masterfeats"), } tableData, err := loadJSONObject(dataset.BasePath) if err != nil { return nativeCollectedDataset{}, err } columns, err := parseColumns(tableData, dataset.Name) if err != nil { return nativeCollectedDataset{}, err } rawRows, ok := tableData["rows"].([]any) if !ok { return nativeCollectedDataset{}, nil } lockData, err := loadLockfile(dataset.LockPath) if err != nil { return nativeCollectedDataset{}, err } usedIDs := map[int]struct{}{} for _, rowID := range lockData { usedIDs[rowID] = struct{}{} } rows := make([]map[string]any, 0, len(rawRows)) explicitID := make([]bool, 0, len(rawRows)) for index, raw := range rawRows { rowObj, ok := raw.(map[string]any) if !ok { continue } rowID := index if value, ok := rowObj["id"]; ok { parsed, err := asInt(value) if err != nil { return nativeCollectedDataset{}, err } rowID = parsed } row := map[string]any{"id": rowID} for _, column := range columns { row[column] = nullValue } for key, value := range rowObj { switch key { case "id": case "key": if keyText, ok := value.(string); ok && keyText != "" { row["key"] = keyText } default: columnName, ok := canonicalColumn(columns, key) if !ok { continue } row[columnName] = value } } rows = append(rows, row) _, hasExplicitID := rowObj["id"] explicitID = append(explicitID, hasExplicitID) if hasExplicitID { usedIDs[rowID] = struct{}{} } } nextID := nextAvailableID(usedIDs) lockModified := false for index, row := range rows { key, hasKey := row["key"].(string) if hasKey && key != "" { if lockedID, ok := lockData[key]; ok { row["id"] = lockedID continue } if explicitID[index] { lockData[key] = row["id"].(int) } else { row["id"] = nextID lockData[key] = nextID usedIDs[nextID] = struct{}{} nextID = nextAvailableID(usedIDs) } lockModified = true continue } if !explicitID[index] { row["id"] = index } } if lockModified { _ = lockModified } return nativeCollectedDataset{ Dataset: nativeDataset{ Name: dataset.Name, OutputName: "masterfeats.2da", Columns: columns, Spec: dataset.Spec, }, Columns: columns, Rows: rows, LockData: lockData, }, nil } func rewriteImportedIDRefs(value any, ids map[string]int) map[string]any { row, ok := rewriteImportedIDRefsValue(value, ids).(map[string]any) if !ok { return map[string]any{} } return row } func rewriteImportedIDRefsValue(value any, ids map[string]int) any { switch typed := value.(type) { case map[string]any: if len(typed) == 1 { if rawID, ok := typed["id"]; ok { if key, ok := rawID.(string); ok { if resolved, ok := ids[key]; ok { return strconv.Itoa(resolved) } } } } out := make(map[string]any, len(typed)) for key, child := range typed { out[key] = rewriteImportedIDRefsValue(child, ids) } return out case []any: out := make([]any, 0, len(typed)) for _, child := range typed { out = append(out, rewriteImportedIDRefsValue(child, ids)) } return out default: return value } }