package topdata import ( "encoding/json" "io/fs" "os" "path/filepath" "sort" "strings" ) type legacyDatasetTransform func(relativePath string, obj map[string]any) func importLegacyDatasetMirror(referenceBuilderDir, dataDir, datasetName, outputName string, transform legacyDatasetTransform) (int, error) { legacyDir := filepath.Join(referenceBuilderDir, "data", datasetName) if _, err := os.Stat(legacyDir); err != nil { if os.IsNotExist(err) { return 0, nil } return 0, err } targetDir := filepath.Join(dataDir, datasetName) baseObj, err := loadJSONObject(filepath.Join(legacyDir, "base.json")) if err != nil { return 0, err } if outputName != "" { baseObj["output"] = outputName } if transform != nil { transform("base.json", baseObj) } lockObj, err := loadJSONObject(filepath.Join(legacyDir, "lock.json")) if err != nil { return 0, err } if transform != nil { transform("lock.json", lockObj) } moduleRelPaths, err := collectJSONRelativePaths(filepath.Join(legacyDir, "modules")) if err != nil { return 0, err } moduleObjs := make(map[string]map[string]any, len(moduleRelPaths)) for _, relPath := range moduleRelPaths { obj, err := loadJSONObject(filepath.Join(legacyDir, "modules", relPath)) if err != nil { return 0, err } normalizedRel := filepath.ToSlash(relPath) if transform != nil { transform(filepath.ToSlash(filepath.Join("modules", normalizedRel)), obj) } moduleObjs[normalizedRel] = obj } if err := os.MkdirAll(targetDir, 0o755); err != nil { return 0, err } updated := 0 for _, write := range []struct { path string obj map[string]any }{ {path: filepath.Join(targetDir, "base.json"), obj: baseObj}, {path: filepath.Join(targetDir, "lock.json"), obj: lockObj}, } { changed, err := writeJSONObjectIfChanged(write.path, write.obj) if err != nil { return 0, err } if changed { updated++ } } targetModulesDir := filepath.Join(targetDir, "modules") if len(moduleObjs) > 0 { if err := os.MkdirAll(targetModulesDir, 0o755); err != nil { return 0, err } } for relPath, obj := range moduleObjs { targetPath := filepath.Join(targetModulesDir, filepath.FromSlash(relPath)) if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { return 0, err } changed, err := writeJSONObjectIfChanged(targetPath, obj) if err != nil { return 0, err } if changed { updated++ } } staleModules, err := collectJSONRelativePaths(targetModulesDir) if err != nil { return 0, err } for _, relPath := range staleModules { normalizedRel := filepath.ToSlash(relPath) if _, ok := moduleObjs[normalizedRel]; ok { continue } if err := os.Remove(filepath.Join(targetModulesDir, filepath.FromSlash(normalizedRel))); err != nil && !os.IsNotExist(err) { return 0, err } updated++ } return updated, nil } func collectJSONRelativePaths(root string) ([]string, error) { if _, err := os.Stat(root); err != nil { if os.IsNotExist(err) { return nil, nil } return nil, err } var relPaths []string err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } if !strings.EqualFold(filepath.Ext(d.Name()), ".json") { return nil } relPath, err := filepath.Rel(root, path) if err != nil { return err } relPaths = append(relPaths, filepath.ToSlash(relPath)) return nil }) if err != nil { return nil, err } sort.Strings(relPaths) return relPaths, nil } func writeJSONObjectIfChanged(path string, obj map[string]any) (bool, error) { raw, err := json.MarshalIndent(obj, "", " ") if err != nil { return false, err } raw = append(raw, '\n') current, err := os.ReadFile(path) if err == nil && string(current) == string(raw) { return false, nil } if err != nil && !os.IsNotExist(err) { return false, err } if err := os.WriteFile(path, raw, 0o644); err != nil { return false, err } return true, nil } func fileExists(path string) bool { _, err := os.Stat(path) return err == nil }