diff --git a/internal/topdata/native.go b/internal/topdata/native.go index d0139a4..66edc85 100644 --- a/internal/topdata/native.go +++ b/internal/topdata/native.go @@ -196,6 +196,14 @@ func BuildNativeWithOptions(p *project.Project, opts NativeBuildOptions, progres } return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount()) } + if warnings := report.WarningCount(); warnings > 0 { + progress(fmt.Sprintf("topdata validation warnings: %d", warnings)) + for _, line := range report.SummaryLines(5) { + if strings.HasPrefix(line, "warning:") { + progress(line) + } + } + } var prebuiltWiki *wikiResult if opts.BuildWiki { if newestWikiSource, _, err := newestRelevantWikiSource(p.TopDataSourceDir()); err == nil && !newestWikiSource.IsZero() { diff --git a/internal/topdata/topdata.go b/internal/topdata/topdata.go index e0c4882..b1404ad 100644 --- a/internal/topdata/topdata.go +++ b/internal/topdata/topdata.go @@ -272,6 +272,7 @@ func ValidateProject(p *project.Project) ValidationReport { _ = validateJSONFile(statePath, "state", &report) } if !report.HasErrors() { + validateNativeAuthoringWarnings(dataDir, &report) validateNativeBuildability(p, &report) } @@ -1137,6 +1138,274 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) { } } +func validateNativeAuthoringWarnings(dataDir string, report *ValidationReport) { + datasets, err := discoverNativeDatasets(dataDir) + if err != nil { + return + } + for _, dataset := range datasets { + if dataset.Kind != nativeDatasetBase { + continue + } + validateDatasetOverrideTargetWarnings(dataset, report) + validateDatasetNormalizedLockWarnings(dataset, report) + validateDatasetGhostRowWarnings(dataset, report) + } +} + +func validateDatasetOverrideTargetWarnings(dataset nativeDataset, report *ValidationReport) { + if dataset.BasePath == "" { + return + } + if _, err := os.Stat(dataset.BasePath); err != nil { + return + } + baseObj, err := loadJSONObject(dataset.BasePath) + if err != nil { + return + } + rawRows, ok := baseObj["rows"].([]any) + if !ok { + return + } + rowIDToKey := map[int]string{} + keyToID := map[string]int{} + usedIDs := map[int]struct{}{} + for index, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + rowID := index + if rawID, ok := row["id"]; ok { + if parsed, err := asInt(rawID); err == nil { + rowID = parsed + } + } + usedIDs[rowID] = struct{}{} + if key, ok := row["key"].(string); ok && key != "" { + rowIDToKey[rowID] = key + keyToID[key] = rowID + } + } + nextID := nextAvailableID(usedIDs) + allocateNextID := func() int { + rowID := nextID + usedIDs[rowID] = struct{}{} + nextID = nextAvailableID(usedIDs) + return rowID + } + modulePaths, err := collectModulePaths(dataset.ModulesDir) + if err != nil { + return + } + for _, path := range modulePaths { + obj, err := loadJSONObject(path) + if err != nil { + continue + } + if rawEntries, ok := obj["entries"].(map[string]any); ok { + for key := range rawEntries { + if key == "" { + continue + } + if _, exists := keyToID[key]; exists { + continue + } + rowID := allocateNextID() + keyToID[key] = rowID + rowIDToKey[rowID] = key + } + } + rawOverrides, ok := obj["overrides"].([]any) + if !ok { + continue + } + for index, raw := range rawOverrides { + override, ok := raw.(map[string]any) + if !ok { + continue + } + keyOnlyTarget := false + keyText, hasKeyText := override["key"].(string) + if _, hasID := override["id"]; !hasID && hasKeyText && keyText != "" { + keyOnlyTarget = true + if _, exists := keyToID[keyText]; !exists { + canonical := resolveCanonicalDatasetKey(keyText, keyToID) + if canonical != keyText { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: path, + Message: fmt.Sprintf("override %d targets key %q, but the live canonical key at that point is %q; this key-only override will create or retarget a new row", index, keyText, canonical), + }) + } else { + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: path, + Message: fmt.Sprintf("override %d targets key %q, but no live row owns that key at that point; this key-only override will create or retarget a new row", index, keyText), + }) + } + } + } + rowID := -1 + if rawID, ok := override["id"]; ok { + parsed, err := asInt(rawID) + if err == nil { + rowID = parsed + } + } else if hasKeyText && keyText != "" { + if mappedID, exists := keyToID[keyText]; exists { + rowID = mappedID + } + } + if rowID < 0 { + if keyOnlyTarget { + rowID = allocateNextID() + keyToID[keyText] = rowID + rowIDToKey[rowID] = keyText + } + continue + } + currentKey := rowIDToKey[rowID] + if overrideRequestsNullRow(override) { + if currentKey != "" { + delete(keyToID, currentKey) + delete(rowIDToKey, rowID) + } + continue + } + if rawKey, present := override["key"]; present { + switch typed := rawKey.(type) { + case nil: + if currentKey != "" { + delete(keyToID, currentKey) + delete(rowIDToKey, rowID) + } + case string: + if typed == "" || strings.TrimSpace(typed) == nullValue { + if currentKey != "" { + delete(keyToID, currentKey) + delete(rowIDToKey, rowID) + } + continue + } + if currentKey != "" && currentKey != typed { + delete(keyToID, currentKey) + } + if previousID, exists := keyToID[typed]; exists && previousID != rowID { + delete(rowIDToKey, previousID) + } + keyToID[typed] = rowID + rowIDToKey[rowID] = typed + } + } + } + } +} + +func validateDatasetNormalizedLockWarnings(dataset nativeDataset, report *ValidationReport) { + lockData, err := loadLockfile(dataset.LockPath) + if err != nil { + return + } + groups := map[string][]string{} + for key := range lockData { + prefix, suffix, ok := strings.Cut(key, ":") + if !ok || prefix == "" || suffix == "" { + continue + } + groups[prefix+":"+normalizeKeyIdentity(suffix)] = append(groups[prefix+":"+normalizeKeyIdentity(suffix)], key) + } + for _, keys := range groups { + if len(keys) < 2 { + continue + } + slices.Sort(keys) + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: dataset.LockPath, + Message: fmt.Sprintf("lockfile keeps multiple normalized-equivalent keys alive: %s", strings.Join(keys, ", ")), + }) + } +} + +func validateDatasetGhostRowWarnings(dataset nativeDataset, report *ValidationReport) { + if dataset.BasePath == "" { + return + } + if _, err := os.Stat(dataset.BasePath); err != nil { + return + } + baseObj, err := loadJSONObject(dataset.BasePath) + if err != nil { + return + } + rawRows, ok := baseObj["rows"].([]any) + if !ok { + return + } + baseKeyedIDs := map[int]struct{}{} + for index, raw := range rawRows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + if key, ok := row["key"].(string); !ok || key == "" { + continue + } + rowID := index + if rawID, ok := row["id"]; ok { + if parsed, err := asInt(rawID); err == nil { + rowID = parsed + } + } + baseKeyedIDs[rowID] = struct{}{} + } + collected, err := collectNativeDataset(dataset) + if err != nil { + return + } + for _, row := range collected.Rows { + if key, ok := row["key"].(string); ok && key != "" { + continue + } + rowID, _ := row["id"].(int) + if _, hadBaseKey := baseKeyedIDs[rowID]; !hadBaseKey { + continue + } + nonNullFields := make([]string, 0, 4) + for field, value := range row { + if field == "id" || field == "key" || field == "meta" { + continue + } + if isEffectivelyNullValue(value) { + continue + } + nonNullFields = append(nonNullFields, field) + } + if len(nonNullFields) == 0 || len(nonNullFields) > 3 { + continue + } + slices.Sort(nonNullFields) + report.Diagnostics = append(report.Diagnostics, Diagnostic{ + Severity: SeverityWarning, + Path: dataset.BasePath, + Message: fmt.Sprintf("row id %d has no canonical key but still carries fields %s; this usually indicates a key-only override hit a retired row or a row was partially de-identified", rowID, strings.Join(nonNullFields, ", ")), + }) + } +} + +func isEffectivelyNullValue(value any) bool { + switch typed := value.(type) { + case nil: + return true + case string: + return strings.TrimSpace(typed) == "" || strings.TrimSpace(typed) == nullValue + default: + return false + } +} + type requiredFeatFamily struct { FamilyKey string Dataset string diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go index 2a745f7..9aa0eff 100644 --- a/internal/topdata/topdata_test.go +++ b/internal/topdata/topdata_test.go @@ -11242,6 +11242,66 @@ func TestBuildPackageIgnoresTemplateSourcesForFreshness(t *testing.T) { } } +func TestValidateProjectWarnsWhenKeyOnlyOverrideMissesCanonicalKeyByCase(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{ + "output": "baseitems.2da", + "columns": ["Label", "MaxRange"], + "rows": [ + {"id": 0, "key": "baseitems:whip", "Label": "Whip", "MaxRange": "3"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:whip":0}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "ovr_baseitems_whip.json"), `{ + "overrides": [ + {"key": "baseitems:Whip", "MaxRange": "4"} + ] +}`+"\n") + + report := ValidateProject(testProject(root)) + if report.HasErrors() { + t.Fatalf("expected warning-only validation result, got errors:\n%s", diagnosticsText(report.Diagnostics)) + } + if report.WarningCount() == 0 { + t.Fatal("expected authoring warning for key-only override case mismatch") + } + text := diagnosticsText(report.Diagnostics) + if !strings.Contains(text, `override 0 targets key "baseitems:Whip", but the live canonical key at that point is "baseitems:whip"`) { + t.Fatalf("expected near-miss override warning, got:\n%s", text) + } +} + +func TestValidateProjectWarnsWhenLockKeepsNormalizedEquivalentKeysAlive(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "feat")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{ + "output": "feat.2da", + "columns": ["LABEL", "FEAT", "DESCRIPTION"], + "rows": [ + {"id": 0, "key": "feat:trackless_step", "LABEL": "TRACKLESS_STEP", "FEAT": "100", "DESCRIPTION": "200"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{ + "feat:trackless_step": 0, + "feat:tracklessstep": 1200 +}`+"\n") + + report := ValidateProject(testProject(root)) + if report.HasErrors() { + t.Fatalf("expected warning-only validation result, got errors:\n%s", diagnosticsText(report.Diagnostics)) + } + if report.WarningCount() == 0 { + t.Fatal("expected normalized-equivalent lock warning") + } + text := diagnosticsText(report.Diagnostics) + if !strings.Contains(text, `lockfile keeps multiple normalized-equivalent keys alive: feat:trackless_step, feat:tracklessstep`) { + t.Fatalf("expected normalized-equivalent key warning, got:\n%s", text) + } +} + func TestValidateProjectErrorsOnTopPackageAssetCollision(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust"))