Validation hardening

2026-05-14 15:01:04 +02:00
parent 651336d48e
commit 2cef662e8e
3 changed files with 337 additions and 0 deletions
@@ -272,6 +272,7 @@ func ValidateProject(p *project.Project) ValidationReport {
 		_ = validateJSONFile(statePath, "state", &report)
 	}
 	if !report.HasErrors() {
+		validateNativeAuthoringWarnings(dataDir, &report)
 		validateNativeBuildability(p, &report)
 	}

@@ -1137,6 +1138,274 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
 	}
 }

+func validateNativeAuthoringWarnings(dataDir string, report *ValidationReport) {
+	datasets, err := discoverNativeDatasets(dataDir)
+	if err != nil {
+		return
+	}
+	for _, dataset := range datasets {
+		if dataset.Kind != nativeDatasetBase {
+			continue
+		}
+		validateDatasetOverrideTargetWarnings(dataset, report)
+		validateDatasetNormalizedLockWarnings(dataset, report)
+		validateDatasetGhostRowWarnings(dataset, report)
+	}
+}
+
+func validateDatasetOverrideTargetWarnings(dataset nativeDataset, report *ValidationReport) {
+	if dataset.BasePath == "" {
+		return
+	}
+	if _, err := os.Stat(dataset.BasePath); err != nil {
+		return
+	}
+	baseObj, err := loadJSONObject(dataset.BasePath)
+	if err != nil {
+		return
+	}
+	rawRows, ok := baseObj["rows"].([]any)
+	if !ok {
+		return
+	}
+	rowIDToKey := map[int]string{}
+	keyToID := map[string]int{}
+	usedIDs := map[int]struct{}{}
+	for index, raw := range rawRows {
+		row, ok := raw.(map[string]any)
+		if !ok {
+			continue
+		}
+		rowID := index
+		if rawID, ok := row["id"]; ok {
+			if parsed, err := asInt(rawID); err == nil {
+				rowID = parsed
+			}
+		}
+		usedIDs[rowID] = struct{}{}
+		if key, ok := row["key"].(string); ok && key != "" {
+			rowIDToKey[rowID] = key
+			keyToID[key] = rowID
+		}
+	}
+	nextID := nextAvailableID(usedIDs)
+	allocateNextID := func() int {
+		rowID := nextID
+		usedIDs[rowID] = struct{}{}
+		nextID = nextAvailableID(usedIDs)
+		return rowID
+	}
+	modulePaths, err := collectModulePaths(dataset.ModulesDir)
+	if err != nil {
+		return
+	}
+	for _, path := range modulePaths {
+		obj, err := loadJSONObject(path)
+		if err != nil {
+			continue
+		}
+		if rawEntries, ok := obj["entries"].(map[string]any); ok {
+			for key := range rawEntries {
+				if key == "" {
+					continue
+				}
+				if _, exists := keyToID[key]; exists {
+					continue
+				}
+				rowID := allocateNextID()
+				keyToID[key] = rowID
+				rowIDToKey[rowID] = key
+			}
+		}
+		rawOverrides, ok := obj["overrides"].([]any)
+		if !ok {
+			continue
+		}
+		for index, raw := range rawOverrides {
+			override, ok := raw.(map[string]any)
+			if !ok {
+				continue
+			}
+			keyOnlyTarget := false
+			keyText, hasKeyText := override["key"].(string)
+			if _, hasID := override["id"]; !hasID && hasKeyText && keyText != "" {
+				keyOnlyTarget = true
+				if _, exists := keyToID[keyText]; !exists {
+					canonical := resolveCanonicalDatasetKey(keyText, keyToID)
+					if canonical != keyText {
+						report.Diagnostics = append(report.Diagnostics, Diagnostic{
+							Severity: SeverityWarning,
+							Path:     path,
+							Message:  fmt.Sprintf("override %d targets key %q, but the live canonical key at that point is %q; this key-only override will create or retarget a new row", index, keyText, canonical),
+						})
+					} else {
+						report.Diagnostics = append(report.Diagnostics, Diagnostic{
+							Severity: SeverityWarning,
+							Path:     path,
+							Message:  fmt.Sprintf("override %d targets key %q, but no live row owns that key at that point; this key-only override will create or retarget a new row", index, keyText),
+						})
+					}
+				}
+			}
+			rowID := -1
+			if rawID, ok := override["id"]; ok {
+				parsed, err := asInt(rawID)
+				if err == nil {
+					rowID = parsed
+				}
+			} else if hasKeyText && keyText != "" {
+				if mappedID, exists := keyToID[keyText]; exists {
+					rowID = mappedID
+				}
+			}
+			if rowID < 0 {
+				if keyOnlyTarget {
+					rowID = allocateNextID()
+					keyToID[keyText] = rowID
+					rowIDToKey[rowID] = keyText
+				}
+				continue
+			}
+			currentKey := rowIDToKey[rowID]
+			if overrideRequestsNullRow(override) {
+				if currentKey != "" {
+					delete(keyToID, currentKey)
+					delete(rowIDToKey, rowID)
+				}
+				continue
+			}
+			if rawKey, present := override["key"]; present {
+				switch typed := rawKey.(type) {
+				case nil:
+					if currentKey != "" {
+						delete(keyToID, currentKey)
+						delete(rowIDToKey, rowID)
+					}
+				case string:
+					if typed == "" || strings.TrimSpace(typed) == nullValue {
+						if currentKey != "" {
+							delete(keyToID, currentKey)
+							delete(rowIDToKey, rowID)
+						}
+						continue
+					}
+					if currentKey != "" && currentKey != typed {
+						delete(keyToID, currentKey)
+					}
+					if previousID, exists := keyToID[typed]; exists && previousID != rowID {
+						delete(rowIDToKey, previousID)
+					}
+					keyToID[typed] = rowID
+					rowIDToKey[rowID] = typed
+				}
+			}
+		}
+	}
+}
+
+func validateDatasetNormalizedLockWarnings(dataset nativeDataset, report *ValidationReport) {
+	lockData, err := loadLockfile(dataset.LockPath)
+	if err != nil {
+		return
+	}
+	groups := map[string][]string{}
+	for key := range lockData {
+		prefix, suffix, ok := strings.Cut(key, ":")
+		if !ok || prefix == "" || suffix == "" {
+			continue
+		}
+		groups[prefix+":"+normalizeKeyIdentity(suffix)] = append(groups[prefix+":"+normalizeKeyIdentity(suffix)], key)
+	}
+	for _, keys := range groups {
+		if len(keys) < 2 {
+			continue
+		}
+		slices.Sort(keys)
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityWarning,
+			Path:     dataset.LockPath,
+			Message:  fmt.Sprintf("lockfile keeps multiple normalized-equivalent keys alive: %s", strings.Join(keys, ", ")),
+		})
+	}
+}
+
+func validateDatasetGhostRowWarnings(dataset nativeDataset, report *ValidationReport) {
+	if dataset.BasePath == "" {
+		return
+	}
+	if _, err := os.Stat(dataset.BasePath); err != nil {
+		return
+	}
+	baseObj, err := loadJSONObject(dataset.BasePath)
+	if err != nil {
+		return
+	}
+	rawRows, ok := baseObj["rows"].([]any)
+	if !ok {
+		return
+	}
+	baseKeyedIDs := map[int]struct{}{}
+	for index, raw := range rawRows {
+		row, ok := raw.(map[string]any)
+		if !ok {
+			continue
+		}
+		if key, ok := row["key"].(string); !ok || key == "" {
+			continue
+		}
+		rowID := index
+		if rawID, ok := row["id"]; ok {
+			if parsed, err := asInt(rawID); err == nil {
+				rowID = parsed
+			}
+		}
+		baseKeyedIDs[rowID] = struct{}{}
+	}
+	collected, err := collectNativeDataset(dataset)
+	if err != nil {
+		return
+	}
+	for _, row := range collected.Rows {
+		if key, ok := row["key"].(string); ok && key != "" {
+			continue
+		}
+		rowID, _ := row["id"].(int)
+		if _, hadBaseKey := baseKeyedIDs[rowID]; !hadBaseKey {
+			continue
+		}
+		nonNullFields := make([]string, 0, 4)
+		for field, value := range row {
+			if field == "id" || field == "key" || field == "meta" {
+				continue
+			}
+			if isEffectivelyNullValue(value) {
+				continue
+			}
+			nonNullFields = append(nonNullFields, field)
+		}
+		if len(nonNullFields) == 0 || len(nonNullFields) > 3 {
+			continue
+		}
+		slices.Sort(nonNullFields)
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityWarning,
+			Path:     dataset.BasePath,
+			Message:  fmt.Sprintf("row id %d has no canonical key but still carries fields %s; this usually indicates a key-only override hit a retired row or a row was partially de-identified", rowID, strings.Join(nonNullFields, ", ")),
+		})
+	}
+}
+
+func isEffectivelyNullValue(value any) bool {
+	switch typed := value.(type) {
+	case nil:
+		return true
+	case string:
+		return strings.TrimSpace(typed) == "" || strings.TrimSpace(typed) == nullValue
+	default:
+		return false
+	}
+}
+
 type requiredFeatFamily struct {
 	FamilyKey string
 	Dataset   string