stricter topdata validation (#24)

makes topdata json validation much stricter, currently still too permissive

Reviewed-on: #24
Reviewed-by: xtul <mpiasecki720@protonmail.com>
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>

internal/topdata/CLASS_FEAT_GLOBAL_INJECTS_CONTRACT.md

@@ -65,6 +65,33 @@ effective output while keeping project policy in topdata authoring files.
 `position` defaults to `append`. The module uses `prepend` for class feat
 injections to keep generated policy rows before per-class authored rows.
 
+## Global Injection Grammar
+
+Each injection entry may use only `row`, `require_present`, `any_present`, and
+`unless_present`. `when_present` is invalid and rejected; it is not an alias.
+
+```json
+{
+  "row": {
+    "FeatIndex": { "id": "feat:example" }
+  },
+  "require_present": [{ "field": "FeatIndex", "id": "feat:required" }],
+  "any_present": [
+    { "field": "FeatIndex", "id": "masterfeats:metamagic" },
+    { "field": "FeatIndex", "id": "masterfeats:combat" }
+  ],
+  "unless_present": [{ "field": "FeatIndex", "id": "feat:example" }]
+}
+```
+
+- `require_present` means every listed reference must exist.
+- `any_present` means at least one listed reference must exist, and the array
+  must not be empty.
+- `unless_present` means none of the listed references may exist.
+- Every present group on an entry must pass.
+- Conditions observe the current rows in injection order, so earlier injections
+  can affect later conditions.
+
 ## Acceptance Criteria
 
 - Class feat output contains the same effective shared feats as the prior YAML

internal/topdata/json_contract.go

@@ -0,0 +1,195 @@
+package topdata
+
+import (
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+)
+
+type globalCondition struct {
+	Field string
+	ID    string
+}
+
+type globalConditionGroups struct {
+	RequirePresent []globalCondition
+	AnyPresent     []globalCondition
+	UnlessPresent  []globalCondition
+}
+
+var (
+	baseOrPlainRootKeys = []string{
+		"output", "key", "columns", "rows", "compare_reference",
+	}
+	canonicalModuleRootKeys = []string{
+		"output", "key", "columns", "entries", "overrides", "rows",
+	}
+	globalRootKeys = []string{
+		"columns", "entries", "overrides", "defaults", "position", "injections",
+	}
+	globalInjectionKeys = []string{
+		"row", "require_present", "any_present", "unless_present",
+	}
+	globalConditionKeys     = []string{"field", "id"}
+	globalDefaultRuleKeys   = []string{"match", "values"}
+	globalDefaultMatchKeys  = []string{"source"}
+	globalDefaultFormatKeys = []string{"format"}
+)
+
+func unsupportedObjectKeys(obj map[string]any, supported ...string) []string {
+	if len(obj) == 0 {
+		return nil
+	}
+	supportedSet := make(map[string]struct{}, len(supported))
+	for _, key := range supported {
+		supportedSet[key] = struct{}{}
+	}
+	unsupported := make([]string, 0, len(obj))
+	for key := range obj {
+		if _, ok := supportedSet[key]; ok {
+			continue
+		}
+		unsupported = append(unsupported, key)
+	}
+	slices.Sort(unsupported)
+	return unsupported
+}
+
+func unsupportedObjectKeysError(label string, obj map[string]any, supported ...string) error {
+	unsupported := unsupportedObjectKeys(obj, supported...)
+	if len(unsupported) == 0 {
+		return nil
+	}
+	return fmt.Errorf("%s contains unsupported key %q (supported keys: %s)", label, unsupported[0], strings.Join(supported, ", "))
+}
+
+type globalConditionParseOptions struct {
+	Label  string
+	Report func(error)
+}
+
+func parseGlobalConditionGroups(injection map[string]any, options ...globalConditionParseOptions) (globalConditionGroups, error) {
+	if injection == nil {
+		return globalConditionGroups{}, nil
+	}
+
+	label := "global injection"
+	var report func(error)
+	if len(options) > 0 {
+		if strings.TrimSpace(options[0].Label) != "" {
+			label = options[0].Label
+		}
+		report = options[0].Report
+	}
+
+	var errs []error
+	addError := func(err error) {
+		if err == nil {
+			return
+		}
+		errs = append(errs, err)
+		if report != nil {
+			report(err)
+		}
+	}
+	if err := unsupportedObjectKeysError(label, injection, globalInjectionKeys...); err != nil {
+		addError(err)
+	}
+
+	var groups globalConditionGroups
+	if raw, ok := injection["require_present"]; ok {
+		parsed, err := parseGlobalConditionList(label+" require_present", raw, false)
+		if err != nil {
+			addError(err)
+		} else {
+			groups.RequirePresent = parsed
+		}
+	}
+	if raw, ok := injection["any_present"]; ok {
+		parsed, err := parseGlobalConditionList(label+" any_present", raw, true)
+		if err != nil {
+			addError(err)
+		} else {
+			groups.AnyPresent = parsed
+		}
+	}
+	if raw, ok := injection["unless_present"]; ok {
+		parsed, err := parseGlobalConditionList(label+" unless_present", raw, false)
+		if err != nil {
+			addError(err)
+		} else {
+			groups.UnlessPresent = parsed
+		}
+	}
+	return groups, errors.Join(errs...)
+}
+
+func parseGlobalConditionList(name string, raw any, requireNonEmpty bool) ([]globalCondition, error) {
+	if raw == nil {
+		return nil, fmt.Errorf("%s must be an array", name)
+	}
+
+	conditions, ok := raw.([]any)
+	if !ok {
+		return nil, fmt.Errorf("%s must be an array", name)
+	}
+	if requireNonEmpty && len(conditions) == 0 {
+		return nil, fmt.Errorf("%s must contain at least one condition", name)
+	}
+
+	parsed := make([]globalCondition, 0, len(conditions))
+	for index, rawCondition := range conditions {
+		condition, ok := rawCondition.(map[string]any)
+		if !ok {
+			return nil, fmt.Errorf("%s[%d] must be an object", name, index)
+		}
+		if err := unsupportedObjectKeysError(fmt.Sprintf("%s[%d]", name, index), condition, globalConditionKeys...); err != nil {
+			return nil, err
+		}
+
+		field, _ := condition["field"].(string)
+		field = strings.TrimSpace(field)
+		if field == "" {
+			return nil, fmt.Errorf("%s[%d].field is required", name, index)
+		}
+
+		id, _ := condition["id"].(string)
+		id = strings.TrimSpace(id)
+		if id == "" {
+			return nil, fmt.Errorf("%s[%d].id is required", name, index)
+		}
+
+		parsed = append(parsed, globalCondition{Field: field, ID: id})
+	}
+	return parsed, nil
+}
+
+func globalConditionGroupsMatch(groups globalConditionGroups, rows []map[string]any) bool {
+	for _, condition := range groups.RequirePresent {
+		if !globalReferencePresent(rows, condition.Field, condition.ID) {
+			return false
+		}
+	}
+	if groups.AnyPresent != nil {
+		if len(groups.AnyPresent) == 0 {
+			return false
+		}
+		matched := false
+		for _, condition := range groups.AnyPresent {
+			if globalReferencePresent(rows, condition.Field, condition.ID) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			return false
+		}
+	}
+	for _, condition := range groups.UnlessPresent {
+		if globalReferencePresent(rows, condition.Field, condition.ID) {
+			return false
+		}
+	}
+	return true
+}

internal/topdata/native.go

@@ -3886,48 +3886,11 @@ func globalManifestPosition(module nativeGeneratedModule) (string, error) {
 }
 
 func globalInjectionConditionsMatch(injection map[string]any, rows []map[string]any) (bool, error) {
-	if rawRequired, ok := injection["require_present"]; ok {
-		matches, err := globalConditionListMatches("require_present", rawRequired, rows, true)
-		if err != nil || !matches {
-			return matches, err
-		}
+	groups, err := parseGlobalConditionGroups(injection)
+	if err != nil {
+		return false, err
 	}
-	if rawBlocked, ok := injection["unless_present"]; ok {
-		matches, err := globalConditionListMatches("unless_present", rawBlocked, rows, false)
-		if err != nil || !matches {
-			return matches, err
-		}
-	}
-	return true, nil
-}
-
-func globalConditionListMatches(name string, raw any, rows []map[string]any, required bool) (bool, error) {
-	conditions, ok := raw.([]any)
-	if !ok {
-		return false, fmt.Errorf("%s must be an array", name)
-	}
-	for index, rawCondition := range conditions {
-		condition, ok := rawCondition.(map[string]any)
-		if !ok {
-			return false, fmt.Errorf("%s[%d] must be an object", name, index)
-		}
-		field, _ := condition["field"].(string)
-		id, _ := condition["id"].(string)
-		if strings.TrimSpace(field) == "" {
-			return false, fmt.Errorf("%s[%d].field is required", name, index)
-		}
-		if strings.TrimSpace(id) == "" {
-			return false, fmt.Errorf("%s[%d].id is required", name, index)
-		}
-		found := globalReferencePresent(rows, field, id)
-		if required && !found {
-			return false, nil
-		}
-		if !required && found {
-			return false, nil
-		}
-	}
-	return true, nil
+	return globalConditionGroupsMatch(groups, rows), nil
 }
 
 func globalRowIdentityPresent(row map[string]any, rows []map[string]any) bool {

internal/topdata/topdata.go

@@ -333,6 +333,17 @@ func validateDuplicateJSONKeys(path string, raw []byte, report *ValidationReport
 	}
 }
 
+func reportTopdataContractError(path string, err error, report *ValidationReport) {
+	if err == nil {
+		return
+	}
+	report.Diagnostics = append(report.Diagnostics, Diagnostic{
+		Severity: SeverityError,
+		Path:     path,
+		Message:  err.Error(),
+	})
+}
+
 func scanJSONValueForDuplicateKeys(decoder *json.Decoder, path string, report *ValidationReport) error {
 	token, err := decoder.Token()
 	if err != nil {
@@ -425,35 +436,40 @@ func validateDataObject(path string, obj map[string]any, report *ValidationRepor
 		return
 	}
 	if base == "base.json" {
+		reportTopdataContractError(path, unsupportedObjectKeysError("base.json root", obj, baseOrPlainRootKeys...), report)
 		validateRowsFile(path, obj, report, true)
 		return
 	}
 	if inModules {
-		if _, hasEntries := obj["entries"]; hasEntries {
-			validateEntriesFile(path, obj, report)
-			return
-		}
-		if _, hasOverrides := obj["overrides"]; hasOverrides {
-			validateOverridesFile(path, obj, report)
-			return
-		}
-		if _, hasRows := obj["rows"]; hasRows {
-			validateRowsFile(path, obj, report, false)
-			return
-		}
-		if _, hasColumns := obj["columns"]; hasColumns {
+		reportTopdataContractError(path, unsupportedObjectKeysError("module root", obj, canonicalModuleRootKeys...), report)
+		_, hasColumns := obj["columns"]
+		_, hasEntries := obj["entries"]
+		_, hasOverrides := obj["overrides"]
+		_, hasRows := obj["rows"]
+		if hasColumns {
 			validateColumnsFile(path, obj, report)
-			return
 		}
-		report.Diagnostics = append(report.Diagnostics, Diagnostic{
-			Severity: SeverityWarning,
-			Path:     path,
-			Message:  "module file does not use a recognized canonical shape yet; expected one of columns, entries, overrides, or rows",
-		})
+		if hasEntries {
+			validateEntriesFileWithColumns(path, obj, report, false)
+		}
+		if hasOverrides {
+			validateOverridesFileWithColumns(path, obj, report, false)
+		}
+		if hasRows {
+			validateRowsFileWithColumns(path, obj, report, false, false)
+		}
+		if !hasColumns && !hasEntries && !hasOverrides && !hasRows {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityWarning,
+				Path:     path,
+				Message:  "module file does not use a recognized canonical shape yet; expected one of columns, entries, overrides, or rows",
+			})
+		}
 		return
 	}
 
 	if _, hasRows := obj["rows"]; hasRows {
+		reportTopdataContractError(path, unsupportedObjectKeysError("plain rows root", obj, baseOrPlainRootKeys...), report)
 		validateRowsFile(path, obj, report, false)
 		return
 	}
@@ -516,6 +532,7 @@ func datasetInvariantForPath(dataPath topdataDataPath) (datasetValidationInvaria
 }
 
 func validateDatasetInvariantBaseFile(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) {
+	reportTopdataContractError(path, unsupportedObjectKeysError("base.json root", obj, baseOrPlainRootKeys...), report)
 	validateRowsFile(path, obj, report, true)
 	validateDerivedBaseOutput(path, obj, invariant.Name, report)
 	validateRequiredColumns(path, obj, invariant, report)
@@ -826,6 +843,10 @@ func validateLockObject(path string, obj map[string]any, report *ValidationRepor
 }
 
 func validateRowsFile(path string, obj map[string]any, report *ValidationReport, requireColumns bool) {
+	validateRowsFileWithColumns(path, obj, report, requireColumns, true)
+}
+
+func validateRowsFileWithColumns(path string, obj map[string]any, report *ValidationReport, requireColumns bool, validateColumns bool) {
 	rows, ok := obj["rows"]
 	if !ok {
 		report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -845,7 +866,7 @@ func validateRowsFile(path string, obj map[string]any, report *ValidationReport,
 	if rowsList, ok := rows.([]any); ok {
 		validateRowCollection(path, obj, rowsList, report)
 	}
-	if _, ok := obj["columns"]; ok {
+	if _, ok := obj["columns"]; ok && validateColumns {
 		validateColumnsFile(path, obj, report)
 	} else if requireColumns {
 		report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -896,6 +917,10 @@ func validateColumnsFile(path string, obj map[string]any, report *ValidationRepo
 }
 
 func validateEntriesFile(path string, obj map[string]any, report *ValidationReport) {
+	validateEntriesFileWithColumns(path, obj, report, true)
+}
+
+func validateEntriesFileWithColumns(path string, obj map[string]any, report *ValidationReport, validateColumns bool) {
 	entries, ok := obj["entries"]
 	if !ok {
 		report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -931,12 +956,16 @@ func validateEntriesFile(path string, obj map[string]any, report *ValidationRepo
 			validateInlineTextUsage(path, key, entry, report)
 		}
 	}
-	if _, ok := obj["columns"]; ok {
+	if _, ok := obj["columns"]; ok && validateColumns {
 		validateColumnsFile(path, obj, report)
 	}
 }
 
 func validateOverridesFile(path string, obj map[string]any, report *ValidationReport) {
+	validateOverridesFileWithColumns(path, obj, report, true)
+}
+
+func validateOverridesFileWithColumns(path string, obj map[string]any, report *ValidationReport, validateColumns bool) {
 	overrides, ok := obj["overrides"]
 	if !ok {
 		report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -957,20 +986,21 @@ func validateOverridesFile(path string, obj map[string]any, report *ValidationRe
 		container := map[string]any{"rows": overrideList}
 		validateRowCollection(path, container, overrideList, report)
 	}
-	if _, ok := obj["columns"]; ok {
+	if _, ok := obj["columns"]; ok && validateColumns {
 		validateColumnsFile(path, obj, report)
 	}
 }
 
 func validateGlobalJSONFile(path string, obj map[string]any, report *ValidationReport) {
+	reportTopdataContractError(path, unsupportedObjectKeysError("global.json root", obj, globalRootKeys...), report)
 	if _, ok := obj["columns"]; ok {
 		validateColumnsFile(path, obj, report)
 	}
 	if _, ok := obj["entries"]; ok {
-		validateEntriesFile(path, obj, report)
+		validateEntriesFileWithColumns(path, obj, report, false)
 	}
 	if _, ok := obj["overrides"]; ok {
-		validateOverridesFile(path, obj, report)
+		validateOverridesFileWithColumns(path, obj, report, false)
 	}
 	if _, ok := obj["defaults"]; ok {
 		validateGlobalDefaults(path, obj, report)
@@ -1029,8 +1059,12 @@ func validateGlobalJSONFile(path string, obj map[string]any, report *ValidationR
 		} else {
 			rows = append(rows, row)
 		}
-		validateGlobalConditionList(path, fmt.Sprintf("global injection %d require_present", index), injection["require_present"], report)
-		validateGlobalConditionList(path, fmt.Sprintf("global injection %d unless_present", index), injection["unless_present"], report)
+		_, _ = parseGlobalConditionGroups(injection, globalConditionParseOptions{
+			Label: fmt.Sprintf("global injection %d", index),
+			Report: func(err error) {
+				reportTopdataContractError(path, err, report)
+			},
+		})
 	}
 	validateRowCollection(path, obj, rows, report)
 }
@@ -1099,6 +1133,7 @@ func validateGlobalDefaults(path string, obj map[string]any, report *ValidationR
 			})
 			continue
 		}
+		reportTopdataContractError(path, unsupportedObjectKeysError(fmt.Sprintf("global default %d", index), defaultRule, globalDefaultRuleKeys...), report)
 		validateGlobalDefaultMatch(path, index, defaultRule["match"], report)
 		rawValues, ok := defaultRule["values"]
 		if !ok {
@@ -1155,14 +1190,12 @@ func validateGlobalDefaultMatch(path string, index int, rawMatch any, report *Va
 		})
 		return
 	}
-	for key := range match {
-		if key != "source" {
-			report.Diagnostics = append(report.Diagnostics, Diagnostic{
-				Severity: SeverityError,
-				Path:     path,
-				Message:  fmt.Sprintf("global default %d match.%s is not supported", index, key),
-			})
-		}
+	for _, key := range unsupportedObjectKeys(match, globalDefaultMatchKeys...) {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  fmt.Sprintf("global default %d match.%s is not supported (unsupported key; supported keys: %s)", index, key, strings.Join(globalDefaultMatchKeys, ", ")),
+		})
 	}
 	source, ok := match["source"].(string)
 	if !ok || strings.TrimSpace(source) == "" {
@@ -1193,14 +1226,7 @@ func validateGlobalDefaultValue(path string, index int, field string, value any,
 	if !hasFormat {
 		return
 	}
-	if len(obj) != 1 {
-		report.Diagnostics = append(report.Diagnostics, Diagnostic{
-			Severity: SeverityError,
-			Path:     path,
-			Message:  fmt.Sprintf("global default %d values.%s format object must contain only format", index, field),
-		})
-		return
-	}
+	reportTopdataContractError(path, unsupportedObjectKeysError(fmt.Sprintf("global default %d values.%s", index, field), obj, globalDefaultFormatKeys...), report)
 	format, ok := rawFormat.(string)
 	if !ok {
 		report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -1220,46 +1246,6 @@ func validateGlobalDefaultValue(path string, index int, field string, value any,
 	}
 }
 
-func validateGlobalConditionList(path, label string, raw any, report *ValidationReport) {
-	if raw == nil {
-		return
-	}
-	conditions, ok := raw.([]any)
-	if !ok {
-		report.Diagnostics = append(report.Diagnostics, Diagnostic{
-			Severity: SeverityError,
-			Path:     path,
-			Message:  fmt.Sprintf("%s must be a JSON array", label),
-		})
-		return
-	}
-	for index, rawCondition := range conditions {
-		condition, ok := rawCondition.(map[string]any)
-		if !ok {
-			report.Diagnostics = append(report.Diagnostics, Diagnostic{
-				Severity: SeverityError,
-				Path:     path,
-				Message:  fmt.Sprintf("%s[%d] must be an object", label, index),
-			})
-			continue
-		}
-		if field, ok := condition["field"].(string); !ok || strings.TrimSpace(field) == "" {
-			report.Diagnostics = append(report.Diagnostics, Diagnostic{
-				Severity: SeverityError,
-				Path:     path,
-				Message:  fmt.Sprintf("%s[%d].field is required", label, index),
-			})
-		}
-		if id, ok := condition["id"].(string); !ok || strings.TrimSpace(id) == "" {
-			report.Diagnostics = append(report.Diagnostics, Diagnostic{
-				Severity: SeverityError,
-				Path:     path,
-				Message:  fmt.Sprintf("%s[%d].id is required", label, index),
-			})
-		}
-	}
-}
-
 func validateStateObject(path string, obj map[string]any, report *ValidationReport) {
 	entries, ok := obj["entries"].(map[string]any)
 	if !ok {