ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

stricter topdata validation (#24)
test / test (push) Successful in 1m23s
Details
build-binaries / build-binaries (push) Successful in 2m13s
Details
build-image / publish (push) Successful in 39s
Details

Browse Source 
makes topdata json validation much stricter, currently still too permissive

Reviewed-on: #24
Reviewed-by: xtul <mpiasecki720@protonmail.com>
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
This commit was merged in pull request #24.
This commit is contained in:
archvillainette
2026-06-25 21:49:03 +00:00
committed by archvillainette
parent da6bf17f69
commit 332a24fd3e
7 changed files with 2773 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/topdata/topdata.go
+68 -82
View File
@@ -333,6 +333,17 @@ func validateDuplicateJSONKeys(path string, raw []byte, report *ValidationReport
}
}
func reportTopdataContractError(path string, err error, report *ValidationReport) {
if err == nil {
return
}
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityError,
Path: path,
Message: err.Error(),
})
}
func scanJSONValueForDuplicateKeys(decoder *json.Decoder, path string, report *ValidationReport) error {
token, err := decoder.Token()
if err != nil {
@@ -425,35 +436,40 @@ func validateDataObject(path string, obj map[string]any, report *ValidationRepor
return
}
if base == "base.json" {
reportTopdataContractError(path, unsupportedObjectKeysError("base.json root", obj, baseOrPlainRootKeys...), report)
validateRowsFile(path, obj, report, true)
return
}
if inModules {
if _, hasEntries := obj["entries"]; hasEntries {
validateEntriesFile(path, obj, report)
return
}
if _, hasOverrides := obj["overrides"]; hasOverrides {
validateOverridesFile(path, obj, report)
return
}
if _, hasRows := obj["rows"]; hasRows {
validateRowsFile(path, obj, report, false)
return
}
if _, hasColumns := obj["columns"]; hasColumns {
reportTopdataContractError(path, unsupportedObjectKeysError("module root", obj, canonicalModuleRootKeys...), report)
_, hasColumns := obj["columns"]
_, hasEntries := obj["entries"]
_, hasOverrides := obj["overrides"]
_, hasRows := obj["rows"]
if hasColumns {
validateColumnsFile(path, obj, report)
return
}
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityWarning,
Path: path,
Message: "module file does not use a recognized canonical shape yet; expected one of columns, entries, overrides, or rows",
})
if hasEntries {
validateEntriesFileWithColumns(path, obj, report, false)
}
if hasOverrides {
validateOverridesFileWithColumns(path, obj, report, false)
}
if hasRows {
validateRowsFileWithColumns(path, obj, report, false, false)
}
if !hasColumns && !hasEntries && !hasOverrides && !hasRows {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityWarning,
Path: path,
Message: "module file does not use a recognized canonical shape yet; expected one of columns, entries, overrides, or rows",
})
}
return
}
if _, hasRows := obj["rows"]; hasRows {
reportTopdataContractError(path, unsupportedObjectKeysError("plain rows root", obj, baseOrPlainRootKeys...), report)
validateRowsFile(path, obj, report, false)
return
}
@@ -516,6 +532,7 @@ func datasetInvariantForPath(dataPath topdataDataPath) (datasetValidationInvaria
}
func validateDatasetInvariantBaseFile(path string, obj map[string]any, invariant datasetValidationInvariant, report *ValidationReport) {
reportTopdataContractError(path, unsupportedObjectKeysError("base.json root", obj, baseOrPlainRootKeys...), report)
validateRowsFile(path, obj, report, true)
validateDerivedBaseOutput(path, obj, invariant.Name, report)
validateRequiredColumns(path, obj, invariant, report)
@@ -826,6 +843,10 @@ func validateLockObject(path string, obj map[string]any, report *ValidationRepor
}
func validateRowsFile(path string, obj map[string]any, report *ValidationReport, requireColumns bool) {
validateRowsFileWithColumns(path, obj, report, requireColumns, true)
}
func validateRowsFileWithColumns(path string, obj map[string]any, report *ValidationReport, requireColumns bool, validateColumns bool) {
rows, ok := obj["rows"]
if !ok {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -845,7 +866,7 @@ func validateRowsFile(path string, obj map[string]any, report *ValidationReport,
if rowsList, ok := rows.([]any); ok {
validateRowCollection(path, obj, rowsList, report)
}
if _, ok := obj["columns"]; ok {
if _, ok := obj["columns"]; ok && validateColumns {
validateColumnsFile(path, obj, report)
} else if requireColumns {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -896,6 +917,10 @@ func validateColumnsFile(path string, obj map[string]any, report *ValidationRepo
}
func validateEntriesFile(path string, obj map[string]any, report *ValidationReport) {
validateEntriesFileWithColumns(path, obj, report, true)
}
func validateEntriesFileWithColumns(path string, obj map[string]any, report *ValidationReport, validateColumns bool) {
entries, ok := obj["entries"]
if !ok {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -931,12 +956,16 @@ func validateEntriesFile(path string, obj map[string]any, report *ValidationRepo
validateInlineTextUsage(path, key, entry, report)
}
}
if _, ok := obj["columns"]; ok {
if _, ok := obj["columns"]; ok && validateColumns {
validateColumnsFile(path, obj, report)
}
}
func validateOverridesFile(path string, obj map[string]any, report *ValidationReport) {
validateOverridesFileWithColumns(path, obj, report, true)
}
func validateOverridesFileWithColumns(path string, obj map[string]any, report *ValidationReport, validateColumns bool) {
overrides, ok := obj["overrides"]
if !ok {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -957,20 +986,21 @@ func validateOverridesFile(path string, obj map[string]any, report *ValidationRe
container := map[string]any{"rows": overrideList}
validateRowCollection(path, container, overrideList, report)
}
if _, ok := obj["columns"]; ok {
if _, ok := obj["columns"]; ok && validateColumns {
validateColumnsFile(path, obj, report)
}
}
func validateGlobalJSONFile(path string, obj map[string]any, report *ValidationReport) {
reportTopdataContractError(path, unsupportedObjectKeysError("global.json root", obj, globalRootKeys...), report)
if _, ok := obj["columns"]; ok {
validateColumnsFile(path, obj, report)
}
if _, ok := obj["entries"]; ok {
validateEntriesFile(path, obj, report)
validateEntriesFileWithColumns(path, obj, report, false)
}
if _, ok := obj["overrides"]; ok {
validateOverridesFile(path, obj, report)
validateOverridesFileWithColumns(path, obj, report, false)
}
if _, ok := obj["defaults"]; ok {
validateGlobalDefaults(path, obj, report)
@@ -1029,8 +1059,12 @@ func validateGlobalJSONFile(path string, obj map[string]any, report *ValidationR
} else {
rows = append(rows, row)
}
validateGlobalConditionList(path, fmt.Sprintf("global injection %d require_present", index), injection["require_present"], report)
validateGlobalConditionList(path, fmt.Sprintf("global injection %d unless_present", index), injection["unless_present"], report)
_, _ = parseGlobalConditionGroups(injection, globalConditionParseOptions{
Label: fmt.Sprintf("global injection %d", index),
Report: func(err error) {
reportTopdataContractError(path, err, report)
},
})
}
validateRowCollection(path, obj, rows, report)
}
@@ -1099,6 +1133,7 @@ func validateGlobalDefaults(path string, obj map[string]any, report *ValidationR
})
continue
}
reportTopdataContractError(path, unsupportedObjectKeysError(fmt.Sprintf("global default %d", index), defaultRule, globalDefaultRuleKeys...), report)
validateGlobalDefaultMatch(path, index, defaultRule["match"], report)
rawValues, ok := defaultRule["values"]
if !ok {
@@ -1155,14 +1190,12 @@ func validateGlobalDefaultMatch(path string, index int, rawMatch any, report *Va
})
return
}
for key := range match {
if key != "source" {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityError,
Path: path,
Message: fmt.Sprintf("global default %d match.%s is not supported", index, key),
})
}
for _, key := range unsupportedObjectKeys(match, globalDefaultMatchKeys...) {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityError,
Path: path,
Message: fmt.Sprintf("global default %d match.%s is not supported (unsupported key; supported keys: %s)", index, key, strings.Join(globalDefaultMatchKeys, ", ")),
})
}
source, ok := match["source"].(string)
if !ok || strings.TrimSpace(source) == "" {
@@ -1193,14 +1226,7 @@ func validateGlobalDefaultValue(path string, index int, field string, value any,
if !hasFormat {
return
}
if len(obj) != 1 {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityError,
Path: path,
Message: fmt.Sprintf("global default %d values.%s format object must contain only format", index, field),
})
return
}
reportTopdataContractError(path, unsupportedObjectKeysError(fmt.Sprintf("global default %d values.%s", index, field), obj, globalDefaultFormatKeys...), report)
format, ok := rawFormat.(string)
if !ok {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -1220,46 +1246,6 @@ func validateGlobalDefaultValue(path string, index int, field string, value any,
}
}
func validateGlobalConditionList(path, label string, raw any, report *ValidationReport) {
if raw == nil {
return
}
conditions, ok := raw.([]any)
if !ok {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityError,
Path: path,
Message: fmt.Sprintf("%s must be a JSON array", label),
})
return
}
for index, rawCondition := range conditions {
condition, ok := rawCondition.(map[string]any)
if !ok {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityError,
Path: path,
Message: fmt.Sprintf("%s[%d] must be an object", label, index),
})
continue
}
if field, ok := condition["field"].(string); !ok || strings.TrimSpace(field) == "" {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityError,
Path: path,
Message: fmt.Sprintf("%s[%d].field is required", label, index),
})
}
if id, ok := condition["id"].(string); !ok || strings.TrimSpace(id) == "" {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityError,
Path: path,
Message: fmt.Sprintf("%s[%d].id is required", label, index),
})
}
}
}
func validateStateObject(path string, obj map[string]any, report *ValidationReport) {
entries, ok := obj["entries"].(map[string]any)
if !ok {