ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Validation hardening

Browse Source
This commit is contained in:
archvillainette
2026-05-14 15:01:04 +02:00
parent 651336d48e
commit 2cef662e8e
3 changed files with 337 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/topdata/native.go
+8
View File
@@ -196,6 +196,14 @@ func BuildNativeWithOptions(p *project.Project, opts NativeBuildOptions, progres
}
return BuildResult{}, fmt.Errorf("topdata validation failed with %d error(s)", report.ErrorCount())
}
if warnings := report.WarningCount(); warnings > 0 {
progress(fmt.Sprintf("topdata validation warnings: %d", warnings))
for _, line := range report.SummaryLines(5) {
if strings.HasPrefix(line, "warning:") {
progress(line)
}
}
}
var prebuiltWiki *wikiResult
if opts.BuildWiki {
if newestWikiSource, _, err := newestRelevantWikiSource(p.TopDataSourceDir()); err == nil && !newestWikiSource.IsZero() {
internal/topdata/topdata.go
+269
View File
@@ -272,6 +272,7 @@ func ValidateProject(p *project.Project) ValidationReport {
_ = validateJSONFile(statePath, "state", &report)
}
if !report.HasErrors() {
validateNativeAuthoringWarnings(dataDir, &report)
validateNativeBuildability(p, &report)
}
@@ -1137,6 +1138,274 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
}
}
func validateNativeAuthoringWarnings(dataDir string, report *ValidationReport) {
datasets, err := discoverNativeDatasets(dataDir)
if err != nil {
return
}
for _, dataset := range datasets {
if dataset.Kind != nativeDatasetBase {
continue
}
validateDatasetOverrideTargetWarnings(dataset, report)
validateDatasetNormalizedLockWarnings(dataset, report)
validateDatasetGhostRowWarnings(dataset, report)
}
}
func validateDatasetOverrideTargetWarnings(dataset nativeDataset, report *ValidationReport) {
if dataset.BasePath == "" {
return
}
if _, err := os.Stat(dataset.BasePath); err != nil {
return
}
baseObj, err := loadJSONObject(dataset.BasePath)
if err != nil {
return
}
rawRows, ok := baseObj["rows"].([]any)
if !ok {
return
}
rowIDToKey := map[int]string{}
keyToID := map[string]int{}
usedIDs := map[int]struct{}{}
for index, raw := range rawRows {
row, ok := raw.(map[string]any)
if !ok {
continue
}
rowID := index
if rawID, ok := row["id"]; ok {
if parsed, err := asInt(rawID); err == nil {
rowID = parsed
}
}
usedIDs[rowID] = struct{}{}
if key, ok := row["key"].(string); ok && key != "" {
rowIDToKey[rowID] = key
keyToID[key] = rowID
}
}
nextID := nextAvailableID(usedIDs)
allocateNextID := func() int {
rowID := nextID
usedIDs[rowID] = struct{}{}
nextID = nextAvailableID(usedIDs)
return rowID
}
modulePaths, err := collectModulePaths(dataset.ModulesDir)
if err != nil {
return
}
for _, path := range modulePaths {
obj, err := loadJSONObject(path)
if err != nil {
continue
}
if rawEntries, ok := obj["entries"].(map[string]any); ok {
for key := range rawEntries {
if key == "" {
continue
}
if _, exists := keyToID[key]; exists {
continue
}
rowID := allocateNextID()
keyToID[key] = rowID
rowIDToKey[rowID] = key
}
}
rawOverrides, ok := obj["overrides"].([]any)
if !ok {
continue
}
for index, raw := range rawOverrides {
override, ok := raw.(map[string]any)
if !ok {
continue
}
keyOnlyTarget := false
keyText, hasKeyText := override["key"].(string)
if _, hasID := override["id"]; !hasID && hasKeyText && keyText != "" {
keyOnlyTarget = true
if _, exists := keyToID[keyText]; !exists {
canonical := resolveCanonicalDatasetKey(keyText, keyToID)
if canonical != keyText {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityWarning,
Path: path,
Message: fmt.Sprintf("override %d targets key %q, but the live canonical key at that point is %q; this key-only override will create or retarget a new row", index, keyText, canonical),
})
} else {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityWarning,
Path: path,
Message: fmt.Sprintf("override %d targets key %q, but no live row owns that key at that point; this key-only override will create or retarget a new row", index, keyText),
})
}
}
}
rowID := -1
if rawID, ok := override["id"]; ok {
parsed, err := asInt(rawID)
if err == nil {
rowID = parsed
}
} else if hasKeyText && keyText != "" {
if mappedID, exists := keyToID[keyText]; exists {
rowID = mappedID
}
}
if rowID < 0 {
if keyOnlyTarget {
rowID = allocateNextID()
keyToID[keyText] = rowID
rowIDToKey[rowID] = keyText
}
continue
}
currentKey := rowIDToKey[rowID]
if overrideRequestsNullRow(override) {
if currentKey != "" {
delete(keyToID, currentKey)
delete(rowIDToKey, rowID)
}
continue
}
if rawKey, present := override["key"]; present {
switch typed := rawKey.(type) {
case nil:
if currentKey != "" {
delete(keyToID, currentKey)
delete(rowIDToKey, rowID)
}
case string:
if typed == "" || strings.TrimSpace(typed) == nullValue {
if currentKey != "" {
delete(keyToID, currentKey)
delete(rowIDToKey, rowID)
}
continue
}
if currentKey != "" && currentKey != typed {
delete(keyToID, currentKey)
}
if previousID, exists := keyToID[typed]; exists && previousID != rowID {
delete(rowIDToKey, previousID)
}
keyToID[typed] = rowID
rowIDToKey[rowID] = typed
}
}
}
}
}
func validateDatasetNormalizedLockWarnings(dataset nativeDataset, report *ValidationReport) {
lockData, err := loadLockfile(dataset.LockPath)
if err != nil {
return
}
groups := map[string][]string{}
for key := range lockData {
prefix, suffix, ok := strings.Cut(key, ":")
if !ok || prefix == "" || suffix == "" {
continue
}
groups[prefix+":"+normalizeKeyIdentity(suffix)] = append(groups[prefix+":"+normalizeKeyIdentity(suffix)], key)
}
for _, keys := range groups {
if len(keys) < 2 {
continue
}
slices.Sort(keys)
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityWarning,
Path: dataset.LockPath,
Message: fmt.Sprintf("lockfile keeps multiple normalized-equivalent keys alive: %s", strings.Join(keys, ", ")),
})
}
}
func validateDatasetGhostRowWarnings(dataset nativeDataset, report *ValidationReport) {
if dataset.BasePath == "" {
return
}
if _, err := os.Stat(dataset.BasePath); err != nil {
return
}
baseObj, err := loadJSONObject(dataset.BasePath)
if err != nil {
return
}
rawRows, ok := baseObj["rows"].([]any)
if !ok {
return
}
baseKeyedIDs := map[int]struct{}{}
for index, raw := range rawRows {
row, ok := raw.(map[string]any)
if !ok {
continue
}
if key, ok := row["key"].(string); !ok || key == "" {
continue
}
rowID := index
if rawID, ok := row["id"]; ok {
if parsed, err := asInt(rawID); err == nil {
rowID = parsed
}
}
baseKeyedIDs[rowID] = struct{}{}
}
collected, err := collectNativeDataset(dataset)
if err != nil {
return
}
for _, row := range collected.Rows {
if key, ok := row["key"].(string); ok && key != "" {
continue
}
rowID, _ := row["id"].(int)
if _, hadBaseKey := baseKeyedIDs[rowID]; !hadBaseKey {
continue
}
nonNullFields := make([]string, 0, 4)
for field, value := range row {
if field == "id" || field == "key" || field == "meta" {
continue
}
if isEffectivelyNullValue(value) {
continue
}
nonNullFields = append(nonNullFields, field)
}
if len(nonNullFields) == 0 || len(nonNullFields) > 3 {
continue
}
slices.Sort(nonNullFields)
report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityWarning,
Path: dataset.BasePath,
Message: fmt.Sprintf("row id %d has no canonical key but still carries fields %s; this usually indicates a key-only override hit a retired row or a row was partially de-identified", rowID, strings.Join(nonNullFields, ", ")),
})
}
}
func isEffectivelyNullValue(value any) bool {
switch typed := value.(type) {
case nil:
return true
case string:
return strings.TrimSpace(typed) == "" || strings.TrimSpace(typed) == nullValue
default:
return false
}
}
type requiredFeatFamily struct {
FamilyKey string
Dataset string
internal/topdata/topdata_test.go
+60
View File
@@ -11242,6 +11242,66 @@ func TestBuildPackageIgnoresTemplateSourcesForFreshness(t *testing.T) {
}
}
func TestValidateProjectWarnsWhenKeyOnlyOverrideMissesCanonicalKeyByCase(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules"))
writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "base.json"), `{
"output": "baseitems.2da",
"columns": ["Label", "MaxRange"],
"rows": [
{"id": 0, "key": "baseitems:whip", "Label": "Whip", "MaxRange": "3"}
]
}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "lock.json"), `{"baseitems:whip":0}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "baseitems", "modules", "ovr_baseitems_whip.json"), `{
"overrides": [
{"key": "baseitems:Whip", "MaxRange": "4"}
]
}`+"\n")
report := ValidateProject(testProject(root))
if report.HasErrors() {
t.Fatalf("expected warning-only validation result, got errors:\n%s", diagnosticsText(report.Diagnostics))
}
if report.WarningCount() == 0 {
t.Fatal("expected authoring warning for key-only override case mismatch")
}
text := diagnosticsText(report.Diagnostics)
if !strings.Contains(text, `override 0 targets key "baseitems:Whip", but the live canonical key at that point is "baseitems:whip"`) {
t.Fatalf("expected near-miss override warning, got:\n%s", text)
}
}
func TestValidateProjectWarnsWhenLockKeepsNormalizedEquivalentKeysAlive(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "feat"))
writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(root, "topdata", "data", "feat", "base.json"), `{
"output": "feat.2da",
"columns": ["LABEL", "FEAT", "DESCRIPTION"],
"rows": [
{"id": 0, "key": "feat:trackless_step", "LABEL": "TRACKLESS_STEP", "FEAT": "100", "DESCRIPTION": "200"}
]
}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "feat", "lock.json"), `{
"feat:trackless_step": 0,
"feat:tracklessstep": 1200
}`+"\n")
report := ValidateProject(testProject(root))
if report.HasErrors() {
t.Fatalf("expected warning-only validation result, got errors:\n%s", diagnosticsText(report.Diagnostics))
}
if report.WarningCount() == 0 {
t.Fatal("expected normalized-equivalent lock warning")
}
text := diagnosticsText(report.Diagnostics)
if !strings.Contains(text, `lockfile keeps multiple normalized-equivalent keys alive: feat:trackless_step, feat:tracklessstep`) {
t.Fatalf("expected normalized-equivalent key warning, got:\n%s", text)
}
}
func TestValidateProjectErrorsOnTopPackageAssetCollision(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust"))