Optimization Pass

internal/topdata/topdata.go

@@ -265,6 +265,7 @@ func ValidateProject(p *project.Project) ValidationReport {
 	validateTopPackageAssets(sourceDir, dataDir, &report)
 	validateNativeOutputCatalog(dataDir, &report)
 	validateNativeLockAllocation(dataDir, &report)
+	validateGeneratedFeatFamilies(dataDir, &report)
 	validateItempropsRegistryGraph(dataDir, &report)
 	if _, err := os.Stat(statePath); err == nil {
 		report.Files++
@@ -289,6 +290,7 @@ func validateJSONFile(path, domain string, report *ValidationReport) error {
 	}
 
 	var payload any
+	validateDuplicateJSONKeys(path, raw, report)
 	if err := json.Unmarshal(raw, &payload); err != nil {
 		report.Diagnostics = append(report.Diagnostics, Diagnostic{
 			Severity: SeverityError,
@@ -321,6 +323,62 @@ func validateJSONFile(path, domain string, report *ValidationReport) error {
 	return nil
 }
 
+func validateDuplicateJSONKeys(path string, raw []byte, report *ValidationReport) {
+	decoder := json.NewDecoder(bytes.NewReader(raw))
+	if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil {
+		return
+	}
+}
+
+func scanJSONValueForDuplicateKeys(decoder *json.Decoder, path string, report *ValidationReport) error {
+	token, err := decoder.Token()
+	if err != nil {
+		return err
+	}
+	switch delim := token.(type) {
+	case json.Delim:
+		switch delim {
+		case '{':
+			seen := map[string]struct{}{}
+			for decoder.More() {
+				rawKey, err := decoder.Token()
+				if err != nil {
+					return err
+				}
+				key, ok := rawKey.(string)
+				if !ok {
+					return fmt.Errorf("expected object key")
+				}
+				if _, ok := seen[key]; ok {
+					report.Diagnostics = append(report.Diagnostics, Diagnostic{
+						Severity: SeverityError,
+						Path:     path,
+						Message:  fmt.Sprintf("duplicate JSON object key %q", key),
+					})
+				}
+				seen[key] = struct{}{}
+				if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil {
+					return err
+				}
+			}
+			_, err := decoder.Token()
+			return err
+		case '[':
+			for decoder.More() {
+				if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil {
+					return err
+				}
+			}
+			_, err := decoder.Token()
+			return err
+		default:
+			return nil
+		}
+	default:
+		return nil
+	}
+}
+
 func validateDataObject(path string, obj map[string]any, report *ValidationReport) {
 	base := filepath.Base(path)
 	slashed := filepath.ToSlash(path)
@@ -689,10 +747,16 @@ func validateLegacyDialogEntries(path string, obj map[string]any, report *Valida
 }
 
 func validateLockObject(path string, obj map[string]any, report *ValidationReport) {
+	ids := map[int]string{}
+	checkDuplicateIDs := !strings.Contains(filepath.ToSlash(path), "/data/itemprops/registry/lock.json")
 	for key, value := range obj {
+		var rowID int
 		switch value.(type) {
 		case float64:
 			// JSON numbers land here.
+			if parsed, err := asInt(value); err == nil {
+				rowID = parsed
+			}
 		default:
 			report.Diagnostics = append(report.Diagnostics, Diagnostic{
 				Severity: SeverityError,
@@ -700,6 +764,19 @@ func validateLockObject(path string, obj map[string]any, report *ValidationRepor
 				Message:  fmt.Sprintf("lock entry %q must be numeric", key),
 			})
 		}
+		if checkDuplicateIDs {
+			if rowID <= 0 {
+				continue
+			}
+			if other, ok := ids[rowID]; ok && other != key {
+				report.Diagnostics = append(report.Diagnostics, Diagnostic{
+					Severity: SeverityError,
+					Path:     path,
+					Message:  fmt.Sprintf("lock id collision between %q and %q at id %d", other, key, rowID),
+				})
+			}
+			ids[rowID] = key
+		}
 	}
 }
 
@@ -1030,6 +1107,304 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
 	}
 }
 
+type requiredFeatFamily struct {
+	FamilyKey string
+	Dataset   string
+	Column    string
+	Predicate string
+}
+
+func validateGeneratedFeatFamilies(dataDir string, report *ValidationReport) {
+	featDataset, ok, err := discoverNamedNativeDataset(dataDir, "feat")
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     dataDir,
+			Message:  fmt.Sprintf("discover feat dataset for generated family validation: %v", err),
+		})
+		return
+	}
+	if !ok {
+		return
+	}
+	lockData, err := loadLockfile(featDataset.LockPath)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     featDataset.LockPath,
+			Message:  fmt.Sprintf("load feat lockfile for generated family validation: %v", err),
+		})
+		return
+	}
+	ctx, err := newFeatGeneratedContext(featDataset, lockData)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     featDataset.GeneratedDir,
+			Message:  fmt.Sprintf("prepare generated family validation: %v", err),
+		})
+		return
+	}
+	specs, specPaths, err := collectGeneratedFamilySpecs(featDataset.GeneratedDir)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     featDataset.GeneratedDir,
+			Message:  fmt.Sprintf("load generated family specs: %v", err),
+		})
+		return
+	}
+	for familyKey, spec := range specs {
+		ctx.familySpecs[familyKey] = spec
+	}
+
+	required := []requiredFeatFamily{
+		{FamilyKey: "skillfocus", Dataset: "skills", Predicate: "accessible"},
+		{FamilyKey: "greaterskillfocus", Dataset: "skills", Predicate: "accessible"},
+		{FamilyKey: "weaponfocus", Dataset: "baseitems", Column: "WeaponFocusFeat"},
+		{FamilyKey: "weaponspecialization", Dataset: "baseitems", Column: "WeaponSpecializationFeat"},
+		{FamilyKey: "improvedcritical", Dataset: "baseitems", Column: "WeaponImprovedCriticalFeat"},
+		{FamilyKey: "overwhelmingcritical", Dataset: "baseitems", Column: "EpicWeaponOverwhelmingCriticalFeat"},
+		{FamilyKey: "greaterweaponfocus", Dataset: "baseitems", Column: "EpicWeaponFocusFeat"},
+		{FamilyKey: "greaterweaponspecialization", Dataset: "baseitems", Column: "EpicWeaponSpecializationFeat"},
+		{FamilyKey: "weaponofchoice", Dataset: "baseitems", Column: "WeaponOfChoiceFeat"},
+	}
+	for _, requirement := range required {
+		spec, ok := specs[requirement.FamilyKey]
+		if !ok {
+			continue
+		}
+		path := specPaths[requirement.FamilyKey]
+		if spec.ChildSource.Dataset != requirement.Dataset ||
+			spec.ChildSource.Column != requirement.Column ||
+			spec.ChildSource.Predicate != requirement.Predicate {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("generated feat family %q must use child source dataset=%q column=%q predicate=%q", requirement.FamilyKey, requirement.Dataset, requirement.Column, requirement.Predicate),
+			})
+			continue
+		}
+		validateGeneratedFeatFamilyCompleteness(path, spec, ctx, report)
+	}
+	validateRequiredProductionGeneratedFamilies(featDataset.GeneratedDir, specs, report)
+}
+
+func validateRequiredProductionGeneratedFamilies(path string, specs map[string]familyExpansionSpec, report *ValidationReport) {
+	weaponFamilies := []string{
+		"weaponfocus",
+		"weaponspecialization",
+		"improvedcritical",
+		"overwhelmingcritical",
+		"greaterweaponfocus",
+		"greaterweaponspecialization",
+	}
+	for _, familyKey := range weaponFamilies {
+		if _, ok := specs[familyKey]; !ok {
+			return
+		}
+	}
+	if _, ok := specs["weaponofchoice"]; !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "missing generated feat family \"weaponofchoice\"",
+		})
+	}
+}
+
+func discoverNamedNativeDataset(dataDir, name string) (nativeDataset, bool, error) {
+	datasets, err := discoverNativeDatasets(dataDir)
+	if err != nil {
+		return nativeDataset{}, false, err
+	}
+	for _, dataset := range datasets {
+		if dataset.Name == name {
+			return dataset, true, nil
+		}
+	}
+	return nativeDataset{}, false, nil
+}
+
+func collectGeneratedFamilySpecs(generatedDir string) (map[string]familyExpansionSpec, map[string]string, error) {
+	paths, err := collectModulePaths(generatedDir)
+	if err != nil {
+		return nil, nil, err
+	}
+	specs := map[string]familyExpansionSpec{}
+	pathsByFamily := map[string]string{}
+	for _, path := range paths {
+		obj, err := loadJSONObject(path)
+		if err != nil {
+			return nil, nil, err
+		}
+		if !isFamilyExpansionObject(obj) {
+			continue
+		}
+		spec, err := parseFamilyExpansionSpec(path, obj)
+		if err != nil {
+			return nil, nil, err
+		}
+		if previous, ok := pathsByFamily[spec.FamilyKey]; ok {
+			return nil, nil, fmt.Errorf("generated family %q is declared by both %s and %s", spec.FamilyKey, previous, path)
+		}
+		specs[spec.FamilyKey] = spec
+		pathsByFamily[spec.FamilyKey] = path
+	}
+	return specs, pathsByFamily, nil
+}
+
+func validateGeneratedFeatFamilyCompleteness(path string, spec familyExpansionSpec, ctx *featGeneratedContext, report *ValidationReport) {
+	sourceRows, err := ctx.datasetRows(spec.ChildSource.Dataset)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  err.Error(),
+		})
+		return
+	}
+	moduleData, err := buildFamilyExpansionGeneratedModule(path, map[string]any{
+		"family":              spec.Family,
+		"family_key":          spec.FamilyKey,
+		"template":            spec.Template,
+		"name_prefix":         spec.NamePrefix,
+		"label_prefix":        spec.LabelPrefix,
+		"constant_prefix":     spec.ConstantPrefix,
+		"template_fields":     validationStringSliceToAny(spec.TemplateFields),
+		"default_fields":      spec.DefaultFields,
+		"apply_after_modules": spec.ApplyAfterModules,
+		"child_ref_field":     spec.ChildRefField,
+		"identity_source":     spec.IdentitySource,
+		"allow_existing_only": spec.AllowExistingOnly,
+		"auto_prereq_fields":  stringMapToAny(spec.AutoPrereqFields),
+		"child_source": map[string]any{
+			"dataset":   spec.ChildSource.Dataset,
+			"column":    spec.ChildSource.Column,
+			"predicate": spec.ChildSource.Predicate,
+		},
+		"overrides": map[string]any{},
+	}, ctx)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  err.Error(),
+		})
+		return
+	}
+	expected := map[string]struct{}{}
+	familyAllowlist := spec.AllowExistingOnly && ctx.familyHasExistingRows(spec.FamilyKey)
+	for sourceKey, row := range sourceRows {
+		include, err := familyExpansionSourceEligible(spec, row)
+		if err != nil {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  err.Error(),
+			})
+			return
+		}
+		if include {
+			if familyAllowlist {
+				slug := strings.TrimPrefix(sourceKey, spec.ChildSource.Dataset+":")
+				featKey, _, _, err := ctx.resolveGeneratedFeatIdentity(spec, slug, row)
+				if err != nil {
+					report.Diagnostics = append(report.Diagnostics, Diagnostic{
+						Severity: SeverityError,
+						Path:     path,
+						Message:  err.Error(),
+					})
+					return
+				}
+				if !ctx.featKeyExists(featKey) {
+					continue
+				}
+			}
+			expected[sourceKey] = struct{}{}
+		}
+	}
+	actual := map[string]string{}
+	overrides, _ := moduleData["overrides"].([]any)
+	for index, raw := range overrides {
+		override, ok := raw.(map[string]any)
+		if !ok {
+			continue
+		}
+		source, ok := familySourceFromMeta(override["meta"])
+		if !ok || source == "" {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("generated family %q row %d is missing family source metadata", spec.FamilyKey, index),
+			})
+			continue
+		}
+		if previousKey, ok := actual[source]; ok {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("generated family %q emits source %q more than once (%s and row %d)", spec.FamilyKey, source, previousKey, index),
+			})
+		}
+		key, _ := override["key"].(string)
+		actual[source] = key
+	}
+	for source := range expected {
+		if _, ok := actual[source]; !ok {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("generated family %q is missing source %q", spec.FamilyKey, source),
+			})
+		}
+	}
+	for source := range actual {
+		if _, ok := expected[source]; !ok {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("generated family %q includes unexpected source %q", spec.FamilyKey, source),
+			})
+		}
+	}
+}
+
+func familySourceFromMeta(raw any) (string, bool) {
+	meta, ok := raw.(map[string]any)
+	if !ok {
+		return "", false
+	}
+	family, ok := meta["family"].(map[string]any)
+	if !ok {
+		return "", false
+	}
+	source, ok := family["source"].(string)
+	return source, ok
+}
+
+func validationStringSliceToAny(items []string) []any {
+	if items == nil {
+		return nil
+	}
+	out := make([]any, 0, len(items))
+	for _, item := range items {
+		out = append(out, item)
+	}
+	return out
+}
+
+func stringMapToAny(items map[string]string) map[string]any {
+	if items == nil {
+		return nil
+	}
+	out := make(map[string]any, len(items))
+	for key, value := range items {
+		out[key] = value
+	}
+	return out
+}
+
 func validateTopPackageAssets(sourceDir, dataDir string, report *ValidationReport) {
 	assetsDir := filepath.Join(sourceDir, "assets")
 	generated2DADir := filepath.Join(assetsDir, "2da")
@@ -1237,6 +1612,8 @@ func restoreLockfiles(snapshot lockfileSnapshot) {
 
 func validateRowCollection(path string, obj map[string]any, rows []any, report *ValidationReport) {
 	columns := extractValidationColumns(obj)
+	declaredKeys := map[string]int{}
+	checkDuplicateKeys := filepath.Base(path) != "base.json"
 	rowHasIdentity := func(row map[string]any, fallbackID int) (string, bool) {
 		if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" {
 			return key, true
@@ -1252,6 +1629,18 @@ func validateRowCollection(path string, obj map[string]any, rows []any, report *
 		if !ok {
 			continue
 		}
+		if checkDuplicateKeys {
+			if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" {
+				if previous, ok := declaredKeys[key]; ok {
+					report.Diagnostics = append(report.Diagnostics, Diagnostic{
+						Severity: SeverityError,
+						Path:     path,
+						Message:  fmt.Sprintf("row %d repeats key %q already declared by row %d", index, key, previous),
+					})
+				}
+				declaredKeys[key] = index
+			}
+		}
 		for key := range row {
 			if isAuthoringOnlyField(key) && !preserveLegacyWikiColumnForPath(path, key) {
 				report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -1604,7 +1993,7 @@ func compareNativeSelfCheck(p *project.Project, actual2DA, actualTLK string, pro
 	if !slices.Equal(actual2DAFiles, expected2DAFiles) {
 		mismatch += symmetricDiffCount(actual2DAFiles, expected2DAFiles)
 	}
-	comparedTLK, tlkMismatch, err := compareExactDirFiles(actualTLK, nativeResult.OutputTLKDir)
+	comparedTLK, tlkMismatch, err := compareExactDirFilesByExtension(actualTLK, nativeResult.OutputTLKDir, ".tlk")
 	if err != nil {
 		return 0, 0, 0, 0, err
 	}
@@ -1732,6 +2121,10 @@ func ioDiscardProgress(progress func(string)) *progressWriter {
 }
 
 func listRelativeFiles(root string) ([]string, error) {
+	return listRelativeFilesByExtension(root, "")
+}
+
+func listRelativeFilesByExtension(root, extension string) ([]string, error) {
 	files := make([]string, 0)
 	if _, err := os.Stat(root); err != nil {
 		if os.IsNotExist(err) {
@@ -1746,6 +2139,9 @@ func listRelativeFiles(root string) ([]string, error) {
 		if d.IsDir() {
 			return nil
 		}
+		if extension != "" && !strings.EqualFold(filepath.Ext(path), extension) {
+			return nil
+		}
 		rel, err := filepath.Rel(root, path)
 		if err != nil {
 			return err
@@ -1784,11 +2180,15 @@ func symmetricDiffCount(left, right []string) int {
 }
 
 func compareExactDirFiles(actualRoot, expectedRoot string) (int, int, error) {
-	actualFiles, err := listRelativeFiles(actualRoot)
+	return compareExactDirFilesByExtension(actualRoot, expectedRoot, "")
+}
+
+func compareExactDirFilesByExtension(actualRoot, expectedRoot, extension string) (int, int, error) {
+	actualFiles, err := listRelativeFilesByExtension(actualRoot, extension)
 	if err != nil {
 		return 0, 0, err
 	}
-	expectedFiles, err := listRelativeFiles(expectedRoot)
+	expectedFiles, err := listRelativeFilesByExtension(expectedRoot, extension)
 	if err != nil {
 		return 0, 0, err
 	}