Optimization Pass

internal/app/app.go

@@ -436,7 +436,12 @@ func runBuildTopData(ctx context) error {
 		return err
 	}
 
-	result, err := topdata.BuildAndPackage(p, func(message string) {
+	opts, err := parseBuildTopDataArgs("build-topdata", ctx.args[1:])
+	if err != nil {
+		return err
+	}
+
+	result, err := topdata.BuildAndPackageWithOptions(p, opts, func(message string) {
 		fmt.Fprintf(ctx.stdout, "[build-topdata] %s\n", message)
 	})
 	if err != nil {
@@ -467,7 +472,12 @@ func runBuildTopPackage(ctx context) error {
 		return err
 	}
 
-	result, err := topdata.BuildAndPackage(p, func(message string) {
+	opts, err := parseBuildTopDataArgs("build-top-package", ctx.args[1:])
+	if err != nil {
+		return err
+	}
+
+	result, err := topdata.BuildAndPackageWithOptions(p, opts, func(message string) {
 		fmt.Fprintf(ctx.stdout, "[build-top-package] %s\n", message)
 	})
 	if err != nil {
@@ -492,6 +502,21 @@ func runBuildTopPackage(ctx context) error {
 	return nil
 }
 
+func parseBuildTopDataArgs(commandName string, args []string) (topdata.BuildAndPackageOptions, error) {
+	opts := topdata.BuildAndPackageOptions{}
+	for _, arg := range args {
+		switch arg {
+		case "--force":
+			opts.Force = true
+		case "-h", "--help":
+			return opts, fmt.Errorf("usage: %s [--force]", commandName)
+		default:
+			return opts, fmt.Errorf("unknown %s argument %q", commandName, arg)
+		}
+	}
+	return opts, nil
+}
+
 func runCompareTopData(ctx context) error {
 	p, err := loadProject(ctx.cwd)
 	if err != nil {

internal/topdata/native.go

@@ -2063,6 +2063,9 @@ func featSourceID(value any) (int, error) {
 }
 
 func (c *featGeneratedContext) resolveGeneratedFeatIdentityBySource(familyKey, slug string, rawSource any) (string, int, bool, error) {
+	if ref, ok := rawSource.(string); ok && strings.HasPrefix(ref, "feat:") {
+		return c.resolveGeneratedFeatIdentityFromRef(familyKey, ref)
+	}
 	if rawSource != nil && rawSource != nullValue {
 		if parsedID, err := featSourceID(rawSource); err == nil && parsedID > 0 {
 			if generatedKey, ok := c.generatedFeatKeyForID(familyKey, parsedID); ok {
@@ -2073,6 +2076,15 @@ func (c *featGeneratedContext) resolveGeneratedFeatIdentityBySource(familyKey, s
 	}
 	if rawMap, ok := rawSource.(map[string]any); ok {
 		if ref, ok := rawMap["id"].(string); ok && strings.HasPrefix(ref, "feat:") {
+			return c.resolveGeneratedFeatIdentityFromRef(familyKey, ref)
+		}
+	}
+	featKey := c.preferredGeneratedFeatKey(familyKey, slug)
+	rowID, hasID, err := c.featID(featKey, rawSource)
+	return featKey, rowID, hasID, err
+}
+
+func (c *featGeneratedContext) resolveGeneratedFeatIdentityFromRef(familyKey, ref string) (string, int, bool, error) {
 	if generatedKey, rowID, ok := c.generatedFeatKeyFromCanonicalAlias(ref); ok {
 		return generatedKey, rowID, true, nil
 	}
@@ -2086,11 +2098,6 @@ func (c *featGeneratedContext) resolveGeneratedFeatIdentityBySource(familyKey, s
 	rowID, hasID, err := c.featID(ref, nil)
 	return ref, rowID, hasID, err
 }
-	}
-	featKey := c.preferredGeneratedFeatKey(familyKey, slug)
-	rowID, hasID, err := c.featID(featKey, rawSource)
-	return featKey, rowID, hasID, err
-}
 
 func (c *featGeneratedContext) resolveGeneratedFeatIdentity(spec familyExpansionSpec, slug string, sourceRow map[string]any) (string, int, bool, error) {
 	if spec.IdentitySource == "child_source_value" {

internal/topdata/top_package.go

@@ -56,9 +56,41 @@ func BuildPackage(p *project.Project, progress func(string)) (PackageResult, err
 }
 
 func BuildAndPackage(p *project.Project, progress func(string)) (PackageResult, error) {
+	return BuildAndPackageWithOptions(p, BuildAndPackageOptions{}, progress)
+}
+
+type BuildAndPackageOptions struct {
+	Force bool
+}
+
+func BuildAndPackageWithOptions(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) {
 	if progress == nil {
 		progress = func(string) {}
 	}
+	if !opts.Force {
+		incrementalResult, ok, err := currentTopPackageResult(p)
+		if err != nil {
+			return PackageResult{}, err
+		}
+		if ok {
+			progress("Topdata outputs are current; skipping native build and top package refresh.")
+			return incrementalResult, nil
+		}
+
+		nativeResult, ok, err := currentCompiledBuildResult(p)
+		if err != nil {
+			return PackageResult{}, err
+		}
+		if ok {
+			progress("Compiled topdata outputs are current; refreshing sow_top.hak and sow_tlk.tlk only.")
+			return packageBuiltTopData(p, nativeResult, progress)
+		}
+	}
+
+	return buildAndPackageFull(p, progress)
+}
+
+func buildAndPackageFull(p *project.Project, progress func(string)) (PackageResult, error) {
 	nativeResult, err := BuildNative(p, progress)
 	if err != nil {
 		return PackageResult{}, err
@@ -323,6 +355,109 @@ func packagedBuildResult(p *project.Project) (BuildResult, error) {
 	}, nil
 }
 
+func currentCompiledBuildResult(p *project.Project) (BuildResult, bool, error) {
+	nativeResult, err := packagedBuildResult(p)
+	if err == nil {
+		return nativeResult, true, nil
+	}
+	if strings.Contains(err.Error(), "run build-topdata first") {
+		return BuildResult{}, false, nil
+	}
+	return BuildResult{}, false, err
+}
+
+func currentTopPackageResult(p *project.Project) (PackageResult, bool, error) {
+	nativeResult, ok, err := currentCompiledBuildResult(p)
+	if err != nil || !ok {
+		return PackageResult{}, ok, err
+	}
+
+	outputHAK := filepath.Join(p.BuildDir(), PackageHAKFileName)
+	outputTLK := filepath.Join(p.BuildDir(), PackageTLKFileName)
+	hakInfo, err := os.Stat(outputHAK)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return PackageResult{}, false, nil
+		}
+		return PackageResult{}, false, fmt.Errorf("stat top package hak %s: %w", outputHAK, err)
+	}
+	tlkInfo, err := os.Stat(outputTLK)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return PackageResult{}, false, nil
+		}
+		return PackageResult{}, false, fmt.Errorf("stat top package tlk %s: %w", outputTLK, err)
+	}
+
+	newestSource, _, err := newestTopDataSource(p.TopDataSourceDir())
+	if err != nil {
+		return PackageResult{}, false, err
+	}
+	newestCompiled, err := newestFileModTime(nativeResult.Output2DADir, ".2da")
+	if err != nil {
+		return PackageResult{}, false, err
+	}
+	compiledTLKInfo, err := os.Stat(compiledTLKOutputPath(p))
+	if err != nil {
+		return PackageResult{}, false, fmt.Errorf("stat compiled tlk output %s: %w", compiledTLKOutputPath(p), err)
+	}
+	if compiledTLKInfo.ModTime().After(newestCompiled) {
+		newestCompiled = compiledTLKInfo.ModTime()
+	}
+	newestInput := newestSource
+	if newestCompiled.After(newestInput) {
+		newestInput = newestCompiled
+	}
+
+	oldestPackage := hakInfo.ModTime()
+	if tlkInfo.ModTime().Before(oldestPackage) {
+		oldestPackage = tlkInfo.ModTime()
+	}
+	if !newestInput.IsZero() && newestInput.After(oldestPackage) {
+		return PackageResult{}, false, nil
+	}
+
+	resources, assetFiles, err := collectTopPackageResources(p, nativeResult.Output2DADir)
+	if err != nil {
+		return PackageResult{}, false, err
+	}
+	return PackageResult{
+		Mode:          "incremental-noop",
+		Output2DADir:  nativeResult.Output2DADir,
+		OutputTLKDir:  nativeResult.OutputTLKDir,
+		OutputHAKPath: outputHAK,
+		OutputTLKPath: outputTLK,
+		Files2DA:      nativeResult.Files2DA,
+		FilesTLK:      nativeResult.FilesTLK,
+		HAKResources:  len(resources),
+		AssetFiles:    assetFiles,
+		WikiOutputDir: nativeResult.WikiOutputDir,
+		WikiPages:     nativeResult.WikiPages,
+		WikiStatus:    nativeResult.WikiStatus,
+	}, true, nil
+}
+
+func newestFileModTime(dir, extension string) (time.Time, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("read output dir %s: %w", dir, err)
+	}
+	newest := time.Time{}
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), extension) {
+			continue
+		}
+		info, err := entry.Info()
+		if err != nil {
+			return time.Time{}, fmt.Errorf("stat output %s: %w", filepath.Join(dir, entry.Name()), err)
+		}
+		if newest.IsZero() || info.ModTime().After(newest) {
+			newest = info.ModTime()
+		}
+	}
+	return newest, nil
+}
+
 func countCompiledFiles(dir, extension string) (int, time.Time, error) {
 	entries, err := os.ReadDir(dir)
 	if err != nil {

internal/topdata/topdata.go

@@ -265,6 +265,7 @@ func ValidateProject(p *project.Project) ValidationReport {
 	validateTopPackageAssets(sourceDir, dataDir, &report)
 	validateNativeOutputCatalog(dataDir, &report)
 	validateNativeLockAllocation(dataDir, &report)
+	validateGeneratedFeatFamilies(dataDir, &report)
 	validateItempropsRegistryGraph(dataDir, &report)
 	if _, err := os.Stat(statePath); err == nil {
 		report.Files++
@@ -289,6 +290,7 @@ func validateJSONFile(path, domain string, report *ValidationReport) error {
 	}
 
 	var payload any
+	validateDuplicateJSONKeys(path, raw, report)
 	if err := json.Unmarshal(raw, &payload); err != nil {
 		report.Diagnostics = append(report.Diagnostics, Diagnostic{
 			Severity: SeverityError,
@@ -321,6 +323,62 @@ func validateJSONFile(path, domain string, report *ValidationReport) error {
 	return nil
 }
 
+func validateDuplicateJSONKeys(path string, raw []byte, report *ValidationReport) {
+	decoder := json.NewDecoder(bytes.NewReader(raw))
+	if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil {
+		return
+	}
+}
+
+func scanJSONValueForDuplicateKeys(decoder *json.Decoder, path string, report *ValidationReport) error {
+	token, err := decoder.Token()
+	if err != nil {
+		return err
+	}
+	switch delim := token.(type) {
+	case json.Delim:
+		switch delim {
+		case '{':
+			seen := map[string]struct{}{}
+			for decoder.More() {
+				rawKey, err := decoder.Token()
+				if err != nil {
+					return err
+				}
+				key, ok := rawKey.(string)
+				if !ok {
+					return fmt.Errorf("expected object key")
+				}
+				if _, ok := seen[key]; ok {
+					report.Diagnostics = append(report.Diagnostics, Diagnostic{
+						Severity: SeverityError,
+						Path:     path,
+						Message:  fmt.Sprintf("duplicate JSON object key %q", key),
+					})
+				}
+				seen[key] = struct{}{}
+				if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil {
+					return err
+				}
+			}
+			_, err := decoder.Token()
+			return err
+		case '[':
+			for decoder.More() {
+				if err := scanJSONValueForDuplicateKeys(decoder, path, report); err != nil {
+					return err
+				}
+			}
+			_, err := decoder.Token()
+			return err
+		default:
+			return nil
+		}
+	default:
+		return nil
+	}
+}
+
 func validateDataObject(path string, obj map[string]any, report *ValidationReport) {
 	base := filepath.Base(path)
 	slashed := filepath.ToSlash(path)
@@ -689,10 +747,16 @@ func validateLegacyDialogEntries(path string, obj map[string]any, report *Valida
 }
 
 func validateLockObject(path string, obj map[string]any, report *ValidationReport) {
+	ids := map[int]string{}
+	checkDuplicateIDs := !strings.Contains(filepath.ToSlash(path), "/data/itemprops/registry/lock.json")
 	for key, value := range obj {
+		var rowID int
 		switch value.(type) {
 		case float64:
 			// JSON numbers land here.
+			if parsed, err := asInt(value); err == nil {
+				rowID = parsed
+			}
 		default:
 			report.Diagnostics = append(report.Diagnostics, Diagnostic{
 				Severity: SeverityError,
@@ -700,6 +764,19 @@ func validateLockObject(path string, obj map[string]any, report *ValidationRepor
 				Message:  fmt.Sprintf("lock entry %q must be numeric", key),
 			})
 		}
+		if checkDuplicateIDs {
+			if rowID <= 0 {
+				continue
+			}
+			if other, ok := ids[rowID]; ok && other != key {
+				report.Diagnostics = append(report.Diagnostics, Diagnostic{
+					Severity: SeverityError,
+					Path:     path,
+					Message:  fmt.Sprintf("lock id collision between %q and %q at id %d", other, key, rowID),
+				})
+			}
+			ids[rowID] = key
+		}
 	}
 }
 
@@ -1030,6 +1107,304 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
 	}
 }
 
+type requiredFeatFamily struct {
+	FamilyKey string
+	Dataset   string
+	Column    string
+	Predicate string
+}
+
+func validateGeneratedFeatFamilies(dataDir string, report *ValidationReport) {
+	featDataset, ok, err := discoverNamedNativeDataset(dataDir, "feat")
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     dataDir,
+			Message:  fmt.Sprintf("discover feat dataset for generated family validation: %v", err),
+		})
+		return
+	}
+	if !ok {
+		return
+	}
+	lockData, err := loadLockfile(featDataset.LockPath)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     featDataset.LockPath,
+			Message:  fmt.Sprintf("load feat lockfile for generated family validation: %v", err),
+		})
+		return
+	}
+	ctx, err := newFeatGeneratedContext(featDataset, lockData)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     featDataset.GeneratedDir,
+			Message:  fmt.Sprintf("prepare generated family validation: %v", err),
+		})
+		return
+	}
+	specs, specPaths, err := collectGeneratedFamilySpecs(featDataset.GeneratedDir)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     featDataset.GeneratedDir,
+			Message:  fmt.Sprintf("load generated family specs: %v", err),
+		})
+		return
+	}
+	for familyKey, spec := range specs {
+		ctx.familySpecs[familyKey] = spec
+	}
+
+	required := []requiredFeatFamily{
+		{FamilyKey: "skillfocus", Dataset: "skills", Predicate: "accessible"},
+		{FamilyKey: "greaterskillfocus", Dataset: "skills", Predicate: "accessible"},
+		{FamilyKey: "weaponfocus", Dataset: "baseitems", Column: "WeaponFocusFeat"},
+		{FamilyKey: "weaponspecialization", Dataset: "baseitems", Column: "WeaponSpecializationFeat"},
+		{FamilyKey: "improvedcritical", Dataset: "baseitems", Column: "WeaponImprovedCriticalFeat"},
+		{FamilyKey: "overwhelmingcritical", Dataset: "baseitems", Column: "EpicWeaponOverwhelmingCriticalFeat"},
+		{FamilyKey: "greaterweaponfocus", Dataset: "baseitems", Column: "EpicWeaponFocusFeat"},
+		{FamilyKey: "greaterweaponspecialization", Dataset: "baseitems", Column: "EpicWeaponSpecializationFeat"},
+		{FamilyKey: "weaponofchoice", Dataset: "baseitems", Column: "WeaponOfChoiceFeat"},
+	}
+	for _, requirement := range required {
+		spec, ok := specs[requirement.FamilyKey]
+		if !ok {
+			continue
+		}
+		path := specPaths[requirement.FamilyKey]
+		if spec.ChildSource.Dataset != requirement.Dataset ||
+			spec.ChildSource.Column != requirement.Column ||
+			spec.ChildSource.Predicate != requirement.Predicate {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("generated feat family %q must use child source dataset=%q column=%q predicate=%q", requirement.FamilyKey, requirement.Dataset, requirement.Column, requirement.Predicate),
+			})
+			continue
+		}
+		validateGeneratedFeatFamilyCompleteness(path, spec, ctx, report)
+	}
+	validateRequiredProductionGeneratedFamilies(featDataset.GeneratedDir, specs, report)
+}
+
+func validateRequiredProductionGeneratedFamilies(path string, specs map[string]familyExpansionSpec, report *ValidationReport) {
+	weaponFamilies := []string{
+		"weaponfocus",
+		"weaponspecialization",
+		"improvedcritical",
+		"overwhelmingcritical",
+		"greaterweaponfocus",
+		"greaterweaponspecialization",
+	}
+	for _, familyKey := range weaponFamilies {
+		if _, ok := specs[familyKey]; !ok {
+			return
+		}
+	}
+	if _, ok := specs["weaponofchoice"]; !ok {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  "missing generated feat family \"weaponofchoice\"",
+		})
+	}
+}
+
+func discoverNamedNativeDataset(dataDir, name string) (nativeDataset, bool, error) {
+	datasets, err := discoverNativeDatasets(dataDir)
+	if err != nil {
+		return nativeDataset{}, false, err
+	}
+	for _, dataset := range datasets {
+		if dataset.Name == name {
+			return dataset, true, nil
+		}
+	}
+	return nativeDataset{}, false, nil
+}
+
+func collectGeneratedFamilySpecs(generatedDir string) (map[string]familyExpansionSpec, map[string]string, error) {
+	paths, err := collectModulePaths(generatedDir)
+	if err != nil {
+		return nil, nil, err
+	}
+	specs := map[string]familyExpansionSpec{}
+	pathsByFamily := map[string]string{}
+	for _, path := range paths {
+		obj, err := loadJSONObject(path)
+		if err != nil {
+			return nil, nil, err
+		}
+		if !isFamilyExpansionObject(obj) {
+			continue
+		}
+		spec, err := parseFamilyExpansionSpec(path, obj)
+		if err != nil {
+			return nil, nil, err
+		}
+		if previous, ok := pathsByFamily[spec.FamilyKey]; ok {
+			return nil, nil, fmt.Errorf("generated family %q is declared by both %s and %s", spec.FamilyKey, previous, path)
+		}
+		specs[spec.FamilyKey] = spec
+		pathsByFamily[spec.FamilyKey] = path
+	}
+	return specs, pathsByFamily, nil
+}
+
+func validateGeneratedFeatFamilyCompleteness(path string, spec familyExpansionSpec, ctx *featGeneratedContext, report *ValidationReport) {
+	sourceRows, err := ctx.datasetRows(spec.ChildSource.Dataset)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  err.Error(),
+		})
+		return
+	}
+	moduleData, err := buildFamilyExpansionGeneratedModule(path, map[string]any{
+		"family":              spec.Family,
+		"family_key":          spec.FamilyKey,
+		"template":            spec.Template,
+		"name_prefix":         spec.NamePrefix,
+		"label_prefix":        spec.LabelPrefix,
+		"constant_prefix":     spec.ConstantPrefix,
+		"template_fields":     validationStringSliceToAny(spec.TemplateFields),
+		"default_fields":      spec.DefaultFields,
+		"apply_after_modules": spec.ApplyAfterModules,
+		"child_ref_field":     spec.ChildRefField,
+		"identity_source":     spec.IdentitySource,
+		"allow_existing_only": spec.AllowExistingOnly,
+		"auto_prereq_fields":  stringMapToAny(spec.AutoPrereqFields),
+		"child_source": map[string]any{
+			"dataset":   spec.ChildSource.Dataset,
+			"column":    spec.ChildSource.Column,
+			"predicate": spec.ChildSource.Predicate,
+		},
+		"overrides": map[string]any{},
+	}, ctx)
+	if err != nil {
+		report.Diagnostics = append(report.Diagnostics, Diagnostic{
+			Severity: SeverityError,
+			Path:     path,
+			Message:  err.Error(),
+		})
+		return
+	}
+	expected := map[string]struct{}{}
+	familyAllowlist := spec.AllowExistingOnly && ctx.familyHasExistingRows(spec.FamilyKey)
+	for sourceKey, row := range sourceRows {
+		include, err := familyExpansionSourceEligible(spec, row)
+		if err != nil {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  err.Error(),
+			})
+			return
+		}
+		if include {
+			if familyAllowlist {
+				slug := strings.TrimPrefix(sourceKey, spec.ChildSource.Dataset+":")
+				featKey, _, _, err := ctx.resolveGeneratedFeatIdentity(spec, slug, row)
+				if err != nil {
+					report.Diagnostics = append(report.Diagnostics, Diagnostic{
+						Severity: SeverityError,
+						Path:     path,
+						Message:  err.Error(),
+					})
+					return
+				}
+				if !ctx.featKeyExists(featKey) {
+					continue
+				}
+			}
+			expected[sourceKey] = struct{}{}
+		}
+	}
+	actual := map[string]string{}
+	overrides, _ := moduleData["overrides"].([]any)
+	for index, raw := range overrides {
+		override, ok := raw.(map[string]any)
+		if !ok {
+			continue
+		}
+		source, ok := familySourceFromMeta(override["meta"])
+		if !ok || source == "" {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("generated family %q row %d is missing family source metadata", spec.FamilyKey, index),
+			})
+			continue
+		}
+		if previousKey, ok := actual[source]; ok {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("generated family %q emits source %q more than once (%s and row %d)", spec.FamilyKey, source, previousKey, index),
+			})
+		}
+		key, _ := override["key"].(string)
+		actual[source] = key
+	}
+	for source := range expected {
+		if _, ok := actual[source]; !ok {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("generated family %q is missing source %q", spec.FamilyKey, source),
+			})
+		}
+	}
+	for source := range actual {
+		if _, ok := expected[source]; !ok {
+			report.Diagnostics = append(report.Diagnostics, Diagnostic{
+				Severity: SeverityError,
+				Path:     path,
+				Message:  fmt.Sprintf("generated family %q includes unexpected source %q", spec.FamilyKey, source),
+			})
+		}
+	}
+}
+
+func familySourceFromMeta(raw any) (string, bool) {
+	meta, ok := raw.(map[string]any)
+	if !ok {
+		return "", false
+	}
+	family, ok := meta["family"].(map[string]any)
+	if !ok {
+		return "", false
+	}
+	source, ok := family["source"].(string)
+	return source, ok
+}
+
+func validationStringSliceToAny(items []string) []any {
+	if items == nil {
+		return nil
+	}
+	out := make([]any, 0, len(items))
+	for _, item := range items {
+		out = append(out, item)
+	}
+	return out
+}
+
+func stringMapToAny(items map[string]string) map[string]any {
+	if items == nil {
+		return nil
+	}
+	out := make(map[string]any, len(items))
+	for key, value := range items {
+		out[key] = value
+	}
+	return out
+}
+
 func validateTopPackageAssets(sourceDir, dataDir string, report *ValidationReport) {
 	assetsDir := filepath.Join(sourceDir, "assets")
 	generated2DADir := filepath.Join(assetsDir, "2da")
@@ -1237,6 +1612,8 @@ func restoreLockfiles(snapshot lockfileSnapshot) {
 
 func validateRowCollection(path string, obj map[string]any, rows []any, report *ValidationReport) {
 	columns := extractValidationColumns(obj)
+	declaredKeys := map[string]int{}
+	checkDuplicateKeys := filepath.Base(path) != "base.json"
 	rowHasIdentity := func(row map[string]any, fallbackID int) (string, bool) {
 		if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" {
 			return key, true
@@ -1252,6 +1629,18 @@ func validateRowCollection(path string, obj map[string]any, rows []any, report *
 		if !ok {
 			continue
 		}
+		if checkDuplicateKeys {
+			if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" {
+				if previous, ok := declaredKeys[key]; ok {
+					report.Diagnostics = append(report.Diagnostics, Diagnostic{
+						Severity: SeverityError,
+						Path:     path,
+						Message:  fmt.Sprintf("row %d repeats key %q already declared by row %d", index, key, previous),
+					})
+				}
+				declaredKeys[key] = index
+			}
+		}
 		for key := range row {
 			if isAuthoringOnlyField(key) && !preserveLegacyWikiColumnForPath(path, key) {
 				report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -1604,7 +1993,7 @@ func compareNativeSelfCheck(p *project.Project, actual2DA, actualTLK string, pro
 	if !slices.Equal(actual2DAFiles, expected2DAFiles) {
 		mismatch += symmetricDiffCount(actual2DAFiles, expected2DAFiles)
 	}
-	comparedTLK, tlkMismatch, err := compareExactDirFiles(actualTLK, nativeResult.OutputTLKDir)
+	comparedTLK, tlkMismatch, err := compareExactDirFilesByExtension(actualTLK, nativeResult.OutputTLKDir, ".tlk")
 	if err != nil {
 		return 0, 0, 0, 0, err
 	}
@@ -1732,6 +2121,10 @@ func ioDiscardProgress(progress func(string)) *progressWriter {
 }
 
 func listRelativeFiles(root string) ([]string, error) {
+	return listRelativeFilesByExtension(root, "")
+}
+
+func listRelativeFilesByExtension(root, extension string) ([]string, error) {
 	files := make([]string, 0)
 	if _, err := os.Stat(root); err != nil {
 		if os.IsNotExist(err) {
@@ -1746,6 +2139,9 @@ func listRelativeFiles(root string) ([]string, error) {
 		if d.IsDir() {
 			return nil
 		}
+		if extension != "" && !strings.EqualFold(filepath.Ext(path), extension) {
+			return nil
+		}
 		rel, err := filepath.Rel(root, path)
 		if err != nil {
 			return err
@@ -1784,11 +2180,15 @@ func symmetricDiffCount(left, right []string) int {
 }
 
 func compareExactDirFiles(actualRoot, expectedRoot string) (int, int, error) {
-	actualFiles, err := listRelativeFiles(actualRoot)
+	return compareExactDirFilesByExtension(actualRoot, expectedRoot, "")
+}
+
+func compareExactDirFilesByExtension(actualRoot, expectedRoot, extension string) (int, int, error) {
+	actualFiles, err := listRelativeFilesByExtension(actualRoot, extension)
 	if err != nil {
 		return 0, 0, err
 	}
-	expectedFiles, err := listRelativeFiles(expectedRoot)
+	expectedFiles, err := listRelativeFilesByExtension(expectedRoot, extension)
 	if err != nil {
 		return 0, 0, err
 	}

internal/topdata/topdata_test.go

@@ -44,6 +44,26 @@ func TestValidateProjectAcceptsInlineTLKSchema(t *testing.T) {
 	}
 }
 
+func TestValidateProjectRejectsDuplicateJSONKeys(t *testing.T) {
+	root := testProjectRoot(t)
+	mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust"))
+	writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{
+  "output": "repadjust.2da",
+  "output": "other.2da",
+  "columns": ["Label"],
+  "rows": [{"id": 0, "Label": "TEST_LABEL"}]
+}`+"\n")
+
+	report := ValidateProject(testProject(root))
+	if !report.HasErrors() {
+		t.Fatal("expected duplicate JSON key validation error")
+	}
+	if !diagnosticsContain(report.Diagnostics, "duplicate JSON object key") {
+		t.Fatalf("expected duplicate JSON object key diagnostic, got %#v", report.Diagnostics)
+	}
+}
+
 func TestResolvedTableRegistryRegistersConsistentTables(t *testing.T) {
 	collected := []nativeCollectedDataset{
 		{
@@ -9085,6 +9105,72 @@ func TestBuildAndPackageIncludesCompiled2DAAndTopAssets(t *testing.T) {
 	}
 }
 
+func TestBuildAndPackageNoOpsWhenOutputsAreCurrent(t *testing.T) {
+	root := topPackageTestProject(t)
+	proj := testProject(root)
+	proj.Config.TopData.ReferenceBuilder = ""
+
+	result, err := BuildAndPackage(proj, nil)
+	if err != nil {
+		t.Fatalf("initial BuildAndPackage failed: %v", err)
+	}
+
+	sourceTime := time.Now().Add(-4 * time.Hour)
+	outputTime := time.Now().Add(-2 * time.Hour)
+	setTopDataSourceTimes(t, root, sourceTime)
+	setBuildOutputTimes(t, result, outputTime)
+
+	beforeHAK := mustStat(t, result.OutputHAKPath).ModTime()
+	beforeTLK := mustStat(t, result.OutputTLKPath).ModTime()
+	incremental, err := BuildAndPackage(proj, nil)
+	if err != nil {
+		t.Fatalf("incremental BuildAndPackage failed: %v", err)
+	}
+	if incremental.Mode != "incremental-noop" {
+		t.Fatalf("expected incremental-noop mode, got %q", incremental.Mode)
+	}
+	if after := mustStat(t, result.OutputHAKPath).ModTime(); !after.Equal(beforeHAK) {
+		t.Fatalf("expected hak mtime to be unchanged, got %s want %s", after, beforeHAK)
+	}
+	if after := mustStat(t, result.OutputTLKPath).ModTime(); !after.Equal(beforeTLK) {
+		t.Fatalf("expected tlk mtime to be unchanged, got %s want %s", after, beforeTLK)
+	}
+}
+
+func TestBuildAndPackageRefreshesPackageWhenCompiledOutputsAreCurrent(t *testing.T) {
+	root := topPackageTestProject(t)
+	proj := testProject(root)
+	proj.Config.TopData.ReferenceBuilder = ""
+
+	result, err := BuildAndPackage(proj, nil)
+	if err != nil {
+		t.Fatalf("initial BuildAndPackage failed: %v", err)
+	}
+
+	sourceTime := time.Now().Add(-6 * time.Hour)
+	compiledTime := time.Now().Add(-4 * time.Hour)
+	packageTime := time.Now().Add(-8 * time.Hour)
+	setTopDataSourceTimes(t, root, sourceTime)
+	setCompiledOutputTimes(t, result, compiledTime)
+	setFileTime(t, result.OutputHAKPath, packageTime)
+
+	compiled2DA := filepath.Join(result.Output2DADir, "repadjust.2da")
+	before2DA := mustStat(t, compiled2DA).ModTime()
+	refreshed, err := BuildAndPackage(proj, nil)
+	if err != nil {
+		t.Fatalf("package refresh BuildAndPackage failed: %v", err)
+	}
+	if refreshed.Mode != "native" {
+		t.Fatalf("expected native mode from packaged compiled output, got %q", refreshed.Mode)
+	}
+	if after := mustStat(t, compiled2DA).ModTime(); !after.Equal(before2DA) {
+		t.Fatalf("expected compiled 2da mtime to be unchanged, got %s want %s", after, before2DA)
+	}
+	if after := mustStat(t, result.OutputHAKPath).ModTime(); !after.After(packageTime) {
+		t.Fatalf("expected hak to be refreshed after %s, got %s", packageTime, after)
+	}
+}
+
 func TestNativeCompileGroupStatsCountSourceFragmentsForBaseDatasets(t *testing.T) {
 	root := testProjectRoot(t)
 	modulesDir := filepath.Join(root, "placeables", "modules")
@@ -9136,6 +9222,80 @@ func TestNativeCompileGroupStatsCountSourceFragmentsForBaseDatasets(t *testing.T
 	}
 }
 
+func topPackageTestProject(t *testing.T) string {
+	t.Helper()
+	root := testProjectRoot(t)
+	mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust"))
+	mkdirAll(t, filepath.Join(root, "topdata", "assets", "gui"))
+	writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
+	writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{
+  "output": "repadjust.2da",
+  "columns": ["Label"],
+  "rows": [{"id": 0, "Label": "TEST_LABEL"}]
+}`+"\n")
+	writeFile(t, filepath.Join(root, "topdata", "assets", "gui", "testicon.png"), "icon-data")
+	return root
+}
+
+func setTopDataSourceTimes(t *testing.T, root string, modTime time.Time) {
+	t.Helper()
+	setTreeFileTimes(t, filepath.Join(root, "topdata"), modTime)
+}
+
+func setBuildOutputTimes(t *testing.T, result PackageResult, modTime time.Time) {
+	t.Helper()
+	setCompiledOutputTimes(t, result, modTime)
+	setFileTime(t, result.OutputHAKPath, modTime)
+	setFileTime(t, result.OutputTLKPath, modTime)
+}
+
+func setCompiledOutputTimes(t *testing.T, result PackageResult, modTime time.Time) {
+	t.Helper()
+	setTreeFileTimes(t, result.Output2DADir, modTime)
+	setTreeFileTimes(t, result.OutputTLKDir, modTime)
+}
+
+func setTreeFileTimes(t *testing.T, root string, modTime time.Time) {
+	t.Helper()
+	err := filepath.WalkDir(root, func(path string, d os.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		if d.IsDir() {
+			return nil
+		}
+		return os.Chtimes(path, modTime, modTime)
+	})
+	if err != nil {
+		t.Fatalf("set file times under %s: %v", root, err)
+	}
+}
+
+func setFileTime(t *testing.T, path string, modTime time.Time) {
+	t.Helper()
+	if err := os.Chtimes(path, modTime, modTime); err != nil {
+		t.Fatalf("set file time %s: %v", path, err)
+	}
+}
+
+func mustStat(t *testing.T, path string) os.FileInfo {
+	t.Helper()
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatalf("stat %s: %v", path, err)
+	}
+	return info
+}
+
+func diagnosticsContain(diagnostics []Diagnostic, text string) bool {
+	for _, diagnostic := range diagnostics {
+		if strings.Contains(diagnostic.Message, text) {
+			return true
+		}
+	}
+	return false
+}
+
 func TestValidateProjectAcceptsPNGTopPackageAsset(t *testing.T) {
 	root := testProjectRoot(t)
 	mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust"))