Sorting changes

internal/topdata/AUTO-INCLUDE_EXISTING_PART_MODELS_IN_2DA_GENERATION_CONTRACT.md

@@ -98,6 +98,10 @@ For rows generated from discovered existing models, set:
 - `COSTMODIFIER = 0`
 - `ACBONUS = 0.00`
 
+Rows without discovered existing models may be intentionally emitted as dense
+2DA null rows when configured. In that mode, missing rows must have
+`COSTMODIFIER = ****` and `ACBONUS = ****` in the final generated 2DA.
+
 ### 5. Override precedence
 
 If file-based overrides exist, they must take precedence over discovered defaults.
@@ -132,9 +136,11 @@ The superseded implementation was correct only if all of the following were true
    - `COSTMODIFIER = 0`
    - `ACBONUS = 0.00`
 
-6. File overrides are applied after discovery and take precedence.
-7. The final output includes all eligible existing models present in the manifest.
-8. Discovery does not require a sibling local `sow-assets` checkout or a Git clone.
+6. When configured, rows without discovered models are emitted as dense null
+   rows with `COSTMODIFIER = ****` and `ACBONUS = ****`.
+7. File overrides are applied after discovery and take precedence.
+8. The final output includes all eligible existing models present in the manifest.
+9. Discovery does not require a sibling local `sow-assets` checkout or a Git clone.
 
 ---
 

internal/topdata/generated_assets.go

@@ -101,7 +101,9 @@ func buildGenerated2DAAssetGroup(p *project.Project, cfg project.GeneratedTopDat
 			return nil, err
 		}
 		outputPath := filepath.Join(outputDir, dataset.Dataset.OutputName)
-		if err := write2DA(compiled, outputPath, dataset.Dataset.Kind == nativeDatasetBase); err != nil {
+		denseRows := dataset.Dataset.Kind == nativeDatasetBase ||
+			(consumer.Mode == "parts_rows" && isPartsDataset(dataset.Dataset.Name))
+		if err := write2DA(compiled, outputPath, denseRows); err != nil {
 			return nil, err
 		}
 		results = append(results, Generated2DAAsset{

internal/topdata/parts_discovery.go

@@ -507,6 +507,12 @@ func configuredPartsRowsACBonusValue(rowID int, datasetName string, cfg project.
 		return "", false
 	}
 	switch strings.TrimSpace(policy.Strategy) {
+	case "ascending_row_id_sort_key":
+		format := strings.TrimSpace(policy.Format)
+		if format == "" {
+			format = "%.2f"
+		}
+		return fmt.Sprintf(format, float64(rowID)/float64(policy.Divisor)), true
 	case "descending_row_id_sort_key":
 		format := strings.TrimSpace(policy.Format)
 		if format == "" {
@@ -655,9 +661,17 @@ func augmentWithAutogeneratedPartsWithConfig(collected []nativeCollectedDataset,
 			}
 		}
 
-		// Add rows for discovered IDs that don't already exist
+		// Add rows for discovered IDs that don't already exist.
 		newRows := make([]map[string]any, 0, len(dataset.Rows))
-		newRows = append(newRows, dataset.Rows...)
+		for _, row := range dataset.Rows {
+			rowID, ok := row["id"].(int)
+			if cfg.NullUndiscoveredRows && ok {
+				if _, discovered := ids[rowID]; !discovered {
+					continue
+				}
+			}
+			newRows = append(newRows, row)
+		}
 
 		for rowID := range ids {
 			if row, exists := existingRows[rowID]; exists {

internal/topdata/top_package.go

@@ -493,6 +493,9 @@ func packagedBuildResult(p *project.Project) (BuildResult, error) {
 	if files2DA == 0 {
 		return BuildResult{}, fmt.Errorf("topdata build output missing: run build-topdata first")
 	}
+	if err := validateCompiled2DAOutputCatalog(p, output2DA); err != nil {
+		return BuildResult{}, err
+	}
 
 	newestSource, newestPath, err := newestTopDataInput(p, time.Now())
 	if err != nil {
@@ -515,6 +518,74 @@ func packagedBuildResult(p *project.Project) (BuildResult, error) {
 	}, nil
 }
 
+func validateCompiled2DAOutputCatalog(p *project.Project, output2DA string) error {
+	expected, err := expectedCompiled2DAOutputs(p)
+	if err != nil {
+		return err
+	}
+	actual, err := compiled2DAOutputNames(output2DA)
+	if err != nil {
+		return err
+	}
+
+	missing := make([]string, 0)
+	for outputName := range expected {
+		if _, ok := actual[outputName]; !ok {
+			missing = append(missing, outputName)
+		}
+	}
+	stale := make([]string, 0)
+	for outputName := range actual {
+		if _, ok := expected[outputName]; !ok {
+			stale = append(stale, outputName)
+		}
+	}
+	slices.Sort(missing)
+	slices.Sort(stale)
+	if len(missing) > 0 {
+		return fmt.Errorf("topdata build output is missing compiled 2da %s; run build-topdata first", strings.Join(missing, ", "))
+	}
+	if len(stale) > 0 {
+		return fmt.Errorf("topdata build output contains stale compiled 2da %s; run build-topdata first", strings.Join(stale, ", "))
+	}
+	return nil
+}
+
+func expectedCompiled2DAOutputs(p *project.Project) (map[string]struct{}, error) {
+	dataDir := filepath.Join(p.TopDataSourceDir(), "data")
+	datasets, err := discoverNativeDatasets(dataDir)
+	if err != nil {
+		return nil, err
+	}
+	registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
+	if err != nil {
+		return nil, err
+	}
+	outputs := make(map[string]struct{}, len(datasets)+len(registryDatasets))
+	for _, dataset := range datasets {
+		outputs[dataset.OutputName] = struct{}{}
+	}
+	for _, dataset := range registryDatasets {
+		outputs[dataset.Dataset.OutputName] = struct{}{}
+	}
+	return outputs, nil
+}
+
+func compiled2DAOutputNames(output2DA string) (map[string]struct{}, error) {
+	entries, err := os.ReadDir(output2DA)
+	if err != nil {
+		return nil, fmt.Errorf("read compiled 2da output dir %s: %w", output2DA, err)
+	}
+	outputs := make(map[string]struct{})
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".2da") {
+			continue
+		}
+		outputs[entry.Name()] = struct{}{}
+	}
+	return outputs, nil
+}
+
 func currentCompiledBuildResult(p *project.Project) (BuildResult, bool, error) {
 	nativeResult, err := packagedBuildResult(p)
 	if err == nil {

internal/topdata/topdata_test.go

@@ -10582,6 +10582,37 @@ func TestNormalizePartsRowsACBonusUsesConfiguredPolicyBeforeOverrides(t *testing
 	}
 }
 
+func TestNormalizePartsRowsACBonusSupportsAscendingRowIDSortKey(t *testing.T) {
+	collected := []nativeCollectedDataset{
+		{
+			Dataset: nativeDataset{Name: "parts/belt", OutputName: "parts_belt.2da"},
+			Columns: []string{"COSTMODIFIER", "ACBONUS"},
+			Rows: []map[string]any{
+				{"id": 1, "COSTMODIFIER": "0", "ACBONUS": "****"},
+				{"id": 117, "COSTMODIFIER": "0", "ACBONUS": "****"},
+			},
+		},
+	}
+	cfg := project.PartsRowsConfig{
+		ACBonus: project.PartsRowsACBonusConfig{
+			Default: project.PartsRowsACBonusPolicy{Strategy: "ascending_row_id_sort_key", Divisor: 100, Format: "%.2f"},
+		},
+	}
+
+	got, err := normalizePartsRowsACBonus(collected, cfg)
+	if err != nil {
+		t.Fatalf("normalizePartsRowsACBonus failed: %v", err)
+	}
+
+	beltRows := rowsByID(got[0].Rows)
+	if got := beltRows[1]["ACBONUS"]; got != "0.01" {
+		t.Fatalf("expected low belt row to get lower sort key 0.01, got %v", got)
+	}
+	if got := beltRows[117]["ACBONUS"]; got != "1.17" {
+		t.Fatalf("expected high belt row to get higher sort key 1.17, got %v", got)
+	}
+}
+
 func rowsByID(rows []map[string]any) map[int]map[string]any {
 	out := map[int]map[string]any{}
 	for _, row := range rows {
@@ -11509,6 +11540,51 @@ func TestBuildAndPackageNoOpsWhenOutputsAreCurrent(t *testing.T) {
 	}
 }
 
+func TestBuildAndPackageRebuildsWhenDatasetWasDeleted(t *testing.T) {
+	root := topPackageTestProject(t)
+	mkdirAll(t, filepath.Join(root, "topdata", "data", "skills"))
+	writeFile(t, filepath.Join(root, "topdata", "data", "skills", "base.json"), `{
+  "output": "skills.2da",
+  "columns": ["Label"],
+  "rows": [{"id": 0, "Label": "ATHLETICS"}]
+}`+"\n")
+
+	proj := testProject(root)
+	proj.Config.TopData.ReferenceBuilder = ""
+	proj.Config.Autogen.Consumers = nil
+
+	result, err := BuildAndPackage(proj, nil)
+	if err != nil {
+		t.Fatalf("initial BuildAndPackage failed: %v", err)
+	}
+	stale2DA := filepath.Join(result.Output2DADir, "repadjust.2da")
+	if _, err := os.Stat(stale2DA); err != nil {
+		t.Fatalf("expected initial repadjust output: %v", err)
+	}
+
+	sourceTime := time.Now().Add(-4 * time.Hour)
+	outputTime := time.Now().Add(-10 * time.Minute)
+	if err := os.RemoveAll(filepath.Join(root, "topdata", "data", "repadjust")); err != nil {
+		t.Fatalf("remove deleted dataset: %v", err)
+	}
+	setTopDataSourceTimes(t, root, sourceTime)
+	setBuildOutputTimes(t, result, outputTime)
+
+	incremental, err := BuildAndPackage(proj, nil)
+	if err != nil {
+		t.Fatalf("incremental BuildAndPackage failed: %v", err)
+	}
+	if incremental.Mode == "incremental-noop" {
+		t.Fatalf("expected deleted dataset to force rebuild, got %q", incremental.Mode)
+	}
+	if _, err := os.Stat(stale2DA); !os.IsNotExist(err) {
+		t.Fatalf("expected stale deleted dataset output to be pruned, got %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(result.Output2DADir, "skills.2da")); err != nil {
+		t.Fatalf("expected remaining dataset output: %v", err)
+	}
+}
+
 func TestBuildPackageFailsWhenAutogenManifestCacheIsStale(t *testing.T) {
 	root := topPackageTestProject(t)
 	proj := testProject(root)