Minimum row

This commit is contained in:
2026-05-25 14:50:01 +02:00
parent a646c905e4
commit 37db91314b
7 changed files with 171 additions and 38 deletions
+4 -1
View File
@@ -246,6 +246,7 @@ topdata:
row_generation: row_generation:
- namespace: portraits - namespace: portraits
mode: first_null_row mode: first_null_row
minimum_row: 15000
class_feat_injections: class_feat_injections:
global_feats: global_feats:
- feat: feat:literate - feat: feat:literate
@@ -312,7 +313,9 @@ topdata:
specific datasets. The default `after_base` mode preserves existing behavior by specific datasets. The default `after_base` mode preserves existing behavior by
allocating after the final imported base row. `first_null_row` fills base rows allocating after the final imported base row. `first_null_row` fills base rows
whose canonical key is empty and whose emitted 2DA columns are all null-like whose canonical key is empty and whose emitted 2DA columns are all null-like
(`null`, empty string, or `****`) before extending the table. (`null`, empty string, or `****`) before extending the table. `minimum_row`
optionally prevents generated allocation below a configured row while preserving
the selected allocation mode at and above that row.
`topdata.class_feat_injections` controls generated rows for every `topdata.class_feat_injections` controls generated rows for every
`classes/feats/*.json` table. `global_feats` injects concrete `feat:*` `classes/feats/*.json` table. `global_feats` injects concrete `feat:*`
+7 -3
View File
@@ -223,9 +223,10 @@ type TopDataConfig struct {
} }
type TopDataRowGenerationConfig struct { type TopDataRowGenerationConfig struct {
Dataset string `json:"dataset,omitempty" yaml:"dataset,omitempty"` Dataset string `json:"dataset,omitempty" yaml:"dataset,omitempty"`
Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"`
Mode string `json:"mode" yaml:"mode"` Mode string `json:"mode" yaml:"mode"`
MinimumRow int `json:"minimum_row,omitempty" yaml:"minimum_row,omitempty"`
} }
type TopDataClassFeatInjectionConfig struct { type TopDataClassFeatInjectionConfig struct {
@@ -860,6 +861,9 @@ func validateTopDataRowGeneration(rules []TopDataRowGenerationConfig) []error {
default: default:
failures = append(failures, fmt.Errorf("%s.mode %q is not supported", prefix, rule.Mode)) failures = append(failures, fmt.Errorf("%s.mode %q is not supported", prefix, rule.Mode))
} }
if rule.MinimumRow < 0 {
failures = append(failures, fmt.Errorf("%s.minimum_row must be zero or greater", prefix))
}
key := filepath.ToSlash(dataset) key := filepath.ToSlash(dataset)
if _, ok := seen[key]; ok { if _, ok := seen[key]; ok {
failures = append(failures, fmt.Errorf("%s duplicates an earlier dataset row generation rule", prefix)) failures = append(failures, fmt.Errorf("%s duplicates an earlier dataset row generation rule", prefix))
+4 -1
View File
@@ -202,6 +202,7 @@ topdata:
row_generation: row_generation:
- namespace: portraits - namespace: portraits
mode: first_null_row mode: first_null_row
minimum_row: 15000
`) `)
proj, err := Load(root) proj, err := Load(root)
@@ -214,7 +215,7 @@ topdata:
t.Fatalf("expected one topdata row generation rule, got %#v", rules) t.Fatalf("expected one topdata row generation rule, got %#v", rules)
} }
got := rules[0] got := rules[0]
if got.Dataset != "portraits" || got.Namespace != "portraits" || got.Mode != "first_null_row" { if got.Dataset != "portraits" || got.Namespace != "portraits" || got.Mode != "first_null_row" || got.MinimumRow != 15000 {
t.Fatalf("unexpected topdata row generation rule: %#v", got) t.Fatalf("unexpected topdata row generation rule: %#v", got)
} }
} }
@@ -285,6 +286,7 @@ func TestValidateLayoutRejectsInvalidTopDataValueEncodings(t *testing.T) {
{Dataset: "portraits", Mode: "after_base"}, {Dataset: "portraits", Mode: "after_base"},
{Namespace: "placeabletypes", Mode: "unsupported"}, {Namespace: "placeabletypes", Mode: "unsupported"},
{Mode: "first_null_row"}, {Mode: "first_null_row"},
{Dataset: "classes", Mode: "first_null_row", MinimumRow: -1},
}, },
}, },
}, },
@@ -308,6 +310,7 @@ func TestValidateLayoutRejectsInvalidTopDataValueEncodings(t *testing.T) {
"topdata.row_generation[1] duplicates an earlier dataset row generation rule", "topdata.row_generation[1] duplicates an earlier dataset row generation rule",
"topdata.row_generation[2].mode \"unsupported\" is not supported", "topdata.row_generation[2].mode \"unsupported\" is not supported",
"topdata.row_generation[3].dataset is required", "topdata.row_generation[3].dataset is required",
"topdata.row_generation[4].minimum_row must be zero or greater",
} { } {
if !strings.Contains(text, want) { if !strings.Contains(text, want) {
t.Fatalf("expected validation error %q, got %v", want, err) t.Fatalf("expected validation error %q, got %v", want, err)
+2 -2
View File
@@ -108,7 +108,7 @@ func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDa
usedKeys[key] = struct{}{} usedKeys[key] = struct{}{}
} }
} }
nextID := nextAvailableID(usedIDs) nextID := nextAvailableIDAtLeast(usedIDs, targetDS.Dataset.RowGenerationMinRow)
for _, targetRow := range targetRows { for _, targetRow := range targetRows {
var rowID int var rowID int
@@ -155,7 +155,7 @@ func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDa
if !hasID { if !hasID {
rowID = nextID rowID = nextID
usedIDs[rowID] = struct{}{} usedIDs[rowID] = struct{}{}
nextID = nextAvailableID(usedIDs) nextID = nextAvailableIDAtLeast(usedIDs, targetDS.Dataset.RowGenerationMinRow)
} else { } else {
if _, exists := usedIDs[rowID]; exists && key == "" { if _, exists := usedIDs[rowID]; exists && key == "" {
return nil, fmt.Errorf("expansion into %s: row id %d already exists", targetDatasetName, rowID) return nil, fmt.Errorf("expansion into %s: row id %d already exists", targetDatasetName, rowID)
+26 -19
View File
@@ -18,20 +18,21 @@ import (
const nullValue = "****" const nullValue = "****"
type nativeDataset struct { type nativeDataset struct {
Kind nativeDatasetKind Kind nativeDatasetKind
Name string Name string
RootPath string RootPath string
BasePath string BasePath string
LockPath string LockPath string
ModulesDir string ModulesDir string
GeneratedDir string GeneratedDir string
OutputName string OutputName string
Spec datasetTextSpec Spec datasetTextSpec
Columns []string Columns []string
ValueEncodings map[string]project.TopDataValueEncodingConfig ValueEncodings map[string]project.TopDataValueEncodingConfig
ValueDefaults map[string]any ValueDefaults map[string]any
RowGeneration string RowGeneration string
CompareReference bool RowGenerationMinRow int
CompareReference bool
} }
type nativeDatasetKind string type nativeDatasetKind string
@@ -470,6 +471,7 @@ func applyTopDataRowGeneration(datasets []nativeDataset, rules []project.TopData
mode = "after_base" mode = "after_base"
} }
out[index].RowGeneration = mode out[index].RowGeneration = mode
out[index].RowGenerationMinRow = rule.MinimumRow
} }
} }
return out return out
@@ -813,6 +815,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
rowByKey := map[string]map[string]any{} rowByKey := map[string]map[string]any{}
usedIDs := map[int]struct{}{} usedIDs := map[int]struct{}{}
firstNullRowMode := dataset.RowGeneration == "first_null_row" firstNullRowMode := dataset.RowGeneration == "first_null_row"
rowGenerationMinRow := dataset.RowGenerationMinRow
lockModified := false lockModified := false
lockAdded := 0 lockAdded := 0
lockPruned := 0 lockPruned := 0
@@ -828,7 +831,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
if firstNullRowMode { if firstNullRowMode {
for key, rowID := range lockData { for key, rowID := range lockData {
if rowID <= baseBoundaryID { if rowID >= rowGenerationMinRow && (rowID <= baseBoundaryID || rowGenerationMinRow > baseBoundaryID) {
continue continue
} }
if _, ok := baseRowKeys[key]; ok { if _, ok := baseRowKeys[key]; ok {
@@ -872,7 +875,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err) return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
} }
rowID := row["id"].(int) rowID := row["id"].(int)
if firstNullRowMode && isNativeAllocationNullRow(row, columns) { if firstNullRowMode && rowID >= rowGenerationMinRow && isNativeAllocationNullRow(row, columns) {
continue continue
} }
rows = append(rows, row) rows = append(rows, row)
@@ -903,11 +906,11 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
usedIDs[rowID] = struct{}{} usedIDs[rowID] = struct{}{}
} }
nextID := nextAvailableID(usedIDs) nextID := nextAvailableIDAtLeast(usedIDs, rowGenerationMinRow)
allocateNextID := func() int { allocateNextID := func() int {
rowID := nextID rowID := nextID
usedIDs[rowID] = struct{}{} usedIDs[rowID] = struct{}{}
nextID = nextAvailableID(usedIDs) nextID = nextAvailableIDAtLeast(usedIDs, rowGenerationMinRow)
return rowID return rowID
} }
lockedIDForKey := func(key string) (int, bool) { lockedIDForKey := func(key string) (int, bool) {
@@ -5669,7 +5672,11 @@ func asInt(value any) (int, error) {
} }
func nextAvailableID(used map[int]struct{}) int { func nextAvailableID(used map[int]struct{}) int {
next := 0 return nextAvailableIDAtLeast(used, 0)
}
func nextAvailableIDAtLeast(used map[int]struct{}, minimum int) int {
next := minimum
for { for {
if _, ok := used[next]; !ok { if _, ok := used[next]; !ok {
return next return next
+18 -12
View File
@@ -1214,27 +1214,33 @@ func validateNativeLockAllocation(dataDir string, rowGeneration []project.TopDat
} }
invalid := 0 invalid := 0
for key, rowID := range lockData { for key, rowID := range lockData {
if rowID > baseBoundaryID {
continue
}
if dataset.RowGeneration == "first_null_row" {
if _, ok := nullBaseIDs[rowID]; ok {
continue
}
}
if _, ok := baseKeys[key]; ok { if _, ok := baseKeys[key]; ok {
continue continue
} }
if explicitID, ok := explicitModuleIDs[key]; ok && explicitID == rowID { if explicitID, ok := explicitModuleIDs[key]; ok && explicitID == rowID {
continue continue
} }
if dataset.RowGenerationMinRow > 0 && rowID < dataset.RowGenerationMinRow {
invalid++
continue
}
if rowID > baseBoundaryID {
continue
}
if dataset.RowGeneration == "first_null_row" {
if _, ok := nullBaseIDs[rowID]; ok {
if rowID >= dataset.RowGenerationMinRow {
continue
}
}
}
invalid++ invalid++
} }
if invalid > 0 { if invalid > 0 {
report.Diagnostics = append(report.Diagnostics, Diagnostic{ report.Diagnostics = append(report.Diagnostics, Diagnostic{
Severity: SeverityWarning, Severity: SeverityWarning,
Path: dataset.LockPath, Path: dataset.LockPath,
Message: fmt.Sprintf("%d lock entrie(s) are inside base id space and will be regenerated above row %d", invalid, baseBoundaryID), Message: fmt.Sprintf("%d lock entrie(s) do not match configured generated-row allocation and will be regenerated", invalid),
}) })
} }
} }
@@ -1286,7 +1292,7 @@ func validateDatasetOverrideTargetWarnings(dataset nativeDataset, report *Valida
rowID = parsed rowID = parsed
} }
} }
if dataset.RowGeneration == "first_null_row" && len(columns) > 0 { if dataset.RowGeneration == "first_null_row" && len(columns) > 0 && rowID >= dataset.RowGenerationMinRow {
canonical, err := canonicalizeBaseRow(dataset, columns, row, index) canonical, err := canonicalizeBaseRow(dataset, columns, row, index)
if err == nil && isNativeAllocationNullRow(canonical, columns) { if err == nil && isNativeAllocationNullRow(canonical, columns) {
continue continue
@@ -1298,11 +1304,11 @@ func validateDatasetOverrideTargetWarnings(dataset nativeDataset, report *Valida
keyToID[key] = rowID keyToID[key] = rowID
} }
} }
nextID := nextAvailableID(usedIDs) nextID := nextAvailableIDAtLeast(usedIDs, dataset.RowGenerationMinRow)
allocateNextID := func() int { allocateNextID := func() int {
rowID := nextID rowID := nextID
usedIDs[rowID] = struct{}{} usedIDs[rowID] = struct{}{}
nextID = nextAvailableID(usedIDs) nextID = nextAvailableIDAtLeast(usedIDs, dataset.RowGenerationMinRow)
return rowID return rowID
} }
modulePaths, err := collectModulePaths(dataset.ModulesDir) modulePaths, err := collectModulePaths(dataset.ModulesDir)
+110
View File
@@ -712,6 +712,73 @@ func TestBuildNativeAllocatesConfiguredDatasetsIntoFirstNullBaseRows(t *testing.
} }
} }
func TestBuildNativeAllocatesFirstNullRowsAtOrAboveConfiguredMinimumRow(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits", "modules"))
writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{
"output": "portraits.2da",
"columns": ["BaseResRef", "Sex"],
"rows": [
{"id": 0, "key": "portraits:none", "BaseResRef": "po_none_", "Sex": 0},
{"id": 1, "BaseResRef": "****", "Sex": "****"},
{"id": 15000, "BaseResRef": "****", "Sex": "****"},
{"id": 15001, "BaseResRef": "****", "Sex": "****"}
]
}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), `{
"portraits:none": 0,
"portraits:first": 1,
"portraits:second": 15002
}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "modules", "custom.json"), `{
"entries": {
"portraits:first": {"BaseResRef": "first_", "Sex": 4},
"portraits:second": {"BaseResRef": "second_", "Sex": 4},
"portraits:third": {"BaseResRef": "third_", "Sex": 4}
}
}`+"\n")
proj := testProject(root)
proj.Config.TopData.RowGeneration = []project.TopDataRowGenerationConfig{
{Dataset: "portraits", Mode: "first_null_row", MinimumRow: 15000},
}
result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil)
if err != nil {
t.Fatalf("BuildNativeWithOptions failed: %v", err)
}
lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "portraits", "lock.json"))
if err != nil {
t.Fatalf("read lockfile: %v", err)
}
lockText := string(lockRaw)
for _, want := range []string{
`"portraits:first": 15000`,
`"portraits:second": 15001`,
`"portraits:third": 15002`,
} {
if !strings.Contains(lockText, want) {
t.Fatalf("expected portrait locks to respect minimum row, missing %s in:\n%s", want, lockText)
}
}
outputRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da"))
if err != nil {
t.Fatalf("read portraits.2da: %v", err)
}
output := string(outputRaw)
for _, want := range []string{
"1\t****\t****",
"15000\tfirst_\t4",
"15001\tsecond_\t4",
"15002\tthird_\t4",
} {
if !strings.Contains(output, want) {
t.Fatalf("expected generated portraits row %q, got:\n%s", want, output)
}
}
}
func TestMergeExpansionDataAllocatesConfiguredDatasetsIntoFirstNullRows(t *testing.T) { func TestMergeExpansionDataAllocatesConfiguredDatasetsIntoFirstNullRows(t *testing.T) {
root := t.TempDir() root := t.TempDir()
lockPath := filepath.Join(root, "portraits-lock.json") lockPath := filepath.Join(root, "portraits-lock.json")
@@ -764,6 +831,49 @@ func TestMergeExpansionDataAllocatesConfiguredDatasetsIntoFirstNullRows(t *testi
} }
} }
func TestMergeExpansionDataAllocatesFirstNullRowsAtOrAboveConfiguredMinimumRow(t *testing.T) {
root := t.TempDir()
lockPath := filepath.Join(root, "portraits-lock.json")
writeFile(t, lockPath, `{
"portraits:existing": 0
}`+"\n")
collected, err := mergeExpansionData([]nativeCollectedDataset{
{
Dataset: nativeDataset{
Kind: nativeDatasetBase,
Name: "appearance",
OutputName: "appearance.2da",
RowGeneration: "after_base",
},
Columns: []string{"LABEL", "PORTRAIT"},
Rows: []map[string]any{{"id": 0, "key": "appearance:source", "PORTRAIT": map[string]any{"value": "po_expanded_", "data": map[string]any{"portraits": map[string]any{"key": "portraits:expanded", "BaseResRef": "expanded_", "Sex": 4}}}}},
LockData: map[string]int{"appearance:source": 0},
},
{
Dataset: nativeDataset{
Kind: nativeDatasetBase,
Name: "portraits",
LockPath: lockPath,
OutputName: "portraits.2da",
RowGeneration: "first_null_row",
RowGenerationMinRow: 15000,
},
Columns: []string{"BaseResRef", "Sex"},
Rows: []map[string]any{{"id": 0, "key": "portraits:existing", "BaseResRef": "existing_", "Sex": 4}, {"id": 1, "BaseResRef": "****", "Sex": "****"}},
LockData: map[string]int{"portraits:existing": 0},
},
})
if err != nil {
t.Fatalf("mergeExpansionData failed: %v", err)
}
portraits := collected[1]
if got := portraits.LockData["portraits:expanded"]; got != 15000 {
t.Fatalf("expected expansion lock to respect minimum row 15000, got %d in %#v", got, portraits.LockData)
}
}
func TestResolvedTableRegistryRejectsDuplicateKeys(t *testing.T) { func TestResolvedTableRegistryRejectsDuplicateKeys(t *testing.T) {
_, err := newResolvedTableRegistry([]nativeCollectedDataset{ _, err := newResolvedTableRegistry([]nativeCollectedDataset{
{ {