ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

"first_null_row" configuration option

Browse Source
This commit is contained in:
archvillainette
2026-05-25 09:47:36 +02:00
parent 34692fa9b4
commit 01ddcf8735
8 changed files with 314 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+9
View File
@@ -243,6 +243,9 @@ topdata:
min: 0
max: 175
hex_width: 2
row_generation:
- namespace: portraits
mode: first_null_row
class_feat_injections:
global_feats:
- feat: feat:literate
@@ -295,6 +298,12 @@ topdata:
value: 0
```
`topdata.row_generation` controls where newly generated rows are allocated for
specific datasets. The default `after_base` mode preserves existing behavior by
allocating after the final imported base row. `first_null_row` fills base rows
whose canonical key is empty and whose emitted 2DA columns are all null-like
(`null`, empty string, or `****`) before extending the table.
`topdata.class_feat_injections` controls generated rows for every
`classes/feats/*.json` table. `global_feats` injects concrete `feat:*`
references unless the class already authors that feat; `require_present` and
internal/project/effective.go
+25
View File
@@ -161,6 +161,7 @@ type EffectiveTopDataConfig struct {
PackageTLK string `json:"package_tlk" yaml:"package_tlk"`
ValueEncodings []TopDataValueEncodingConfig `json:"value_encodings" yaml:"value_encodings"`
ValueDefaults []TopDataValueDefaultConfig `json:"value_defaults" yaml:"value_defaults"`
RowGeneration []TopDataRowGenerationConfig `json:"row_generation" yaml:"row_generation"`
ClassFeatInjections TopDataClassFeatInjectionConfig `json:"class_feat_injections" yaml:"class_feat_injections"`
Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"`
}
@@ -242,6 +243,7 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
PackageTLK: defaultString(p.Config.TopData.PackageTLK, DefaultTopDataPackageTLK),
ValueEncodings: cloneTopDataValueEncodings(p.Config.TopData.ValueEncodings),
ValueDefaults: cloneTopDataValueDefaults(p.Config.TopData.ValueDefaults),
RowGeneration: cloneTopDataRowGeneration(p.Config.TopData.RowGeneration),
ClassFeatInjections: cloneTopDataClassFeatInjections(p.Config.TopData.ClassFeatInjections),
Wiki: TopDataWikiConfig{
OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot),
@@ -431,6 +433,29 @@ func cloneTopDataValueDefaults(values []TopDataValueDefaultConfig) []TopDataValu
return out
}
func cloneTopDataRowGeneration(values []TopDataRowGenerationConfig) []TopDataRowGenerationConfig {
if len(values) == 0 {
return nil
}
out := make([]TopDataRowGenerationConfig, len(values))
copy(out, values)
for index := range out {
if strings.TrimSpace(out[index].Dataset) == "" {
out[index].Dataset = strings.TrimSpace(out[index].Namespace)
}
if strings.TrimSpace(out[index].Namespace) == "" {
out[index].Namespace = strings.TrimSpace(out[index].Dataset)
}
if strings.TrimSpace(out[index].Mode) == "" {
out[index].Mode = "after_base"
}
out[index].Dataset = filepath.ToSlash(strings.TrimSpace(out[index].Dataset))
out[index].Namespace = filepath.ToSlash(strings.TrimSpace(out[index].Namespace))
out[index].Mode = strings.TrimSpace(out[index].Mode)
}
return out
}
func cloneTopDataClassFeatInjections(value TopDataClassFeatInjectionConfig) TopDataClassFeatInjectionConfig {
out := TopDataClassFeatInjectionConfig{
GlobalFeats: slices.Clone(value.GlobalFeats),
internal/project/project.go
+35
View File
@@ -217,10 +217,17 @@ type TopDataConfig struct {
PackageTLK string `json:"package_tlk" yaml:"package_tlk"`
ValueEncodings []TopDataValueEncodingConfig `json:"value_encodings" yaml:"value_encodings"`
ValueDefaults []TopDataValueDefaultConfig `json:"value_defaults" yaml:"value_defaults"`
RowGeneration []TopDataRowGenerationConfig `json:"row_generation" yaml:"row_generation"`
ClassFeatInjections TopDataClassFeatInjectionConfig `json:"class_feat_injections" yaml:"class_feat_injections"`
Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"`
}
type TopDataRowGenerationConfig struct {
Dataset string `json:"dataset,omitempty" yaml:"dataset,omitempty"`
Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"`
Mode string `json:"mode" yaml:"mode"`
}
type TopDataClassFeatInjectionConfig struct {
GlobalFeats []TopDataClassFeatGlobalRule `json:"global_feats" yaml:"global_feats"`
ClassSkillMasterfeats []TopDataClassFeatMasterfeatRule `json:"class_skill_masterfeats" yaml:"class_skill_masterfeats"`
@@ -606,6 +613,7 @@ func (p *Project) ValidateLayout() error {
failures = append(failures, validateGeneratedConfig(effective.Generated)...)
failures = append(failures, validateTopDataValueEncodings(effective.TopData.ValueEncodings)...)
failures = append(failures, validateTopDataValueDefaults(effective.TopData.ValueDefaults)...)
failures = append(failures, validateTopDataRowGeneration(effective.TopData.RowGeneration)...)
failures = append(failures, validateTopDataClassFeatInjections(effective.TopData.ClassFeatInjections)...)
failures = append(failures, validateRelativePath("scripts.cache", effective.Scripts.Cache)...)
failures = append(failures, validateRelativePath("scripts.source_dir", effective.Scripts.SourceDir)...)
@@ -805,6 +813,33 @@ func validateTopDataValueDefaults(defaults []TopDataValueDefaultConfig) []error
return failures
}
func validateTopDataRowGeneration(rules []TopDataRowGenerationConfig) []error {
failures := []error{}
seen := map[string]struct{}{}
for index, rule := range rules {
prefix := fmt.Sprintf("topdata.row_generation[%d]", index)
dataset := strings.TrimSpace(rule.Dataset)
namespace := strings.TrimSpace(rule.Namespace)
if dataset == "" {
dataset = namespace
}
if dataset == "" {
failures = append(failures, fmt.Errorf("%s.dataset is required", prefix))
}
switch strings.TrimSpace(rule.Mode) {
case "", "after_base", "first_null_row":
default:
failures = append(failures, fmt.Errorf("%s.mode %q is not supported", prefix, rule.Mode))
}
key := filepath.ToSlash(dataset)
if _, ok := seen[key]; ok {
failures = append(failures, fmt.Errorf("%s duplicates an earlier dataset row generation rule", prefix))
}
seen[key] = struct{}{}
}
return failures
}
func validateTopDataClassFeatInjections(config TopDataClassFeatInjectionConfig) []error {
failures := []error{}
for index, rule := range config.GlobalFeats {
internal/project/project_test.go
+37
View File
@@ -191,6 +191,34 @@ topdata:
}
}
func TestEffectiveConfigIncludesTopDataRowGeneration(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `
module:
name: Test Module
resref: testmod
topdata:
source: topdata
row_generation:
- namespace: portraits
mode: first_null_row
`)
proj, err := Load(root)
if err != nil {
t.Fatalf("Load returned error: %v", err)
}
rules := proj.EffectiveConfig().TopData.RowGeneration
if len(rules) != 1 {
t.Fatalf("expected one topdata row generation rule, got %#v", rules)
}
got := rules[0]
if got.Dataset != "portraits" || got.Namespace != "portraits" || got.Mode != "first_null_row" {
t.Fatalf("unexpected topdata row generation rule: %#v", got)
}
}
func TestEffectiveConfigIncludesTopDataClassFeatInjections(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `
@@ -252,6 +280,12 @@ func TestValidateLayoutRejectsInvalidTopDataValueEncodings(t *testing.T) {
{Dataset: "racialtypes/core", Column: "ECL", Value: 1},
{Dataset: "", Column: "", Value: 0},
},
RowGeneration: []TopDataRowGenerationConfig{
{Dataset: "portraits", Mode: "first_null_row"},
{Dataset: "portraits", Mode: "after_base"},
{Namespace: "placeabletypes", Mode: "unsupported"},
{Mode: "first_null_row"},
},
},
},
}
@@ -271,6 +305,9 @@ func TestValidateLayoutRejectsInvalidTopDataValueEncodings(t *testing.T) {
"topdata.value_defaults[1] duplicates an earlier dataset/column default",
"topdata.value_defaults[2].dataset is required",
"topdata.value_defaults[2].column is required",
"topdata.row_generation[1] duplicates an earlier dataset row generation rule",
"topdata.row_generation[2].mode \"unsupported\" is not supported",
"topdata.row_generation[3].dataset is required",
} {
if !strings.Contains(text, want) {
t.Fatalf("expected validation error %q, got %v", want, err)
internal/topdata/expansion_native.go
+1 -1
View File
@@ -139,7 +139,7 @@ func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDa
if existingID, exists := targetDS.LockData[key]; exists {
rowID = existingID
hasID = true
} else if existingID, exists := originalLockData[key]; exists {
} else if existingID, exists := originalLockData[key]; exists && targetDS.Dataset.RowGeneration != "first_null_row" {
rowID = existingID
hasID = true
targetDS.LockData[key] = existingID
internal/topdata/native.go
+67 -3
View File
@@ -30,6 +30,7 @@ type nativeDataset struct {
Columns []string
ValueEncodings map[string]project.TopDataValueEncodingConfig
ValueDefaults map[string]any
RowGeneration string
CompareReference bool
}
@@ -240,6 +241,7 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
}
datasets = applyTopDataValueEncodings(datasets, p.Config.TopData.ValueEncodings)
datasets = applyTopDataValueDefaults(datasets, p.Config.TopData.ValueDefaults)
datasets = applyTopDataRowGeneration(datasets, p.Config.TopData.RowGeneration)
registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
if err != nil {
return BuildResult{}, err
@@ -448,6 +450,30 @@ func applyTopDataValueDefaults(datasets []nativeDataset, defaults []project.TopD
return out
}
func applyTopDataRowGeneration(datasets []nativeDataset, rules []project.TopDataRowGenerationConfig) []nativeDataset {
if len(rules) == 0 {
return datasets
}
out := append([]nativeDataset(nil), datasets...)
for index := range out {
for _, rule := range rules {
datasetName := strings.TrimSpace(rule.Dataset)
if datasetName == "" {
datasetName = strings.TrimSpace(rule.Namespace)
}
if filepath.ToSlash(datasetName) != out[index].Name {
continue
}
mode := strings.TrimSpace(rule.Mode)
if mode == "" {
mode = "after_base"
}
out[index].RowGeneration = mode
}
}
return out
}
func nativeCompileGroup(datasetName string) string {
head, _, ok := strings.Cut(datasetName, "/")
if !ok {
@@ -785,6 +811,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
rowByID := map[int]map[string]any{}
rowByKey := map[string]map[string]any{}
usedIDs := map[int]struct{}{}
firstNullRowMode := dataset.RowGeneration == "first_null_row"
lockModified := false
lockAdded := 0
lockPruned := 0
@@ -798,8 +825,28 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
}
}
if firstNullRowMode {
for key, rowID := range lockData {
if rowID <= baseBoundaryID {
continue
}
if _, ok := baseRowKeys[key]; ok {
continue
}
if explicitID, ok := explicitModuleIDs[key]; ok && explicitID == rowID {
continue
}
delete(lockData, key)
lockModified = true
lockPruned++
}
}
for key, rowID := range lockData {
if rowID <= baseBoundaryID {
if firstNullRowMode {
continue
}
if _, ok := baseRowKeys[key]; ok {
continue
}
@@ -823,8 +870,11 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
if err != nil {
return nativeCollectedDataset{}, fmt.Errorf("dataset %s: %w", dataset.Name, err)
}
rows = append(rows, row)
rowID := row["id"].(int)
if firstNullRowMode && isNativeAllocationNullRow(row, columns) {
continue
}
rows = append(rows, row)
if key, ok := row["key"].(string); ok && key != "" {
if baseRowKeyCounts[key] > 1 {
if lockedID, ok := lockData[key]; !ok || lockedID != rowID {
@@ -843,8 +893,10 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) {
}
usedIDs[rowID] = struct{}{}
}
for rowID := 0; rowID <= baseBoundaryID; rowID++ {
usedIDs[rowID] = struct{}{}
if !firstNullRowMode {
for rowID := 0; rowID <= baseBoundaryID; rowID++ {
usedIDs[rowID] = struct{}{}
}
}
for _, rowID := range explicitModuleIDs {
usedIDs[rowID] = struct{}{}
@@ -4457,6 +4509,18 @@ func ensureRowsExposeColumns(rows []map[string]any, columns []string) {
}
}
func isNativeAllocationNullRow(row map[string]any, columns []string) bool {
if key, ok := row["key"].(string); ok && strings.TrimSpace(key) != "" && strings.TrimSpace(key) != nullValue {
return false
}
for _, column := range columns {
if !isNullLikeValue(row[column]) {
return false
}
}
return true
}
func canonicalizeBaseRow(dataset nativeDataset, columns []string, raw map[string]any, index int) (map[string]any, error) {
rowID := index
if value, ok := raw["id"]; ok {
internal/topdata/topdata.go
+26 -4
View File
@@ -265,7 +265,7 @@ func ValidateProject(p *project.Project) ValidationReport {
validateTopPackageAssets(sourceDir, dataDir, &report)
validateNativeOutputCatalog(dataDir, &report)
validateNativeEntryKeyUniqueness(dataDir, &report)
validateNativeLockAllocation(dataDir, &report)
validateNativeLockAllocation(dataDir, p.EffectiveConfig().TopData.RowGeneration, &report)
validateGeneratedFeatFamilies(dataDir, &report)
validateItempropsRegistryGraph(dataDir, &report)
if _, err := os.Stat(statePath); err == nil {
@@ -273,7 +273,7 @@ func ValidateProject(p *project.Project) ValidationReport {
_ = validateJSONFile(statePath, "state", &report)
}
if !report.HasErrors() {
validateNativeAuthoringWarnings(dataDir, &report)
validateNativeAuthoringWarnings(dataDir, p.EffectiveConfig().TopData.RowGeneration, &report)
validateNativeBuildability(p, &report)
}
@@ -1089,7 +1089,7 @@ func validateNativeEntryKeyUniqueness(dataDir string, report *ValidationReport)
}
}
func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
func validateNativeLockAllocation(dataDir string, rowGeneration []project.TopDataRowGenerationConfig, report *ValidationReport) {
datasets, err := discoverNativeDatasets(dataDir)
if err != nil {
report.Diagnostics = append(report.Diagnostics, Diagnostic{
@@ -1099,6 +1099,7 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
})
return
}
datasets = applyTopDataRowGeneration(datasets, rowGeneration)
for _, dataset := range datasets {
if dataset.Kind != nativeDatasetBase {
continue
@@ -1118,6 +1119,8 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
}
baseBoundaryID := len(rawRows) - 1
baseKeys := map[string]struct{}{}
nullBaseIDs := map[int]struct{}{}
columns, _ := parseColumns(baseObj, dataset.Name)
for index, raw := range rawRows {
row, ok := raw.(map[string]any)
if !ok {
@@ -1133,6 +1136,12 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
if key, ok := row["key"].(string); ok && key != "" {
baseKeys[key] = struct{}{}
}
if len(columns) > 0 {
canonical, err := canonicalizeBaseRow(dataset, columns, row, index)
if err == nil && isNativeAllocationNullRow(canonical, columns) {
nullBaseIDs[rowID] = struct{}{}
}
}
}
explicitModuleIDs := map[string]int{}
for _, dir := range []string{dataset.ModulesDir, dataset.GeneratedDir} {
@@ -1208,6 +1217,11 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
if rowID > baseBoundaryID {
continue
}
if dataset.RowGeneration == "first_null_row" {
if _, ok := nullBaseIDs[rowID]; ok {
continue
}
}
if _, ok := baseKeys[key]; ok {
continue
}
@@ -1226,11 +1240,12 @@ func validateNativeLockAllocation(dataDir string, report *ValidationReport) {
}
}
func validateNativeAuthoringWarnings(dataDir string, report *ValidationReport) {
func validateNativeAuthoringWarnings(dataDir string, rowGeneration []project.TopDataRowGenerationConfig, report *ValidationReport) {
datasets, err := discoverNativeDatasets(dataDir)
if err != nil {
return
}
datasets = applyTopDataRowGeneration(datasets, rowGeneration)
for _, dataset := range datasets {
if dataset.Kind != nativeDatasetBase {
continue
@@ -1256,6 +1271,7 @@ func validateDatasetOverrideTargetWarnings(dataset nativeDataset, report *Valida
if !ok {
return
}
columns, _ := parseColumns(baseObj, dataset.Name)
rowIDToKey := map[int]string{}
keyToID := map[string]int{}
usedIDs := map[int]struct{}{}
@@ -1270,6 +1286,12 @@ func validateDatasetOverrideTargetWarnings(dataset nativeDataset, report *Valida
rowID = parsed
}
}
if dataset.RowGeneration == "first_null_row" && len(columns) > 0 {
canonical, err := canonicalizeBaseRow(dataset, columns, row, index)
if err == nil && isNativeAllocationNullRow(canonical, columns) {
continue
}
}
usedIDs[rowID] = struct{}{}
if key, ok := row["key"].(string); ok && key != "" {
rowIDToKey[rowID] = key
internal/topdata/topdata_test.go
+114
View File
@@ -646,6 +646,120 @@ func TestSaveLockfilePreservesExistingOrderAndAppendsNewKeys(t *testing.T) {
}
}
func TestBuildNativeAllocatesConfiguredDatasetsIntoFirstNullBaseRows(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits", "modules"))
writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{
"output": "portraits.2da",
"columns": ["BaseResRef", "Sex"],
"rows": [
{"id": 0, "key": "portraits:none", "BaseResRef": "****", "Sex": 4},
{"id": 1, "BaseResRef": "****", "Sex": "****"},
{"id": 2, "BaseResRef": null, "Sex": null},
{"id": 3, "key": "portraits:existing", "BaseResRef": "existing_", "Sex": 1}
]
}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), `{
"portraits:none": 0,
"portraits:existing": 3,
"portraits:first": 4,
"portraits:second": 5
}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "modules", "custom.json"), `{
"entries": {
"portraits:first": {"BaseResRef": "first_", "Sex": 4},
"portraits:second": {"BaseResRef": "second_", "Sex": 4}
}
}`+"\n")
proj := testProject(root)
proj.Config.TopData.RowGeneration = []project.TopDataRowGenerationConfig{
{Dataset: "portraits", Mode: "first_null_row"},
}
result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil)
if err != nil {
t.Fatalf("BuildNativeWithOptions failed: %v", err)
}
lockRaw, err := os.ReadFile(filepath.Join(root, "topdata", "data", "portraits", "lock.json"))
if err != nil {
t.Fatalf("read lockfile: %v", err)
}
lockText := string(lockRaw)
if !strings.Contains(lockText, `"portraits:first": 1`) || !strings.Contains(lockText, `"portraits:second": 2`) {
t.Fatalf("expected new portrait locks to use first null base rows, got:\n%s", lockText)
}
outputRaw, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da"))
if err != nil {
t.Fatalf("read portraits.2da: %v", err)
}
output := string(outputRaw)
for _, want := range []string{
"1\tfirst_\t4",
"2\tsecond_\t4",
} {
if !strings.Contains(output, want) {
t.Fatalf("expected generated portraits row %q, got:\n%s", want, output)
}
}
if strings.Contains(output, "4\tfirst_") || strings.Contains(output, "5\tsecond_") {
t.Fatalf("expected portraits not to allocate after the base boundary, got:\n%s", output)
}
}
func TestMergeExpansionDataAllocatesConfiguredDatasetsIntoFirstNullRows(t *testing.T) {
root := t.TempDir()
lockPath := filepath.Join(root, "portraits-lock.json")
writeFile(t, lockPath, `{
"portraits:existing": 0,
"portraits:expanded": 5
}`+"\n")
collected, err := mergeExpansionData([]nativeCollectedDataset{
{
Dataset: nativeDataset{
Kind: nativeDatasetBase,
Name: "appearance",
OutputName: "appearance.2da",
RowGeneration: "after_base",
},
Columns: []string{"LABEL", "PORTRAIT"},
Rows: []map[string]any{{"id": 0, "key": "appearance:source", "PORTRAIT": map[string]any{"value": "po_expanded_", "data": map[string]any{"portraits": map[string]any{"key": "portraits:expanded", "BaseResRef": "expanded_", "Sex": 4}}}}},
LockData: map[string]int{"appearance:source": 0},
},
{
Dataset: nativeDataset{
Kind: nativeDatasetBase,
Name: "portraits",
LockPath: lockPath,
OutputName: "portraits.2da",
RowGeneration: "first_null_row",
},
Columns: []string{"BaseResRef", "Sex"},
Rows: []map[string]any{{"id": 0, "key": "portraits:existing", "BaseResRef": "existing_", "Sex": 4}},
LockData: map[string]int{"portraits:existing": 0},
},
})
if err != nil {
t.Fatalf("mergeExpansionData failed: %v", err)
}
portraits := collected[1]
if got := portraits.LockData["portraits:expanded"]; got != 1 {
t.Fatalf("expected expansion lock to use first null row 1, got %d in %#v", got, portraits.LockData)
}
found := false
for _, row := range portraits.Rows {
if row["key"] == "portraits:expanded" && row["id"] == 1 {
found = true
}
}
if !found {
t.Fatalf("expected expanded row at id 1, got %#v", portraits.Rows)
}
}
func TestResolvedTableRegistryRejectsDuplicateKeys(t *testing.T) {
_, err := newResolvedTableRegistry([]nativeCollectedDataset{
{