Automatic spellknown/spellgain null-filling (#12)

Implemented on branches `feature/topdata-row-extensions` in both `toolkit` and `module`.

**Scope Audited**
- Toolkit native topdata collection/build path.
- Project config/effective config validation.
- Module `nwn-tool.yaml` topdata config and topdata README.

**Changes Made**
- Added `topdata.row_extensions` config with `repeat_last` support.
- Added glob dataset matching and build-time row extension for plain native datasets in [native.go](/home/vicky/Projects/nwnee-shadowsoverwestgate/toolkit/internal/topdata/native.go:641).
- Extension copies emitted columns from the highest authored `Level`, increments `Level`, and assigns sequential `id`s through target level.
- Enabled rules for `classes/spellsgained/*` and `classes/spellsknown/*` to level 60 in [nwn-tool.yaml](/home/vicky/Projects/nwnee-shadowsoverwestgate/module/nwn-tool.yaml:90).
- Updated toolkit/module docs.

**Configuration / Compatibility**
- New public config type: `TopDataRowExtensionConfig` in [project.go](/home/vicky/Projects/nwnee-shadowsoverwestgate/toolkit/internal/project/project.go:233).
- Existing behavior is unchanged unless a repo opts into `topdata.row_extensions`.
- Source JSON and lockfiles are not updated for generated extension rows.

**Tests / Validation**
- Added project config tests for loading and validation.
- Added native topdata tests for extension behavior, glob application, preserving manually authored rows, unmatched datasets, and failure cases.
- Ran:
  - `go test ./internal/project ./internal/topdata`
  - `go test ./internal/topdata -run TestBuildNativeKeepsGeneratedIDsStableAcrossRepeatedBuilds`
  - `/tmp/sow-toolkit-rowext config validate` in `module`
  - `/tmp/sow-toolkit-rowext validate-topdata` in `module`
  - `/tmp/sow-toolkit-rowext build-topdata --force`, then inspected generated rows: `cls_spgn_acolyte.2da` now reaches row `59` / level `60`.

**Remaining Notes**
- `validate-topdata` still reports the pre-existing portrait lock warnings for `cotbl/cotbl_` and `lantern/lantern_`.
- `build-topdata --force` briefly rewrote three spellbook JSON files as a side effect; I reverted those, leaving only the intended YAML/docs changes in `module`.

Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/12
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>

internal/topdata/native.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
 	"slices"
 	"strconv"
@@ -452,6 +453,10 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
 	if err != nil {
 		return BuildResult{}, err
 	}
+	collected, err = applyTopDataRowExtensions(collected, p.EffectiveConfig().TopData.RowExtensions)
+	if err != nil {
+		return BuildResult{}, err
+	}
 
 	tableRegistry, err := newResolvedTableRegistry(collected)
 	if err != nil {
@@ -633,6 +638,131 @@ func applyTopDataRowGeneration(datasets []nativeDataset, rules []project.TopData
 	return out
 }
 
+func applyTopDataRowExtensions(collected []nativeCollectedDataset, rules []project.TopDataRowExtensionConfig) ([]nativeCollectedDataset, error) {
+	if len(rules) == 0 {
+		return collected, nil
+	}
+	out := append([]nativeCollectedDataset(nil), collected...)
+	for index := range out {
+		dataset := &out[index]
+		rule, ok, err := topDataRowExtensionForDataset(dataset.Dataset.Name, rules)
+		if err != nil {
+			return nil, err
+		}
+		if !ok {
+			continue
+		}
+		extended, err := extendCollectedRowsByRepeatingLastLevel(*dataset, rule)
+		if err != nil {
+			return nil, err
+		}
+		out[index] = extended
+	}
+	return out, nil
+}
+
+func topDataRowExtensionForDataset(datasetName string, rules []project.TopDataRowExtensionConfig) (project.TopDataRowExtensionConfig, bool, error) {
+	name := filepath.ToSlash(strings.TrimSpace(datasetName))
+	for _, rule := range rules {
+		pattern := filepath.ToSlash(strings.TrimSpace(rule.Dataset))
+		matched, err := path.Match(pattern, name)
+		if err != nil {
+			return project.TopDataRowExtensionConfig{}, false, fmt.Errorf("topdata row extension pattern %q is invalid: %w", rule.Dataset, err)
+		}
+		if matched {
+			return rule, true, nil
+		}
+	}
+	return project.TopDataRowExtensionConfig{}, false, nil
+}
+
+func extendCollectedRowsByRepeatingLastLevel(dataset nativeCollectedDataset, rule project.TopDataRowExtensionConfig) (nativeCollectedDataset, error) {
+	if dataset.Dataset.Kind != nativeDatasetPlain {
+		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: topdata row extension supports only plain datasets", dataset.Dataset.Name)
+	}
+	if strings.TrimSpace(rule.Mode) != "repeat_last" {
+		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: topdata row extension mode %q is not supported", dataset.Dataset.Name, rule.Mode)
+	}
+	levelColumn, ok := datasetColumnName(dataset.Columns, rule.LevelColumn)
+	if !ok {
+		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: topdata row extension level_column %q is not present in columns", dataset.Dataset.Name, rule.LevelColumn)
+	}
+	if len(dataset.Rows) == 0 {
+		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: topdata row extension requires at least one authored row", dataset.Dataset.Name)
+	}
+
+	maxLevel := 0
+	maxID := -1
+	var reference map[string]any
+	usedIDs := map[int]struct{}{}
+	for index, row := range dataset.Rows {
+		rowID, err := asInt(row["id"])
+		if err != nil {
+			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d id must be numeric for topdata row extension", dataset.Dataset.Name, index)
+		}
+		usedIDs[rowID] = struct{}{}
+		if rowID > maxID {
+			maxID = rowID
+		}
+		rawLevel, ok := lookupField(row, levelColumn)
+		if !ok {
+			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d missing level_column %q for topdata row extension", dataset.Dataset.Name, index, levelColumn)
+		}
+		level, err := asInt(rawLevel)
+		if err != nil {
+			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: row %d level_column %q must be numeric for topdata row extension", dataset.Dataset.Name, index, levelColumn)
+		}
+		if reference == nil || level > maxLevel {
+			maxLevel = level
+			reference = row
+		}
+	}
+	if rule.TargetLevel < maxLevel {
+		return nativeCollectedDataset{}, fmt.Errorf("dataset %s: topdata row extension target_level %d is below highest authored %s %d", dataset.Dataset.Name, rule.TargetLevel, levelColumn, maxLevel)
+	}
+	if rule.TargetLevel == maxLevel {
+		return dataset, nil
+	}
+
+	rows := append([]map[string]any(nil), dataset.Rows...)
+	nextID := maxID + 1
+	for level := maxLevel + 1; level <= rule.TargetLevel; level++ {
+		if _, exists := usedIDs[nextID]; exists {
+			return nativeCollectedDataset{}, fmt.Errorf("dataset %s: generated topdata row id %d already exists", dataset.Dataset.Name, nextID)
+		}
+		row := make(map[string]any, len(dataset.Columns)+1)
+		row["id"] = nextID
+		for _, column := range dataset.Columns {
+			if column == levelColumn {
+				row[column] = level
+				continue
+			}
+			if value, ok := lookupField(reference, column); ok {
+				row[column] = cloneAuthoringValue(value)
+			}
+		}
+		rows = append(rows, row)
+		usedIDs[nextID] = struct{}{}
+		nextID++
+	}
+	slices.SortFunc(rows, func(a, b map[string]any) int {
+		left, _ := asInt(a["id"])
+		right, _ := asInt(b["id"])
+		return left - right
+	})
+	dataset.Rows = rows
+	return dataset, nil
+}
+
+func datasetColumnName(columns []string, name string) (string, bool) {
+	for _, column := range columns {
+		if strings.EqualFold(column, name) {
+			return column, true
+		}
+	}
+	return "", false
+}
+
 func nativeCompileGroup(datasetName string) string {
 	head, _, ok := strings.Cut(datasetName, "/")
 	if !ok {