ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

List/range support

Browse Source
This commit is contained in:
archvillainette
2026-05-24 14:11:15 +02:00
parent 91ab34ddff
commit 0e1ebf552d
6 changed files with 414 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/project/effective.go
+20 -9
View File
@@ -151,15 +151,16 @@ type EffectiveMusicDataset struct {
}
type EffectiveTopDataConfig struct {
Source string `json:"source" yaml:"source"`
Build string `json:"build" yaml:"build"`
ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"`
Assets string `json:"assets" yaml:"assets"`
Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"`
CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"`
PackageHAK string `json:"package_hak" yaml:"package_hak"`
PackageTLK string `json:"package_tlk" yaml:"package_tlk"`
Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"`
Source string `json:"source" yaml:"source"`
Build string `json:"build" yaml:"build"`
ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"`
Assets string `json:"assets" yaml:"assets"`
Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"`
CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"`
PackageHAK string `json:"package_hak" yaml:"package_hak"`
PackageTLK string `json:"package_tlk" yaml:"package_tlk"`
ValueEncodings []TopDataValueEncodingConfig `json:"value_encodings" yaml:"value_encodings"`
Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"`
}
type EffectiveAutogenConfig struct {
@@ -237,6 +238,7 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
CompiledTLK: defaultString(p.Config.TopData.CompiledTLK, DefaultTopDataCompiledTLK),
PackageHAK: defaultString(p.Config.TopData.PackageHAK, DefaultTopDataPackageHAK),
PackageTLK: defaultString(p.Config.TopData.PackageTLK, DefaultTopDataPackageTLK),
ValueEncodings: cloneTopDataValueEncodings(p.Config.TopData.ValueEncodings),
Wiki: TopDataWikiConfig{
OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot),
PagesDir: defaultString(p.Config.TopData.Wiki.PagesDir, DefaultTopDataWikiPagesDir),
@@ -406,6 +408,15 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
return effective
}
func cloneTopDataValueEncodings(values []TopDataValueEncodingConfig) []TopDataValueEncodingConfig {
if len(values) == 0 {
return nil
}
out := make([]TopDataValueEncodingConfig, len(values))
copy(out, values)
return out
}
func (p *Project) EffectiveConfigJSON() ([]byte, error) {
return json.MarshalIndent(p.EffectiveConfig(), "", " ")
}
internal/project/project.go
+66 -9
View File
@@ -206,15 +206,25 @@ type MusicDatasetConfig struct {
}
type TopDataConfig struct {
Source string `json:"source" yaml:"source"`
Build string `json:"build" yaml:"build"`
ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"`
Assets string `json:"assets" yaml:"assets"`
Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"`
CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"`
PackageHAK string `json:"package_hak" yaml:"package_hak"`
PackageTLK string `json:"package_tlk" yaml:"package_tlk"`
Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"`
Source string `json:"source" yaml:"source"`
Build string `json:"build" yaml:"build"`
ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"`
Assets string `json:"assets" yaml:"assets"`
Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"`
CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"`
PackageHAK string `json:"package_hak" yaml:"package_hak"`
PackageTLK string `json:"package_tlk" yaml:"package_tlk"`
ValueEncodings []TopDataValueEncodingConfig `json:"value_encodings" yaml:"value_encodings"`
Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"`
}
type TopDataValueEncodingConfig struct {
Dataset string `json:"dataset" yaml:"dataset"`
Column string `json:"column" yaml:"column"`
Mode string `json:"mode" yaml:"mode"`
Min int `json:"min" yaml:"min"`
Max int `json:"max" yaml:"max"`
HexWidth int `json:"hex_width" yaml:"hex_width"`
}
type TopDataWikiConfig struct {
@@ -559,6 +569,7 @@ func (p *Project) ValidateLayout() error {
failures = append(failures, validateRelativePath("outputs.hak_manifest", effective.Outputs.HAKManifest)...)
failures = append(failures, validateRelativePath("outputs.hak_archive", effective.Outputs.HAKArchive)...)
failures = append(failures, validateGeneratedConfig(effective.Generated)...)
failures = append(failures, validateTopDataValueEncodings(effective.TopData.ValueEncodings)...)
failures = append(failures, validateRelativePath("scripts.cache", effective.Scripts.Cache)...)
failures = append(failures, validateRelativePath("scripts.source_dir", effective.Scripts.SourceDir)...)
failures = append(failures, validateRelativePath("topdata.compiled_2da_dir", effective.TopData.Compiled2DADir)...)
@@ -697,6 +708,52 @@ func (p *Project) ValidateLayout() error {
return nil
}
func validateTopDataValueEncodings(encodings []TopDataValueEncodingConfig) []error {
failures := []error{}
seen := map[string]struct{}{}
for index, encoding := range encodings {
prefix := fmt.Sprintf("topdata.value_encodings[%d]", index)
dataset := strings.TrimSpace(encoding.Dataset)
column := strings.TrimSpace(encoding.Column)
if dataset == "" {
failures = append(failures, fmt.Errorf("%s.dataset is required", prefix))
}
if column == "" {
failures = append(failures, fmt.Errorf("%s.column is required", prefix))
}
switch strings.TrimSpace(encoding.Mode) {
case "packed_hex_list":
default:
failures = append(failures, fmt.Errorf("%s.mode %q is not supported", prefix, encoding.Mode))
}
if encoding.Min < 0 {
failures = append(failures, fmt.Errorf("%s.min must be zero or greater", prefix))
}
if encoding.Max < encoding.Min {
failures = append(failures, fmt.Errorf("%s.max must be greater than or equal to min", prefix))
}
if encoding.HexWidth <= 0 {
failures = append(failures, fmt.Errorf("%s.hex_width must be greater than zero", prefix))
} else if encoding.Max >= intPow(16, encoding.HexWidth) {
failures = append(failures, fmt.Errorf("%s.max does not fit in hex_width %d", prefix, encoding.HexWidth))
}
key := filepath.ToSlash(dataset) + "\x00" + strings.ToLower(column)
if _, ok := seen[key]; ok {
failures = append(failures, fmt.Errorf("%s duplicates an earlier dataset/column encoding", prefix))
}
seen[key] = struct{}{}
}
return failures
}
func intPow(base, exponent int) int {
result := 1
for i := 0; i < exponent; i++ {
result *= base
}
return result
}
func validateExtractMergeConfig(config ExtractMergeConfig) []error {
var failures []error
seenTargets := map[string]struct{}{}
internal/project/project_test.go
+77
View File
@@ -125,6 +125,83 @@ validation:
}
}
func TestEffectiveConfigIncludesTopDataValueEncodings(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `
module:
name: Test Module
resref: testmod
topdata:
source: topdata
value_encodings:
- dataset: racialtypes/core
column: AvailableSkinColors
mode: packed_hex_list
min: 0
max: 175
hex_width: 2
`)
proj, err := Load(root)
if err != nil {
t.Fatalf("Load returned error: %v", err)
}
encodings := proj.EffectiveConfig().TopData.ValueEncodings
if len(encodings) != 1 {
t.Fatalf("expected one topdata value encoding, got %#v", encodings)
}
got := encodings[0]
if got.Dataset != "racialtypes/core" ||
got.Column != "AvailableSkinColors" ||
got.Mode != "packed_hex_list" ||
got.Min != 0 ||
got.Max != 175 ||
got.HexWidth != 2 {
t.Fatalf("unexpected topdata value encoding: %#v", got)
}
}
func TestValidateLayoutRejectsInvalidTopDataValueEncodings(t *testing.T) {
root := t.TempDir()
mkdirAll(t, filepath.Join(root, "topdata"))
mkdirAll(t, filepath.Join(root, "build"))
proj := &Project{
Root: root,
Config: Config{
Module: ModuleConfig{Name: "Test", ResRef: "test"},
Paths: PathConfig{Build: "build"},
TopData: TopDataConfig{
Source: "topdata",
ValueEncodings: []TopDataValueEncodingConfig{
{Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 256, HexWidth: 2},
{Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 175, HexWidth: 2},
{Dataset: "", Column: "AvailableHeadsMale", Mode: "unknown", Min: -1, Max: 999, HexWidth: 0},
},
},
},
}
err := proj.ValidateLayout()
if err == nil {
t.Fatal("expected invalid topdata value encoding validation error")
}
text := err.Error()
for _, want := range []string{
"topdata.value_encodings[0].max does not fit in hex_width 2",
"topdata.value_encodings[1] duplicates an earlier dataset/column encoding",
"topdata.value_encodings[2].dataset is required",
"topdata.value_encodings[2].mode \"unknown\" is not supported",
"topdata.value_encodings[2].min must be zero or greater",
"topdata.value_encodings[2].hex_width must be greater than zero",
} {
if !strings.Contains(text, want) {
t.Fatalf("expected validation error %q, got %v", want, err)
}
}
}
func TestScanUsesYAMLMusicConvertExtensionsWithinConfiguredDatasetRoots(t *testing.T) {
root := t.TempDir()
if err := os.MkdirAll(filepath.Join(root, "assets", "audio", "westgate"), 0o755); err != nil {
internal/topdata/native.go
+133
View File
@@ -28,6 +28,7 @@ type nativeDataset struct {
OutputName string
Spec datasetTextSpec
Columns []string
ValueEncodings map[string]project.TopDataValueEncodingConfig
CompareReference bool
}
@@ -236,6 +237,7 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
if err != nil {
return BuildResult{}, err
}
datasets = applyTopDataValueEncodings(datasets, p.Config.TopData.ValueEncodings)
registryDatasets, err := collectGeneratedRegistryDatasets(dataDir)
if err != nil {
return BuildResult{}, err
@@ -406,6 +408,25 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
}, nil
}
func applyTopDataValueEncodings(datasets []nativeDataset, encodings []project.TopDataValueEncodingConfig) []nativeDataset {
if len(encodings) == 0 {
return datasets
}
out := append([]nativeDataset(nil), datasets...)
for index := range out {
for _, encoding := range encodings {
if filepath.ToSlash(strings.TrimSpace(encoding.Dataset)) != out[index].Name {
continue
}
if out[index].ValueEncodings == nil {
out[index].ValueEncodings = map[string]project.TopDataValueEncodingConfig{}
}
out[index].ValueEncodings[strings.ToLower(strings.TrimSpace(encoding.Column))] = encoding
}
}
return out
}
func nativeCompileGroup(datasetName string) string {
head, _, ok := strings.Cut(datasetName, "/")
if !ok {
@@ -3694,6 +3715,15 @@ func (r *valueResolver) resolveValue(row map[string]any, field string, value any
switch typed := value.(type) {
case map[string]any:
if encoding, ok := r.valueEncodingForField(field); ok {
if _, hasList := typed["list"]; hasList {
value, err := encodeConfiguredValueList(typed, encoding)
if err != nil {
return tlkCompiledValue{}, fmt.Errorf("field %s: %w", field, err)
}
return tlkCompiledValue{Value: value}, nil
}
}
if refID, ok := typed["id"]; ok {
if refKey, ok := refID.(string); ok && strings.Contains(refKey, ":") {
id, ok := r.keyToID[refKey]
@@ -3750,6 +3780,109 @@ func (r *valueResolver) resolveValue(row map[string]any, field string, value any
}
}
func (r *valueResolver) valueEncodingForField(field string) (project.TopDataValueEncodingConfig, bool) {
if len(r.dataset.ValueEncodings) == 0 {
return project.TopDataValueEncodingConfig{}, false
}
encoding, ok := r.dataset.ValueEncodings[strings.ToLower(strings.TrimSpace(field))]
return encoding, ok
}
func encodeConfiguredValueList(obj map[string]any, encoding project.TopDataValueEncodingConfig) (string, error) {
if strings.TrimSpace(encoding.Mode) != "packed_hex_list" {
return "", fmt.Errorf("unsupported value encoding mode %q", encoding.Mode)
}
for key := range obj {
if key != "list" {
return "", fmt.Errorf("list encoding object contains unsupported key %q", key)
}
}
rawList, ok := obj["list"].([]any)
if !ok {
return "", fmt.Errorf("list must be an array")
}
values := make([]int, 0, len(rawList))
for _, raw := range rawList {
expanded, err := expandConfiguredValueListItem(raw, encoding)
if err != nil {
return "", err
}
values = append(values, expanded...)
}
if len(values) == 0 {
return "", nil
}
var builder strings.Builder
builder.WriteString("0x")
for _, value := range values {
builder.WriteString(fmt.Sprintf("%0*X", encoding.HexWidth, value))
}
return builder.String(), nil
}
func expandConfiguredValueListItem(raw any, encoding project.TopDataValueEncodingConfig) ([]int, error) {
if obj, ok := raw.(map[string]any); ok {
rawRange, ok := obj["range"]
if !ok {
return nil, fmt.Errorf("list object item must contain range")
}
if len(obj) != 1 {
return nil, fmt.Errorf("range item contains unsupported keys")
}
rangeList, ok := rawRange.([]any)
if !ok || len(rangeList) != 2 {
return nil, fmt.Errorf("range must be an array with two values")
}
start, err := parseConfiguredListValue(rangeList[0], encoding)
if err != nil {
return nil, err
}
end, err := parseConfiguredListValue(rangeList[1], encoding)
if err != nil {
return nil, err
}
if end < start {
return nil, fmt.Errorf("range %d-%d ends before it starts", start, end)
}
values := make([]int, 0, end-start+1)
for value := start; value <= end; value++ {
values = append(values, value)
}
return values, nil
}
value, err := parseConfiguredListValue(raw, encoding)
if err != nil {
return nil, err
}
return []int{value}, nil
}
func parseConfiguredListValue(raw any, encoding project.TopDataValueEncodingConfig) (int, error) {
var value int
switch typed := raw.(type) {
case int:
value = typed
case float64:
if typed != float64(int(typed)) {
return 0, fmt.Errorf("list value %v must be an integer", typed)
}
value = int(typed)
case string:
trimmed := strings.TrimSpace(typed)
parsed, err := strconv.Atoi(trimmed)
if err != nil {
return 0, fmt.Errorf("list value %q must be numeric", typed)
}
value = parsed
default:
return 0, fmt.Errorf("list value %v must be numeric", raw)
}
if value < encoding.Min || value > encoding.Max {
return 0, fmt.Errorf("value %d outside range %d-%d", value, encoding.Min, encoding.Max)
}
return value, nil
}
func (r *valueResolver) resolveLiteralFeatObject(row map[string]any, field string, value any) (string, bool) {
if r.dataset.Name != "feat" {
return "", false
internal/topdata/topdata_test.go
+106
View File
@@ -179,6 +179,112 @@ func TestValidateProjectRejectsInvalidModuleColumnExpansionFile(t *testing.T) {
}
}
func TestBuildNativeEncodesConfiguredPackedHexLists(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules"))
writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{
"output": "racialtypes.2da",
"columns": ["Label", "StartLanguages", "BonusLanguages", "AvailableHeadsMale", "AvailableHeadsFemale", "AvailableSkinColors"],
"rows": [
{
"id": 6,
"key": "racialtypes:human",
"Label": "Human",
"StartLanguages": "regional",
"BonusLanguages": "all"
}
]
}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "10_human.json"), `{
"overrides": [
{
"key": "racialtypes:human",
"AvailableHeadsMale": {"list": [1, 10, 999]},
"AvailableHeadsFemale": {"list": [{"range": [1, 3]}]},
"AvailableSkinColors": {
"list": [
{"range": [0, 12]},
24,
74,
{"range": [116, 119]},
{"range": [128, 131]},
156,
157,
168,
170,
174
]
}
}
]
}`+"\n")
proj := testProject(root)
proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{
{Dataset: "racialtypes/core", Column: "AvailableHeadsMale", Mode: "packed_hex_list", Min: 0, Max: 999, HexWidth: 3},
{Dataset: "racialtypes/core", Column: "AvailableHeadsFemale", Mode: "packed_hex_list", Min: 0, Max: 999, HexWidth: 3},
{Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 175, HexWidth: 2},
}
result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil)
if err != nil {
t.Fatalf("BuildNativeWithOptions failed: %v", err)
}
raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "racialtypes.2da"))
if err != nil {
t.Fatalf("read racialtypes.2da: %v", err)
}
got := string(raw)
for _, want := range []string{
"regional",
"all",
"0x00100A3E7",
"0x001002003",
"0x000102030405060708090A0B0C184A74757677808182839C9DA8AAAE",
} {
if !strings.Contains(got, want) {
t.Fatalf("expected compiled racialtypes.2da to contain %q, got:\n%s", want, got)
}
}
}
func TestValidateProjectRejectsInvalidConfiguredPackedHexList(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules"))
writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{
"output": "racialtypes.2da",
"columns": ["Label", "AvailableSkinColors"],
"rows": [
{"id": 6, "key": "racialtypes:human", "Label": "Human"}
]
}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6}`+"\n")
writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "10_human.json"), `{
"overrides": [
{
"key": "racialtypes:human",
"AvailableSkinColors": {"list": [{"range": [12, 0]}]}
}
]
}`+"\n")
proj := testProject(root)
proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{
{Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 175, HexWidth: 2},
}
report := ValidateProject(proj)
if !report.HasErrors() {
t.Fatalf("expected invalid packed hex list to fail validation, got:\n%s", diagnosticsText(report.Diagnostics))
}
if !diagnosticsContain(report.Diagnostics, "range 12-0 ends before it starts") {
t.Fatalf("expected reversed range diagnostic, got:\n%s", diagnosticsText(report.Diagnostics))
}
}
func TestValidateProjectRejectsDuplicateEntryKeysAcrossModules(t *testing.T) {
root := testProjectRoot(t)
mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules"))