From 0e1ebf552d5fcf4cdde1da3ebaea80eda80a488c Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Sun, 24 May 2026 14:11:15 +0200 Subject: [PATCH] List/range support --- README.md | 12 +++ internal/project/effective.go | 29 ++++--- internal/project/project.go | 75 ++++++++++++++--- internal/project/project_test.go | 77 ++++++++++++++++++ internal/topdata/native.go | 133 +++++++++++++++++++++++++++++++ internal/topdata/topdata_test.go | 106 ++++++++++++++++++++++++ 6 files changed, 414 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8d3536e..c6ebc92 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,13 @@ topdata: compiled_tlk: sow_tlk.tlk package_hak: sow_top.hak package_tlk: sow_tlk.tlk + value_encodings: + - dataset: racialtypes/core + column: AvailableSkinColors + mode: packed_hex_list + min: 0 + max: 175 + hex_width: 2 wiki: output_root: wiki pages_dir: pages @@ -258,6 +265,11 @@ autogen: repo_env: SOW_ASSETS_REPO ``` +`topdata.value_encodings` enables configured authoring sugar for compact 2DA +cell values. The `packed_hex_list` mode accepts authored values such as +`{"list": [0, {"range": [10, 12]}]}` and emits a single `0x...` string with +fixed-width hex chunks. + `paths.source` and `paths.assets` must name real repository subtrees when they are configured. They may be omitted for repositories that do not own that class of source, but they must not resolve to the repository root (`.`). Output and diff --git a/internal/project/effective.go b/internal/project/effective.go index d66e0ac..6ba6bf9 100644 --- a/internal/project/effective.go +++ b/internal/project/effective.go @@ -151,15 +151,16 @@ type EffectiveMusicDataset struct { } type EffectiveTopDataConfig struct { - Source string `json:"source" yaml:"source"` - Build string `json:"build" yaml:"build"` - ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"` - Assets string `json:"assets" yaml:"assets"` - Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"` - CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"` - PackageHAK string `json:"package_hak" yaml:"package_hak"` - PackageTLK string `json:"package_tlk" yaml:"package_tlk"` - Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"` + Source string `json:"source" yaml:"source"` + Build string `json:"build" yaml:"build"` + ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"` + Assets string `json:"assets" yaml:"assets"` + Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"` + CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"` + PackageHAK string `json:"package_hak" yaml:"package_hak"` + PackageTLK string `json:"package_tlk" yaml:"package_tlk"` + ValueEncodings []TopDataValueEncodingConfig `json:"value_encodings" yaml:"value_encodings"` + Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"` } type EffectiveAutogenConfig struct { @@ -237,6 +238,7 @@ func (p *Project) EffectiveConfig() EffectiveConfig { CompiledTLK: defaultString(p.Config.TopData.CompiledTLK, DefaultTopDataCompiledTLK), PackageHAK: defaultString(p.Config.TopData.PackageHAK, DefaultTopDataPackageHAK), PackageTLK: defaultString(p.Config.TopData.PackageTLK, DefaultTopDataPackageTLK), + ValueEncodings: cloneTopDataValueEncodings(p.Config.TopData.ValueEncodings), Wiki: TopDataWikiConfig{ OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot), PagesDir: defaultString(p.Config.TopData.Wiki.PagesDir, DefaultTopDataWikiPagesDir), @@ -406,6 +408,15 @@ func (p *Project) EffectiveConfig() EffectiveConfig { return effective } +func cloneTopDataValueEncodings(values []TopDataValueEncodingConfig) []TopDataValueEncodingConfig { + if len(values) == 0 { + return nil + } + out := make([]TopDataValueEncodingConfig, len(values)) + copy(out, values) + return out +} + func (p *Project) EffectiveConfigJSON() ([]byte, error) { return json.MarshalIndent(p.EffectiveConfig(), "", " ") } diff --git a/internal/project/project.go b/internal/project/project.go index d68543e..d39e9c5 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -206,15 +206,25 @@ type MusicDatasetConfig struct { } type TopDataConfig struct { - Source string `json:"source" yaml:"source"` - Build string `json:"build" yaml:"build"` - ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"` - Assets string `json:"assets" yaml:"assets"` - Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"` - CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"` - PackageHAK string `json:"package_hak" yaml:"package_hak"` - PackageTLK string `json:"package_tlk" yaml:"package_tlk"` - Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"` + Source string `json:"source" yaml:"source"` + Build string `json:"build" yaml:"build"` + ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"` + Assets string `json:"assets" yaml:"assets"` + Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"` + CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"` + PackageHAK string `json:"package_hak" yaml:"package_hak"` + PackageTLK string `json:"package_tlk" yaml:"package_tlk"` + ValueEncodings []TopDataValueEncodingConfig `json:"value_encodings" yaml:"value_encodings"` + Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"` +} + +type TopDataValueEncodingConfig struct { + Dataset string `json:"dataset" yaml:"dataset"` + Column string `json:"column" yaml:"column"` + Mode string `json:"mode" yaml:"mode"` + Min int `json:"min" yaml:"min"` + Max int `json:"max" yaml:"max"` + HexWidth int `json:"hex_width" yaml:"hex_width"` } type TopDataWikiConfig struct { @@ -559,6 +569,7 @@ func (p *Project) ValidateLayout() error { failures = append(failures, validateRelativePath("outputs.hak_manifest", effective.Outputs.HAKManifest)...) failures = append(failures, validateRelativePath("outputs.hak_archive", effective.Outputs.HAKArchive)...) failures = append(failures, validateGeneratedConfig(effective.Generated)...) + failures = append(failures, validateTopDataValueEncodings(effective.TopData.ValueEncodings)...) failures = append(failures, validateRelativePath("scripts.cache", effective.Scripts.Cache)...) failures = append(failures, validateRelativePath("scripts.source_dir", effective.Scripts.SourceDir)...) failures = append(failures, validateRelativePath("topdata.compiled_2da_dir", effective.TopData.Compiled2DADir)...) @@ -697,6 +708,52 @@ func (p *Project) ValidateLayout() error { return nil } +func validateTopDataValueEncodings(encodings []TopDataValueEncodingConfig) []error { + failures := []error{} + seen := map[string]struct{}{} + for index, encoding := range encodings { + prefix := fmt.Sprintf("topdata.value_encodings[%d]", index) + dataset := strings.TrimSpace(encoding.Dataset) + column := strings.TrimSpace(encoding.Column) + if dataset == "" { + failures = append(failures, fmt.Errorf("%s.dataset is required", prefix)) + } + if column == "" { + failures = append(failures, fmt.Errorf("%s.column is required", prefix)) + } + switch strings.TrimSpace(encoding.Mode) { + case "packed_hex_list": + default: + failures = append(failures, fmt.Errorf("%s.mode %q is not supported", prefix, encoding.Mode)) + } + if encoding.Min < 0 { + failures = append(failures, fmt.Errorf("%s.min must be zero or greater", prefix)) + } + if encoding.Max < encoding.Min { + failures = append(failures, fmt.Errorf("%s.max must be greater than or equal to min", prefix)) + } + if encoding.HexWidth <= 0 { + failures = append(failures, fmt.Errorf("%s.hex_width must be greater than zero", prefix)) + } else if encoding.Max >= intPow(16, encoding.HexWidth) { + failures = append(failures, fmt.Errorf("%s.max does not fit in hex_width %d", prefix, encoding.HexWidth)) + } + key := filepath.ToSlash(dataset) + "\x00" + strings.ToLower(column) + if _, ok := seen[key]; ok { + failures = append(failures, fmt.Errorf("%s duplicates an earlier dataset/column encoding", prefix)) + } + seen[key] = struct{}{} + } + return failures +} + +func intPow(base, exponent int) int { + result := 1 + for i := 0; i < exponent; i++ { + result *= base + } + return result +} + func validateExtractMergeConfig(config ExtractMergeConfig) []error { var failures []error seenTargets := map[string]struct{}{} diff --git a/internal/project/project_test.go b/internal/project/project_test.go index 45db165..552c690 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -125,6 +125,83 @@ validation: } } +func TestEffectiveConfigIncludesTopDataValueEncodings(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +topdata: + source: topdata + value_encodings: + - dataset: racialtypes/core + column: AvailableSkinColors + mode: packed_hex_list + min: 0 + max: 175 + hex_width: 2 +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + encodings := proj.EffectiveConfig().TopData.ValueEncodings + if len(encodings) != 1 { + t.Fatalf("expected one topdata value encoding, got %#v", encodings) + } + got := encodings[0] + if got.Dataset != "racialtypes/core" || + got.Column != "AvailableSkinColors" || + got.Mode != "packed_hex_list" || + got.Min != 0 || + got.Max != 175 || + got.HexWidth != 2 { + t.Fatalf("unexpected topdata value encoding: %#v", got) + } +} + +func TestValidateLayoutRejectsInvalidTopDataValueEncodings(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "topdata")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Build: "build"}, + TopData: TopDataConfig{ + Source: "topdata", + ValueEncodings: []TopDataValueEncodingConfig{ + {Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 256, HexWidth: 2}, + {Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 175, HexWidth: 2}, + {Dataset: "", Column: "AvailableHeadsMale", Mode: "unknown", Min: -1, Max: 999, HexWidth: 0}, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected invalid topdata value encoding validation error") + } + text := err.Error() + for _, want := range []string{ + "topdata.value_encodings[0].max does not fit in hex_width 2", + "topdata.value_encodings[1] duplicates an earlier dataset/column encoding", + "topdata.value_encodings[2].dataset is required", + "topdata.value_encodings[2].mode \"unknown\" is not supported", + "topdata.value_encodings[2].min must be zero or greater", + "topdata.value_encodings[2].hex_width must be greater than zero", + } { + if !strings.Contains(text, want) { + t.Fatalf("expected validation error %q, got %v", want, err) + } + } +} + func TestScanUsesYAMLMusicConvertExtensionsWithinConfiguredDatasetRoots(t *testing.T) { root := t.TempDir() if err := os.MkdirAll(filepath.Join(root, "assets", "audio", "westgate"), 0o755); err != nil { diff --git a/internal/topdata/native.go b/internal/topdata/native.go index ab65f0a..d088d7d 100644 --- a/internal/topdata/native.go +++ b/internal/topdata/native.go @@ -28,6 +28,7 @@ type nativeDataset struct { OutputName string Spec datasetTextSpec Columns []string + ValueEncodings map[string]project.TopDataValueEncodingConfig CompareReference bool } @@ -236,6 +237,7 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress if err != nil { return BuildResult{}, err } + datasets = applyTopDataValueEncodings(datasets, p.Config.TopData.ValueEncodings) registryDatasets, err := collectGeneratedRegistryDatasets(dataDir) if err != nil { return BuildResult{}, err @@ -406,6 +408,25 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress }, nil } +func applyTopDataValueEncodings(datasets []nativeDataset, encodings []project.TopDataValueEncodingConfig) []nativeDataset { + if len(encodings) == 0 { + return datasets + } + out := append([]nativeDataset(nil), datasets...) + for index := range out { + for _, encoding := range encodings { + if filepath.ToSlash(strings.TrimSpace(encoding.Dataset)) != out[index].Name { + continue + } + if out[index].ValueEncodings == nil { + out[index].ValueEncodings = map[string]project.TopDataValueEncodingConfig{} + } + out[index].ValueEncodings[strings.ToLower(strings.TrimSpace(encoding.Column))] = encoding + } + } + return out +} + func nativeCompileGroup(datasetName string) string { head, _, ok := strings.Cut(datasetName, "/") if !ok { @@ -3694,6 +3715,15 @@ func (r *valueResolver) resolveValue(row map[string]any, field string, value any switch typed := value.(type) { case map[string]any: + if encoding, ok := r.valueEncodingForField(field); ok { + if _, hasList := typed["list"]; hasList { + value, err := encodeConfiguredValueList(typed, encoding) + if err != nil { + return tlkCompiledValue{}, fmt.Errorf("field %s: %w", field, err) + } + return tlkCompiledValue{Value: value}, nil + } + } if refID, ok := typed["id"]; ok { if refKey, ok := refID.(string); ok && strings.Contains(refKey, ":") { id, ok := r.keyToID[refKey] @@ -3750,6 +3780,109 @@ func (r *valueResolver) resolveValue(row map[string]any, field string, value any } } +func (r *valueResolver) valueEncodingForField(field string) (project.TopDataValueEncodingConfig, bool) { + if len(r.dataset.ValueEncodings) == 0 { + return project.TopDataValueEncodingConfig{}, false + } + encoding, ok := r.dataset.ValueEncodings[strings.ToLower(strings.TrimSpace(field))] + return encoding, ok +} + +func encodeConfiguredValueList(obj map[string]any, encoding project.TopDataValueEncodingConfig) (string, error) { + if strings.TrimSpace(encoding.Mode) != "packed_hex_list" { + return "", fmt.Errorf("unsupported value encoding mode %q", encoding.Mode) + } + for key := range obj { + if key != "list" { + return "", fmt.Errorf("list encoding object contains unsupported key %q", key) + } + } + rawList, ok := obj["list"].([]any) + if !ok { + return "", fmt.Errorf("list must be an array") + } + values := make([]int, 0, len(rawList)) + for _, raw := range rawList { + expanded, err := expandConfiguredValueListItem(raw, encoding) + if err != nil { + return "", err + } + values = append(values, expanded...) + } + if len(values) == 0 { + return "", nil + } + var builder strings.Builder + builder.WriteString("0x") + for _, value := range values { + builder.WriteString(fmt.Sprintf("%0*X", encoding.HexWidth, value)) + } + return builder.String(), nil +} + +func expandConfiguredValueListItem(raw any, encoding project.TopDataValueEncodingConfig) ([]int, error) { + if obj, ok := raw.(map[string]any); ok { + rawRange, ok := obj["range"] + if !ok { + return nil, fmt.Errorf("list object item must contain range") + } + if len(obj) != 1 { + return nil, fmt.Errorf("range item contains unsupported keys") + } + rangeList, ok := rawRange.([]any) + if !ok || len(rangeList) != 2 { + return nil, fmt.Errorf("range must be an array with two values") + } + start, err := parseConfiguredListValue(rangeList[0], encoding) + if err != nil { + return nil, err + } + end, err := parseConfiguredListValue(rangeList[1], encoding) + if err != nil { + return nil, err + } + if end < start { + return nil, fmt.Errorf("range %d-%d ends before it starts", start, end) + } + values := make([]int, 0, end-start+1) + for value := start; value <= end; value++ { + values = append(values, value) + } + return values, nil + } + value, err := parseConfiguredListValue(raw, encoding) + if err != nil { + return nil, err + } + return []int{value}, nil +} + +func parseConfiguredListValue(raw any, encoding project.TopDataValueEncodingConfig) (int, error) { + var value int + switch typed := raw.(type) { + case int: + value = typed + case float64: + if typed != float64(int(typed)) { + return 0, fmt.Errorf("list value %v must be an integer", typed) + } + value = int(typed) + case string: + trimmed := strings.TrimSpace(typed) + parsed, err := strconv.Atoi(trimmed) + if err != nil { + return 0, fmt.Errorf("list value %q must be numeric", typed) + } + value = parsed + default: + return 0, fmt.Errorf("list value %v must be numeric", raw) + } + if value < encoding.Min || value > encoding.Max { + return 0, fmt.Errorf("value %d outside range %d-%d", value, encoding.Min, encoding.Max) + } + return value, nil +} + func (r *valueResolver) resolveLiteralFeatObject(row map[string]any, field string, value any) (string, bool) { if r.dataset.Name != "feat" { return "", false diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go index 08f584c..3e0ef7c 100644 --- a/internal/topdata/topdata_test.go +++ b/internal/topdata/topdata_test.go @@ -179,6 +179,112 @@ func TestValidateProjectRejectsInvalidModuleColumnExpansionFile(t *testing.T) { } } +func TestBuildNativeEncodesConfiguredPackedHexLists(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "StartLanguages", "BonusLanguages", "AvailableHeadsMale", "AvailableHeadsFemale", "AvailableSkinColors"], + "rows": [ + { + "id": 6, + "key": "racialtypes:human", + "Label": "Human", + "StartLanguages": "regional", + "BonusLanguages": "all" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "10_human.json"), `{ + "overrides": [ + { + "key": "racialtypes:human", + "AvailableHeadsMale": {"list": [1, 10, 999]}, + "AvailableHeadsFemale": {"list": [{"range": [1, 3]}]}, + "AvailableSkinColors": { + "list": [ + {"range": [0, 12]}, + 24, + 74, + {"range": [116, 119]}, + {"range": [128, 131]}, + 156, + 157, + 168, + 170, + 174 + ] + } + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "racialtypes/core", Column: "AvailableHeadsMale", Mode: "packed_hex_list", Min: 0, Max: 999, HexWidth: 3}, + {Dataset: "racialtypes/core", Column: "AvailableHeadsFemale", Mode: "packed_hex_list", Min: 0, Max: 999, HexWidth: 3}, + {Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 175, HexWidth: 2}, + } + + result, err := BuildNativeWithOptions(proj, NativeBuildOptions{BuildWiki: false}, nil) + if err != nil { + t.Fatalf("BuildNativeWithOptions failed: %v", err) + } + raw, err := os.ReadFile(filepath.Join(result.Output2DADir, "racialtypes.2da")) + if err != nil { + t.Fatalf("read racialtypes.2da: %v", err) + } + got := string(raw) + for _, want := range []string{ + "regional", + "all", + "0x00100A3E7", + "0x001002003", + "0x000102030405060708090A0B0C184A74757677808182839C9DA8AAAE", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected compiled racialtypes.2da to contain %q, got:\n%s", want, got) + } + } +} + +func TestValidateProjectRejectsInvalidConfiguredPackedHexList(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "base.json"), `{ + "output": "racialtypes.2da", + "columns": ["Label", "AvailableSkinColors"], + "rows": [ + {"id": 6, "key": "racialtypes:human", "Label": "Human"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "lock.json"), `{"racialtypes:human":6}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "racialtypes", "core", "modules", "10_human.json"), `{ + "overrides": [ + { + "key": "racialtypes:human", + "AvailableSkinColors": {"list": [{"range": [12, 0]}]} + } + ] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ValueEncodings = []project.TopDataValueEncodingConfig{ + {Dataset: "racialtypes/core", Column: "AvailableSkinColors", Mode: "packed_hex_list", Min: 0, Max: 175, HexWidth: 2}, + } + + report := ValidateProject(proj) + if !report.HasErrors() { + t.Fatalf("expected invalid packed hex list to fail validation, got:\n%s", diagnosticsText(report.Diagnostics)) + } + if !diagnosticsContain(report.Diagnostics, "range 12-0 ends before it starts") { + t.Fatalf("expected reversed range diagnostic, got:\n%s", diagnosticsText(report.Diagnostics)) + } +} + func TestValidateProjectRejectsDuplicateEntryKeysAcrossModules(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data", "baseitems", "modules"))