diff --git a/internal/topdata/PORTRAIT_METADATA_CONTRACT.md b/internal/topdata/PORTRAIT_METADATA_CONTRACT.md index 7ab7ce1..6a7899e 100644 --- a/internal/topdata/PORTRAIT_METADATA_CONTRACT.md +++ b/internal/topdata/PORTRAIT_METADATA_CONTRACT.md @@ -2,7 +2,7 @@ ## Objective -Define the canonical behavior for portrait metadata embedded in topdata rows, enabling cross-dataset portrait injection into the portraits.2da table. +Define the canonical behavior for portrait metadata embedded in topdata rows, enabling explicit opt-in portrait injection into the portraits.2da table. --- @@ -46,7 +46,13 @@ Field names are case-insensitive with hyphen/underscore normalization: ### Identity Matching -Portrait entries are matched by `resref`. When any row in any dataset carries `meta.portrait.resref`: +Portrait entries are matched by `resref`. Injection is only enabled when all of the following are true: + +1. The row carries `meta.portrait.resref` +2. The row comes from a JSON module under `topdata/data//modules` +3. The dataset is one of `appearance`, `genericdoors`, or `placeables` + +When those conditions are met: 1. If a portraits.2da row with matching `BaseResRef` already exists → merge into that row 2. If no matching row exists → create a new row with auto-generated identity `portraits:auto:` @@ -91,14 +97,15 @@ For example: `portraits:auto:nw2book1_` ## Cross-Dataset Injection -Any dataset may contribute portrait metadata. The build pipeline: +Only module-authored rows from `appearance`, `genericdoors`, and `placeables` may contribute portrait metadata. The build pipeline: 1. Collects all datasets 2. Runs `mergePortraitMetadata()` after dataset collection 3. Injects portrait rows into the portraits.2da dataset 4. Updates lock data if new auto-generated rows were created +5. Prunes stale `portraits:auto:*` lock entries when no active auto-injected portrait row remains -This enables non-portrait datasets (e.g., placeables, genericdoors) to define portrait associations without modifying the portraits dataset directly. +This enables supported non-portrait datasets to define portrait associations without modifying the portraits dataset directly, while making the behavior explicit and removable with the originating module. --- @@ -107,8 +114,9 @@ This enables non-portrait datasets (e.g., placeables, genericdoors) to define po 1. `resref` is required — missing resref produces a parse error 2. Unknown metadata fields in `meta.portrait` produce a parse error 3. Unknown top-level keys in `meta` produce a parse error (allowed keys: `family`, `wiki`, `portrait`) -4. Conflicting portrait field values from different contributors produce a build error -5. Portrait metadata is excluded from emitted 2DA output — it exists only at authoring/build time +4. Base rows do not inject portraits, even if they carry `meta.portrait` +5. Conflicting portrait field values from different contributors produce a build error +6. Portrait metadata is excluded from emitted 2DA output — it exists only at authoring/build time --- @@ -117,9 +125,10 @@ This enables non-portrait datasets (e.g., placeables, genericdoors) to define po Implementation is correct only if all of the following are true: 1. Portrait metadata is parsed from `meta.portrait` with case-insensitive field names -2. Rows with portrait metadata inject into portraits.2da by `resref` matching -3. Auto-generated rows receive `portraits:auto:` lock keys -4. First-contributor-wins precedence is enforced per column -5. Conflicting values produce build errors with clear diagnostic messages -6. Portrait metadata does not leak into emitted 2DA column output -7. Lock data is persisted when new auto-generated rows are created +2. Only module-authored rows from `appearance`, `genericdoors`, and `placeables` inject into portraits.2da by `resref` matching +3. Base rows do not trigger portrait injection +4. Auto-generated rows receive `portraits:auto:` lock keys +5. First-contributor-wins precedence is enforced per column +6. Conflicting values produce build errors with clear diagnostic messages +7. Portrait metadata does not leak into emitted 2DA column output +8. Lock data is persisted when new auto-generated rows are created and stale auto-generated keys are pruned diff --git a/internal/topdata/convert.go b/internal/topdata/convert.go index 6222731..d3bca09 100644 --- a/internal/topdata/convert.go +++ b/internal/topdata/convert.go @@ -21,6 +21,7 @@ type convertOptions struct { CollisionMode string BaseDialog string Type string + Name string } func RunConvertCommand(args []string, stdout io.Writer) error { @@ -49,7 +50,7 @@ func RunConvertCommand(args []string, stdout io.Writer) error { return writeJSONFile(output, result) case "2da-to-module": if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") { - _, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-module --namespace [--type entries|override] [flags] [output.json]") + _, _ = fmt.Fprintln(stdout, "usage: convert-topdata 2da-to-module [--namespace ] [--name ] [--type entries|override] [flags] [output.json]") return nil } opts, input, output, err := parseConvertArgs(args[1:], true) @@ -68,17 +69,13 @@ func RunConvertCommand(args []string, stdout io.Writer) error { _, _ = fmt.Fprintf(stdout, "converted overrides: %d\n", len(result["overrides"].([]map[string]any))) return nil } - result, skipped, err := convert2DAToModule(data, output, opts) + result, err := convert2DAToModule(data, output, opts) if err != nil { return err } if err := writeJSONFile(output, result); err != nil { return err } - if skipped > 0 { - _, _ = fmt.Fprintf(stdout, "converted entries: %d, skipped unkeyed: %d\n", len(result["entries"].(map[string]any)), skipped) - return nil - } _, _ = fmt.Fprintf(stdout, "converted entries: %d\n", len(result["entries"].(map[string]any))) return nil case "json-to-2da": @@ -106,6 +103,11 @@ func printConvertUsage(stdout io.Writer) { _, _ = fmt.Fprintln(stdout, " 2da-to-json Convert a 2DA file into canonical JSON rows") _, _ = fmt.Fprintln(stdout, " 2da-to-module Convert a 2DA file into module entries or override JSON") _, _ = fmt.Fprintln(stdout, " json-to-2da Convert canonical JSON rows into a 2DA file") + _, _ = fmt.Fprintln(stdout, "") + _, _ = fmt.Fprintln(stdout, "2da-to-module notes:") + _, _ = fmt.Fprintln(stdout, " - --namespace is optional when topdata/templates/config.json declares it for the input table") + _, _ = fmt.Fprintln(stdout, " - --name controls inferred output naming: add__.json") + _, _ = fmt.Fprintln(stdout, " - unkeyed rows now fail conversion instead of being skipped") } type parsed2DA struct { @@ -115,7 +117,7 @@ type parsed2DA struct { func parseConvertArgs(args []string, allowFormat bool) (convertOptions, string, string, error) { opts := convertOptions{ - CollisionMode: "prompt", + CollisionMode: "error", Type: "entries", } positional := make([]string, 0, 2) @@ -134,6 +136,12 @@ func parseConvertArgs(args []string, allowFormat bool) (convertOptions, string, return opts, "", "", errors.New("--key-field requires a value") } opts.KeyFields = append(opts.KeyFields, args[index]) + case "--name": + index++ + if index >= len(args) { + return opts, "", "", errors.New("--name requires a value") + } + opts.Name = args[index] case "--collision": index++ if index >= len(args) { @@ -163,22 +171,29 @@ func parseConvertArgs(args []string, allowFormat bool) (convertOptions, string, } } if allowFormat { - if strings.TrimSpace(opts.Namespace) == "" { - return opts, "", "", errors.New("2da-to-module requires --namespace") - } if len(positional) < 1 || len(positional) > 2 { - return opts, "", "", errors.New("usage: convert-topdata 2da-to-module --namespace [--type entries|override] [flags] [output.json]") + return opts, "", "", errors.New("usage: convert-topdata 2da-to-module [--namespace ] [--name ] [--type entries|override] [flags] [output.json]") } } else if len(positional) != 2 { return opts, "", "", errors.New("usage: convert-topdata 2da-to-json [flags] ") } - if opts.CollisionMode == "prompt" { - opts.CollisionMode = "suffix" - } if opts.CollisionMode != "suffix" && opts.CollisionMode != "error" { return opts, "", "", fmt.Errorf("unsupported collision mode %q", opts.CollisionMode) } if allowFormat { + ctx, err := resolveConvertContext(positional[0]) + if err != nil { + return opts, "", "", err + } + if strings.TrimSpace(opts.Namespace) == "" { + opts.Namespace = ctx.Template.Namespace + } + if len(opts.KeyFields) == 0 && len(ctx.Template.KeyFields) > 0 { + opts.KeyFields = append(opts.KeyFields, ctx.Template.KeyFields...) + } + if strings.TrimSpace(opts.Namespace) == "" { + return opts, "", "", errors.New("2da-to-module requires --namespace or a topdata/templates/config.json entry for the input table") + } switch opts.Type { case "entry": opts.Type = "entries" @@ -192,7 +207,7 @@ func parseConvertArgs(args []string, allowFormat bool) (convertOptions, string, if len(positional) == 2 { output = positional[1] } else { - resolved, err := defaultModuleOutputPath(positional[0], opts) + resolved, err := defaultModuleOutputPath(positional[0], opts, ctx.Project) if err != nil { return opts, "", "", err } @@ -213,7 +228,7 @@ func positionalSafe(args []string) string { func parse2DAFile(path string) (parsed2DA, error) { raw, err := os.ReadFile(path) if err != nil { - return parsed2DA{}, err + return parsed2DA{}, fmt.Errorf("read %s: %w", path, err) } lines := strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") trimmed := make([]string, 0, len(lines)) @@ -224,10 +239,14 @@ func parse2DAFile(path string) (parsed2DA, error) { } } if len(trimmed) < 2 || !strings.HasPrefix(trimmed[0], "2DA") { - return parsed2DA{}, errors.New("invalid 2DA file") + return parsed2DA{}, fmt.Errorf("%s: invalid 2DA file", path) } columns := split2DALine(trimmed[1]) + if len(columns) == 0 { + return parsed2DA{}, fmt.Errorf("%s: missing 2DA columns", path) + } rows := make([]map[string]any, 0, max(0, len(trimmed)-2)) + seenIDs := map[int]struct{}{} for _, line := range trimmed[2:] { fields := split2DALine(line) if len(fields) == 0 { @@ -235,8 +254,15 @@ func parse2DAFile(path string) (parsed2DA, error) { } rowID, err := strconv.Atoi(fields[0]) if err != nil { - return parsed2DA{}, fmt.Errorf("invalid 2DA row id %q", fields[0]) + return parsed2DA{}, fmt.Errorf("%s: invalid 2DA row id %q", path, fields[0]) } + if len(fields)-1 > len(columns) { + return parsed2DA{}, fmt.Errorf("%s: row %d has %d values but only %d columns are declared", path, rowID, len(fields)-1, len(columns)) + } + if _, ok := seenIDs[rowID]; ok { + return parsed2DA{}, fmt.Errorf("%s: duplicate 2DA row id %d", path, rowID) + } + seenIDs[rowID] = struct{}{} row := map[string]any{"id": rowID} for index, column := range columns { if index+1 < len(fields) { @@ -302,17 +328,17 @@ func convert2DAToJSON(data parsed2DA, outputPath string, opts convertOptions) (m return map[string]any{"columns": data.Columns, "rows": outRows}, nil } -func convert2DAToModule(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, int, error) { +func convert2DAToModule(data parsed2DA, outputPath string, opts convertOptions) (map[string]any, error) { rows, err := assignConvertedRowKeys(data.Rows, outputPath, opts) if err != nil { - return nil, 0, err + return nil, err } entries := map[string]any{} - skipped := 0 + missing := []string{} for _, row := range rows { key, _ := row["key"].(string) if key == "" { - skipped++ + missing = append(missing, strconv.Itoa(row["id"].(int))) continue } entry := map[string]any{} @@ -325,7 +351,10 @@ func convert2DAToModule(data parsed2DA, outputPath string, opts convertOptions) } entries[key] = entry } - return map[string]any{"entries": entries}, skipped, nil + if len(missing) > 0 { + return nil, fmt.Errorf("unable to generate keys for row ids %s; declare --key-field or configure key_fields in topdata/templates/config.json", strings.Join(missing, ", ")) + } + return map[string]any{"entries": entries}, nil } func toOverridesRows(rows []map[string]any) []map[string]any { @@ -457,27 +486,151 @@ func normalizeConvertedKeyText(text string) string { return text } -func defaultModuleOutputPath(input string, opts convertOptions) (string, error) { +type convertTemplateTable struct { + Namespace string `json:"namespace"` + KeyFields []string `json:"key_fields"` +} + +type convertTemplateConfig struct { + Tables map[string]convertTemplateTable `json:"tables"` +} + +type convertContext struct { + Project *project.Project + Template convertTemplateTable +} + +func resolveConvertContext(input string) (convertContext, error) { cwd, err := os.Getwd() if err != nil { - return "", err + return convertContext{}, err } root, err := project.FindRoot(cwd) if err != nil { - return "", errors.New("2da-to-module requires an explicit output path when not run inside a project root") + return convertContext{}, nil } p, err := project.Load(root) if err != nil { - return "", err + return convertContext{}, err + } + if !p.HasTopData() { + return convertContext{Project: p}, nil + } + table, err := loadTemplateTableConfig(p, input) + if err != nil { + return convertContext{}, err + } + return convertContext{ + Project: p, + Template: table, + }, nil +} + +func loadTemplateTableConfig(p *project.Project, input string) (convertTemplateTable, error) { + templatesDir := filepath.Join(p.TopDataSourceDir(), "templates") + inputAbs, err := filepath.Abs(input) + if err != nil { + return convertTemplateTable{}, err + } + templatesAbs, err := filepath.Abs(templatesDir) + if err != nil { + return convertTemplateTable{}, err + } + rel, err := filepath.Rel(templatesAbs, inputAbs) + if err != nil || strings.HasPrefix(rel, "..") || rel == "." { + return convertTemplateTable{}, nil + } + + configPath := filepath.Join(templatesDir, "config.json") + raw, err := os.ReadFile(configPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return convertTemplateTable{}, nil + } + return convertTemplateTable{}, fmt.Errorf("read %s: %w", configPath, err) + } + + var config convertTemplateConfig + if err := json.Unmarshal(raw, &config); err != nil { + return convertTemplateTable{}, fmt.Errorf("parse %s: %w", configPath, err) + } + if len(config.Tables) == 0 { + return convertTemplateTable{}, nil + } + + rel = filepath.ToSlash(rel) + base := filepath.Base(rel) + stem := strings.TrimSuffix(base, filepath.Ext(base)) + for _, key := range []string{rel, base, stem} { + if table, ok := config.Tables[key]; ok { + return table, nil + } + } + return convertTemplateTable{}, nil +} + +func defaultModuleOutputPath(input string, opts convertOptions, p *project.Project) (string, error) { + if p == nil { + return "", errors.New("2da-to-module requires an explicit output path when not run inside a project root") } if !p.HasTopData() { return "", errors.New("2da-to-module requires topdata to be configured when inferring an output path") } - filename := strings.TrimSuffix(filepath.Base(input), filepath.Ext(input)) + ".json" + filename, err := defaultModuleFileName(input, opts, p) + if err != nil { + return "", err + } modulesDir := filepath.Join(p.TopDataSourceDir(), "data", filepath.FromSlash(opts.Namespace), "modules") return filepath.Join(modulesDir, filename), nil } +func defaultModuleFileName(input string, opts convertOptions, p *project.Project) (string, error) { + if strings.TrimSpace(opts.Name) != "" { + name := normalizeConvertedKeyText(opts.Name) + if name == "" { + return "", fmt.Errorf("invalid --name %q", opts.Name) + } + namespace := normalizeModuleFilenameNamespace(opts.Namespace) + return fmt.Sprintf("add_%s_%s.json", namespace, name), nil + } + if isTemplateInput(p, input) { + return "", errors.New("2da-to-module requires --name when converting from topdata/templates without an explicit output path") + } + return strings.TrimSuffix(filepath.Base(input), filepath.Ext(input)) + ".json", nil +} + +func normalizeModuleFilenameNamespace(namespace string) string { + namespace = strings.TrimSpace(strings.ToLower(namespace)) + namespace = strings.ReplaceAll(namespace, "/", "_") + namespace = strings.ReplaceAll(namespace, "\\", "_") + namespace = convertKeyWhitespace.ReplaceAllString(namespace, "_") + namespace = convertKeyCleaner.ReplaceAllString(namespace, "_") + namespace = strings.Trim(namespace, "_") + if namespace == "" { + return "module" + } + return namespace +} + +func isTemplateInput(p *project.Project, input string) bool { + if p == nil || !p.HasTopData() { + return false + } + templatesAbs, err := filepath.Abs(filepath.Join(p.TopDataSourceDir(), "templates")) + if err != nil { + return false + } + inputAbs, err := filepath.Abs(input) + if err != nil { + return false + } + rel, err := filepath.Rel(templatesAbs, inputAbs) + if err != nil { + return false + } + return rel != "." && !strings.HasPrefix(rel, "..") +} + func readCanonicalJSON(path string) (parsed2DA, error) { raw, err := os.ReadFile(path) if err != nil { diff --git a/internal/topdata/convert_test.go b/internal/topdata/convert_test.go new file mode 100644 index 0000000..a0fcb60 --- /dev/null +++ b/internal/topdata/convert_test.go @@ -0,0 +1,128 @@ +package topdata + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunConvertCommandInfersTemplateNamespaceAndOutput(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": {"name": "Test", "resref": "test"}, + "paths": {"source": "src", "assets": "assets", "build": "build"}, + "topdata": {"source": "topdata", "build": "build/topdata"} +}`+"\n") + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance", "modules")) + writeFile(t, filepath.Join(root, "topdata", "templates", "config.json"), `{ + "tables": { + "template_appearance": { + "namespace": "appearance", + "key_fields": ["LABEL"] + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "templates", "template_appearance.2da"), "2DA V2.0\n\nLABEL STRING_REF NAME\n960 \"Zombie, Ash, Hot\" **** Zombie_Ash_Hot\n961 \"Zombie, Ash, Done\" **** Zombie_Ash_Done\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + var stdout bytes.Buffer + if err := RunConvertCommand([]string{ + "2da-to-module", + "--name", "ashzombies", + "topdata/templates/template_appearance.2da", + }, &stdout); err != nil { + t.Fatalf("RunConvertCommand failed: %v", err) + } + + outputPath := filepath.Join(root, "topdata", "data", "appearance", "modules", "add_appearance_ashzombies.json") + raw, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("read output: %v", err) + } + text := string(raw) + if !strings.Contains(text, `"appearance:zombie_ash_hot"`) || !strings.Contains(text, `"appearance:zombie_ash_done"`) { + t.Fatalf("unexpected output:\n%s", text) + } + if !strings.Contains(stdout.String(), "converted entries: 2") { + t.Fatalf("unexpected stdout: %s", stdout.String()) + } +} + +func TestRunConvertCommandFailsOnUnkeyedRows(t *testing.T) { + root := testProjectRoot(t) + writeFile(t, filepath.Join(root, "nwn-tool.json"), `{ + "module": {"name": "Test", "resref": "test"}, + "paths": {"source": "src", "assets": "assets", "build": "build"}, + "topdata": {"source": "topdata", "build": "build/topdata"} +}`+"\n") + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + writeFile(t, filepath.Join(root, "topdata", "templates", "config.json"), `{ + "tables": { + "template_appearance": { + "namespace": "appearance", + "key_fields": ["STRING_REF"] + } + } +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "templates", "template_appearance.2da"), "2DA V2.0\n\nLABEL STRING_REF NAME\n960 \"Zombie, Ash, Hot\" **** Zombie_Ash_Hot\n") + + oldCwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + t.Cleanup(func() { + _ = os.Chdir(oldCwd) + }) + if err := os.Chdir(root); err != nil { + t.Fatalf("chdir: %v", err) + } + + err = RunConvertCommand([]string{ + "2da-to-module", + "--name", "ashzombies", + "topdata/templates/template_appearance.2da", + }, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "unable to generate keys for row ids 960") { + t.Fatalf("expected unkeyed row failure, got %v", err) + } +} + +func TestConvert2DAToModuleFailsOnDuplicateGeneratedKeysByDefault(t *testing.T) { + _, err := convert2DAToModule(parsed2DA{ + Columns: []string{"LABEL"}, + Rows: []map[string]any{ + {"id": 1, "LABEL": "Zombie"}, + {"id": 2, "LABEL": "Zombie"}, + }, + }, "topdata/data/appearance/modules/add_appearance_ashzombies.json", convertOptions{ + Namespace: "appearance", + CollisionMode: "error", + }) + if err == nil || !strings.Contains(err.Error(), `duplicate generated key "appearance:zombie"`) { + t.Fatalf("expected duplicate generated key error, got %v", err) + } +} + +func TestParse2DAFileRejectsRowsWithTooManyFields(t *testing.T) { + root := t.TempDir() + inputPath := filepath.Join(root, "bad.2da") + writeFile(t, inputPath, "2DA V2.0\n\nLABEL NAME\n0 one two three\n") + + _, err := parse2DAFile(inputPath) + if err == nil || !strings.Contains(err.Error(), "has 3 values but only 2 columns are declared") { + t.Fatalf("expected row width error, got %v", err) + } +} diff --git a/internal/topdata/native.go b/internal/topdata/native.go index 32fdbae..6d46ec0 100644 --- a/internal/topdata/native.go +++ b/internal/topdata/native.go @@ -16,6 +16,7 @@ import ( ) const nullValue = "****" +const internalPortraitModuleAuthoredKey = "__topdata_portrait_module_authored" type nativeDataset struct { Kind nativeDatasetKind @@ -568,7 +569,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { } nextID := nextAvailableID(usedIDs) - applyModuleData := func(sourceLabel string, moduleData map[string]any) error { + applyModuleData := func(sourceLabel string, moduleData map[string]any, allowPortraitMetadataInjection bool) error { if rawEntries, ok := moduleData["entries"]; ok { entries, ok := rawEntries.(map[string]any) if !ok { @@ -625,6 +626,9 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { } existing[field] = value } + if allowPortraitMetadataInjection && rowHasPortraitMetadata(existing) { + existing[internalPortraitModuleAuthoredKey] = true + } continue } rowID, ok := lockData[key] @@ -642,6 +646,9 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { rows = append(rows, row) rowByID[rowID] = row rowByKey[key] = row + if allowPortraitMetadataInjection && rowHasPortraitMetadata(row) { + row[internalPortraitModuleAuthoredKey] = true + } } } @@ -747,6 +754,9 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { } row[field] = value } + if allowPortraitMetadataInjection && rowHasPortraitMetadata(row) { + row[internalPortraitModuleAuthoredKey] = true + } } } return nil @@ -757,7 +767,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { return nativeCollectedDataset{}, err } for _, module := range generatedModules { - if err := applyModuleData(module.Name, module.Data); err != nil { + if err := applyModuleData(module.Name, module.Data, false); err != nil { return nativeCollectedDataset{}, err } } @@ -771,7 +781,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { if err != nil { return nativeCollectedDataset{}, err } - if err := applyModuleData(path, moduleData); err != nil { + if err := applyModuleData(path, moduleData, true); err != nil { return nativeCollectedDataset{}, err } } diff --git a/internal/topdata/portraits_native.go b/internal/topdata/portraits_native.go index a88bd1f..5fb1a40 100644 --- a/internal/topdata/portraits_native.go +++ b/internal/topdata/portraits_native.go @@ -7,6 +7,11 @@ import ( ) var portraitColumns = []string{"BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"} +var portraitMetadataInjectionDatasets = map[string]struct{}{ + "appearance": {}, + "genericdoors": {}, + "placeables": {}, +} func parseRowMetadata(raw any) (map[string]any, error) { if raw == nil { @@ -101,6 +106,19 @@ func portraitMetaFromRow(row map[string]any) (map[string]any, bool) { return portrait, true } +func rowHasPortraitMetadata(row map[string]any) bool { + _, ok := portraitMetaFromRow(row) + return ok +} + +func portraitMetadataMayInject(datasetName string, row map[string]any) bool { + if _, ok := portraitMetadataInjectionDatasets[datasetName]; !ok { + return false + } + optIn, _ := row[internalPortraitModuleAuthoredKey].(bool) + return optIn && rowHasPortraitMetadata(row) +} + func mergePortraitMetadata(collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) { portraitIndex := -1 for i, dataset := range collected { @@ -135,17 +153,22 @@ func mergePortraitMetadata(collected []nativeCollectedDataset) ([]nativeCollecte } nextID := nextAvailableID(usedIDs) lockModified := false + activeAutoLockKeys := map[string]struct{}{} for _, dataset := range collected { + if _, ok := portraitMetadataInjectionDatasets[dataset.Dataset.Name]; !ok { + continue + } for _, row := range dataset.Rows { - portrait, ok := portraitMetaFromRow(row) - if !ok { + if !portraitMetadataMayInject(dataset.Dataset.Name, row) { continue } + portrait, _ := portraitMetaFromRow(row) resref := portrait["resref"].(string) target, ok := rowByResref[resref] if !ok { lockKey := "portraits:auto:" + resref + activeAutoLockKeys[lockKey] = struct{}{} rowID, found := portraits.LockData[lockKey] if !found { rowID = nextID @@ -171,6 +194,17 @@ func mergePortraitMetadata(collected []nativeCollectedDataset) ([]nativeCollecte } } + for lockKey := range portraits.LockData { + if !strings.HasPrefix(lockKey, "portraits:auto:") { + continue + } + if _, ok := activeAutoLockKeys[lockKey]; ok { + continue + } + delete(portraits.LockData, lockKey) + lockModified = true + } + if lockModified && portraits.Dataset.LockPath != "" { if err := saveLockfile(portraits.Dataset.LockPath, portraits.LockData); err != nil { return nil, fmt.Errorf("dataset %s: %w", portraits.Dataset.Name, err) diff --git a/internal/topdata/top_package.go b/internal/topdata/top_package.go index 375928b..5dfdd7f 100644 --- a/internal/topdata/top_package.go +++ b/internal/topdata/top_package.go @@ -306,10 +306,14 @@ func countCompiledFiles(dir, extension string) (int, time.Time, error) { func newestTopDataSource(sourceDir string) (time.Time, string, error) { newest := time.Time{} newestPath := "" + templatesDir := filepath.Join(sourceDir, "templates") err := filepath.WalkDir(sourceDir, func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } + if d.IsDir() && path == templatesDir { + return filepath.SkipDir + } if d.IsDir() { return nil } diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go index 8706b6d..570a082 100644 --- a/internal/topdata/topdata_test.go +++ b/internal/topdata/topdata_test.go @@ -6001,6 +6001,7 @@ func TestBuildDenseGenerationDefaultsMissingColumnsAndPortraitFields(t *testing. mkdirAll(t, filepath.Join(root, "topdata", "data", "plain")) mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits")) mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables", "modules")) writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ "output": "dense.2da", @@ -6037,7 +6038,15 @@ func TestBuildDenseGenerationDefaultsMissingColumnsAndPortraitFields(t *testing. { "key": "placeables:bookshelf_special", "id": 10, - "Label": "BookshelfSpecial", + "Label": "BookshelfSpecial" + } + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "lock.json"), `{"placeables:bookshelf_special":10}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "modules", "add_bookshelf_special.json"), `{ + "overrides": [ + { + "key": "placeables:bookshelf_special", "meta": { "portrait": { "resref": "nw2book1_", @@ -6048,7 +6057,6 @@ func TestBuildDenseGenerationDefaultsMissingColumnsAndPortraitFields(t *testing. } ] }`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "lock.json"), `{"placeables:bookshelf_special":10}`+"\n") mkdirAll(t, filepath.Join(root, "reference")) writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") @@ -6180,6 +6188,8 @@ func TestBuildRejectsConflictingPortraitMetadata(t *testing.T) { mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits")) mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables")) mkdirAll(t, filepath.Join(root, "topdata", "data", "genericdoors")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "genericdoors", "modules")) writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ "output": "portraits.2da", @@ -6191,14 +6201,26 @@ func TestBuildRejectsConflictingPortraitMetadata(t *testing.T) { "output": "placeables.2da", "columns": ["Label"], "rows": [ - {"id": 1, "Label": "Shelf", "meta": {"portrait": {"resref": "nw2book1_", "sex": 4}}} + {"key": "placeables:shelf", "id": 1, "Label": "Shelf"} ] }`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "lock.json"), `{"placeables:shelf":1}`+"\n") writeFile(t, filepath.Join(root, "topdata", "data", "genericdoors", "base.json"), `{ "output": "genericdoors.2da", "columns": ["Label"], "rows": [ - {"id": 2, "Label": "Door", "meta": {"portrait": {"resref": "nw2book1_", "sex": 1}}} + {"key": "genericdoors:door", "id": 2, "Label": "Door"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "genericdoors", "lock.json"), `{"genericdoors:door":2}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "modules", "ovr_shelf.json"), `{ + "overrides": [ + {"key": "placeables:shelf", "meta": {"portrait": {"resref": "nw2book1_", "sex": 4}}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "genericdoors", "modules", "ovr_door.json"), `{ + "overrides": [ + {"key": "genericdoors:door", "meta": {"portrait": {"resref": "nw2book1_", "sex": 1}}} ] }`+"\n") mkdirAll(t, filepath.Join(root, "reference")) @@ -7569,6 +7591,7 @@ func TestBuildSupportsCanonicalPortraitsFromDatasetAndMetadata(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits", "modules")) mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables", "modules")) writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ "output": "portraits.2da", @@ -7592,7 +7615,13 @@ func TestBuildSupportsCanonicalPortraitsFromDatasetAndMetadata(t *testing.T) { "output": "placeables.2da", "columns": ["Label"], "rows": [ - {"id": 1, "Label": "Bookshelf", "meta": {"portrait": {"resref": "nw2book1_", "sex": 4, "inanimate_type": 4}}} + {"key": "placeables:bookshelf", "id": 1, "Label": "Bookshelf"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "lock.json"), `{"placeables:bookshelf":1}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "modules", "ovr_bookshelf.json"), `{ + "overrides": [ + {"key": "placeables:bookshelf", "meta": {"portrait": {"resref": "nw2book1_", "sex": 4, "inanimate_type": 4}}} ] }`+"\n") mkdirAll(t, filepath.Join(root, "reference")) @@ -7615,6 +7644,51 @@ func TestBuildSupportsCanonicalPortraitsFromDatasetAndMetadata(t *testing.T) { } } +func TestBuildIgnoresBasePortraitMetadataWithoutModuleOptInAndPrunesAutoLocks(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ + "output": "portraits.2da", + "columns": ["BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"], + "rows": [ + {"id": 0, "BaseResRef": "****", "Sex": 4} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), `{"portraits:auto:nw2book1_":16004}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "base.json"), `{ + "output": "placeables.2da", + "columns": ["Label"], + "rows": [ + {"key": "placeables:bookshelf", "id": 1, "Label": "Bookshelf", "meta": {"portrait": {"resref": "nw2book1_", "sex": 4, "inanimate_type": 4}}} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "lock.json"), `{"placeables:bookshelf":1}`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + got, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da")) + if err != nil { + t.Fatalf("read portraits.2da: %v", err) + } + want := "2DA V2.0\n\nBaseResRef\tSex\tRace\tInanimateType\tPlot\tLowGore\n0\t****\t4\t****\t****\t****\t****\n" + if string(got) != want { + t.Fatalf("unexpected portraits.2da output:\n%s", string(got)) + } + lockBytes, err := os.ReadFile(filepath.Join(root, "topdata", "data", "portraits", "lock.json")) + if err != nil { + t.Fatalf("read portraits lock: %v", err) + } + if strings.TrimSpace(string(lockBytes)) != "{}" { + t.Fatalf("expected stale auto portrait lock to be pruned, got:\n%s", string(lockBytes)) + } +} + func TestNormalizeProjectImportsRacialtypesRegistry(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data")) @@ -8472,6 +8546,36 @@ func TestBuildPackageFailsWhenCompiledOutputIsStale(t *testing.T) { } } +func TestBuildPackageIgnoresTemplateSourcesForFreshness(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust")) + mkdirAll(t, filepath.Join(root, "topdata", "templates")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + writeFile(t, filepath.Join(root, "topdata", "data", "repadjust", "base.json"), `{ + "output": "repadjust.2da", + "columns": ["Label"], + "rows": [{"id": 0, "Label": "TEST_LABEL"}] +}`+"\n") + + proj := testProject(root) + proj.Config.TopData.ReferenceBuilder = "" + + if _, err := BuildNative(proj, nil); err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + templatePath := filepath.Join(root, "topdata", "templates", "scratch.2da") + writeFile(t, templatePath, "2DA V2.0\n\nLabel\n0\tScratch\n") + newTime := time.Now().Add(2 * time.Second) + if err := os.Chtimes(templatePath, newTime, newTime); err != nil { + t.Fatalf("touch topdata template: %v", err) + } + + if _, err := BuildPackage(proj, nil); err != nil { + t.Fatalf("expected template sources to be ignored for freshness, got %v", err) + } +} + func TestValidateProjectErrorsOnTopPackageAssetCollision(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data", "repadjust"))