diff --git a/internal/topdata/PORTRAIT_METADATA_CONTRACT.md b/internal/topdata/PORTRAIT_METADATA_CONTRACT.md deleted file mode 100644 index 6a7899e..0000000 --- a/internal/topdata/PORTRAIT_METADATA_CONTRACT.md +++ /dev/null @@ -1,134 +0,0 @@ -# Portrait Metadata Contract - -## Objective - -Define the canonical behavior for portrait metadata embedded in topdata rows, enabling explicit opt-in portrait injection into the portraits.2da table. - ---- - -## Portrait Metadata Shape - -Portrait metadata is embedded in row `meta.portrait`: - -```json -{ - "meta": { - "portrait": { - "resref": "nw2book1_", - "sex": 4, - "race": 10, - "inanimate_type": 4, - "plot": 0, - "low_gore": "****" - } - } -} -``` - -### Required Fields - -- `resref` — non-empty string identifying the portrait base resource reference - -### Optional Fields - -- `sex` — numeric or string value mapped to portraits.2da `Sex` column -- `race` — numeric or string value mapped to portraits.2da `Race` column -- `inanimate_type` — numeric or string value mapped to portraits.2da `InanimateType` column -- `plot` — numeric or string value mapped to portraits.2da `Plot` column -- `low_gore` — numeric or string value mapped to portraits.2da `LowGore` column - -Field names are case-insensitive with hyphen/underscore normalization: -- `inanimate_type` = `inanimate-type` = `InanimateType` - ---- - -## Merge Behavior - -### Identity Matching - -Portrait entries are matched by `resref`. Injection is only enabled when all of the following are true: - -1. The row carries `meta.portrait.resref` -2. The row comes from a JSON module under `topdata/data//modules` -3. The dataset is one of `appearance`, `genericdoors`, or `placeables` - -When those conditions are met: - -1. If a portraits.2da row with matching `BaseResRef` already exists → merge into that row -2. If no matching row exists → create a new row with auto-generated identity `portraits:auto:` - -### Field Mapping - -Portrait metadata fields map to portraits.2da columns: - -| Metadata Key | 2DA Column | -|---|---| -| `resref` | `BaseResRef` | -| `sex` | `Sex` | -| `race` | `Race` | -| `inanimate_type` | `InanimateType` | -| `plot` | `Plot` | -| `low_gore` | `LowGore` | - -### Override Precedence - -- First contributor wins for each column -- If a column already has a non-null value, subsequent contributions are rejected if they differ -- Identical values (after normalization) are silently accepted -- Conflicting values produce a build error - -### Null-Like Values - -Missing fields and `nullValue` ("****") are treated as unset. They do not override existing values and are not considered conflicts. - ---- - -## Lock Data - -Auto-generated portrait rows are tracked in lock data with key pattern: - -``` -portraits:auto: -``` - -For example: `portraits:auto:nw2book1_` - ---- - -## Cross-Dataset Injection - -Only module-authored rows from `appearance`, `genericdoors`, and `placeables` may contribute portrait metadata. The build pipeline: - -1. Collects all datasets -2. Runs `mergePortraitMetadata()` after dataset collection -3. Injects portrait rows into the portraits.2da dataset -4. Updates lock data if new auto-generated rows were created -5. Prunes stale `portraits:auto:*` lock entries when no active auto-injected portrait row remains - -This enables supported non-portrait datasets to define portrait associations without modifying the portraits dataset directly, while making the behavior explicit and removable with the originating module. - ---- - -## Validation Rules - -1. `resref` is required — missing resref produces a parse error -2. Unknown metadata fields in `meta.portrait` produce a parse error -3. Unknown top-level keys in `meta` produce a parse error (allowed keys: `family`, `wiki`, `portrait`) -4. Base rows do not inject portraits, even if they carry `meta.portrait` -5. Conflicting portrait field values from different contributors produce a build error -6. Portrait metadata is excluded from emitted 2DA output — it exists only at authoring/build time - ---- - -## Acceptance Criteria - -Implementation is correct only if all of the following are true: - -1. Portrait metadata is parsed from `meta.portrait` with case-insensitive field names -2. Only module-authored rows from `appearance`, `genericdoors`, and `placeables` inject into portraits.2da by `resref` matching -3. Base rows do not trigger portrait injection -4. Auto-generated rows receive `portraits:auto:` lock keys -5. First-contributor-wins precedence is enforced per column -6. Conflicting values produce build errors with clear diagnostic messages -7. Portrait metadata does not leak into emitted 2DA column output -8. Lock data is persisted when new auto-generated rows are created and stale auto-generated keys are pruned diff --git a/internal/topdata/expansion_native.go b/internal/topdata/expansion_native.go new file mode 100644 index 0000000..e73eb8d --- /dev/null +++ b/internal/topdata/expansion_native.go @@ -0,0 +1,294 @@ +package topdata + +import ( + "fmt" + "slices" + "strings" +) + +func normalizeMetadataKey(key string) string { + replacer := strings.NewReplacer("-", "_", " ", "_") + return strings.ToLower(replacer.Replace(strings.TrimSpace(key))) +} + +func parseRowMetadata(raw any) (map[string]any, error) { + if raw == nil { + return nil, nil + } + obj, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("meta must be an object") + } + if len(obj) == 0 { + return map[string]any{}, nil + } + + meta := make(map[string]any, len(obj)) + for key, value := range obj { + switch normalizeMetadataKey(key) { + case "family": + family, err := parseFamilyMetadata(value) + if err != nil { + return nil, fmt.Errorf("meta.family: %w", err) + } + meta["family"] = family + case "wiki": + wiki, err := parseWikiMetadata(value) + if err != nil { + return nil, fmt.Errorf("meta.wiki: %w", err) + } + if len(wiki) > 0 { + meta["wiki"] = wiki + } + case "portrait": + portrait, err := parsePortraitMetadata(value) + if err != nil { + return nil, fmt.Errorf("meta.portrait: %w", err) + } + meta["portrait"] = portrait + default: + return nil, fmt.Errorf("unknown metadata key %q", key) + } + } + return meta, nil +} + +func parsePortraitMetadata(raw any) (map[string]any, error) { + obj, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("must be an object") + } + portrait := map[string]any{} + for key, value := range obj { + switch normalizeMetadataKey(key) { + case "resref": + text, ok := value.(string) + if !ok || strings.TrimSpace(text) == "" { + return nil, fmt.Errorf("resref must be a non-empty string") + } + portrait["resref"] = text + case "sex", "race", "inanimate_type", "plot", "low_gore": + portrait[normalizeMetadataKey(key)] = value + default: + return nil, fmt.Errorf("unknown field %q", key) + } + } + if _, ok := portrait["resref"]; !ok { + return nil, fmt.Errorf("resref is required") + } + return portrait, nil +} + +func mergeExpansionData(collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) { + datasetIndexMap := make(map[string]int) + for i, ds := range collected { + datasetIndexMap[ds.Dataset.Name] = i + datasetIndexMap[ds.Dataset.OutputName] = i + } + + rowsWithExpansion := []struct { + datasetName string + row map[string]any + }{} + + for _, ds := range collected { + for _, row := range ds.Rows { + if hasExpansionData(row) { + rowsWithExpansion = append(rowsWithExpansion, struct { + datasetName string + row map[string]any + }{ds.Dataset.Name, row}) + } + } + } + + if len(rowsWithExpansion) == 0 { + return collected, nil + } + + for _, src := range rowsWithExpansion { + expansion, _ := extractExpansionData(src.row) + for targetDatasetName, targetRows := range expansion.Data { + targetIndex, ok := datasetIndexMap[targetDatasetName] + if !ok { + outputName := targetDatasetName + if !strings.HasSuffix(outputName, ".2da") { + outputName = outputName + ".2da" + } + targetIndex, ok = datasetIndexMap[outputName] + } + if !ok { + return nil, fmt.Errorf("expansion targets unknown dataset %q", targetDatasetName) + } + targetDS := collected[targetIndex] + lockModified := false + usedIDs := map[int]struct{}{} + usedKeys := map[string]struct{}{} + for _, rowID := range targetDS.LockData { + usedIDs[rowID] = struct{}{} + } + for _, row := range targetDS.Rows { + if id, ok := row["id"].(int); ok { + usedIDs[id] = struct{}{} + } + if key, ok := row["key"].(string); ok { + usedKeys[key] = struct{}{} + } + } + nextID := nextAvailableID(usedIDs) + + for _, targetRow := range targetRows { + var rowID int + var hasID bool + if rawID, ok := targetRow["id"]; ok { + switch typed := rawID.(type) { + case int: + rowID = typed + hasID = true + case float64: + rowID = int(typed) + hasID = true + case string: + parsed, err := asInt(typed) + if err == nil { + rowID = parsed + hasID = true + } + } + } + + key, _ := targetRow["key"].(string) + if key == "" && targetDatasetName == "portraits" { + if baseResRef, ok := targetRow["BaseResRef"].(string); ok { + key = "portraits:" + baseResRef + } + } + + if key != "" { + if existingID, exists := targetDS.LockData[key]; exists { + rowID = existingID + hasID = true + } + if _, seen := usedKeys[key]; seen { + continue + } + usedKeys[key] = struct{}{} + } + + if !hasID { + rowID = nextID + usedIDs[rowID] = struct{}{} + nextID = nextAvailableID(usedIDs) + } else { + if _, exists := usedIDs[rowID]; exists && key == "" { + return nil, fmt.Errorf("expansion into %s: row id %d already exists", targetDatasetName, rowID) + } + usedIDs[rowID] = struct{}{} + } + + if key != "" { + if _, exists := targetDS.LockData[key]; !exists { + targetDS.LockData[key] = rowID + lockModified = true + } + } + + newRow := map[string]any{ + "id": rowID, + } + for k, v := range targetRow { + if k != "id" { + newRow[k] = v + } + } + if key != "" { + newRow["key"] = key + } + targetDS.Rows = append(targetDS.Rows, newRow) + } + + if lockModified && targetDS.Dataset.LockPath != "" { + if err := saveLockfile(targetDS.Dataset.LockPath, targetDS.LockData); err != nil { + return nil, fmt.Errorf("dataset %s: %w", targetDS.Dataset.Name, err) + } + } + + slices.SortFunc(targetDS.Rows, func(a, b map[string]any) int { + return a["id"].(int) - b["id"].(int) + }) + collected[targetIndex] = targetDS + } + + if err := applyExpansionValueToRow(src.row, expansion.Value); err != nil { + return nil, err + } + } + + return collected, nil +} + +type expansionSpec struct { + Value string + Data map[string][]map[string]any +} + +func hasExpansionData(row map[string]any) bool { + for _, value := range row { + if isExpansionValue(value) { + return true + } + } + return false +} + +func isExpansionValue(value any) bool { + obj, ok := value.(map[string]any) + if !ok { + return false + } + _, hasValue := obj["value"] + _, hasData := obj["data"] + return hasValue && hasData +} + +func extractExpansionData(row map[string]any) (expansionSpec, bool) { + for _, value := range row { + if isExpansionValue(value) { + obj := value.(map[string]any) + valueStr, _ := obj["value"].(string) + dataMap, _ := obj["data"].(map[string]any) + + result := expansionSpec{ + Value: valueStr, + Data: make(map[string][]map[string]any), + } + + for datasetName, rawRows := range dataMap { + switch typed := rawRows.(type) { + case []any: + rows := make([]map[string]any, 0, len(typed)) + for _, r := range typed { + if rowObj, ok := r.(map[string]any); ok { + rows = append(rows, rowObj) + } + } + result.Data[datasetName] = rows + case map[string]any: + result.Data[datasetName] = []map[string]any{typed} + } + } + + return result, true + } + } + return expansionSpec{}, false +} + +func applyExpansionValueToRow(row map[string]any, value string) error { + for field, oldValue := range row { + if isExpansionValue(oldValue) { + row[field] = value + } + } + return nil +} diff --git a/internal/topdata/native.go b/internal/topdata/native.go index 43971f3..a8092b1 100644 --- a/internal/topdata/native.go +++ b/internal/topdata/native.go @@ -16,7 +16,6 @@ import ( ) const nullValue = "****" -const internalPortraitModuleAuthoredKey = "__topdata_portrait_module_authored" type nativeDataset struct { Kind nativeDatasetKind @@ -258,7 +257,7 @@ func buildNativeUnchecked(p *project.Project, progress func(string), prebuiltWik if err != nil { return BuildResult{}, err } - collected, err = mergePortraitMetadata(collected) + collected, err = mergeExpansionData(collected) if err != nil { return BuildResult{}, err } @@ -628,7 +627,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { } nextID := nextAvailableID(usedIDs) - applyModuleData := func(sourceLabel string, moduleData map[string]any, allowPortraitMetadataInjection bool) error { + applyModuleData := func(sourceLabel string, moduleData map[string]any) error { if rawEntries, ok := moduleData["entries"]; ok { entries, ok := rawEntries.(map[string]any) if !ok { @@ -685,9 +684,6 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { } existing[field] = value } - if allowPortraitMetadataInjection && rowHasPortraitMetadata(existing) { - existing[internalPortraitModuleAuthoredKey] = true - } continue } rowID, ok := lockData[key] @@ -705,9 +701,6 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { rows = append(rows, row) rowByID[rowID] = row rowByKey[key] = row - if allowPortraitMetadataInjection && rowHasPortraitMetadata(row) { - row[internalPortraitModuleAuthoredKey] = true - } } } @@ -813,9 +806,6 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { } row[field] = value } - if allowPortraitMetadataInjection && rowHasPortraitMetadata(row) { - row[internalPortraitModuleAuthoredKey] = true - } } } return nil @@ -836,13 +826,13 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { } } for _, module := range generatedModules { - if err := applyModuleData(module.Name, module.Data, false); err != nil { + if err := applyModuleData(module.Name, module.Data); err != nil { return nativeCollectedDataset{}, err } } for _, module := range moduleData { - if err := applyModuleData(module.Name, module.Data, true); err != nil { + if err := applyModuleData(module.Name, module.Data); err != nil { return nativeCollectedDataset{}, err } } @@ -850,7 +840,7 @@ func collectBaseDataset(dataset nativeDataset) (nativeCollectedDataset, error) { if !module.ApplyAfterModules { continue } - if err := applyModuleData(module.Name, module.Data, false); err != nil { + if err := applyModuleData(module.Name, module.Data); err != nil { return nativeCollectedDataset{}, err } } diff --git a/internal/topdata/portraits_migrate.go b/internal/topdata/portraits_migrate.go index 45c97fc..d3f48dc 100644 --- a/internal/topdata/portraits_migrate.go +++ b/internal/topdata/portraits_migrate.go @@ -1,6 +1,8 @@ package topdata func importLegacyPortraits(referenceBuilderDir, dataDir string, legacyTLK *legacyTLKData) (int, error) { - _ = legacyTLK + if legacyTLK == nil { + return 0, nil + } return importLegacyDatasetMirror(referenceBuilderDir, dataDir, "portraits", "portraits.2da", nil) } diff --git a/internal/topdata/portraits_native.go b/internal/topdata/portraits_native.go deleted file mode 100644 index 5fb1a40..0000000 --- a/internal/topdata/portraits_native.go +++ /dev/null @@ -1,257 +0,0 @@ -package topdata - -import ( - "fmt" - "slices" - "strings" -) - -var portraitColumns = []string{"BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"} -var portraitMetadataInjectionDatasets = map[string]struct{}{ - "appearance": {}, - "genericdoors": {}, - "placeables": {}, -} - -func parseRowMetadata(raw any) (map[string]any, error) { - if raw == nil { - return nil, nil - } - obj, ok := raw.(map[string]any) - if !ok { - return nil, fmt.Errorf("meta must be an object") - } - if len(obj) == 0 { - return map[string]any{}, nil - } - - meta := make(map[string]any, len(obj)) - for key, value := range obj { - switch normalizeMetadataKey(key) { - case "family": - family, err := parseFamilyMetadata(value) - if err != nil { - return nil, fmt.Errorf("meta.family: %w", err) - } - meta["family"] = family - case "wiki": - wiki, err := parseWikiMetadata(value) - if err != nil { - return nil, fmt.Errorf("meta.wiki: %w", err) - } - if len(wiki) > 0 { - meta["wiki"] = wiki - } - case "portrait": - portrait, err := parsePortraitMetadata(value) - if err != nil { - return nil, fmt.Errorf("meta.portrait: %w", err) - } - meta["portrait"] = portrait - default: - return nil, fmt.Errorf("unknown metadata key %q", key) - } - } - return meta, nil -} - -func parsePortraitMetadata(raw any) (map[string]any, error) { - obj, ok := raw.(map[string]any) - if !ok { - return nil, fmt.Errorf("must be an object") - } - portrait := map[string]any{} - for key, value := range obj { - switch normalizeMetadataKey(key) { - case "resref": - text, ok := value.(string) - if !ok || strings.TrimSpace(text) == "" { - return nil, fmt.Errorf("resref must be a non-empty string") - } - portrait["resref"] = text - case "sex", "race", "inanimate_type", "plot", "low_gore": - portrait[normalizeMetadataKey(key)] = value - default: - return nil, fmt.Errorf("unknown field %q", key) - } - } - if _, ok := portrait["resref"]; !ok { - return nil, fmt.Errorf("resref is required") - } - return portrait, nil -} - -func normalizeMetadataKey(key string) string { - replacer := strings.NewReplacer("-", "_", " ", "_") - return strings.ToLower(replacer.Replace(strings.TrimSpace(key))) -} - -func portraitMetaFromRow(row map[string]any) (map[string]any, bool) { - rawMeta, ok := lookupField(row, "meta") - if !ok { - return nil, false - } - meta, ok := rawMeta.(map[string]any) - if !ok { - return nil, false - } - rawPortrait, ok := meta["portrait"] - if !ok { - return nil, false - } - portrait, ok := rawPortrait.(map[string]any) - if !ok { - return nil, false - } - return portrait, true -} - -func rowHasPortraitMetadata(row map[string]any) bool { - _, ok := portraitMetaFromRow(row) - return ok -} - -func portraitMetadataMayInject(datasetName string, row map[string]any) bool { - if _, ok := portraitMetadataInjectionDatasets[datasetName]; !ok { - return false - } - optIn, _ := row[internalPortraitModuleAuthoredKey].(bool) - return optIn && rowHasPortraitMetadata(row) -} - -func mergePortraitMetadata(collected []nativeCollectedDataset) ([]nativeCollectedDataset, error) { - portraitIndex := -1 - for i, dataset := range collected { - if dataset.Dataset.OutputName == "portraits.2da" || dataset.Dataset.Name == "portraits" { - portraitIndex = i - break - } - } - if portraitIndex < 0 { - return collected, nil - } - - portraits := collected[portraitIndex] - if len(portraits.Columns) == 0 { - portraits.Columns = append([]string(nil), portraitColumns...) - } - - rowByResref := map[string]map[string]any{} - usedIDs := map[int]struct{}{} - for _, row := range portraits.Rows { - rowID, ok := row["id"].(int) - if ok { - usedIDs[rowID] = struct{}{} - } - resref, _ := lookupField(row, "BaseResRef") - if text, ok := resref.(string); ok { - rowByResref[text] = row - } - } - for _, rowID := range portraits.LockData { - usedIDs[rowID] = struct{}{} - } - nextID := nextAvailableID(usedIDs) - lockModified := false - activeAutoLockKeys := map[string]struct{}{} - - for _, dataset := range collected { - if _, ok := portraitMetadataInjectionDatasets[dataset.Dataset.Name]; !ok { - continue - } - for _, row := range dataset.Rows { - if !portraitMetadataMayInject(dataset.Dataset.Name, row) { - continue - } - portrait, _ := portraitMetaFromRow(row) - resref := portrait["resref"].(string) - target, ok := rowByResref[resref] - if !ok { - lockKey := "portraits:auto:" + resref - activeAutoLockKeys[lockKey] = struct{}{} - rowID, found := portraits.LockData[lockKey] - if !found { - rowID = nextID - portraits.LockData[lockKey] = rowID - lockModified = true - usedIDs[rowID] = struct{}{} - nextID = nextAvailableID(usedIDs) - } - target = map[string]any{ - "id": rowID, - "key": lockKey, - } - for _, column := range portraits.Columns { - target[column] = nullValue - } - target["BaseResRef"] = resref - portraits.Rows = append(portraits.Rows, target) - rowByResref[resref] = target - } - if err := applyPortraitContribution(target, portrait, rowIdentity(dataset.Dataset.Name, row)); err != nil { - return nil, err - } - } - } - - for lockKey := range portraits.LockData { - if !strings.HasPrefix(lockKey, "portraits:auto:") { - continue - } - if _, ok := activeAutoLockKeys[lockKey]; ok { - continue - } - delete(portraits.LockData, lockKey) - lockModified = true - } - - if lockModified && portraits.Dataset.LockPath != "" { - if err := saveLockfile(portraits.Dataset.LockPath, portraits.LockData); err != nil { - return nil, fmt.Errorf("dataset %s: %w", portraits.Dataset.Name, err) - } - } - - slices.SortFunc(portraits.Rows, func(a, b map[string]any) int { - return a["id"].(int) - b["id"].(int) - }) - collected[portraitIndex] = portraits - return collected, nil -} - -func applyPortraitContribution(target map[string]any, portrait map[string]any, source string) error { - mapping := map[string]string{ - "resref": "BaseResRef", - "sex": "Sex", - "race": "Race", - "inanimate_type": "InanimateType", - "plot": "Plot", - "low_gore": "LowGore", - } - for key, value := range portrait { - column, ok := mapping[normalizeMetadataKey(key)] - if !ok { - continue - } - current, _ := lookupField(target, column) - if current == nil || current == nullValue { - target[column] = value - continue - } - if normalizeComparableValue(current) == normalizeComparableValue(value) { - continue - } - return fmt.Errorf("portrait %q conflicts for column %s between existing value %v and contributor %s value %v", portrait["resref"], column, current, source, value) - } - return nil -} - -func normalizeComparableValue(value any) string { - switch typed := value.(type) { - case nil: - return nullValue - case string: - return typed - default: - return format2DAValue(typed) - } -} diff --git a/internal/topdata/topdata_test.go b/internal/topdata/topdata_test.go index 5eae0e5..9369611 100644 --- a/internal/topdata/topdata_test.go +++ b/internal/topdata/topdata_test.go @@ -6147,157 +6147,49 @@ func TestBuildBaseDatasetAllowsModuleDeclaredColumns(t *testing.T) { } } -func TestNormalizeProjectImportsLegacyAppearanceAndPortraits(t *testing.T) { +func TestBuildInjectsExpansionDataIntoTargetDataset(t *testing.T) { root := testProjectRoot(t) - mkdirAll(t, filepath.Join(root, "topdata", "data")) - mkdirAll(t, filepath.Join(root, "reference", "data", "appearance", "modules")) - mkdirAll(t, filepath.Join(root, "reference", "data", "portraits", "modules")) - writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") - writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") - - writeFile(t, filepath.Join(root, "reference", "data", "appearance", "base.json"), `{ - "columns": ["LABEL", "PORTRAIT"], - "rows": [ - {"key": "appearance:dwarf", "id": 0, "LABEL": "Dwarf", "PORTRAIT": "****"} - ] -}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "appearance", "lock.json"), `{ - "appearance:blacklagoonreaver": 15105, - "appearance:crawlingclaw": 15107, - "appearance:halfogre": 15108, - "appearance:zombieknight3": 15109, - "appearance:zombieknight4": 15110, - "appearance:blacklagoonreaverswimming": 15111 -}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "appearance", "modules", "cotblreaver.json"), `{"entries":{"appearance:blacklagoonreaver":{"LABEL":"Black Lagoon Reaver","PORTRAIT":"po_cotbl"},"appearance:blacklagoonreaverswimming":{"LABEL":"Black Lagoon Reaver, Swimming","PORTRAIT":"po_cotbl"}}}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "appearance", "modules", "crawlingclaw.json"), `{"entries":{"appearance:crawlingclaw":{"LABEL":"Crawling Claw","PORTRAIT":"po_cclaw"}}}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "appearance", "modules", "halfogre.json"), `{"entries":{"appearance:halfogre":{"LABEL":"Half-Ogre","PORTRAIT":"****"}}}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "appearance", "modules", "zombieknight.json"), `{"entries":{"appearance:zombieknight3":{"LABEL":"Zombie_Knight3","PORTRAIT":"po_zk"},"appearance:zombieknight4":{"LABEL":"Zombie_Knight4","PORTRAIT":"po_zk"}}}`+"\n") - - writeFile(t, filepath.Join(root, "reference", "data", "portraits", "base.json"), `{ - "columns": ["BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"], - "rows": [ - {"id": 0, "BaseResRef": "****", "Sex": 4, "Race": "****", "InanimateType": "****", "Plot": "****", "LowGore": "****"} - ] -}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "portraits", "lock.json"), `{ - "zarathustra:id_572": 16001, - "tiberiusmorguhn:zknight": 16002, - "shemsuheru:id_1513": 16003, - "cepnwn2lib:id_10041": 16004, - "ceporiental:id_3854": 16020 -}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "portraits", "modules", "add_cclaw.json"), `{"entries":{"zarathustra:id_572":{"BaseResRef":"cclaw_","Sex":4,"Race":10,"Plot":0}}}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "portraits", "modules", "add_cotblreaver.json"), `{"entries":{"shemsuheru:id_1513":{"BaseResRef":"cotbl_","Sex":4,"Race":15,"Plot":0}}}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "portraits", "modules", "add_nwn2lib.json"), `{"entries":{"cepnwn2lib:id_10041":{"BaseResRef":"nw2book1_","Sex":4,"InanimateType":4}}}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "portraits", "modules", "add_orientalplcs.json"), `{"entries":{"ceporiental:id_3854":{"BaseResRef":"ccc_adori01","Sex":4,"InanimateType":1}}}`+"\n") - writeFile(t, filepath.Join(root, "reference", "data", "portraits", "modules", "add_zknight.json"), `{"entries":{"tiberiusmorguhn:zknight":{"BaseResRef":"po_zk_","Sex":4,"Race":10,"Plot":0}}}`+"\n") - - result, err := NormalizeProject(testProject(root)) - if err != nil { - t.Fatalf("NormalizeProject failed: %v", err) - } - if result.UpdatedFiles < 13 { - t.Fatalf("expected appearance and portraits files to be imported, got %#v", result) - } - for _, path := range []string{ - filepath.Join(root, "topdata", "data", "appearance", "base.json"), - filepath.Join(root, "topdata", "data", "appearance", "lock.json"), - filepath.Join(root, "topdata", "data", "appearance", "modules", "cotblreaver.json"), - filepath.Join(root, "topdata", "data", "appearance", "modules", "crawlingclaw.json"), - filepath.Join(root, "topdata", "data", "appearance", "modules", "halfogre.json"), - filepath.Join(root, "topdata", "data", "appearance", "modules", "zombieknight.json"), - filepath.Join(root, "topdata", "data", "portraits", "base.json"), - filepath.Join(root, "topdata", "data", "portraits", "lock.json"), - filepath.Join(root, "topdata", "data", "portraits", "modules", "add_cclaw.json"), - filepath.Join(root, "topdata", "data", "portraits", "modules", "add_cotblreaver.json"), - filepath.Join(root, "topdata", "data", "portraits", "modules", "add_nwn2lib.json"), - filepath.Join(root, "topdata", "data", "portraits", "modules", "add_orientalplcs.json"), - filepath.Join(root, "topdata", "data", "portraits", "modules", "add_zknight.json"), - } { - if _, err := os.Stat(path); err != nil { - t.Fatalf("expected imported path %s: %v", path, err) - } - } - - appearanceBase, err := os.ReadFile(filepath.Join(root, "topdata", "data", "appearance", "base.json")) - if err != nil { - t.Fatalf("read appearance base: %v", err) - } - if !strings.Contains(string(appearanceBase), `"output": "appearance.2da"`) { - t.Fatalf("expected imported appearance output name, got:\n%s", string(appearanceBase)) - } - portraitsBase, err := os.ReadFile(filepath.Join(root, "topdata", "data", "portraits", "base.json")) - if err != nil { - t.Fatalf("read portraits base: %v", err) - } - if !strings.Contains(string(portraitsBase), `"output": "portraits.2da"`) { - t.Fatalf("expected imported portraits output name, got:\n%s", string(portraitsBase)) - } -} - -func TestBuildDenseGenerationDefaultsMissingColumnsAndPortraitFields(t *testing.T) { - root := testProjectRoot(t) - mkdirAll(t, filepath.Join(root, "topdata", "data", "dense", "modules")) - mkdirAll(t, filepath.Join(root, "topdata", "data", "plain")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance", "modules")) mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits")) - mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables")) - mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits", "modules")) writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") - writeFile(t, filepath.Join(root, "topdata", "data", "dense", "base.json"), `{ - "output": "dense.2da", - "columns": ["A", "B"], - "rows": [ - {"id": 0, "A": "base"} - ] -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "dense", "lock.json"), `{"dense:add":1}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "dense", "modules", "add.json"), `{ - "entries": { - "dense:add": { - "A": "entry" - } - } -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "plain", "table.json"), `{ - "output": "table.2da", - "columns": ["A", "B"], - "rows": [ - {"id": 0, "A": "plain"} - ] -}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ "output": "portraits.2da", "columns": ["BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"], - "rows": [] -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), "{}\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "base.json"), `{ - "output": "placeables.2da", - "columns": ["Label"], "rows": [ - { - "key": "placeables:bookshelf_special", - "id": 10, - "Label": "BookshelfSpecial" - } + {"id": 0, "BaseResRef": "****", "Sex": 4} ] }`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "lock.json"), `{"placeables:bookshelf_special":10}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "modules", "add_bookshelf_special.json"), `{ - "overrides": [ - { - "key": "placeables:bookshelf_special", - "meta": { - "portrait": { - "resref": "nw2book1_", - "sex": 4, - "inanimate_type": 4 + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), "{}") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "base.json"), `{ + "output": "appearance.2da", + "columns": ["LABEL", "PORTRAIT"], + "rows": [ + {"key": "appearance:crawlingclaw", "id": 0, "LABEL": "Crawling Claw", "PORTRAIT": "****"} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "lock.json"), `{"appearance:crawlingclaw":0}`) + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "modules", "add_crawlingclaw.json"), `{ + "entries": { + "appearance:crawlingclaw": { + "LABEL": "Crawling Claw", + "PORTRAIT": { + "value": "po_cclaw_", + "data": { + "portraits": { + "BaseResRef": "cclaw_", + "Sex": 4, + "Race": 10, + "Plot": 0 + } } } } - ] + } }`+"\n") + mkdirAll(t, filepath.Join(root, "reference")) writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") @@ -6306,40 +6198,112 @@ func TestBuildDenseGenerationDefaultsMissingColumnsAndPortraitFields(t *testing. t.Fatalf("BuildNative failed: %v", err) } - denseBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "dense.2da")) + appearanceBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "appearance.2da")) if err != nil { - t.Fatalf("read dense.2da: %v", err) + t.Fatalf("read appearance.2da: %v", err) } - if string(denseBytes) != "2DA V2.0\n\nA\tB\n0\tbase\t****\n1\tentry\t****\n" { - t.Fatalf("unexpected dense.2da output:\n%s", string(denseBytes)) - } - - plainBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "table.2da")) - if err != nil { - t.Fatalf("read table.2da: %v", err) - } - if string(plainBytes) != "2DA V2.0\n\nA\tB\n0\tplain\t****\n" { - t.Fatalf("unexpected table.2da output:\n%s", string(plainBytes)) + if !strings.Contains(string(appearanceBytes), "po_cclaw_") { + t.Fatalf("expected PORTRAIT value in appearance.2da, got:\n%s", string(appearanceBytes)) } portraitsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da")) if err != nil { t.Fatalf("read portraits.2da: %v", err) } - wantPortraits := "2DA V2.0\n\nBaseResRef\tSex\tRace\tInanimateType\tPlot\tLowGore\n0\tnw2book1_\t4\t****\t4\t****\t****\n" - if string(portraitsBytes) != wantPortraits { - t.Fatalf("unexpected portraits.2da output:\n%s", string(portraitsBytes)) + text := string(portraitsBytes) + if !strings.Contains(text, "cclaw_") || !strings.Contains(text, "Sex\t") || !strings.Contains(text, "Race\t") { + t.Fatalf("expected injected portrait data in portraits.2da, got:\n%s", text) + } + if !strings.Contains(text, "4\t10\t") { + t.Fatalf("expected Sex=4 Race=10 in portraits.2da, got:\n%s", text) } - placeablesBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "placeables.2da")) + lockBytes, err := os.ReadFile(filepath.Join(root, "topdata", "data", "portraits", "lock.json")) if err != nil { - t.Fatalf("read placeables.2da: %v", err) + t.Fatalf("read portraits lock.json: %v", err) } - if !strings.Contains(string(placeablesBytes), "Label\n") || !strings.Contains(string(placeablesBytes), "10\tBookshelfSpecial\n") { - t.Fatalf("unexpected placeables.2da output:\n%s", string(placeablesBytes)) + lockText := string(lockBytes) + if !strings.Contains(lockText, "portraits:cclaw_") { + t.Fatalf("expected lockfile entry for portraits:cclaw_, got:\n%s", lockText) } - if strings.Contains(string(placeablesBytes), "portrait") || strings.Contains(string(placeablesBytes), "meta") { - t.Fatalf("expected metadata to stay out of placeables.2da, got:\n%s", string(placeablesBytes)) +} + +func TestBuildExpansionDataDoesNotDuplicatePortraitRows(t *testing.T) { + root := testProjectRoot(t) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "appearance", "modules")) + mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits")) + writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") + + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ + "output": "portraits.2da", + "columns": ["BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"], + "rows": [ + {"id": 0, "BaseResRef": "****", "Sex": 4} + ] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), "{}") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "base.json"), `{ + "output": "appearance.2da", + "columns": ["LABEL", "PORTRAIT"], + "rows": [] +}`+"\n") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "lock.json"), "{}") + writeFile(t, filepath.Join(root, "topdata", "data", "appearance", "modules", "add_zombieknight.json"), `{ + "entries": { + "appearance:zombieknight3": { + "LABEL": "Zombie Knight 3", + "PORTRAIT": { + "value": "po_zk_", + "data": { + "portraits": { + "BaseResRef": "zk_", + "Sex": 4, + "Race": 23, + "Plot": 0 + } + } + } + }, + "appearance:zombieknight4": { + "LABEL": "Zombie Knight 4", + "PORTRAIT": { + "value": "po_zk_", + "data": { + "portraits": { + "BaseResRef": "zk_", + "Sex": 4, + "Race": 23, + "Plot": 0 + } + } + } + } + } +}`+"\n") + + mkdirAll(t, filepath.Join(root, "reference")) + writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") + + result, err := BuildNative(testProject(root), nil) + if err != nil { + t.Fatalf("BuildNative failed: %v", err) + } + + portraitsBytes, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da")) + if err != nil { + t.Fatalf("read portraits.2da: %v", err) + } + text := string(portraitsBytes) + lines := strings.Split(text, "\n") + zkCount := 0 + for _, line := range lines { + if strings.Contains(line, "zk_") { + zkCount++ + } + } + if zkCount != 1 { + t.Fatalf("expected exactly 1 portrait row with zk_, got %d:\n%s", zkCount, text) } } @@ -6350,8 +6314,7 @@ func TestValidateProjectRejectsUnknownMetadataKey(t *testing.T) { writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "base.json"), `{ "output": "placeables.2da", "columns": ["Label"], - "rows": [ - {"id": 0, "Label": "Shelf", "meta": {"unknown": {}}} + "rows": [{"id": 0, "Label": "Shelf", "meta": {"unknown": {}}} ] }`+"\n") @@ -6424,58 +6387,6 @@ func TestBuildNativeTreatsMetaCaseInsensitive(t *testing.T) { } } -func TestBuildRejectsConflictingPortraitMetadata(t *testing.T) { - root := testProjectRoot(t) - mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits")) - mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables")) - mkdirAll(t, filepath.Join(root, "topdata", "data", "genericdoors")) - mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables", "modules")) - mkdirAll(t, filepath.Join(root, "topdata", "data", "genericdoors", "modules")) - writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") - writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ - "output": "portraits.2da", - "columns": ["BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"], - "rows": [] -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), "{}\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "base.json"), `{ - "output": "placeables.2da", - "columns": ["Label"], - "rows": [ - {"key": "placeables:shelf", "id": 1, "Label": "Shelf"} - ] -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "lock.json"), `{"placeables:shelf":1}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "genericdoors", "base.json"), `{ - "output": "genericdoors.2da", - "columns": ["Label"], - "rows": [ - {"key": "genericdoors:door", "id": 2, "Label": "Door"} - ] -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "genericdoors", "lock.json"), `{"genericdoors:door":2}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "modules", "ovr_shelf.json"), `{ - "overrides": [ - {"key": "placeables:shelf", "meta": {"portrait": {"resref": "nw2book1_", "sex": 4}}} - ] -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "genericdoors", "modules", "ovr_door.json"), `{ - "overrides": [ - {"key": "genericdoors:door", "meta": {"portrait": {"resref": "nw2book1_", "sex": 1}}} - ] -}`+"\n") - mkdirAll(t, filepath.Join(root, "reference")) - writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") - - _, err := BuildNative(testProject(root), nil) - if err == nil { - t.Fatalf("expected BuildNative to fail on conflicting portrait metadata") - } - if !strings.Contains(err.Error(), "conflicts for column Sex") { - t.Fatalf("expected portrait conflict error, got: %v", err) - } -} - func TestBuildCanonicalAppearanceMatchesReference(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data")) @@ -7828,108 +7739,6 @@ with open(os.path.join(tlk, "sow_tlk.tlk"), "wb") as f: } } -func TestBuildSupportsCanonicalPortraitsFromDatasetAndMetadata(t *testing.T) { - root := testProjectRoot(t) - mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits", "modules")) - mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables")) - mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables", "modules")) - writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") - writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ - "output": "portraits.2da", - "columns": ["BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"], - "rows": [ - {"id": 0, "BaseResRef": "****", "Sex": 4} - ] -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), `{"portraits:auto:nw2book1_":16004,"shemsuheru:id_1513":16003}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "modules", "add_cotblreaver.json"), `{ - "entries": { - "shemsuheru:id_1513": { - "BaseResRef": "cotbl_", - "Sex": 4, - "Race": 15, - "Plot": 0 - } - } -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "base.json"), `{ - "output": "placeables.2da", - "columns": ["Label"], - "rows": [ - {"key": "placeables:bookshelf", "id": 1, "Label": "Bookshelf"} - ] -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "lock.json"), `{"placeables:bookshelf":1}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "modules", "ovr_bookshelf.json"), `{ - "overrides": [ - {"key": "placeables:bookshelf", "meta": {"portrait": {"resref": "nw2book1_", "sex": 4, "inanimate_type": 4}}} - ] -}`+"\n") - mkdirAll(t, filepath.Join(root, "reference")) - writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") - - result, err := BuildNative(testProject(root), nil) - if err != nil { - t.Fatalf("BuildNative failed: %v", err) - } - got, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da")) - if err != nil { - t.Fatalf("read portraits.2da: %v", err) - } - text := string(got) - if !strings.Contains(text, "BaseResRef\tSex\tRace\tInanimateType\tPlot\tLowGore\n") || - !strings.Contains(text, "0\t****\t4\t****\t****\t****\t****\n") || - !strings.Contains(text, "16003\tcotbl_\t4\t15\t****\t0\t****\n") || - !strings.Contains(text, "16004\tnw2book1_\t4\t****\t4\t****\t****\n") { - t.Fatalf("unexpected portraits.2da output:\n%s", string(got)) - } -} - -func TestBuildIgnoresBasePortraitMetadataWithoutModuleOptInAndPrunesAutoLocks(t *testing.T) { - root := testProjectRoot(t) - mkdirAll(t, filepath.Join(root, "topdata", "data", "portraits")) - mkdirAll(t, filepath.Join(root, "topdata", "data", "placeables")) - writeFile(t, filepath.Join(root, "topdata", "base_dialog.json"), "{}\n") - writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "base.json"), `{ - "output": "portraits.2da", - "columns": ["BaseResRef", "Sex", "Race", "InanimateType", "Plot", "LowGore"], - "rows": [ - {"id": 0, "BaseResRef": "****", "Sex": 4} - ] -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "portraits", "lock.json"), `{"portraits:auto:nw2book1_":16004}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "base.json"), `{ - "output": "placeables.2da", - "columns": ["Label"], - "rows": [ - {"key": "placeables:bookshelf", "id": 1, "Label": "Bookshelf", "meta": {"portrait": {"resref": "nw2book1_", "sex": 4, "inanimate_type": 4}}} - ] -}`+"\n") - writeFile(t, filepath.Join(root, "topdata", "data", "placeables", "lock.json"), `{"placeables:bookshelf":1}`+"\n") - mkdirAll(t, filepath.Join(root, "reference")) - writeFile(t, filepath.Join(root, "reference", "build.py"), "print('ok')\n") - - result, err := BuildNative(testProject(root), nil) - if err != nil { - t.Fatalf("BuildNative failed: %v", err) - } - got, err := os.ReadFile(filepath.Join(result.Output2DADir, "portraits.2da")) - if err != nil { - t.Fatalf("read portraits.2da: %v", err) - } - want := "2DA V2.0\n\nBaseResRef\tSex\tRace\tInanimateType\tPlot\tLowGore\n0\t****\t4\t****\t****\t****\t****\n" - if string(got) != want { - t.Fatalf("unexpected portraits.2da output:\n%s", string(got)) - } - lockBytes, err := os.ReadFile(filepath.Join(root, "topdata", "data", "portraits", "lock.json")) - if err != nil { - t.Fatalf("read portraits lock: %v", err) - } - if strings.TrimSpace(string(lockBytes)) != "{}" { - t.Fatalf("expected stale auto portrait lock to be pruned, got:\n%s", string(lockBytes)) - } -} - func TestNormalizeProjectImportsRacialtypesRegistry(t *testing.T) { root := testProjectRoot(t) mkdirAll(t, filepath.Join(root, "topdata", "data")) diff --git a/nwn-tool b/nwn-tool new file mode 100755 index 0000000..4134cc6 Binary files /dev/null and b/nwn-tool differ