diff --git a/README.md b/README.md index de40591..7a4302b 100644 --- a/README.md +++ b/README.md @@ -304,6 +304,36 @@ setting remains as a module-only compatibility alias when `consume_archives` is not set. Stale cleanup is scoped to roots actually touched by extracted resources, so the default module-only extraction does not prune assets. +GFF JSON extraction can merge selected fields and lists from the existing source +file instead of replacing the whole extracted document. Rules are matched by +`target` relative to `paths.source`; files without a matching rule keep the +default overwrite behavior. For example: + +```yaml +extract: + merge: + gff_json: + - target: module/module.ifo.json + preserve_fields: + - Mod_Entry_Area + - Mod_Entry_X + - Mod_Entry_Y + - Mod_Entry_Z + - Mod_Entry_Dir_X + - Mod_Entry_Dir_Y + merge_lists: + - field: Mod_Area_list + key_field: Area_Name + strategy: preserve_existing_order_append_new +``` + +`preserve_fields` copies matching root fields from the existing source document +into the extracted document before writing. The +`preserve_existing_order_append_new` list strategy keeps existing keyed list +order for entries still present in the extracted archive, uses extracted entries +as the value source, appends archive-only entries in archive order, and removes +existing entries that are no longer present in the archive. + Music defaults (used when not explicitly configured): ```yaml diff --git a/internal/pipeline/extract.go b/internal/pipeline/extract.go index e49e9d9..6f47c28 100644 --- a/internal/pipeline/extract.go +++ b/internal/pipeline/extract.go @@ -300,15 +300,18 @@ func extractedFile(p *project.Project, resource erf.Resource, extension string) if err != nil { return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err) } + target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json") + if err != nil { + return "", nil, err + } + if err := mergeExtractedGFFJSON(p, target, &document); err != nil { + return "", nil, fmt.Errorf("merge extracted gff json %s.%s: %w", resource.Name, extension, err) + } formatted, err := json.MarshalIndent(document, "", " ") if err != nil { return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err) } formatted = append(formatted, '\n') - target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json") - if err != nil { - return "", nil, err - } return target, formatted, nil default: target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension) @@ -340,6 +343,146 @@ func extractionTarget(p *project.Project, field, configured, root string, parts return target, nil } +func mergeExtractedGFFJSON(p *project.Project, target string, extracted *gff.Document) error { + rule, ok, err := extractGFFJSONMergeRule(p, target) + if err != nil || !ok { + return err + } + + raw, err := os.ReadFile(target) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("read existing source %s: %w", target, err) + } + var existing gff.Document + if err := json.Unmarshal(raw, &existing); err != nil { + return fmt.Errorf("parse existing source %s: %w", target, err) + } + + for _, label := range rule.PreserveFields { + if field, ok := gffField(existing.Root, label); ok { + setGFFField(&extracted.Root, field) + } + } + for _, listRule := range rule.MergeLists { + if err := mergeGFFListByKey(&extracted.Root, existing.Root, listRule); err != nil { + return err + } + } + return nil +} + +func extractGFFJSONMergeRule(p *project.Project, target string) (project.ExtractGFFJSONMergeRule, bool, error) { + rel, err := filepath.Rel(filepath.Clean(p.SourceDir()), filepath.Clean(target)) + if err != nil { + return project.ExtractGFFJSONMergeRule{}, false, fmt.Errorf("resolve source-relative target %s: %w", target, err) + } + rel = filepath.ToSlash(rel) + if rel == ".." || strings.HasPrefix(rel, "../") { + return project.ExtractGFFJSONMergeRule{}, false, nil + } + for _, rule := range p.EffectiveConfig().Extract.Merge.GFFJSON { + if rule.Target == rel { + return rule, true, nil + } + } + return project.ExtractGFFJSONMergeRule{}, false, nil +} + +func gffField(s gff.Struct, label string) (gff.Field, bool) { + for _, field := range s.Fields { + if field.Label == label { + return field, true + } + } + return gff.Field{}, false +} + +func setGFFField(s *gff.Struct, replacement gff.Field) { + for index, field := range s.Fields { + if field.Label == replacement.Label { + s.Fields[index] = replacement + return + } + } + s.Fields = append(s.Fields, replacement) +} + +func mergeGFFListByKey(extracted *gff.Struct, existing gff.Struct, rule project.ExtractListMergeRule) error { + extractedField, ok := gffField(*extracted, rule.Field) + if !ok { + return fmt.Errorf("extracted field %q not found", rule.Field) + } + extractedList, ok := extractedField.Value.(gff.ListValue) + if !ok { + return fmt.Errorf("extracted field %q is %s, not List", rule.Field, extractedField.Type) + } + existingField, ok := gffField(existing, rule.Field) + if !ok { + return nil + } + existingList, ok := existingField.Value.(gff.ListValue) + if !ok { + return fmt.Errorf("existing field %q is %s, not List", rule.Field, existingField.Type) + } + + extractedByKey := map[string]gff.Struct{} + extractedOrder := make([]string, 0, len(extractedList)) + for _, item := range extractedList { + key, err := gffStructKey(item, rule.KeyField) + if err != nil { + return fmt.Errorf("extracted field %q: %w", rule.Field, err) + } + if _, exists := extractedByKey[key]; exists { + return fmt.Errorf("extracted field %q has duplicate %s key %q", rule.Field, rule.KeyField, key) + } + extractedByKey[key] = item + extractedOrder = append(extractedOrder, key) + } + + merged := make(gff.ListValue, 0, len(extractedList)) + seen := map[string]struct{}{} + for _, item := range existingList { + key, err := gffStructKey(item, rule.KeyField) + if err != nil { + return fmt.Errorf("existing field %q: %w", rule.Field, err) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("existing field %q has duplicate %s key %q", rule.Field, rule.KeyField, key) + } + if extractedItem, exists := extractedByKey[key]; exists { + merged = append(merged, extractedItem) + seen[key] = struct{}{} + } + } + for _, key := range extractedOrder { + if _, exists := seen[key]; exists { + continue + } + merged = append(merged, extractedByKey[key]) + } + + setGFFField(extracted, gff.NewField(rule.Field, merged)) + return nil +} + +func gffStructKey(s gff.Struct, keyField string) (string, error) { + field, ok := gffField(s, keyField) + if !ok { + return "", fmt.Errorf("key field %q not found", keyField) + } + switch value := field.Value.(type) { + case gff.ResRefValue: + return string(value), nil + case gff.StringValue: + return string(value), nil + default: + return "", fmt.Errorf("key field %q is %s, not ResRef or CExoString", keyField, field.Type) + } +} + type writeState int const ( diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index 11a67e0..fab49fa 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -444,6 +444,7 @@ extract: func TestExtractConsumeArchivesFalseOverridesLegacyDeleteFlag(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "src", "areas")) mustMkdir(t, filepath.Join(root, "assets")) mustMkdir(t, filepath.Join(root, "build")) @@ -474,7 +475,6 @@ extract: } } `) - p, err := project.Load(root) if err != nil { t.Fatalf("load project: %v", err) @@ -3253,6 +3253,197 @@ func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) { } } +func TestExtractMergesConfiguredGFFJSONFieldsAndLists(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src", "module")) + mustMkdir(t, filepath.Join(root, "src", "areas")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +extract: + merge: + gff_json: + - target: module/module.ifo.json + preserve_fields: + - Mod_Entry_Area + - Mod_Entry_X + merge_lists: + - field: Mod_Area_list + key_field: Area_Name + strategy: preserve_existing_order_append_new +`) + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Name", + "type": "CExoString", + "value": "Test Module" + }, + { + "label": "Mod_Entry_Area", + "type": "ResRef", + "value": "area_a" + }, + { + "label": "Mod_Entry_X", + "type": "Float", + "value": 1 + }, + { + "label": "Mod_Area_list", + "type": "List", + "value": [ + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "area_a" + } + ] + }, + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "area_b" + } + ] + }, + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "area_c" + } + ] + } + ] + } + ] + } +} +`) + for _, area := range []string{"area_a", "area_b", "area_c"} { + mustWriteFile(t, filepath.Join(root, "src", "areas", area+".are.json"), `{ + "file_type": "ARE ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [] + } +} +`) + } + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + if err := p.Scan(); err != nil { + t.Fatalf("scan: %v", err) + } + if _, err := BuildModule(p); err != nil { + t.Fatalf("build module: %v", err) + } + + mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{ + "file_type": "IFO ", + "file_version": "V3.2", + "root": { + "struct_type": 0, + "fields": [ + { + "label": "Mod_Entry_Area", + "type": "ResRef", + "value": "area_b" + }, + { + "label": "Mod_Entry_X", + "type": "Float", + "value": 99 + }, + { + "label": "Mod_Area_list", + "type": "List", + "value": [ + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "area_b" + } + ] + }, + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "area_a" + } + ] + }, + { + "struct_type": 6, + "fields": [ + { + "label": "Area_Name", + "type": "ResRef", + "value": "removed_area" + } + ] + } + ] + } + ] + } +} +`) + + if err := p.Scan(); err != nil { + t.Fatalf("rescan before extract: %v", err) + } + if _, err := Extract(p); err != nil { + t.Fatalf("extract: %v", err) + } + + document := readGFFJSON(t, filepath.Join(root, "src", "module", "module.ifo.json")) + if got, want := fieldValue(t, document.Root, "Mod_Entry_Area"), gff.ResRefValue("area_b"); got != want { + t.Fatalf("expected preserved Mod_Entry_Area %#v, got %#v", want, got) + } + if got, want := fieldValue(t, document.Root, "Mod_Entry_X"), gff.FloatValue(99); got != want { + t.Fatalf("expected preserved Mod_Entry_X %#v, got %#v", want, got) + } + if got, want := keyedListValues(t, document.Root, "Mod_Area_list", "Area_Name"), []string{"area_b", "area_a", "area_c"}; !slices.Equal(got, want) { + t.Fatalf("expected merged area order %#v, got %#v", want, got) + } +} + func TestExtractNormalizesResourceNamesToLowercase(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src", "blueprints", "items")) @@ -3423,6 +3614,51 @@ func mustWriteFile(t *testing.T, path, data string) { } } +func readGFFJSON(t *testing.T, path string) gff.Document { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + var document gff.Document + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + return document +} + +func fieldValue(t *testing.T, s gff.Struct, label string) gff.Value { + t.Helper() + for _, field := range s.Fields { + if field.Label == label { + return field.Value + } + } + t.Fatalf("field %s not found", label) + return nil +} + +func keyedListValues(t *testing.T, s gff.Struct, listLabel, keyLabel string) []string { + t.Helper() + list, ok := fieldValue(t, s, listLabel).(gff.ListValue) + if !ok { + t.Fatalf("field %s is not a list", listLabel) + } + values := make([]string, 0, len(list)) + for _, item := range list { + value := fieldValue(t, item, keyLabel) + switch typed := value.(type) { + case gff.ResRefValue: + values = append(values, string(typed)) + case gff.StringValue: + values = append(values, string(typed)) + default: + t.Fatalf("field %s has unsupported key type %T", keyLabel, value) + } + } + return values +} + func minimalGFF(t *testing.T, fileType string) []byte { t.Helper() document := gff.Document{ diff --git a/internal/project/project.go b/internal/project/project.go index 3088db9..b1c2e94 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -247,13 +247,30 @@ type TopDataWikiManagedRegionConfig struct { } type ExtractConfig struct { - IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"` - Archives []string `json:"archives,omitempty" yaml:"archives,omitempty"` - Layout string `json:"layout" yaml:"layout"` - HAKDiscovery string `json:"hak_discovery" yaml:"hak_discovery"` - CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"` - ConsumeArchives *bool `json:"consume_archives,omitempty" yaml:"consume_archives,omitempty"` - DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"` + IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"` + Archives []string `json:"archives,omitempty" yaml:"archives,omitempty"` + Layout string `json:"layout" yaml:"layout"` + HAKDiscovery string `json:"hak_discovery" yaml:"hak_discovery"` + CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"` + ConsumeArchives *bool `json:"consume_archives,omitempty" yaml:"consume_archives,omitempty"` + DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"` + Merge ExtractMergeConfig `json:"merge" yaml:"merge"` +} + +type ExtractMergeConfig struct { + GFFJSON []ExtractGFFJSONMergeRule `json:"gff_json" yaml:"gff_json"` +} + +type ExtractGFFJSONMergeRule struct { + Target string `json:"target" yaml:"target"` + PreserveFields []string `json:"preserve_fields" yaml:"preserve_fields"` + MergeLists []ExtractListMergeRule `json:"merge_lists" yaml:"merge_lists"` +} + +type ExtractListMergeRule struct { + Field string `json:"field" yaml:"field"` + KeyField string `json:"key_field" yaml:"key_field"` + Strategy string `json:"strategy" yaml:"strategy"` } type AutogenConfig struct { @@ -598,6 +615,7 @@ func (p *Project) ValidateLayout() error { failures = append(failures, fmt.Errorf("extract.hak_discovery %q is not supported", effective.Extract.HAKDiscovery)) } failures = append(failures, validateGlobList("extract.archives", effective.Extract.Archives)...) + failures = append(failures, validateExtractMergeConfig(effective.Extract.Merge)...) if _, err := time.ParseDuration(effective.Autogen.Cache.MaxAge); err != nil { failures = append(failures, fmt.Errorf("autogen.cache.max_age must be a duration: %w", err)) } @@ -669,6 +687,49 @@ func (p *Project) ValidateLayout() error { return nil } +func validateExtractMergeConfig(config ExtractMergeConfig) []error { + var failures []error + seenTargets := map[string]struct{}{} + for ruleIndex, rule := range config.GFFJSON { + prefix := fmt.Sprintf("extract.merge.gff_json[%d]", ruleIndex) + target := strings.TrimSpace(rule.Target) + switch { + case target == "": + failures = append(failures, fmt.Errorf("%s.target must not be empty", prefix)) + case filepath.IsAbs(filepath.FromSlash(target)): + failures = append(failures, fmt.Errorf("%s.target %q must be relative to paths.source", prefix, rule.Target)) + default: + clean := filepath.Clean(filepath.FromSlash(target)) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + failures = append(failures, fmt.Errorf("%s.target %q must not escape paths.source", prefix, rule.Target)) + } + normalized := filepath.ToSlash(clean) + if _, exists := seenTargets[normalized]; exists { + failures = append(failures, fmt.Errorf("extract.merge.gff_json target %q is configured more than once", normalized)) + } + seenTargets[normalized] = struct{}{} + } + + for listIndex, listRule := range rule.MergeLists { + listPrefix := fmt.Sprintf("%s.merge_lists[%d]", prefix, listIndex) + if strings.TrimSpace(listRule.Field) == "" { + failures = append(failures, fmt.Errorf("%s.field must not be empty", listPrefix)) + } + if strings.TrimSpace(listRule.KeyField) == "" { + failures = append(failures, fmt.Errorf("%s.key_field must not be empty", listPrefix)) + } + switch listRule.Strategy { + case "preserve_existing_order_append_new": + case "": + failures = append(failures, fmt.Errorf("%s.strategy must not be empty", listPrefix)) + default: + failures = append(failures, fmt.Errorf("%s.strategy %q is not supported", listPrefix, listRule.Strategy)) + } + } + } + return failures +} + func (p *Project) Scan() error { var sourceFiles []string var sourceExts []string @@ -1100,6 +1161,19 @@ func normalizeConfig(cfg *Config) { } cfg.Extract.IgnoreExtensions = normalizeStringSlice(cfg.Extract.IgnoreExtensions) cfg.Extract.Archives = normalizeStringSlice(cfg.Extract.Archives) + for i := range cfg.Extract.Merge.GFFJSON { + target := strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].Target) + if target != "" { + target = filepath.ToSlash(filepath.Clean(filepath.FromSlash(target))) + } + cfg.Extract.Merge.GFFJSON[i].Target = target + cfg.Extract.Merge.GFFJSON[i].PreserveFields = normalizeStringSlice(cfg.Extract.Merge.GFFJSON[i].PreserveFields) + for j := range cfg.Extract.Merge.GFFJSON[i].MergeLists { + cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Field = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Field) + cfg.Extract.Merge.GFFJSON[i].MergeLists[j].KeyField = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].KeyField) + cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Strategy = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Strategy) + } + } for i := range cfg.Autogen.Producers { cfg.Autogen.Producers[i].Include = normalizeStringSlice(cfg.Autogen.Producers[i].Include) } diff --git a/internal/project/project_test.go b/internal/project/project_test.go index 004dc72..07d97e9 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "reflect" "slices" "strings" "testing" @@ -254,6 +255,15 @@ extract: layout: nwn_canonical_json hak_discovery: configured_haks cleanup_stale: false + merge: + gff_json: + - target: module/module.ifo.json + preserve_fields: + - Mod_Entry_Area + merge_lists: + - field: Mod_Area_list + key_field: Area_Name + strategy: preserve_existing_order_append_new topdata: wiki: source: topdata/wiki @@ -305,6 +315,19 @@ autogen: if effective.Extract.CleanupStale == nil || *effective.Extract.CleanupStale { t.Fatalf("expected cleanup_stale false, got %#v", effective.Extract.CleanupStale) } + if got, want := len(effective.Extract.Merge.GFFJSON), 1; got != want { + t.Fatalf("expected %d gff json merge rule, got %d", want, got) + } + rule := effective.Extract.Merge.GFFJSON[0] + if got, want := rule.Target, "module/module.ifo.json"; got != want { + t.Fatalf("expected merge target %q, got %q", want, got) + } + if got, want := rule.PreserveFields, []string{"Mod_Entry_Area"}; !reflect.DeepEqual(got, want) { + t.Fatalf("expected preserve fields %#v, got %#v", want, got) + } + if got, want := rule.MergeLists[0].Strategy, "preserve_existing_order_append_new"; got != want { + t.Fatalf("expected merge list strategy %q, got %q", want, got) + } if got, want := effective.TopData.Wiki.DeployManifest, "wiki-manifest.json"; got != want { t.Fatalf("expected wiki deploy manifest %q, got %q", want, got) } @@ -346,6 +369,91 @@ autogen: } } +func TestValidateLayoutRejectsInvalidExtractMergeRules(t *testing.T) { + tests := []struct { + name string + config string + wantErr string + }{ + { + name: "empty target", + config: ` +extract: + merge: + gff_json: + - target: "" + preserve_fields: [Mod_Entry_Area] +`, + wantErr: "extract.merge.gff_json[0].target must not be empty", + }, + { + name: "duplicate target", + config: ` +extract: + merge: + gff_json: + - target: module/module.ifo.json + - target: module/module.ifo.json +`, + wantErr: "extract.merge.gff_json target \"module/module.ifo.json\" is configured more than once", + }, + { + name: "empty merge list field", + config: ` +extract: + merge: + gff_json: + - target: module/module.ifo.json + merge_lists: + - field: "" + key_field: Area_Name + strategy: preserve_existing_order_append_new +`, + wantErr: "extract.merge.gff_json[0].merge_lists[0].field must not be empty", + }, + { + name: "unsupported strategy", + config: ` +extract: + merge: + gff_json: + - target: module/module.ifo.json + merge_lists: + - field: Mod_Area_list + key_field: Area_Name + strategy: replace +`, + wantErr: "extract.merge.gff_json[0].merge_lists[0].strategy \"replace\" is not supported", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +`+tt.config) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + err = proj.ValidateLayout() + if err == nil { + t.Fatalf("expected validation error containing %q", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected validation error containing %q, got %v", tt.wantErr, err) + } + }) + } +} + func TestActiveOverridesAreVisibleAndSensitiveValuesAreMasked(t *testing.T) { root := t.TempDir() writeProjectFile(t, filepath.Join(root, ConfigFile), `