Immutable Extracts
This commit is contained in:
@@ -304,6 +304,36 @@ setting remains as a module-only compatibility alias when `consume_archives` is
|
|||||||
not set. Stale cleanup is scoped to roots actually touched by extracted
|
not set. Stale cleanup is scoped to roots actually touched by extracted
|
||||||
resources, so the default module-only extraction does not prune assets.
|
resources, so the default module-only extraction does not prune assets.
|
||||||
|
|
||||||
|
GFF JSON extraction can merge selected fields and lists from the existing source
|
||||||
|
file instead of replacing the whole extracted document. Rules are matched by
|
||||||
|
`target` relative to `paths.source`; files without a matching rule keep the
|
||||||
|
default overwrite behavior. For example:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
extract:
|
||||||
|
merge:
|
||||||
|
gff_json:
|
||||||
|
- target: module/module.ifo.json
|
||||||
|
preserve_fields:
|
||||||
|
- Mod_Entry_Area
|
||||||
|
- Mod_Entry_X
|
||||||
|
- Mod_Entry_Y
|
||||||
|
- Mod_Entry_Z
|
||||||
|
- Mod_Entry_Dir_X
|
||||||
|
- Mod_Entry_Dir_Y
|
||||||
|
merge_lists:
|
||||||
|
- field: Mod_Area_list
|
||||||
|
key_field: Area_Name
|
||||||
|
strategy: preserve_existing_order_append_new
|
||||||
|
```
|
||||||
|
|
||||||
|
`preserve_fields` copies matching root fields from the existing source document
|
||||||
|
into the extracted document before writing. The
|
||||||
|
`preserve_existing_order_append_new` list strategy keeps existing keyed list
|
||||||
|
order for entries still present in the extracted archive, uses extracted entries
|
||||||
|
as the value source, appends archive-only entries in archive order, and removes
|
||||||
|
existing entries that are no longer present in the archive.
|
||||||
|
|
||||||
Music defaults (used when not explicitly configured):
|
Music defaults (used when not explicitly configured):
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
|
|||||||
@@ -300,15 +300,18 @@ func extractedFile(p *project.Project, resource erf.Resource, extension string)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err)
|
return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err)
|
||||||
}
|
}
|
||||||
|
target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json")
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
if err := mergeExtractedGFFJSON(p, target, &document); err != nil {
|
||||||
|
return "", nil, fmt.Errorf("merge extracted gff json %s.%s: %w", resource.Name, extension, err)
|
||||||
|
}
|
||||||
formatted, err := json.MarshalIndent(document, "", " ")
|
formatted, err := json.MarshalIndent(document, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err)
|
return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err)
|
||||||
}
|
}
|
||||||
formatted = append(formatted, '\n')
|
formatted = append(formatted, '\n')
|
||||||
target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json")
|
|
||||||
if err != nil {
|
|
||||||
return "", nil, err
|
|
||||||
}
|
|
||||||
return target, formatted, nil
|
return target, formatted, nil
|
||||||
default:
|
default:
|
||||||
target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension)
|
target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension)
|
||||||
@@ -340,6 +343,146 @@ func extractionTarget(p *project.Project, field, configured, root string, parts
|
|||||||
return target, nil
|
return target, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func mergeExtractedGFFJSON(p *project.Project, target string, extracted *gff.Document) error {
|
||||||
|
rule, ok, err := extractGFFJSONMergeRule(p, target)
|
||||||
|
if err != nil || !ok {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := os.ReadFile(target)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("read existing source %s: %w", target, err)
|
||||||
|
}
|
||||||
|
var existing gff.Document
|
||||||
|
if err := json.Unmarshal(raw, &existing); err != nil {
|
||||||
|
return fmt.Errorf("parse existing source %s: %w", target, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, label := range rule.PreserveFields {
|
||||||
|
if field, ok := gffField(existing.Root, label); ok {
|
||||||
|
setGFFField(&extracted.Root, field)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, listRule := range rule.MergeLists {
|
||||||
|
if err := mergeGFFListByKey(&extracted.Root, existing.Root, listRule); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractGFFJSONMergeRule(p *project.Project, target string) (project.ExtractGFFJSONMergeRule, bool, error) {
|
||||||
|
rel, err := filepath.Rel(filepath.Clean(p.SourceDir()), filepath.Clean(target))
|
||||||
|
if err != nil {
|
||||||
|
return project.ExtractGFFJSONMergeRule{}, false, fmt.Errorf("resolve source-relative target %s: %w", target, err)
|
||||||
|
}
|
||||||
|
rel = filepath.ToSlash(rel)
|
||||||
|
if rel == ".." || strings.HasPrefix(rel, "../") {
|
||||||
|
return project.ExtractGFFJSONMergeRule{}, false, nil
|
||||||
|
}
|
||||||
|
for _, rule := range p.EffectiveConfig().Extract.Merge.GFFJSON {
|
||||||
|
if rule.Target == rel {
|
||||||
|
return rule, true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return project.ExtractGFFJSONMergeRule{}, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func gffField(s gff.Struct, label string) (gff.Field, bool) {
|
||||||
|
for _, field := range s.Fields {
|
||||||
|
if field.Label == label {
|
||||||
|
return field, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return gff.Field{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func setGFFField(s *gff.Struct, replacement gff.Field) {
|
||||||
|
for index, field := range s.Fields {
|
||||||
|
if field.Label == replacement.Label {
|
||||||
|
s.Fields[index] = replacement
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.Fields = append(s.Fields, replacement)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeGFFListByKey(extracted *gff.Struct, existing gff.Struct, rule project.ExtractListMergeRule) error {
|
||||||
|
extractedField, ok := gffField(*extracted, rule.Field)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("extracted field %q not found", rule.Field)
|
||||||
|
}
|
||||||
|
extractedList, ok := extractedField.Value.(gff.ListValue)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("extracted field %q is %s, not List", rule.Field, extractedField.Type)
|
||||||
|
}
|
||||||
|
existingField, ok := gffField(existing, rule.Field)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
existingList, ok := existingField.Value.(gff.ListValue)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("existing field %q is %s, not List", rule.Field, existingField.Type)
|
||||||
|
}
|
||||||
|
|
||||||
|
extractedByKey := map[string]gff.Struct{}
|
||||||
|
extractedOrder := make([]string, 0, len(extractedList))
|
||||||
|
for _, item := range extractedList {
|
||||||
|
key, err := gffStructKey(item, rule.KeyField)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("extracted field %q: %w", rule.Field, err)
|
||||||
|
}
|
||||||
|
if _, exists := extractedByKey[key]; exists {
|
||||||
|
return fmt.Errorf("extracted field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
|
||||||
|
}
|
||||||
|
extractedByKey[key] = item
|
||||||
|
extractedOrder = append(extractedOrder, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
merged := make(gff.ListValue, 0, len(extractedList))
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
for _, item := range existingList {
|
||||||
|
key, err := gffStructKey(item, rule.KeyField)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("existing field %q: %w", rule.Field, err)
|
||||||
|
}
|
||||||
|
if _, duplicate := seen[key]; duplicate {
|
||||||
|
return fmt.Errorf("existing field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
|
||||||
|
}
|
||||||
|
if extractedItem, exists := extractedByKey[key]; exists {
|
||||||
|
merged = append(merged, extractedItem)
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range extractedOrder {
|
||||||
|
if _, exists := seen[key]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
merged = append(merged, extractedByKey[key])
|
||||||
|
}
|
||||||
|
|
||||||
|
setGFFField(extracted, gff.NewField(rule.Field, merged))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func gffStructKey(s gff.Struct, keyField string) (string, error) {
|
||||||
|
field, ok := gffField(s, keyField)
|
||||||
|
if !ok {
|
||||||
|
return "", fmt.Errorf("key field %q not found", keyField)
|
||||||
|
}
|
||||||
|
switch value := field.Value.(type) {
|
||||||
|
case gff.ResRefValue:
|
||||||
|
return string(value), nil
|
||||||
|
case gff.StringValue:
|
||||||
|
return string(value), nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("key field %q is %s, not ResRef or CExoString", keyField, field.Type)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type writeState int
|
type writeState int
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|||||||
@@ -444,6 +444,7 @@ extract:
|
|||||||
func TestExtractConsumeArchivesFalseOverridesLegacyDeleteFlag(t *testing.T) {
|
func TestExtractConsumeArchivesFalseOverridesLegacyDeleteFlag(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
mustMkdir(t, filepath.Join(root, "src", "module"))
|
mustMkdir(t, filepath.Join(root, "src", "module"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "src", "areas"))
|
||||||
mustMkdir(t, filepath.Join(root, "assets"))
|
mustMkdir(t, filepath.Join(root, "assets"))
|
||||||
mustMkdir(t, filepath.Join(root, "build"))
|
mustMkdir(t, filepath.Join(root, "build"))
|
||||||
|
|
||||||
@@ -474,7 +475,6 @@ extract:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
`)
|
`)
|
||||||
|
|
||||||
p, err := project.Load(root)
|
p, err := project.Load(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("load project: %v", err)
|
t.Fatalf("load project: %v", err)
|
||||||
@@ -3253,6 +3253,197 @@ func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExtractMergesConfiguredGFFJSONFieldsAndLists(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
mustMkdir(t, filepath.Join(root, "src", "module"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "src", "areas"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "assets"))
|
||||||
|
mustMkdir(t, filepath.Join(root, "build"))
|
||||||
|
|
||||||
|
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
|
||||||
|
module:
|
||||||
|
name: Test Module
|
||||||
|
resref: testmod
|
||||||
|
paths:
|
||||||
|
source: src
|
||||||
|
assets: assets
|
||||||
|
build: build
|
||||||
|
extract:
|
||||||
|
merge:
|
||||||
|
gff_json:
|
||||||
|
- target: module/module.ifo.json
|
||||||
|
preserve_fields:
|
||||||
|
- Mod_Entry_Area
|
||||||
|
- Mod_Entry_X
|
||||||
|
merge_lists:
|
||||||
|
- field: Mod_Area_list
|
||||||
|
key_field: Area_Name
|
||||||
|
strategy: preserve_existing_order_append_new
|
||||||
|
`)
|
||||||
|
|
||||||
|
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
|
||||||
|
"file_type": "IFO ",
|
||||||
|
"file_version": "V3.2",
|
||||||
|
"root": {
|
||||||
|
"struct_type": 0,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"label": "Mod_Name",
|
||||||
|
"type": "CExoString",
|
||||||
|
"value": "Test Module"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Mod_Entry_Area",
|
||||||
|
"type": "ResRef",
|
||||||
|
"value": "area_a"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Mod_Entry_X",
|
||||||
|
"type": "Float",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Mod_Area_list",
|
||||||
|
"type": "List",
|
||||||
|
"value": [
|
||||||
|
{
|
||||||
|
"struct_type": 6,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"label": "Area_Name",
|
||||||
|
"type": "ResRef",
|
||||||
|
"value": "area_a"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"struct_type": 6,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"label": "Area_Name",
|
||||||
|
"type": "ResRef",
|
||||||
|
"value": "area_b"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"struct_type": 6,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"label": "Area_Name",
|
||||||
|
"type": "ResRef",
|
||||||
|
"value": "area_c"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`)
|
||||||
|
for _, area := range []string{"area_a", "area_b", "area_c"} {
|
||||||
|
mustWriteFile(t, filepath.Join(root, "src", "areas", area+".are.json"), `{
|
||||||
|
"file_type": "ARE ",
|
||||||
|
"file_version": "V3.2",
|
||||||
|
"root": {
|
||||||
|
"struct_type": 0,
|
||||||
|
"fields": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
|
p, err := project.Load(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load project: %v", err)
|
||||||
|
}
|
||||||
|
if err := p.ValidateLayout(); err != nil {
|
||||||
|
t.Fatalf("validate layout: %v", err)
|
||||||
|
}
|
||||||
|
if err := p.Scan(); err != nil {
|
||||||
|
t.Fatalf("scan: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := BuildModule(p); err != nil {
|
||||||
|
t.Fatalf("build module: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
|
||||||
|
"file_type": "IFO ",
|
||||||
|
"file_version": "V3.2",
|
||||||
|
"root": {
|
||||||
|
"struct_type": 0,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"label": "Mod_Entry_Area",
|
||||||
|
"type": "ResRef",
|
||||||
|
"value": "area_b"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Mod_Entry_X",
|
||||||
|
"type": "Float",
|
||||||
|
"value": 99
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Mod_Area_list",
|
||||||
|
"type": "List",
|
||||||
|
"value": [
|
||||||
|
{
|
||||||
|
"struct_type": 6,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"label": "Area_Name",
|
||||||
|
"type": "ResRef",
|
||||||
|
"value": "area_b"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"struct_type": 6,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"label": "Area_Name",
|
||||||
|
"type": "ResRef",
|
||||||
|
"value": "area_a"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"struct_type": 6,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"label": "Area_Name",
|
||||||
|
"type": "ResRef",
|
||||||
|
"value": "removed_area"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`)
|
||||||
|
|
||||||
|
if err := p.Scan(); err != nil {
|
||||||
|
t.Fatalf("rescan before extract: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := Extract(p); err != nil {
|
||||||
|
t.Fatalf("extract: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
document := readGFFJSON(t, filepath.Join(root, "src", "module", "module.ifo.json"))
|
||||||
|
if got, want := fieldValue(t, document.Root, "Mod_Entry_Area"), gff.ResRefValue("area_b"); got != want {
|
||||||
|
t.Fatalf("expected preserved Mod_Entry_Area %#v, got %#v", want, got)
|
||||||
|
}
|
||||||
|
if got, want := fieldValue(t, document.Root, "Mod_Entry_X"), gff.FloatValue(99); got != want {
|
||||||
|
t.Fatalf("expected preserved Mod_Entry_X %#v, got %#v", want, got)
|
||||||
|
}
|
||||||
|
if got, want := keyedListValues(t, document.Root, "Mod_Area_list", "Area_Name"), []string{"area_b", "area_a", "area_c"}; !slices.Equal(got, want) {
|
||||||
|
t.Fatalf("expected merged area order %#v, got %#v", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExtractNormalizesResourceNamesToLowercase(t *testing.T) {
|
func TestExtractNormalizesResourceNamesToLowercase(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
mustMkdir(t, filepath.Join(root, "src", "blueprints", "items"))
|
mustMkdir(t, filepath.Join(root, "src", "blueprints", "items"))
|
||||||
@@ -3423,6 +3614,51 @@ func mustWriteFile(t *testing.T, path, data string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func readGFFJSON(t *testing.T, path string) gff.Document {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", path, err)
|
||||||
|
}
|
||||||
|
var document gff.Document
|
||||||
|
if err := json.Unmarshal(raw, &document); err != nil {
|
||||||
|
t.Fatalf("parse %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return document
|
||||||
|
}
|
||||||
|
|
||||||
|
func fieldValue(t *testing.T, s gff.Struct, label string) gff.Value {
|
||||||
|
t.Helper()
|
||||||
|
for _, field := range s.Fields {
|
||||||
|
if field.Label == label {
|
||||||
|
return field.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("field %s not found", label)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func keyedListValues(t *testing.T, s gff.Struct, listLabel, keyLabel string) []string {
|
||||||
|
t.Helper()
|
||||||
|
list, ok := fieldValue(t, s, listLabel).(gff.ListValue)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("field %s is not a list", listLabel)
|
||||||
|
}
|
||||||
|
values := make([]string, 0, len(list))
|
||||||
|
for _, item := range list {
|
||||||
|
value := fieldValue(t, item, keyLabel)
|
||||||
|
switch typed := value.(type) {
|
||||||
|
case gff.ResRefValue:
|
||||||
|
values = append(values, string(typed))
|
||||||
|
case gff.StringValue:
|
||||||
|
values = append(values, string(typed))
|
||||||
|
default:
|
||||||
|
t.Fatalf("field %s has unsupported key type %T", keyLabel, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
func minimalGFF(t *testing.T, fileType string) []byte {
|
func minimalGFF(t *testing.T, fileType string) []byte {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
document := gff.Document{
|
document := gff.Document{
|
||||||
|
|||||||
@@ -254,6 +254,23 @@ type ExtractConfig struct {
|
|||||||
CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"`
|
CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"`
|
||||||
ConsumeArchives *bool `json:"consume_archives,omitempty" yaml:"consume_archives,omitempty"`
|
ConsumeArchives *bool `json:"consume_archives,omitempty" yaml:"consume_archives,omitempty"`
|
||||||
DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"`
|
DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"`
|
||||||
|
Merge ExtractMergeConfig `json:"merge" yaml:"merge"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ExtractMergeConfig struct {
|
||||||
|
GFFJSON []ExtractGFFJSONMergeRule `json:"gff_json" yaml:"gff_json"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ExtractGFFJSONMergeRule struct {
|
||||||
|
Target string `json:"target" yaml:"target"`
|
||||||
|
PreserveFields []string `json:"preserve_fields" yaml:"preserve_fields"`
|
||||||
|
MergeLists []ExtractListMergeRule `json:"merge_lists" yaml:"merge_lists"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ExtractListMergeRule struct {
|
||||||
|
Field string `json:"field" yaml:"field"`
|
||||||
|
KeyField string `json:"key_field" yaml:"key_field"`
|
||||||
|
Strategy string `json:"strategy" yaml:"strategy"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type AutogenConfig struct {
|
type AutogenConfig struct {
|
||||||
@@ -598,6 +615,7 @@ func (p *Project) ValidateLayout() error {
|
|||||||
failures = append(failures, fmt.Errorf("extract.hak_discovery %q is not supported", effective.Extract.HAKDiscovery))
|
failures = append(failures, fmt.Errorf("extract.hak_discovery %q is not supported", effective.Extract.HAKDiscovery))
|
||||||
}
|
}
|
||||||
failures = append(failures, validateGlobList("extract.archives", effective.Extract.Archives)...)
|
failures = append(failures, validateGlobList("extract.archives", effective.Extract.Archives)...)
|
||||||
|
failures = append(failures, validateExtractMergeConfig(effective.Extract.Merge)...)
|
||||||
if _, err := time.ParseDuration(effective.Autogen.Cache.MaxAge); err != nil {
|
if _, err := time.ParseDuration(effective.Autogen.Cache.MaxAge); err != nil {
|
||||||
failures = append(failures, fmt.Errorf("autogen.cache.max_age must be a duration: %w", err))
|
failures = append(failures, fmt.Errorf("autogen.cache.max_age must be a duration: %w", err))
|
||||||
}
|
}
|
||||||
@@ -669,6 +687,49 @@ func (p *Project) ValidateLayout() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateExtractMergeConfig(config ExtractMergeConfig) []error {
|
||||||
|
var failures []error
|
||||||
|
seenTargets := map[string]struct{}{}
|
||||||
|
for ruleIndex, rule := range config.GFFJSON {
|
||||||
|
prefix := fmt.Sprintf("extract.merge.gff_json[%d]", ruleIndex)
|
||||||
|
target := strings.TrimSpace(rule.Target)
|
||||||
|
switch {
|
||||||
|
case target == "":
|
||||||
|
failures = append(failures, fmt.Errorf("%s.target must not be empty", prefix))
|
||||||
|
case filepath.IsAbs(filepath.FromSlash(target)):
|
||||||
|
failures = append(failures, fmt.Errorf("%s.target %q must be relative to paths.source", prefix, rule.Target))
|
||||||
|
default:
|
||||||
|
clean := filepath.Clean(filepath.FromSlash(target))
|
||||||
|
if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||||
|
failures = append(failures, fmt.Errorf("%s.target %q must not escape paths.source", prefix, rule.Target))
|
||||||
|
}
|
||||||
|
normalized := filepath.ToSlash(clean)
|
||||||
|
if _, exists := seenTargets[normalized]; exists {
|
||||||
|
failures = append(failures, fmt.Errorf("extract.merge.gff_json target %q is configured more than once", normalized))
|
||||||
|
}
|
||||||
|
seenTargets[normalized] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
for listIndex, listRule := range rule.MergeLists {
|
||||||
|
listPrefix := fmt.Sprintf("%s.merge_lists[%d]", prefix, listIndex)
|
||||||
|
if strings.TrimSpace(listRule.Field) == "" {
|
||||||
|
failures = append(failures, fmt.Errorf("%s.field must not be empty", listPrefix))
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(listRule.KeyField) == "" {
|
||||||
|
failures = append(failures, fmt.Errorf("%s.key_field must not be empty", listPrefix))
|
||||||
|
}
|
||||||
|
switch listRule.Strategy {
|
||||||
|
case "preserve_existing_order_append_new":
|
||||||
|
case "":
|
||||||
|
failures = append(failures, fmt.Errorf("%s.strategy must not be empty", listPrefix))
|
||||||
|
default:
|
||||||
|
failures = append(failures, fmt.Errorf("%s.strategy %q is not supported", listPrefix, listRule.Strategy))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return failures
|
||||||
|
}
|
||||||
|
|
||||||
func (p *Project) Scan() error {
|
func (p *Project) Scan() error {
|
||||||
var sourceFiles []string
|
var sourceFiles []string
|
||||||
var sourceExts []string
|
var sourceExts []string
|
||||||
@@ -1100,6 +1161,19 @@ func normalizeConfig(cfg *Config) {
|
|||||||
}
|
}
|
||||||
cfg.Extract.IgnoreExtensions = normalizeStringSlice(cfg.Extract.IgnoreExtensions)
|
cfg.Extract.IgnoreExtensions = normalizeStringSlice(cfg.Extract.IgnoreExtensions)
|
||||||
cfg.Extract.Archives = normalizeStringSlice(cfg.Extract.Archives)
|
cfg.Extract.Archives = normalizeStringSlice(cfg.Extract.Archives)
|
||||||
|
for i := range cfg.Extract.Merge.GFFJSON {
|
||||||
|
target := strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].Target)
|
||||||
|
if target != "" {
|
||||||
|
target = filepath.ToSlash(filepath.Clean(filepath.FromSlash(target)))
|
||||||
|
}
|
||||||
|
cfg.Extract.Merge.GFFJSON[i].Target = target
|
||||||
|
cfg.Extract.Merge.GFFJSON[i].PreserveFields = normalizeStringSlice(cfg.Extract.Merge.GFFJSON[i].PreserveFields)
|
||||||
|
for j := range cfg.Extract.Merge.GFFJSON[i].MergeLists {
|
||||||
|
cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Field = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Field)
|
||||||
|
cfg.Extract.Merge.GFFJSON[i].MergeLists[j].KeyField = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].KeyField)
|
||||||
|
cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Strategy = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Strategy)
|
||||||
|
}
|
||||||
|
}
|
||||||
for i := range cfg.Autogen.Producers {
|
for i := range cfg.Autogen.Producers {
|
||||||
cfg.Autogen.Producers[i].Include = normalizeStringSlice(cfg.Autogen.Producers[i].Include)
|
cfg.Autogen.Producers[i].Include = normalizeStringSlice(cfg.Autogen.Producers[i].Include)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -254,6 +255,15 @@ extract:
|
|||||||
layout: nwn_canonical_json
|
layout: nwn_canonical_json
|
||||||
hak_discovery: configured_haks
|
hak_discovery: configured_haks
|
||||||
cleanup_stale: false
|
cleanup_stale: false
|
||||||
|
merge:
|
||||||
|
gff_json:
|
||||||
|
- target: module/module.ifo.json
|
||||||
|
preserve_fields:
|
||||||
|
- Mod_Entry_Area
|
||||||
|
merge_lists:
|
||||||
|
- field: Mod_Area_list
|
||||||
|
key_field: Area_Name
|
||||||
|
strategy: preserve_existing_order_append_new
|
||||||
topdata:
|
topdata:
|
||||||
wiki:
|
wiki:
|
||||||
source: topdata/wiki
|
source: topdata/wiki
|
||||||
@@ -305,6 +315,19 @@ autogen:
|
|||||||
if effective.Extract.CleanupStale == nil || *effective.Extract.CleanupStale {
|
if effective.Extract.CleanupStale == nil || *effective.Extract.CleanupStale {
|
||||||
t.Fatalf("expected cleanup_stale false, got %#v", effective.Extract.CleanupStale)
|
t.Fatalf("expected cleanup_stale false, got %#v", effective.Extract.CleanupStale)
|
||||||
}
|
}
|
||||||
|
if got, want := len(effective.Extract.Merge.GFFJSON), 1; got != want {
|
||||||
|
t.Fatalf("expected %d gff json merge rule, got %d", want, got)
|
||||||
|
}
|
||||||
|
rule := effective.Extract.Merge.GFFJSON[0]
|
||||||
|
if got, want := rule.Target, "module/module.ifo.json"; got != want {
|
||||||
|
t.Fatalf("expected merge target %q, got %q", want, got)
|
||||||
|
}
|
||||||
|
if got, want := rule.PreserveFields, []string{"Mod_Entry_Area"}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("expected preserve fields %#v, got %#v", want, got)
|
||||||
|
}
|
||||||
|
if got, want := rule.MergeLists[0].Strategy, "preserve_existing_order_append_new"; got != want {
|
||||||
|
t.Fatalf("expected merge list strategy %q, got %q", want, got)
|
||||||
|
}
|
||||||
if got, want := effective.TopData.Wiki.DeployManifest, "wiki-manifest.json"; got != want {
|
if got, want := effective.TopData.Wiki.DeployManifest, "wiki-manifest.json"; got != want {
|
||||||
t.Fatalf("expected wiki deploy manifest %q, got %q", want, got)
|
t.Fatalf("expected wiki deploy manifest %q, got %q", want, got)
|
||||||
}
|
}
|
||||||
@@ -346,6 +369,91 @@ autogen:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateLayoutRejectsInvalidExtractMergeRules(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
config string
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty target",
|
||||||
|
config: `
|
||||||
|
extract:
|
||||||
|
merge:
|
||||||
|
gff_json:
|
||||||
|
- target: ""
|
||||||
|
preserve_fields: [Mod_Entry_Area]
|
||||||
|
`,
|
||||||
|
wantErr: "extract.merge.gff_json[0].target must not be empty",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "duplicate target",
|
||||||
|
config: `
|
||||||
|
extract:
|
||||||
|
merge:
|
||||||
|
gff_json:
|
||||||
|
- target: module/module.ifo.json
|
||||||
|
- target: module/module.ifo.json
|
||||||
|
`,
|
||||||
|
wantErr: "extract.merge.gff_json target \"module/module.ifo.json\" is configured more than once",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty merge list field",
|
||||||
|
config: `
|
||||||
|
extract:
|
||||||
|
merge:
|
||||||
|
gff_json:
|
||||||
|
- target: module/module.ifo.json
|
||||||
|
merge_lists:
|
||||||
|
- field: ""
|
||||||
|
key_field: Area_Name
|
||||||
|
strategy: preserve_existing_order_append_new
|
||||||
|
`,
|
||||||
|
wantErr: "extract.merge.gff_json[0].merge_lists[0].field must not be empty",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unsupported strategy",
|
||||||
|
config: `
|
||||||
|
extract:
|
||||||
|
merge:
|
||||||
|
gff_json:
|
||||||
|
- target: module/module.ifo.json
|
||||||
|
merge_lists:
|
||||||
|
- field: Mod_Area_list
|
||||||
|
key_field: Area_Name
|
||||||
|
strategy: replace
|
||||||
|
`,
|
||||||
|
wantErr: "extract.merge.gff_json[0].merge_lists[0].strategy \"replace\" is not supported",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
writeProjectFile(t, filepath.Join(root, ConfigFile), `
|
||||||
|
module:
|
||||||
|
name: Test Module
|
||||||
|
resref: testmod
|
||||||
|
paths:
|
||||||
|
source: src
|
||||||
|
build: build
|
||||||
|
`+tt.config)
|
||||||
|
|
||||||
|
proj, err := Load(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load returned error: %v", err)
|
||||||
|
}
|
||||||
|
err = proj.ValidateLayout()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected validation error containing %q", tt.wantErr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("expected validation error containing %q, got %v", tt.wantErr, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestActiveOverridesAreVisibleAndSensitiveValuesAreMasked(t *testing.T) {
|
func TestActiveOverridesAreVisibleAndSensitiveValuesAreMasked(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
writeProjectFile(t, filepath.Join(root, ConfigFile), `
|
writeProjectFile(t, filepath.Join(root, ConfigFile), `
|
||||||
|
|||||||
Reference in New Issue
Block a user