Immutable Extracts

internal/pipeline/extract.go

@@ -300,15 +300,18 @@ func extractedFile(p *project.Project, resource erf.Resource, extension string)
 		if err != nil {
 			return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err)
 		}
+		target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json")
+		if err != nil {
+			return "", nil, err
+		}
+		if err := mergeExtractedGFFJSON(p, target, &document); err != nil {
+			return "", nil, fmt.Errorf("merge extracted gff json %s.%s: %w", resource.Name, extension, err)
+		}
 		formatted, err := json.MarshalIndent(document, "", "  ")
 		if err != nil {
 			return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err)
 		}
 		formatted = append(formatted, '\n')
-		target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json")
-		if err != nil {
-			return "", nil, err
-		}
 		return target, formatted, nil
 	default:
 		target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension)
@@ -340,6 +343,146 @@ func extractionTarget(p *project.Project, field, configured, root string, parts
 	return target, nil
 }
 
+func mergeExtractedGFFJSON(p *project.Project, target string, extracted *gff.Document) error {
+	rule, ok, err := extractGFFJSONMergeRule(p, target)
+	if err != nil || !ok {
+		return err
+	}
+
+	raw, err := os.ReadFile(target)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil
+		}
+		return fmt.Errorf("read existing source %s: %w", target, err)
+	}
+	var existing gff.Document
+	if err := json.Unmarshal(raw, &existing); err != nil {
+		return fmt.Errorf("parse existing source %s: %w", target, err)
+	}
+
+	for _, label := range rule.PreserveFields {
+		if field, ok := gffField(existing.Root, label); ok {
+			setGFFField(&extracted.Root, field)
+		}
+	}
+	for _, listRule := range rule.MergeLists {
+		if err := mergeGFFListByKey(&extracted.Root, existing.Root, listRule); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func extractGFFJSONMergeRule(p *project.Project, target string) (project.ExtractGFFJSONMergeRule, bool, error) {
+	rel, err := filepath.Rel(filepath.Clean(p.SourceDir()), filepath.Clean(target))
+	if err != nil {
+		return project.ExtractGFFJSONMergeRule{}, false, fmt.Errorf("resolve source-relative target %s: %w", target, err)
+	}
+	rel = filepath.ToSlash(rel)
+	if rel == ".." || strings.HasPrefix(rel, "../") {
+		return project.ExtractGFFJSONMergeRule{}, false, nil
+	}
+	for _, rule := range p.EffectiveConfig().Extract.Merge.GFFJSON {
+		if rule.Target == rel {
+			return rule, true, nil
+		}
+	}
+	return project.ExtractGFFJSONMergeRule{}, false, nil
+}
+
+func gffField(s gff.Struct, label string) (gff.Field, bool) {
+	for _, field := range s.Fields {
+		if field.Label == label {
+			return field, true
+		}
+	}
+	return gff.Field{}, false
+}
+
+func setGFFField(s *gff.Struct, replacement gff.Field) {
+	for index, field := range s.Fields {
+		if field.Label == replacement.Label {
+			s.Fields[index] = replacement
+			return
+		}
+	}
+	s.Fields = append(s.Fields, replacement)
+}
+
+func mergeGFFListByKey(extracted *gff.Struct, existing gff.Struct, rule project.ExtractListMergeRule) error {
+	extractedField, ok := gffField(*extracted, rule.Field)
+	if !ok {
+		return fmt.Errorf("extracted field %q not found", rule.Field)
+	}
+	extractedList, ok := extractedField.Value.(gff.ListValue)
+	if !ok {
+		return fmt.Errorf("extracted field %q is %s, not List", rule.Field, extractedField.Type)
+	}
+	existingField, ok := gffField(existing, rule.Field)
+	if !ok {
+		return nil
+	}
+	existingList, ok := existingField.Value.(gff.ListValue)
+	if !ok {
+		return fmt.Errorf("existing field %q is %s, not List", rule.Field, existingField.Type)
+	}
+
+	extractedByKey := map[string]gff.Struct{}
+	extractedOrder := make([]string, 0, len(extractedList))
+	for _, item := range extractedList {
+		key, err := gffStructKey(item, rule.KeyField)
+		if err != nil {
+			return fmt.Errorf("extracted field %q: %w", rule.Field, err)
+		}
+		if _, exists := extractedByKey[key]; exists {
+			return fmt.Errorf("extracted field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
+		}
+		extractedByKey[key] = item
+		extractedOrder = append(extractedOrder, key)
+	}
+
+	merged := make(gff.ListValue, 0, len(extractedList))
+	seen := map[string]struct{}{}
+	for _, item := range existingList {
+		key, err := gffStructKey(item, rule.KeyField)
+		if err != nil {
+			return fmt.Errorf("existing field %q: %w", rule.Field, err)
+		}
+		if _, duplicate := seen[key]; duplicate {
+			return fmt.Errorf("existing field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
+		}
+		if extractedItem, exists := extractedByKey[key]; exists {
+			merged = append(merged, extractedItem)
+			seen[key] = struct{}{}
+		}
+	}
+	for _, key := range extractedOrder {
+		if _, exists := seen[key]; exists {
+			continue
+		}
+		merged = append(merged, extractedByKey[key])
+	}
+
+	setGFFField(extracted, gff.NewField(rule.Field, merged))
+	return nil
+}
+
+func gffStructKey(s gff.Struct, keyField string) (string, error) {
+	field, ok := gffField(s, keyField)
+	if !ok {
+		return "", fmt.Errorf("key field %q not found", keyField)
+	}
+	switch value := field.Value.(type) {
+	case gff.ResRefValue:
+		return string(value), nil
+	case gff.StringValue:
+		return string(value), nil
+	default:
+		return "", fmt.Errorf("key field %q is %s, not ResRef or CExoString", keyField, field.Type)
+	}
+}
+
 type writeState int
 
 const (