ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Immutable Extracts

Browse Source
This commit is contained in:
archvillainette
2026-05-20 20:20:07 +02:00
parent b1b7a08848
commit c37b27b720
5 changed files with 603 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/pipeline/extract.go
+147 -4
View File
@@ -300,15 +300,18 @@ func extractedFile(p *project.Project, resource erf.Resource, extension string)
if err != nil {
return "", nil, fmt.Errorf("decode gff %s.%s: %w", resource.Name, extension, err)
}
target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json")
if err != nil {
return "", nil, err
}
if err := mergeExtractedGFFJSON(p, target, &document); err != nil {
return "", nil, fmt.Errorf("merge extracted gff json %s.%s: %w", resource.Name, extension, err)
}
formatted, err := json.MarshalIndent(document, "", " ")
if err != nil {
return "", nil, fmt.Errorf("marshal json %s.%s: %w", resource.Name, extension, err)
}
formatted = append(formatted, '\n')
target, err := extractionTarget(p, "paths.source", effective.Paths.Source, p.SourceDir(), sourceSubdir(extension), resref+"."+extension+".json")
if err != nil {
return "", nil, err
}
return target, formatted, nil
default:
target, err := extractionTarget(p, "paths.assets", effective.Paths.Assets, p.AssetsDir(), extension, resref+"."+extension)
@@ -340,6 +343,146 @@ func extractionTarget(p *project.Project, field, configured, root string, parts
return target, nil
}
func mergeExtractedGFFJSON(p *project.Project, target string, extracted *gff.Document) error {
rule, ok, err := extractGFFJSONMergeRule(p, target)
if err != nil || !ok {
return err
}
raw, err := os.ReadFile(target)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return fmt.Errorf("read existing source %s: %w", target, err)
}
var existing gff.Document
if err := json.Unmarshal(raw, &existing); err != nil {
return fmt.Errorf("parse existing source %s: %w", target, err)
}
for _, label := range rule.PreserveFields {
if field, ok := gffField(existing.Root, label); ok {
setGFFField(&extracted.Root, field)
}
}
for _, listRule := range rule.MergeLists {
if err := mergeGFFListByKey(&extracted.Root, existing.Root, listRule); err != nil {
return err
}
}
return nil
}
func extractGFFJSONMergeRule(p *project.Project, target string) (project.ExtractGFFJSONMergeRule, bool, error) {
rel, err := filepath.Rel(filepath.Clean(p.SourceDir()), filepath.Clean(target))
if err != nil {
return project.ExtractGFFJSONMergeRule{}, false, fmt.Errorf("resolve source-relative target %s: %w", target, err)
}
rel = filepath.ToSlash(rel)
if rel == ".." || strings.HasPrefix(rel, "../") {
return project.ExtractGFFJSONMergeRule{}, false, nil
}
for _, rule := range p.EffectiveConfig().Extract.Merge.GFFJSON {
if rule.Target == rel {
return rule, true, nil
}
}
return project.ExtractGFFJSONMergeRule{}, false, nil
}
func gffField(s gff.Struct, label string) (gff.Field, bool) {
for _, field := range s.Fields {
if field.Label == label {
return field, true
}
}
return gff.Field{}, false
}
func setGFFField(s *gff.Struct, replacement gff.Field) {
for index, field := range s.Fields {
if field.Label == replacement.Label {
s.Fields[index] = replacement
return
}
}
s.Fields = append(s.Fields, replacement)
}
func mergeGFFListByKey(extracted *gff.Struct, existing gff.Struct, rule project.ExtractListMergeRule) error {
extractedField, ok := gffField(*extracted, rule.Field)
if !ok {
return fmt.Errorf("extracted field %q not found", rule.Field)
}
extractedList, ok := extractedField.Value.(gff.ListValue)
if !ok {
return fmt.Errorf("extracted field %q is %s, not List", rule.Field, extractedField.Type)
}
existingField, ok := gffField(existing, rule.Field)
if !ok {
return nil
}
existingList, ok := existingField.Value.(gff.ListValue)
if !ok {
return fmt.Errorf("existing field %q is %s, not List", rule.Field, existingField.Type)
}
extractedByKey := map[string]gff.Struct{}
extractedOrder := make([]string, 0, len(extractedList))
for _, item := range extractedList {
key, err := gffStructKey(item, rule.KeyField)
if err != nil {
return fmt.Errorf("extracted field %q: %w", rule.Field, err)
}
if _, exists := extractedByKey[key]; exists {
return fmt.Errorf("extracted field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
}
extractedByKey[key] = item
extractedOrder = append(extractedOrder, key)
}
merged := make(gff.ListValue, 0, len(extractedList))
seen := map[string]struct{}{}
for _, item := range existingList {
key, err := gffStructKey(item, rule.KeyField)
if err != nil {
return fmt.Errorf("existing field %q: %w", rule.Field, err)
}
if _, duplicate := seen[key]; duplicate {
return fmt.Errorf("existing field %q has duplicate %s key %q", rule.Field, rule.KeyField, key)
}
if extractedItem, exists := extractedByKey[key]; exists {
merged = append(merged, extractedItem)
seen[key] = struct{}{}
}
}
for _, key := range extractedOrder {
if _, exists := seen[key]; exists {
continue
}
merged = append(merged, extractedByKey[key])
}
setGFFField(extracted, gff.NewField(rule.Field, merged))
return nil
}
func gffStructKey(s gff.Struct, keyField string) (string, error) {
field, ok := gffField(s, keyField)
if !ok {
return "", fmt.Errorf("key field %q not found", keyField)
}
switch value := field.Value.(type) {
case gff.ResRefValue:
return string(value), nil
case gff.StringValue:
return string(value), nil
default:
return "", fmt.Errorf("key field %q is %s, not ResRef or CExoString", keyField, field.Type)
}
}
type writeState int
const (
internal/pipeline/pipeline_test.go
+237 -1
View File
@@ -444,6 +444,7 @@ extract:
func TestExtractConsumeArchivesFalseOverridesLegacyDeleteFlag(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "src", "areas"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
@@ -474,7 +475,6 @@ extract:
}
}
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
@@ -3253,6 +3253,197 @@ func TestExtractOverwritesAndRemovesStaleFiles(t *testing.T) {
}
}
func TestExtractMergesConfiguredGFFJSONFieldsAndLists(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
mustMkdir(t, filepath.Join(root, "src", "areas"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Module
resref: testmod
paths:
source: src
assets: assets
build: build
extract:
merge:
gff_json:
- target: module/module.ifo.json
preserve_fields:
- Mod_Entry_Area
- Mod_Entry_X
merge_lists:
- field: Mod_Area_list
key_field: Area_Name
strategy: preserve_existing_order_append_new
`)
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Name",
"type": "CExoString",
"value": "Test Module"
},
{
"label": "Mod_Entry_Area",
"type": "ResRef",
"value": "area_a"
},
{
"label": "Mod_Entry_X",
"type": "Float",
"value": 1
},
{
"label": "Mod_Area_list",
"type": "List",
"value": [
{
"struct_type": 6,
"fields": [
{
"label": "Area_Name",
"type": "ResRef",
"value": "area_a"
}
]
},
{
"struct_type": 6,
"fields": [
{
"label": "Area_Name",
"type": "ResRef",
"value": "area_b"
}
]
},
{
"struct_type": 6,
"fields": [
{
"label": "Area_Name",
"type": "ResRef",
"value": "area_c"
}
]
}
]
}
]
}
}
`)
for _, area := range []string{"area_a", "area_b", "area_c"} {
mustWriteFile(t, filepath.Join(root, "src", "areas", area+".are.json"), `{
"file_type": "ARE ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": []
}
}
`)
}
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
if err := p.Scan(); err != nil {
t.Fatalf("scan: %v", err)
}
if _, err := BuildModule(p); err != nil {
t.Fatalf("build module: %v", err)
}
mustWriteFile(t, filepath.Join(root, "src", "module", "module.ifo.json"), `{
"file_type": "IFO ",
"file_version": "V3.2",
"root": {
"struct_type": 0,
"fields": [
{
"label": "Mod_Entry_Area",
"type": "ResRef",
"value": "area_b"
},
{
"label": "Mod_Entry_X",
"type": "Float",
"value": 99
},
{
"label": "Mod_Area_list",
"type": "List",
"value": [
{
"struct_type": 6,
"fields": [
{
"label": "Area_Name",
"type": "ResRef",
"value": "area_b"
}
]
},
{
"struct_type": 6,
"fields": [
{
"label": "Area_Name",
"type": "ResRef",
"value": "area_a"
}
]
},
{
"struct_type": 6,
"fields": [
{
"label": "Area_Name",
"type": "ResRef",
"value": "removed_area"
}
]
}
]
}
]
}
}
`)
if err := p.Scan(); err != nil {
t.Fatalf("rescan before extract: %v", err)
}
if _, err := Extract(p); err != nil {
t.Fatalf("extract: %v", err)
}
document := readGFFJSON(t, filepath.Join(root, "src", "module", "module.ifo.json"))
if got, want := fieldValue(t, document.Root, "Mod_Entry_Area"), gff.ResRefValue("area_b"); got != want {
t.Fatalf("expected preserved Mod_Entry_Area %#v, got %#v", want, got)
}
if got, want := fieldValue(t, document.Root, "Mod_Entry_X"), gff.FloatValue(99); got != want {
t.Fatalf("expected preserved Mod_Entry_X %#v, got %#v", want, got)
}
if got, want := keyedListValues(t, document.Root, "Mod_Area_list", "Area_Name"), []string{"area_b", "area_a", "area_c"}; !slices.Equal(got, want) {
t.Fatalf("expected merged area order %#v, got %#v", want, got)
}
}
func TestExtractNormalizesResourceNamesToLowercase(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "blueprints", "items"))
@@ -3423,6 +3614,51 @@ func mustWriteFile(t *testing.T, path, data string) {
}
}
func readGFFJSON(t *testing.T, path string) gff.Document {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
var document gff.Document
if err := json.Unmarshal(raw, &document); err != nil {
t.Fatalf("parse %s: %v", path, err)
}
return document
}
func fieldValue(t *testing.T, s gff.Struct, label string) gff.Value {
t.Helper()
for _, field := range s.Fields {
if field.Label == label {
return field.Value
}
}
t.Fatalf("field %s not found", label)
return nil
}
func keyedListValues(t *testing.T, s gff.Struct, listLabel, keyLabel string) []string {
t.Helper()
list, ok := fieldValue(t, s, listLabel).(gff.ListValue)
if !ok {
t.Fatalf("field %s is not a list", listLabel)
}
values := make([]string, 0, len(list))
for _, item := range list {
value := fieldValue(t, item, keyLabel)
switch typed := value.(type) {
case gff.ResRefValue:
values = append(values, string(typed))
case gff.StringValue:
values = append(values, string(typed))
default:
t.Fatalf("field %s has unsupported key type %T", keyLabel, value)
}
}
return values
}
func minimalGFF(t *testing.T, fileType string) []byte {
t.Helper()
document := gff.Document{