ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Immutable Extracts

Browse Source
This commit is contained in:
archvillainette
2026-05-20 20:20:07 +02:00
parent b1b7a08848
commit c37b27b720
5 changed files with 603 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/project/project.go
+81 -7
View File
@@ -247,13 +247,30 @@ type TopDataWikiManagedRegionConfig struct {
}
type ExtractConfig struct {
IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"`
Archives []string `json:"archives,omitempty" yaml:"archives,omitempty"`
Layout string `json:"layout" yaml:"layout"`
HAKDiscovery string `json:"hak_discovery" yaml:"hak_discovery"`
CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"`
ConsumeArchives *bool `json:"consume_archives,omitempty" yaml:"consume_archives,omitempty"`
DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"`
IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"`
Archives []string `json:"archives,omitempty" yaml:"archives,omitempty"`
Layout string `json:"layout" yaml:"layout"`
HAKDiscovery string `json:"hak_discovery" yaml:"hak_discovery"`
CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"`
ConsumeArchives *bool `json:"consume_archives,omitempty" yaml:"consume_archives,omitempty"`
DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"`
Merge ExtractMergeConfig `json:"merge" yaml:"merge"`
}
type ExtractMergeConfig struct {
GFFJSON []ExtractGFFJSONMergeRule `json:"gff_json" yaml:"gff_json"`
}
type ExtractGFFJSONMergeRule struct {
Target string `json:"target" yaml:"target"`
PreserveFields []string `json:"preserve_fields" yaml:"preserve_fields"`
MergeLists []ExtractListMergeRule `json:"merge_lists" yaml:"merge_lists"`
}
type ExtractListMergeRule struct {
Field string `json:"field" yaml:"field"`
KeyField string `json:"key_field" yaml:"key_field"`
Strategy string `json:"strategy" yaml:"strategy"`
}
type AutogenConfig struct {
@@ -598,6 +615,7 @@ func (p *Project) ValidateLayout() error {
failures = append(failures, fmt.Errorf("extract.hak_discovery %q is not supported", effective.Extract.HAKDiscovery))
}
failures = append(failures, validateGlobList("extract.archives", effective.Extract.Archives)...)
failures = append(failures, validateExtractMergeConfig(effective.Extract.Merge)...)
if _, err := time.ParseDuration(effective.Autogen.Cache.MaxAge); err != nil {
failures = append(failures, fmt.Errorf("autogen.cache.max_age must be a duration: %w", err))
}
@@ -669,6 +687,49 @@ func (p *Project) ValidateLayout() error {
return nil
}
func validateExtractMergeConfig(config ExtractMergeConfig) []error {
var failures []error
seenTargets := map[string]struct{}{}
for ruleIndex, rule := range config.GFFJSON {
prefix := fmt.Sprintf("extract.merge.gff_json[%d]", ruleIndex)
target := strings.TrimSpace(rule.Target)
switch {
case target == "":
failures = append(failures, fmt.Errorf("%s.target must not be empty", prefix))
case filepath.IsAbs(filepath.FromSlash(target)):
failures = append(failures, fmt.Errorf("%s.target %q must be relative to paths.source", prefix, rule.Target))
default:
clean := filepath.Clean(filepath.FromSlash(target))
if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
failures = append(failures, fmt.Errorf("%s.target %q must not escape paths.source", prefix, rule.Target))
}
normalized := filepath.ToSlash(clean)
if _, exists := seenTargets[normalized]; exists {
failures = append(failures, fmt.Errorf("extract.merge.gff_json target %q is configured more than once", normalized))
}
seenTargets[normalized] = struct{}{}
}
for listIndex, listRule := range rule.MergeLists {
listPrefix := fmt.Sprintf("%s.merge_lists[%d]", prefix, listIndex)
if strings.TrimSpace(listRule.Field) == "" {
failures = append(failures, fmt.Errorf("%s.field must not be empty", listPrefix))
}
if strings.TrimSpace(listRule.KeyField) == "" {
failures = append(failures, fmt.Errorf("%s.key_field must not be empty", listPrefix))
}
switch listRule.Strategy {
case "preserve_existing_order_append_new":
case "":
failures = append(failures, fmt.Errorf("%s.strategy must not be empty", listPrefix))
default:
failures = append(failures, fmt.Errorf("%s.strategy %q is not supported", listPrefix, listRule.Strategy))
}
}
}
return failures
}
func (p *Project) Scan() error {
var sourceFiles []string
var sourceExts []string
@@ -1100,6 +1161,19 @@ func normalizeConfig(cfg *Config) {
}
cfg.Extract.IgnoreExtensions = normalizeStringSlice(cfg.Extract.IgnoreExtensions)
cfg.Extract.Archives = normalizeStringSlice(cfg.Extract.Archives)
for i := range cfg.Extract.Merge.GFFJSON {
target := strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].Target)
if target != "" {
target = filepath.ToSlash(filepath.Clean(filepath.FromSlash(target)))
}
cfg.Extract.Merge.GFFJSON[i].Target = target
cfg.Extract.Merge.GFFJSON[i].PreserveFields = normalizeStringSlice(cfg.Extract.Merge.GFFJSON[i].PreserveFields)
for j := range cfg.Extract.Merge.GFFJSON[i].MergeLists {
cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Field = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Field)
cfg.Extract.Merge.GFFJSON[i].MergeLists[j].KeyField = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].KeyField)
cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Strategy = strings.TrimSpace(cfg.Extract.Merge.GFFJSON[i].MergeLists[j].Strategy)
}
}
for i := range cfg.Autogen.Producers {
cfg.Autogen.Producers[i].Include = normalizeStringSlice(cfg.Autogen.Producers[i].Include)
}
internal/project/project_test.go
+108
View File
@@ -5,6 +5,7 @@ import (
"errors"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
@@ -254,6 +255,15 @@ extract:
layout: nwn_canonical_json
hak_discovery: configured_haks
cleanup_stale: false
merge:
gff_json:
- target: module/module.ifo.json
preserve_fields:
- Mod_Entry_Area
merge_lists:
- field: Mod_Area_list
key_field: Area_Name
strategy: preserve_existing_order_append_new
topdata:
wiki:
source: topdata/wiki
@@ -305,6 +315,19 @@ autogen:
if effective.Extract.CleanupStale == nil || *effective.Extract.CleanupStale {
t.Fatalf("expected cleanup_stale false, got %#v", effective.Extract.CleanupStale)
}
if got, want := len(effective.Extract.Merge.GFFJSON), 1; got != want {
t.Fatalf("expected %d gff json merge rule, got %d", want, got)
}
rule := effective.Extract.Merge.GFFJSON[0]
if got, want := rule.Target, "module/module.ifo.json"; got != want {
t.Fatalf("expected merge target %q, got %q", want, got)
}
if got, want := rule.PreserveFields, []string{"Mod_Entry_Area"}; !reflect.DeepEqual(got, want) {
t.Fatalf("expected preserve fields %#v, got %#v", want, got)
}
if got, want := rule.MergeLists[0].Strategy, "preserve_existing_order_append_new"; got != want {
t.Fatalf("expected merge list strategy %q, got %q", want, got)
}
if got, want := effective.TopData.Wiki.DeployManifest, "wiki-manifest.json"; got != want {
t.Fatalf("expected wiki deploy manifest %q, got %q", want, got)
}
@@ -346,6 +369,91 @@ autogen:
}
}
func TestValidateLayoutRejectsInvalidExtractMergeRules(t *testing.T) {
tests := []struct {
name string
config string
wantErr string
}{
{
name: "empty target",
config: `
extract:
merge:
gff_json:
- target: ""
preserve_fields: [Mod_Entry_Area]
`,
wantErr: "extract.merge.gff_json[0].target must not be empty",
},
{
name: "duplicate target",
config: `
extract:
merge:
gff_json:
- target: module/module.ifo.json
- target: module/module.ifo.json
`,
wantErr: "extract.merge.gff_json target \"module/module.ifo.json\" is configured more than once",
},
{
name: "empty merge list field",
config: `
extract:
merge:
gff_json:
- target: module/module.ifo.json
merge_lists:
- field: ""
key_field: Area_Name
strategy: preserve_existing_order_append_new
`,
wantErr: "extract.merge.gff_json[0].merge_lists[0].field must not be empty",
},
{
name: "unsupported strategy",
config: `
extract:
merge:
gff_json:
- target: module/module.ifo.json
merge_lists:
- field: Mod_Area_list
key_field: Area_Name
strategy: replace
`,
wantErr: "extract.merge.gff_json[0].merge_lists[0].strategy \"replace\" is not supported",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `
module:
name: Test Module
resref: testmod
paths:
source: src
build: build
`+tt.config)
proj, err := Load(root)
if err != nil {
t.Fatalf("Load returned error: %v", err)
}
err = proj.ValidateLayout()
if err == nil {
t.Fatalf("expected validation error containing %q", tt.wantErr)
}
if !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("expected validation error containing %q, got %v", tt.wantErr, err)
}
})
}
}
func TestActiveOverridesAreVisibleAndSensitiveValuesAreMasked(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `