ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Autogen configuration enabled

Browse Source
This commit is contained in:
archvillainette
2026-05-29 22:13:17 +02:00
parent 311bb8a3d0
commit a5dbad4ff9
5 changed files with 119 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6 -5
View File
@@ -405,11 +405,12 @@ discovered model stems from the configured include globs to the selected
`cachedmodels` dataset without `.mdl` extensions and skips models already `cachedmodels` dataset without `.mdl` extensions and skips models already
present in authored rows. present in authored rows.
For `head_visualeffects` autogen consumers, repositories can configure the For `head_visualeffects` autogen, producers can publish the group-to-row policy
group-to-row policy in `autogen.consumers[].head_visualeffects`. The toolkit in `autogen.producers[].head_visualeffects`, and consumers can override or
uses compatibility defaults for older configs, but YAML can own the group extend it in `autogen.consumers[].head_visualeffects`. The toolkit uses
prefixes, model stem prefixes to strip, key/label formats, model column, and compatibility defaults for older configs and manifests, but YAML can own the
default row values. Individual groups can override the model column with group prefixes, model stem prefixes to strip, key/label formats, model column,
and default row values. Individual groups can override the model column with
`model_column` or write the discovered model stem to several columns with `model_column` or write the discovered model stem to several columns with
`model_columns`, which is useful for root-node visualeffects. Supported format `model_columns`, which is useful for root-node visualeffects. Supported format
tokens are `{dataset}`, `{group}`, `{prefix}`, `{stem}`, `{stem_upper}`, and tokens are `{dataset}`, `{group}`, `{prefix}`, `{stem}`, `{stem_upper}`, and
internal/pipeline/pipeline_test.go
+11
View File
@@ -2209,6 +2209,14 @@ func TestBuildHAKsWritesConfiguredHeadVisualeffectsAutogenManifest(t *testing.T)
"release_tag": "head-vfx-manifest-current", "release_tag": "head-vfx-manifest-current",
"asset_name": "sow-head-vfx-manifest.json", "asset_name": "sow-head-vfx-manifest.json",
"cache_name": "sow-head-vfx-manifest.json" "cache_name": "sow-head-vfx-manifest.json"
},
"head_visualeffects": {
"groups": {
"head_accessories": {
"prefix": "headaccessory",
"model_columns": ["Imp_HeadCon_Node", "Imp_Root_H_Node"]
}
}
} }
} }
] ]
@@ -2263,6 +2271,9 @@ func TestBuildHAKsWritesConfiguredHeadVisualeffectsAutogenManifest(t *testing.T)
`"model_stem": "tfx_sash"`, `"model_stem": "tfx_sash"`,
`"group": "head_accessories"`, `"group": "head_accessories"`,
`"model_stem": "hfx_bandana"`, `"model_stem": "hfx_bandana"`,
`"head_visualeffects": {`,
`"model_columns": [`,
`"Imp_Root_H_Node"`,
`"group": "head_decorations"`, `"group": "head_decorations"`,
`"model_stem": "hfx_laurel"`, `"model_stem": "hfx_laurel"`,
`"group": "head_features"`, `"group": "head_features"`,
internal/project/project.go
+18 -14
View File
@@ -363,11 +363,12 @@ type AutogenReleaseSourceConfig struct {
} }
type AutogenProducerConfig struct { type AutogenProducerConfig struct {
ID string `json:"id" yaml:"id"` ID string `json:"id" yaml:"id"`
Root string `json:"root" yaml:"root"` Root string `json:"root" yaml:"root"`
Include []string `json:"include" yaml:"include"` Include []string `json:"include" yaml:"include"`
Derive AutogenDeriveConfig `json:"derive" yaml:"derive"` Derive AutogenDeriveConfig `json:"derive" yaml:"derive"`
Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"` Manifest AutogenManifestConfig `json:"manifest" yaml:"manifest"`
HeadVisualeffects HeadVisualeffectsConfig `json:"head_visualeffects" yaml:"head_visualeffects"`
} }
type AutogenConsumerConfig struct { type AutogenConsumerConfig struct {
@@ -386,19 +387,19 @@ type AutogenConsumerConfig struct {
} }
type HeadVisualeffectsConfig struct { type HeadVisualeffectsConfig struct {
Groups map[string]HeadVisualeffectGroupConfig `json:"groups" yaml:"groups"` Groups map[string]HeadVisualeffectGroupConfig `json:"groups,omitempty" yaml:"groups"`
StripModelPrefixes []string `json:"strip_model_prefixes" yaml:"strip_model_prefixes"` StripModelPrefixes []string `json:"strip_model_prefixes,omitempty" yaml:"strip_model_prefixes"`
KeyFormat string `json:"key_format" yaml:"key_format"` KeyFormat string `json:"key_format,omitempty" yaml:"key_format"`
LabelFormat string `json:"label_format" yaml:"label_format"` LabelFormat string `json:"label_format,omitempty" yaml:"label_format"`
ModelColumn string `json:"model_column" yaml:"model_column"` ModelColumn string `json:"model_column,omitempty" yaml:"model_column"`
RowDefaults map[string]string `json:"row_defaults" yaml:"row_defaults"` RowDefaults map[string]string `json:"row_defaults,omitempty" yaml:"row_defaults"`
} }
type HeadVisualeffectGroupConfig struct { type HeadVisualeffectGroupConfig struct {
Prefix string `json:"prefix" yaml:"prefix"` Prefix string `json:"prefix" yaml:"prefix"`
ModelColumn string `json:"model_column" yaml:"model_column"` ModelColumn string `json:"model_column,omitempty" yaml:"model_column"`
ModelColumns []string `json:"model_columns" yaml:"model_columns"` ModelColumns []string `json:"model_columns,omitempty" yaml:"model_columns"`
RowDefaults map[string]string `json:"row_defaults" yaml:"row_defaults"` RowDefaults map[string]string `json:"row_defaults,omitempty" yaml:"row_defaults"`
} }
type PartsRowsConfig struct { type PartsRowsConfig struct {
@@ -1693,6 +1694,9 @@ func validateAutogenConfig(cfg AutogenConfig) []error {
failures = append(failures, validateGlobList(fieldPrefix+".include", producer.Include)...) failures = append(failures, validateGlobList(fieldPrefix+".include", producer.Include)...)
failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", producer.Derive)...) failures = append(failures, validateAutogenDeriveConfig(fieldPrefix+".derive", producer.Derive)...)
failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", producer.Manifest)...) failures = append(failures, validateAutogenManifestConfig(fieldPrefix+".manifest", producer.Manifest)...)
if headVisualeffectsConfigConfigured(producer.HeadVisualeffects) {
failures = append(failures, validateHeadVisualeffectsConfig(fieldPrefix+".head_visualeffects", producer.HeadVisualeffects)...)
}
} }
for index, consumer := range cfg.Consumers { for index, consumer := range cfg.Consumers {
internal/topdata/autogen.go
+59 -22
View File
@@ -17,11 +17,12 @@ import (
const autogenManifestCacheMaxAge = time.Hour const autogenManifestCacheMaxAge = time.Hour
type autogenManifest struct { type autogenManifest struct {
ID string `json:"id"` ID string `json:"id"`
Repo string `json:"repo"` Repo string `json:"repo"`
Ref string `json:"ref"` Ref string `json:"ref"`
GeneratedAt string `json:"generated_at"` GeneratedAt string `json:"generated_at"`
Entries []autogenManifestEntry `json:"entries"` HeadVisualeffects *project.HeadVisualeffectsConfig `json:"head_visualeffects,omitempty"`
Entries []autogenManifestEntry `json:"entries"`
} }
type autogenManifestEntry struct { type autogenManifestEntry struct {
@@ -57,7 +58,7 @@ func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDatase
if !autogenConsumerTargetsCollectedDataset(result, consumer) { if !autogenConsumerTargetsCollectedDataset(result, consumer) {
continue continue
} }
entries, err := resolveAutogenConsumerEntries(p, consumer, progress) manifest, err := resolveAutogenConsumerManifest(p, consumer, progress)
if err != nil { if err != nil {
if consumer.Optional && isIgnorableOptionalAutogenError(err) { if consumer.Optional && isIgnorableOptionalAutogenError(err) {
if progress != nil { if progress != nil {
@@ -67,6 +68,7 @@ func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDatase
} }
return nil, err return nil, err
} }
entries := manifest.Entries
if len(entries) == 0 { if len(entries) == 0 {
continue continue
} }
@@ -74,7 +76,7 @@ func applyAutogenConsumers(p *project.Project, collected []nativeCollectedDatase
case "parts_rows": case "parts_rows":
result = augmentWithAutogeneratedParts(result, autogenPartsInventory(entries)) result = augmentWithAutogeneratedParts(result, autogenPartsInventory(entries))
case "head_visualeffects": case "head_visualeffects":
result, err = augmentWithAutogeneratedHeadVisualeffects(result, entries, consumer) result, err = augmentWithAutogeneratedHeadVisualeffects(result, entries, consumer, manifest.HeadVisualeffects)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -115,7 +117,7 @@ func isIgnorableOptionalAutogenError(err error) bool {
strings.Contains(message, "entries is empty") strings.Contains(message, "entries is empty")
} }
func resolveAutogenConsumerEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) { func resolveAutogenConsumerManifest(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) (*autogenManifest, error) {
overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer) overrideRoot, err := resolveAutogenLocalOverrideRoot(p, consumer)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -124,9 +126,13 @@ func resolveAutogenConsumerEntries(p *project.Project, consumer project.AutogenC
if progress != nil { if progress != nil {
progress(fmt.Sprintf("Scanning local autogen override for %s from %s...", consumer.ID, overrideRoot)) progress(fmt.Sprintf("Scanning local autogen override for %s from %s...", consumer.ID, overrideRoot))
} }
return scanLocalAutogenEntries(overrideRoot, consumer) entries, err := scanLocalAutogenEntries(overrideRoot, consumer)
if err != nil {
return nil, err
}
return &autogenManifest{ID: consumer.Producer, Entries: entries}, nil
} }
return resolveReleasedAutogenManifestEntries(p, consumer, progress) return resolveReleasedAutogenManifest(p, consumer, progress)
} }
func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.AutogenConsumerConfig) (string, error) { func resolveAutogenLocalOverrideRoot(p *project.Project, consumer project.AutogenConsumerConfig) (string, error) {
@@ -331,7 +337,7 @@ func deriveAutogenManifestEntry(rel string, derive project.AutogenDeriveConfig)
return entry, true return entry, true
} }
func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) { func resolveReleasedAutogenManifest(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) (*autogenManifest, error) {
cachePath := p.AutogenCachePath(consumer.Manifest.CacheName) cachePath := p.AutogenCachePath(consumer.Manifest.CacheName)
if strings.TrimSpace(os.Getenv(p.AutogenRefreshEnv())) == "" { if strings.TrimSpace(os.Getenv(p.AutogenRefreshEnv())) == "" {
manifest, fresh, err := readFreshAutogenManifestCache(cachePath, p.AutogenCacheMaxAge()) manifest, fresh, err := readFreshAutogenManifestCache(cachePath, p.AutogenCacheMaxAge())
@@ -344,12 +350,12 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.
if progress != nil { if progress != nil {
progress(fmt.Sprintf("Using cached released autogen manifest %s (remote check failed: %v)...", cachePath, err)) progress(fmt.Sprintf("Using cached released autogen manifest %s (remote check failed: %v)...", cachePath, err))
} }
return manifest.Entries, nil return manifest, nil
} else if current { } else if current {
if progress != nil { if progress != nil {
progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath)) progress(fmt.Sprintf("Using cached released autogen manifest %s...", cachePath))
} }
return manifest.Entries, nil return manifest, nil
} else if progress != nil { } else if progress != nil {
progress(fmt.Sprintf("Refreshing released autogen manifest %s; remote asset changed at %s.", consumer.Manifest.CacheName, remoteTime.Format(time.RFC3339))) progress(fmt.Sprintf("Refreshing released autogen manifest %s; remote asset changed at %s.", consumer.Manifest.CacheName, remoteTime.Format(time.RFC3339)))
} }
@@ -383,6 +389,14 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.
if err := writeAutogenManifestCache(cachePath, manifest); err != nil { if err := writeAutogenManifestCache(cachePath, manifest); err != nil {
return nil, err return nil, err
} }
return manifest, nil
}
func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) {
manifest, err := resolveReleasedAutogenManifest(p, consumer, progress)
if err != nil {
return nil, err
}
return manifest.Entries, nil return manifest.Entries, nil
} }
@@ -660,12 +674,12 @@ func augmentWithAutogeneratedCachedModels(collected []nativeCollectedDataset, en
return result, nil return result, nil
} }
func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDataset, entries []autogenManifestEntry, consumer project.AutogenConsumerConfig) ([]nativeCollectedDataset, error) { func augmentWithAutogeneratedHeadVisualeffects(collected []nativeCollectedDataset, entries []autogenManifestEntry, consumer project.AutogenConsumerConfig, manifestPolicy *project.HeadVisualeffectsConfig) ([]nativeCollectedDataset, error) {
if len(entries) == 0 { if len(entries) == 0 {
return collected, nil return collected, nil
} }
policy := resolveHeadVisualeffectsPolicy(consumer) policy := resolveHeadVisualeffectsPolicy(consumer, manifestPolicy)
result := append([]nativeCollectedDataset(nil), collected...) result := append([]nativeCollectedDataset(nil), collected...)
for i, dataset := range result { for i, dataset := range result {
if dataset.Dataset.Name != "visualeffects" { if dataset.Dataset.Name != "visualeffects" {
@@ -771,20 +785,31 @@ type headVisualeffectsGroupPolicy struct {
RowDefaults map[string]string RowDefaults map[string]string
} }
func resolveHeadVisualeffectsPolicy(consumer project.AutogenConsumerConfig) headVisualeffectsPolicy { func resolveHeadVisualeffectsPolicy(consumer project.AutogenConsumerConfig, manifestPolicy *project.HeadVisualeffectsConfig) headVisualeffectsPolicy {
policy := defaultHeadVisualeffectsPolicy() policy := defaultHeadVisualeffectsPolicy()
cfg := consumer.HeadVisualeffects if manifestPolicy != nil {
applyHeadVisualeffectsConfig(&policy, *manifestPolicy)
}
applyHeadVisualeffectsConfig(&policy, consumer.HeadVisualeffects)
return policy
}
func applyHeadVisualeffectsConfig(policy *headVisualeffectsPolicy, cfg project.HeadVisualeffectsConfig) {
for group, groupCfg := range cfg.Groups { for group, groupCfg := range cfg.Groups {
group = strings.TrimSpace(group) group = strings.TrimSpace(group)
prefix := strings.TrimSpace(groupCfg.Prefix) prefix := strings.TrimSpace(groupCfg.Prefix)
if group == "" || prefix == "" { if group == "" || prefix == "" {
continue continue
} }
groupPolicy := headVisualeffectsGroupPolicy{ groupPolicy := policy.Groups[group]
Prefix: prefix, groupPolicy.Prefix = prefix
ModelColumns: normalizeHeadVisualeffectsModelColumns(groupCfg.ModelColumn, groupCfg.ModelColumns), if columns := normalizeHeadVisualeffectsModelColumns(groupCfg.ModelColumn, groupCfg.ModelColumns); len(columns) > 0 {
RowDefaults: normalizeHeadVisualeffectsRowDefaults(groupCfg.RowDefaults), groupPolicy.ModelColumns = columns
} }
if groupPolicy.RowDefaults == nil {
groupPolicy.RowDefaults = map[string]string{}
}
mergeStringMap(groupPolicy.RowDefaults, normalizeHeadVisualeffectsRowDefaults(groupCfg.RowDefaults))
policy.Groups[group] = groupPolicy policy.Groups[group] = groupPolicy
} }
if len(cfg.StripModelPrefixes) > 0 { if len(cfg.StripModelPrefixes) > 0 {
@@ -800,7 +825,6 @@ func resolveHeadVisualeffectsPolicy(consumer project.AutogenConsumerConfig) head
policy.ModelColumn = strings.TrimSpace(cfg.ModelColumn) policy.ModelColumn = strings.TrimSpace(cfg.ModelColumn)
} }
mergeStringMap(policy.RowDefaults, normalizeHeadVisualeffectsRowDefaults(cfg.RowDefaults)) mergeStringMap(policy.RowDefaults, normalizeHeadVisualeffectsRowDefaults(cfg.RowDefaults))
return policy
} }
func defaultHeadVisualeffectsPolicy() headVisualeffectsPolicy { func defaultHeadVisualeffectsPolicy() headVisualeffectsPolicy {
@@ -964,9 +988,22 @@ func formatAutogenManifest(root string, producer project.AutogenProducerConfig,
GeneratedAt: time.Now().UTC().Format(time.RFC3339), GeneratedAt: time.Now().UTC().Format(time.RFC3339),
Entries: entries, Entries: entries,
} }
if autogenHeadVisualeffectsConfigConfigured(producer.HeadVisualeffects) {
cfg := producer.HeadVisualeffects
manifest.HeadVisualeffects = &cfg
}
return json.MarshalIndent(manifest, "", " ") return json.MarshalIndent(manifest, "", " ")
} }
func autogenHeadVisualeffectsConfigConfigured(cfg project.HeadVisualeffectsConfig) bool {
return len(cfg.Groups) > 0 ||
len(cfg.StripModelPrefixes) > 0 ||
strings.TrimSpace(cfg.KeyFormat) != "" ||
strings.TrimSpace(cfg.LabelFormat) != "" ||
strings.TrimSpace(cfg.ModelColumn) != "" ||
len(cfg.RowDefaults) > 0
}
func parseAutogenManifestEntryList(raw []byte) ([]autogenManifestEntry, error) { func parseAutogenManifestEntryList(raw []byte) ([]autogenManifestEntry, error) {
var manifest autogenManifest var manifest autogenManifest
if err := json.Unmarshal(raw, &manifest); err != nil { if err := json.Unmarshal(raw, &manifest); err != nil {
internal/topdata/topdata_test.go
+25 -5
View File
@@ -12666,9 +12666,9 @@ func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) {
writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n") writeFile(t, filepath.Join(projRoot, "topdata", "base_dialog.json"), "{}\n")
writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "base.json"), `{ writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "base.json"), `{
"output": "visualeffects.2da", "output": "visualeffects.2da",
"columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "OrientWithObject"], "columns": ["Label", "Type_FD", "OrientWithGround", "Imp_HeadCon_Node", "Imp_Root_H_Node", "OrientWithObject"],
"rows": [ "rows": [
{"key": "visualeffects:existing", "id": 17, "Label": "EXISTING", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "OrientWithObject": "0"} {"key": "visualeffects:existing", "id": 17, "Label": "EXISTING", "Type_FD": "F", "OrientWithGround": "0", "Imp_HeadCon_Node": "****", "Imp_Root_H_Node": "****", "OrientWithObject": "0"}
] ]
}`+"\n") }`+"\n")
writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "lock.json"), `{ writeFile(t, filepath.Join(projRoot, "topdata", "data", "visualeffects", "lock.json"), `{
@@ -12682,6 +12682,26 @@ func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) {
"repo": "ShadowsOverWestgate/sow-assets", "repo": "ShadowsOverWestgate/sow-assets",
"ref": "test-manifest", "ref": "test-manifest",
"generated_at": "2026-04-25T00:00:00Z", "generated_at": "2026-04-25T00:00:00Z",
"head_visualeffects": {
"groups": {
"head_accessories": {
"prefix": "headaccessory",
"model_columns": ["Imp_HeadCon_Node", "Imp_Root_H_Node"]
},
"head_features": {
"prefix": "headfeature",
"model_columns": ["Imp_HeadCon_Node", "Imp_Root_H_Node"]
}
},
"strip_model_prefixes": ["hfx_"],
"key_format": "{dataset}:{prefix}_{stem}",
"label_format": "{prefix}_{stem_upper}",
"row_defaults": {
"Type_FD": "D",
"OrientWithGround": "0",
"OrientWithObject": "1"
}
},
"entries": [ "entries": [
{ {
"group": "head_accessories", "group": "head_accessories",
@@ -12758,9 +12778,9 @@ func TestBuildNativeWithReleasedHeadVisualeffectsManifest(t *testing.T) {
} }
text := string(visualeffectsBytes) text := string(visualeffectsBytes)
for _, want := range []string{ for _, want := range []string{
"0\theadaccessory_BANDANA\tD\t0\thfx_bandana\t1", "0\theadaccessory_BANDANA\tD\t0\thfx_bandana\thfx_bandana\t1",
"1\theadfeature_HAIR_BANGS\tD\t0\thfx_hair_bangs\t1", "1\theadfeature_HAIR_BANGS\tD\t0\thfx_hair_bangs\thfx_hair_bangs\t1",
"17\tEXISTING\tF\t0\t****\t0", "17\tEXISTING\tF\t0\t****\t****\t0",
} { } {
if !strings.Contains(text, want) { if !strings.Contains(text, want) {
t.Fatalf("expected visualeffects output to contain %q, got:\n%s", want, text) t.Fatalf("expected visualeffects output to contain %q, got:\n%s", want, text)