Files
sow-tools/internal/project/effective.go
T
archvillainette b39bdf13e8 Configuration Hardening 2
What changed:

  - Added visible runtime override reporting with masking for sensitive values.
  - Expanded effective config for build cleanup, script compiler discovery/env, extract layout/cleanup/
    HAK discovery, wiki deploy defaults, autogen cache policy, and release-source env names.
  - Wired configured values into build, script compiler resolution, extract, autogen cache refresh/max
    age, parts manifest release source, wiki generation/deploy, and TLK output naming.
  - Added config sources override output and validation-rule hints in config explain.
  - Added CONFIGURATION_HARDENING_AUDIT.md.
  - Updated README.md with config commands, schema defaults, and JSON artifact boundaries.
  - Added regression tests for override visibility/masking, compiler/extract/wiki/autogen config,
    configured HAK discovery, and config sources output.
2026-05-07 14:22:47 +02:00

467 lines
20 KiB
Go

package project
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
)
const (
DefaultBuildPath = "build"
DefaultCachePath = ".cache"
DefaultToolsPath = "tools"
DefaultModuleArchiveTemplate = "{module.resref}.mod"
DefaultHAKManifest = "haks.json"
DefaultHAKArchiveTemplate = "{hak.name}.hak"
DefaultSourceJSONPattern = "{resref}.{extension}.json"
DefaultScriptsSourceDir = "scripts"
DefaultScriptsCache = "{paths.cache}/ncs"
DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER"
DefaultMusicStageRoot = "{paths.cache}/music"
DefaultCreditsRoot = "{paths.cache}/credits"
DefaultCreditsOverlayFile = "CREDITS.md"
DefaultTopDataBuild = "{paths.build}/topdata"
DefaultTopDataCompiled2DADir = "2da"
DefaultTopDataCompiledTLK = "sow_tlk.tlk"
DefaultTopDataWikiOutputRoot = "wiki"
DefaultTopDataWikiPagesDir = "pages"
DefaultTopDataWikiStateFile = "state.json"
DefaultWikiDeployManifest = ".wiki_deploy_manifest.json"
DefaultWikiDeployEditSummary = "Auto-generated from native builder"
DefaultTopDataPackageHAK = "sow_top.hak"
DefaultTopDataPackageTLK = "sow_tlk.tlk"
DefaultAutogenCacheRoot = "{paths.cache}"
DefaultAutogenCacheMaxAge = time.Hour
DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH"
DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH"
DefaultAutogenReleaseProvider = "gitea"
DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL"
DefaultAutogenRepoEnv = "SOW_ASSETS_REPO"
DefaultExtractLayout = "nwn_canonical_json"
DefaultExtractHAKDiscovery = "build_glob"
)
type ConfigProvenance map[string]ConfigValueProvenance
type ConfigValueProvenance struct {
Source string `json:"source" yaml:"source"`
Detail string `json:"detail,omitempty" yaml:"detail,omitempty"`
Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"`
}
type EffectiveConfig struct {
ConfigSource ConfigSource `json:"config_source" yaml:"config_source"`
Module ModuleConfig `json:"module" yaml:"module"`
Paths EffectivePathConfig `json:"paths" yaml:"paths"`
Outputs EffectiveOutputConfig `json:"outputs" yaml:"outputs"`
Build BuildConfig `json:"build" yaml:"build"`
Inventory InventoryConfig `json:"inventory" yaml:"inventory"`
Scripts EffectiveScriptsConfig `json:"scripts" yaml:"scripts"`
Music EffectiveMusicConfig `json:"music" yaml:"music"`
TopData EffectiveTopDataConfig `json:"topdata" yaml:"topdata"`
Extract ExtractConfig `json:"extract" yaml:"extract"`
Autogen EffectiveAutogenConfig `json:"autogen" yaml:"autogen"`
HAKs []HAKConfig `json:"haks" yaml:"haks"`
Provenance ConfigProvenance `json:"provenance" yaml:"provenance"`
Overrides []ConfigOverride `json:"overrides,omitempty" yaml:"overrides,omitempty"`
}
type EffectivePathConfig struct {
Source string `json:"source" yaml:"source"`
Assets string `json:"assets" yaml:"assets"`
Build string `json:"build" yaml:"build"`
Cache string `json:"cache" yaml:"cache"`
Tools string `json:"tools" yaml:"tools"`
}
type EffectiveOutputConfig struct {
ModuleArchive string `json:"module_archive" yaml:"module_archive"`
HAKManifest string `json:"hak_manifest" yaml:"hak_manifest"`
HAKArchive string `json:"hak_archive" yaml:"hak_archive"`
}
type EffectiveScriptsConfig struct {
SourceDir string `json:"source_dir" yaml:"source_dir"`
Cache string `json:"cache" yaml:"cache"`
Compiler EffectiveScriptCompilerConfig `json:"compiler" yaml:"compiler"`
}
type EffectiveScriptCompilerConfig struct {
Path string `json:"path" yaml:"path"`
Search []string `json:"search" yaml:"search"`
Env EffectiveScriptCompilerEnvConfig `json:"env" yaml:"env"`
}
type EffectiveScriptCompilerEnvConfig struct {
Path string `json:"path" yaml:"path"`
NWNRoot []string `json:"nwn_root" yaml:"nwn_root"`
NWNUserDirectory []string `json:"nwn_user_directory" yaml:"nwn_user_directory"`
}
type EffectiveMusicConfig struct {
Prefixes map[string]string `json:"prefixes" yaml:"prefixes"`
StageRoot string `json:"stage_root" yaml:"stage_root"`
CreditsRoot string `json:"credits_root" yaml:"credits_root"`
CreditsOverlay string `json:"credits_overlay" yaml:"credits_overlay"`
MaxStemLength int `json:"max_stem_length" yaml:"max_stem_length"`
}
type EffectiveTopDataConfig struct {
Source string `json:"source" yaml:"source"`
Build string `json:"build" yaml:"build"`
ReferenceBuilder string `json:"reference_builder" yaml:"reference_builder"`
Assets string `json:"assets" yaml:"assets"`
Compiled2DADir string `json:"compiled_2da_dir" yaml:"compiled_2da_dir"`
CompiledTLK string `json:"compiled_tlk" yaml:"compiled_tlk"`
PackageHAK string `json:"package_hak" yaml:"package_hak"`
PackageTLK string `json:"package_tlk" yaml:"package_tlk"`
Wiki TopDataWikiConfig `json:"wiki" yaml:"wiki"`
}
type EffectiveAutogenConfig struct {
Producers []AutogenProducerConfig `json:"producers" yaml:"producers"`
Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"`
Cache EffectiveAutogenCacheConfig `json:"cache" yaml:"cache"`
ReleaseSource EffectiveAutogenReleaseSourceConfig `json:"release_source" yaml:"release_source"`
}
type EffectiveAutogenCacheConfig struct {
Root string `json:"root" yaml:"root"`
MaxAge string `json:"max_age" yaml:"max_age"`
RefreshEnv string `json:"refresh_env" yaml:"refresh_env"`
PartsManifestRefreshEnv string `json:"parts_manifest_refresh_env" yaml:"parts_manifest_refresh_env"`
}
type EffectiveAutogenReleaseSourceConfig struct {
Provider string `json:"provider" yaml:"provider"`
ServerURLEnv string `json:"server_url_env" yaml:"server_url_env"`
RepoEnv string `json:"repo_env" yaml:"repo_env"`
}
type ConfigOverride struct {
Key string `json:"key" yaml:"key"`
Value string `json:"value" yaml:"value"`
Source string `json:"source" yaml:"source"`
}
func (p *Project) EffectiveConfig() EffectiveConfig {
provenance := cloneProvenance(p.Provenance)
markMissingDefaults(provenance)
paths := EffectivePathConfig{
Source: strings.TrimSpace(p.Config.Paths.Source),
Assets: strings.TrimSpace(p.Config.Paths.Assets),
Build: defaultString(p.Config.Paths.Build, DefaultBuildPath),
Cache: defaultString(p.Config.Paths.Cache, DefaultCachePath),
Tools: defaultString(p.Config.Paths.Tools, DefaultToolsPath),
}
outputs := EffectiveOutputConfig{
ModuleArchive: defaultString(p.Config.Outputs.ModuleArchive, DefaultModuleArchiveTemplate),
HAKManifest: defaultString(p.Config.Outputs.HAKManifest, DefaultHAKManifest),
HAKArchive: defaultString(p.Config.Outputs.HAKArchive, DefaultHAKArchiveTemplate),
}
inventory := InventoryConfig{
SourceExtensions: defaultStringSlice(p.Config.Inventory.SourceExtensions, SourceExtensions),
AssetExtensions: defaultStringSlice(p.Config.Inventory.AssetExtensions, AssetExtensions),
SourceJSONPattern: defaultString(p.Config.Inventory.SourceJSONPattern, DefaultSourceJSONPattern),
}
scripts := EffectiveScriptsConfig{
SourceDir: defaultString(p.Config.Scripts.SourceDir, DefaultScriptsSourceDir),
Cache: expandPathTemplate(defaultString(p.Config.Scripts.Cache, DefaultScriptsCache), paths),
Compiler: EffectiveScriptCompilerConfig{
Path: strings.TrimSpace(p.Config.Scripts.Compiler.Path),
Search: expandPathTemplates(defaultStringSlice(p.Config.Scripts.Compiler.Search, defaultScriptCompilerSearch()), paths),
Env: EffectiveScriptCompilerEnvConfig{
Path: defaultString(p.Config.Scripts.Compiler.Env.Path, DefaultScriptCompilerEnvPath),
NWNRoot: defaultStringSlice(p.Config.Scripts.Compiler.Env.NWNRoot, []string{"SOW_NWN_ROOT", "NWN_ROOT"}),
NWNUserDirectory: defaultStringSlice(p.Config.Scripts.Compiler.Env.NWNUserDirectory, []string{"SOW_NWN_USER_DIRECTORY", "NWN_HOME", "NWN_USER_DIRECTORY"}),
},
},
}
topBuild := expandPathTemplate(defaultString(p.Config.TopData.Build, DefaultTopDataBuild), paths)
top := EffectiveTopDataConfig{
Source: strings.TrimSpace(p.Config.TopData.Source),
Build: topBuild,
ReferenceBuilder: strings.TrimSpace(p.Config.TopData.ReferenceBuilder),
Assets: strings.TrimSpace(p.Config.TopData.Assets),
Compiled2DADir: defaultString(p.Config.TopData.Compiled2DADir, DefaultTopDataCompiled2DADir),
CompiledTLK: defaultString(p.Config.TopData.CompiledTLK, DefaultTopDataCompiledTLK),
PackageHAK: defaultString(p.Config.TopData.PackageHAK, DefaultTopDataPackageHAK),
PackageTLK: defaultString(p.Config.TopData.PackageTLK, DefaultTopDataPackageTLK),
Wiki: TopDataWikiConfig{
OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot),
PagesDir: defaultString(p.Config.TopData.Wiki.PagesDir, DefaultTopDataWikiPagesDir),
StateFile: defaultString(p.Config.TopData.Wiki.StateFile, DefaultTopDataWikiStateFile),
ManagedNamespaces: defaultStringSlice(p.Config.TopData.Wiki.ManagedNamespaces, []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}),
DeployManifest: defaultString(p.Config.TopData.Wiki.DeployManifest, DefaultWikiDeployManifest),
DeployEditSummary: defaultString(p.Config.TopData.Wiki.DeployEditSummary, DefaultWikiDeployEditSummary),
},
}
extract := p.Config.Extract
extract.Layout = defaultString(extract.Layout, DefaultExtractLayout)
extract.HAKDiscovery = defaultString(extract.HAKDiscovery, DefaultExtractHAKDiscovery)
if extract.CleanupStale == nil {
defaultCleanup := true
extract.CleanupStale = &defaultCleanup
}
effective := EffectiveConfig{
ConfigSource: p.ConfigSource,
Module: p.Config.Module,
Paths: paths,
Outputs: outputs,
Build: p.Config.Build,
Inventory: inventory,
Scripts: scripts,
Music: EffectiveMusicConfig{
Prefixes: cloneStringMap(p.Config.Music.Prefixes),
StageRoot: expandPathTemplate(DefaultMusicStageRoot, paths),
CreditsRoot: expandPathTemplate(DefaultCreditsRoot, paths),
CreditsOverlay: DefaultCreditsOverlayFile,
MaxStemLength: 16,
},
TopData: top,
Extract: extract,
Autogen: EffectiveAutogenConfig{
Producers: slices.Clone(p.Config.Autogen.Producers),
Consumers: slices.Clone(p.Config.Autogen.Consumers),
Cache: EffectiveAutogenCacheConfig{
Root: expandPathTemplate(defaultString(p.Config.Autogen.Cache.Root, DefaultAutogenCacheRoot), paths),
MaxAge: defaultString(p.Config.Autogen.Cache.MaxAge, DefaultAutogenCacheMaxAge.String()),
RefreshEnv: defaultString(p.Config.Autogen.Cache.RefreshEnv, DefaultAutogenRefreshEnv),
PartsManifestRefreshEnv: DefaultPartsManifestRefreshEnv,
},
ReleaseSource: EffectiveAutogenReleaseSourceConfig{
Provider: defaultString(p.Config.Autogen.ReleaseSource.Provider, DefaultAutogenReleaseProvider),
ServerURLEnv: defaultString(p.Config.Autogen.ReleaseSource.ServerURLEnv, DefaultAutogenServerURLEnv),
RepoEnv: defaultString(p.Config.Autogen.ReleaseSource.RepoEnv, DefaultAutogenRepoEnv),
},
},
HAKs: slices.Clone(p.Config.HAKs),
Provenance: provenance,
}
effective.Overrides = activeOverrides(effective)
return effective
}
func (p *Project) EffectiveConfigJSON() ([]byte, error) {
return json.MarshalIndent(p.EffectiveConfig(), "", " ")
}
func (p *Project) ExplainConfig(key string) (any, ConfigValueProvenance, bool) {
effective := p.EffectiveConfig()
value, ok := lookupEffectiveValue(effective, key)
if !ok {
return nil, ConfigValueProvenance{}, false
}
prov := effective.Provenance[key]
if prov.Source == "" {
prov = ConfigValueProvenance{Source: "yaml", Detail: p.ConfigSource.Name}
}
return value, prov, true
}
func lookupEffectiveValue(effective EffectiveConfig, key string) (any, bool) {
raw, err := json.Marshal(effective)
if err != nil {
return nil, false
}
var data any
if err := json.Unmarshal(raw, &data); err != nil {
return nil, false
}
current := data
for _, part := range strings.Split(key, ".") {
object, ok := current.(map[string]any)
if !ok {
return nil, false
}
current, ok = object[part]
if !ok {
return nil, false
}
}
return current, true
}
func markMissingDefaults(provenance ConfigProvenance) {
defaults := map[string]string{
"paths.build": "build",
"paths.cache": ".cache",
"paths.tools": "tools",
"outputs.module_archive": DefaultModuleArchiveTemplate,
"outputs.hak_manifest": DefaultHAKManifest,
"outputs.hak_archive": DefaultHAKArchiveTemplate,
"inventory.source_extensions": "NWN source resource extensions",
"inventory.asset_extensions": "NWN asset resource extensions",
"inventory.source_json_pattern": DefaultSourceJSONPattern,
"scripts.source_dir": DefaultScriptsSourceDir,
"scripts.cache": DefaultScriptsCache,
"scripts.compiler.env.path": DefaultScriptCompilerEnvPath,
"scripts.compiler.search": "repo tools and PATH:nwn_script_comp",
"music.stage_root": DefaultMusicStageRoot,
"music.credits_root": DefaultCreditsRoot,
"music.credits_overlay": DefaultCreditsOverlayFile,
"music.max_stem_length": "16",
"topdata.build": DefaultTopDataBuild,
"topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir,
"topdata.compiled_tlk": DefaultTopDataCompiledTLK,
"topdata.package_hak": DefaultTopDataPackageHAK,
"topdata.package_tlk": DefaultTopDataPackageTLK,
"topdata.wiki.output_root": DefaultTopDataWikiOutputRoot,
"topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir,
"topdata.wiki.state_file": DefaultTopDataWikiStateFile,
"topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces",
"topdata.wiki.deploy_manifest": DefaultWikiDeployManifest,
"topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary,
"extract.layout": DefaultExtractLayout,
"extract.hak_discovery": DefaultExtractHAKDiscovery,
"extract.cleanup_stale": "true",
"autogen.cache.root": DefaultAutogenCacheRoot,
"autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(),
"autogen.cache.refresh_env": DefaultAutogenRefreshEnv,
"autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv,
"autogen.release_source.provider": DefaultAutogenReleaseProvider,
"autogen.release_source.server_url_env": DefaultAutogenServerURLEnv,
"autogen.release_source.repo_env": DefaultAutogenRepoEnv,
}
for key, detail := range defaults {
if _, ok := provenance[key]; !ok {
provenance[key] = ConfigValueProvenance{Source: "toolkit default", Detail: detail}
}
}
}
func cloneProvenance(input ConfigProvenance) ConfigProvenance {
out := ConfigProvenance{}
for key, value := range input {
out[key] = value
}
return out
}
func defaultString(value, fallback string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return fallback
}
return trimmed
}
func defaultStringSlice(value, fallback []string) []string {
if len(value) == 0 {
return slices.Clone(fallback)
}
return slices.Clone(value)
}
func cloneStringMap(input map[string]string) map[string]string {
if len(input) == 0 {
return nil
}
out := make(map[string]string, len(input))
for key, value := range input {
out[key] = value
}
return out
}
func expandPathTemplate(template string, paths EffectivePathConfig) string {
replacer := strings.NewReplacer(
"{paths.source}", paths.Source,
"{paths.assets}", paths.Assets,
"{paths.build}", paths.Build,
"{paths.cache}", paths.Cache,
"{paths.tools}", paths.Tools,
)
return filepath.ToSlash(replacer.Replace(template))
}
func expandPathTemplates(templates []string, paths EffectivePathConfig) []string {
out := make([]string, 0, len(templates))
for _, template := range templates {
out = append(out, expandPathTemplate(template, paths))
}
return out
}
func defaultScriptCompilerSearch() []string {
return []string{
"{paths.tools}/script-compiler/nwn_script_comp",
"{paths.tools}/script-compiler/nwn_script_comp.exe",
"{paths.tools}/nwn_script_comp",
"{paths.tools}/nwn_script_comp.exe",
"PATH:nwn_script_comp",
"PATH:nwn_script_comp.exe",
}
}
func (p *Project) ActiveOverrides() []ConfigOverride {
return activeOverrides(p.EffectiveConfig())
}
func activeOverrides(effective EffectiveConfig) []ConfigOverride {
envs := map[string]string{
"build.keep_existing_haks": effectiveBuildKeepExistingEnv,
"scripts.compiler.path": effective.Scripts.Compiler.Env.Path,
"autogen.cache.refresh": effective.Autogen.Cache.RefreshEnv,
"autogen.cache.parts_manifest_refresh": effective.Autogen.Cache.PartsManifestRefreshEnv,
"autogen.release_source.server_url": effective.Autogen.ReleaseSource.ServerURLEnv,
"autogen.release_source.repo": effective.Autogen.ReleaseSource.RepoEnv,
"music.ffmpeg": "SOW_FFMPEG",
"music.ffprobe": "SOW_FFPROBE",
"wiki.endpoint": "NODEBB_API_ENDPOINT",
"wiki.token": "NODEBB_API_TOKEN",
"wiki.categories": "NODEBB_WIKI_CATEGORIES",
"release.version": "GITHUB_REF_NAME",
}
for _, key := range effective.Scripts.Compiler.Env.NWNRoot {
envs["scripts.compiler.env.nwn_root."+key] = key
}
for _, key := range effective.Scripts.Compiler.Env.NWNUserDirectory {
envs["scripts.compiler.env.nwn_user_directory."+key] = key
}
keys := make([]string, 0, len(envs))
for key := range envs {
keys = append(keys, key)
}
slices.Sort(keys)
overrides := make([]ConfigOverride, 0)
for _, key := range keys {
env := envs[key]
if value := strings.TrimSpace(os.Getenv(env)); value != "" {
if isSensitiveOverrideKey(key) || isSensitiveOverrideKey(env) {
value = "<set>"
}
overrides = append(overrides, ConfigOverride{Key: key, Value: value, Source: "env:" + env})
}
}
return overrides
}
const effectiveBuildKeepExistingEnv = "SOW_BUILD_HAKS_KEEP_EXISTING"
func isSensitiveOverrideKey(key string) bool {
lower := strings.ToLower(key)
return strings.Contains(lower, "token") || strings.Contains(lower, "password")
}
func (o EffectiveOutputConfig) ModuleArchiveName(module ModuleConfig) string {
return strings.NewReplacer("{module.resref}", strings.TrimSpace(module.ResRef)).Replace(o.ModuleArchive)
}
func (o EffectiveOutputConfig) HAKArchiveName(name string) string {
return strings.NewReplacer("{hak.name}", strings.TrimSpace(name)).Replace(o.HAKArchive)
}
func (p ConfigValueProvenance) String() string {
if p.Detail == "" {
return p.Source
}
return fmt.Sprintf("%s (%s)", p.Source, p.Detail)
}