ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Configuration Hardening 2

Browse Source 
What changed:

  - Added visible runtime override reporting with masking for sensitive values.
  - Expanded effective config for build cleanup, script compiler discovery/env, extract layout/cleanup/
    HAK discovery, wiki deploy defaults, autogen cache policy, and release-source env names.
  - Wired configured values into build, script compiler resolution, extract, autogen cache refresh/max
    age, parts manifest release source, wiki generation/deploy, and TLK output naming.
  - Added config sources override output and validation-rule hints in config explain.
  - Added CONFIGURATION_HARDENING_AUDIT.md.
  - Updated README.md with config commands, schema defaults, and JSON artifact boundaries.
  - Added regression tests for override visibility/masking, compiler/extract/wiki/autogen config,
    configured HAK discovery, and config sources output.
This commit is contained in:
archvillainette
2026-05-07 14:22:47 +02:00
parent 60d2de9f2d
commit b39bdf13e8
17 changed files with 877 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/app/app.go
+35 -2
View File
@@ -149,7 +149,7 @@ var commands = []command{
},
{
name: "extract",
description: "Extract a module archive back into the src/ tree.",
description: "Extract built archives back into the configured source and asset trees.",
run: runExtract,
},
{
@@ -169,7 +169,7 @@ var commands = []command{
},
{
name: "apply-hak-manifest",
description: "Apply a generated HAK manifest to src/module/module.ifo.json.",
description: "Apply a generated HAK manifest to the configured module source.",
run: runApplyHAKManifest,
},
{
@@ -749,6 +749,9 @@ func runConfig(ctx context) error {
if provenance.Deprecated {
fmt.Fprintf(ctx.stdout, "deprecated: true\n")
}
if rules := configValidationRules(key); len(rules) > 0 {
fmt.Fprintf(ctx.stdout, "validation: %s\n", strings.Join(rules, "; "))
}
return nil
case "sources":
if len(ctx.args) > 2 {
@@ -759,12 +762,42 @@ func runConfig(ctx context) error {
fmt.Fprintf(ctx.stdout, "format: %s\n", p.ConfigSource.Format)
fmt.Fprintf(ctx.stdout, "legacy: %t\n", p.ConfigSource.Legacy)
fmt.Fprintf(ctx.stdout, "candidates: %s\n", strings.Join(project.ConfigFileCandidates, ", "))
overrides := p.ActiveOverrides()
if len(overrides) == 0 {
fmt.Fprintln(ctx.stdout, "active overrides: none")
return nil
}
fmt.Fprintln(ctx.stdout, "active overrides:")
for _, override := range overrides {
fmt.Fprintf(ctx.stdout, " %s=%s (%s)\n", override.Key, override.Value, override.Source)
}
return nil
default:
return fmt.Errorf("unknown config subcommand %q", ctx.args[1])
}
}
func configValidationRules(key string) []string {
switch {
case strings.HasPrefix(key, "paths."), strings.Contains(key, ".root"), strings.Contains(key, ".dir"), strings.Contains(key, ".cache"):
return []string{"relative paths must not escape the repository root unless explicitly documented"}
case strings.HasPrefix(key, "outputs."):
return []string{"generated output names must use the configured extension and must not escape the repository root"}
case key == "extract.layout":
return []string{"supported value: nwn_canonical_json"}
case key == "extract.hak_discovery":
return []string{"supported values: build_glob, configured_haks"}
case key == "autogen.cache.max_age":
return []string{"Go duration string, for example 1h or 30m"}
case strings.HasPrefix(key, "topdata.package_hak"):
return []string{"must be a .hak file name"}
case strings.HasPrefix(key, "topdata.package_tlk"), strings.HasPrefix(key, "topdata.compiled_tlk"):
return []string{"must be a .tlk file name"}
default:
return nil
}
}
func parseConfigFormatArgs(commandName string, args []string) (string, error) {
format := "yaml"
for _, arg := range args {
internal/app/app_test.go
+31
View File
@@ -286,6 +286,37 @@ paths:
}
}
func TestRunConfigSourcesListsActiveOverrides(t *testing.T) {
root := t.TempDir()
writeFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Module
resref: testmod
paths:
source: src
`)
t.Setenv("SOW_BUILD_HAKS_KEEP_EXISTING", "1")
var stdout bytes.Buffer
ctx := context{
stdout: &stdout,
stderr: &bytes.Buffer{},
cwd: root,
args: []string{"config", "sources"},
}
if err := runConfig(ctx); err != nil {
t.Fatalf("runConfig failed: %v", err)
}
output := stdout.String()
if !strings.Contains(output, "active overrides:") {
t.Fatalf("expected active overrides section, got %q", output)
}
if !strings.Contains(output, "build.keep_existing_haks=1") {
t.Fatalf("expected keep existing override, got %q", output)
}
}
func mkdirAll(t *testing.T, path string) {
t.Helper()
if err := os.MkdirAll(path, 0o755); err != nil {
internal/pipeline/build.go
+25 -25
View File
@@ -215,7 +215,7 @@ func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) {
}
func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string) (result BuildResult, err error) {
preserveExistingHAKs := envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
preserveExistingHAKs := p.EffectiveConfig().Build.KeepExistingHAKs || envBool("SOW_BUILD_HAKS_KEEP_EXISTING")
progressf(progress, "Validating project...")
if err := validateForBuild(p); err != nil {
@@ -556,7 +556,7 @@ func compileReferencedScripts(p *project.Project) ([]erf.Resource, error) {
if err != nil {
return nil, err
}
compilerEnv, err := resolveScriptCompilerEnvironment()
compilerEnv, err := resolveScriptCompilerEnvironment(p)
if err != nil {
return nil, err
}
@@ -630,40 +630,40 @@ func referencedScriptResrefs(p *project.Project) ([]string, error) {
}
func resolveScriptCompiler(p *project.Project) (string, error) {
if configured := strings.TrimSpace(os.Getenv("SOW_NWN_SCRIPT_COMPILER")); configured != "" {
if configured := p.ScriptCompilerPath(); configured != "" {
return configured, nil
}
if configured := strings.TrimSpace(os.Getenv(p.ScriptCompilerPathEnv())); configured != "" {
return configured, nil
}
candidates := []string{
filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp"),
filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp.exe"),
filepath.Join(p.Root, "tools", "nwn_script_comp"),
filepath.Join(p.Root, "tools", "nwn_script_comp.exe"),
}
candidates := p.ScriptCompilerSearchPaths()
if runtime.GOOS == "windows" {
candidates = []string{
filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp.exe"),
filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp"),
filepath.Join(p.Root, "tools", "nwn_script_comp.exe"),
filepath.Join(p.Root, "tools", "nwn_script_comp"),
}
slices.SortStableFunc(candidates, func(a, b string) int {
aExe := strings.HasSuffix(strings.ToLower(a), ".exe")
bExe := strings.HasSuffix(strings.ToLower(b), ".exe")
if aExe == bExe {
return strings.Compare(a, b)
}
if aExe {
return -1
}
return 1
})
}
for _, candidate := range candidates {
if binary, ok := strings.CutPrefix(candidate, "PATH:"); ok {
if compiler, err := exec.LookPath(binary); err == nil {
return compiler, nil
}
continue
}
if info, err := os.Stat(candidate); err == nil && !info.IsDir() {
return candidate, nil
}
}
if compiler, err := exec.LookPath("nwn_script_comp"); err == nil {
return compiler, nil
}
if runtime.GOOS == "windows" {
if compiler, err := exec.LookPath("nwn_script_comp.exe"); err == nil {
return compiler, nil
}
}
return "", errors.New("script compiler not found; install nwn_script_comp into ./tools or set SOW_NWN_SCRIPT_COMPILER")
return "", fmt.Errorf("script compiler not found; configure scripts.compiler.path or set %s", p.ScriptCompilerPathEnv())
}
func validatorLoadDocument(path string) (gff.Document, string, string, error) {
internal/pipeline/extract.go
+31 -5
View File
@@ -57,7 +57,7 @@ func Extract(p *project.Project, files ...string) (ExtractResult, error) {
failures = append(failures, errs...)
}
hakPaths, err := filepath.Glob(filepath.Join(p.BuildDir(), "*.hak"))
hakPaths, err := extractHAKPaths(p)
if err != nil {
return result, fmt.Errorf("scan hak archives: %w", err)
}
@@ -88,9 +88,11 @@ func Extract(p *project.Project, files ...string) (ExtractResult, error) {
failures = append(failures, errs...)
}
removed, errs := cleanupStaleFiles(p, desired)
result.Removed = removed
failures = append(failures, errs...)
if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale {
removed, errs := cleanupStaleFiles(p, desired)
result.Removed = removed
failures = append(failures, errs...)
}
if len(failures) > 0 {
return result, errors.Join(failures...)
@@ -157,7 +159,7 @@ func extractedFile(p *project.Project, resource erf.Resource, extension string)
switch extension {
case "nss":
return filepath.Join(p.SourceDir(), "scripts", resref+".nss"), resource.Data, nil
return filepath.Join(p.ScriptSourceDir(), resref+".nss"), resource.Data, nil
case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw",
"are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl":
document, err := gff.Read(bytes.NewReader(resource.Data))
@@ -175,6 +177,30 @@ func extractedFile(p *project.Project, resource erf.Resource, extension string)
}
}
func extractHAKPaths(p *project.Project) ([]string, error) {
switch p.EffectiveConfig().Extract.HAKDiscovery {
case "configured_haks":
paths := make([]string, 0, len(p.Config.HAKs))
for _, hak := range p.Config.HAKs {
path := p.HAKArchivePath(hak.Name)
if _, err := os.Stat(path); err == nil {
paths = append(paths, path)
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
}
slices.Sort(paths)
return paths, nil
default:
hakPaths, err := filepath.Glob(filepath.Join(p.BuildDir(), "*.hak"))
if err != nil {
return nil, err
}
slices.Sort(hakPaths)
return hakPaths, nil
}
}
type writeState int
const (
internal/pipeline/pipeline_test.go
+71
View File
@@ -175,6 +175,77 @@ func TestExtractReadsHAKAssets(t *testing.T) {
}
}
func TestExtractConfiguredHAKDiscoveryIgnoresUnconfiguredArchives(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src"))
mustMkdir(t, filepath.Join(root, "assets"))
mustMkdir(t, filepath.Join(root, "build"))
mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), `
module:
name: Test Module
resref: testmod
paths:
source: src
assets: assets
build: build
extract:
hak_discovery: configured_haks
haks:
- name: wanted
priority: 1
include:
- wanted/**
`)
p, err := project.Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := p.ValidateLayout(); err != nil {
t.Fatalf("validate layout: %v", err)
}
modFile, err := os.Create(p.ModuleArchivePath())
if err != nil {
t.Fatalf("create mod: %v", err)
}
if err := erf.Write(modFile, erf.New("MOD ", nil)); err != nil {
t.Fatalf("write mod: %v", err)
}
if err := modFile.Close(); err != nil {
t.Fatalf("close mod: %v", err)
}
for _, name := range []string{"wanted", "ignored"} {
hakFile, err := os.Create(filepath.Join(root, "build", name+".hak"))
if err != nil {
t.Fatalf("create hak: %v", err)
}
if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{
{Name: name + "_asset", Type: 0x07D2, Data: []byte("mdl-data")},
})); err != nil {
t.Fatalf("write hak: %v", err)
}
if err := hakFile.Close(); err != nil {
t.Fatalf("close hak: %v", err)
}
}
result, err := Extract(p)
if err != nil {
t.Fatalf("extract: %v", err)
}
if len(result.HAKPaths) != 1 || filepath.Base(result.HAKPaths[0]) != "wanted.hak" {
t.Fatalf("expected only configured hak, got %#v", result.HAKPaths)
}
if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "wanted_asset.mdl")); err != nil {
t.Fatalf("expected wanted asset: %v", err)
}
if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "ignored_asset.mdl")); !os.IsNotExist(err) {
t.Fatalf("expected ignored asset to remain absent, err=%v", err)
}
}
func TestExtractDeletesConsumedModuleArchiveAfterSuccessWhenConfigured(t *testing.T) {
root := t.TempDir()
mustMkdir(t, filepath.Join(root, "src", "module"))
internal/pipeline/script_compiler_env.go
+9 -7
View File
@@ -9,19 +9,21 @@ import (
"regexp"
"runtime"
"strings"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
)
func resolveScriptCompilerEnvironment() ([]string, error) {
func resolveScriptCompilerEnvironment(p *project.Project) ([]string, error) {
env := os.Environ()
userDir, err := resolveNWNUserDirectory()
userDir, err := resolveNWNUserDirectory(p.ScriptCompilerNWNUserDirectoryEnvKeys())
if err != nil {
return nil, err
}
env = upsertEnv(env, "NWN_HOME", userDir)
env = upsertEnv(env, "NWN_USER_DIRECTORY", userDir)
root, err := resolveNWNRoot()
root, err := resolveNWNRoot(p.ScriptCompilerNWNRootEnvKeys())
if err != nil {
return nil, err
}
@@ -32,8 +34,8 @@ func resolveScriptCompilerEnvironment() ([]string, error) {
return env, nil
}
func resolveNWNUserDirectory() (string, error) {
for _, key := range []string{"SOW_NWN_USER_DIRECTORY", "NWN_HOME", "NWN_USER_DIRECTORY"} {
func resolveNWNUserDirectory(envKeys []string) (string, error) {
for _, key := range envKeys {
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
path := filepath.Clean(expandUserPath(value))
if err := os.MkdirAll(path, 0o755); err != nil {
@@ -71,8 +73,8 @@ func defaultNWNUserDirectory() (string, error) {
}
}
func resolveNWNRoot() (string, error) {
for _, key := range []string{"SOW_NWN_ROOT", "NWN_ROOT"} {
func resolveNWNRoot(envKeys []string) (string, error) {
for _, key := range envKeys {
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
path := filepath.Clean(expandUserPath(value))
if !looksLikeNWNInstall(path) {
internal/project/effective.go
+213 -65
View File
@@ -3,6 +3,7 @@ package project
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
@@ -10,29 +11,38 @@ import (
)
const (
DefaultBuildPath = "build"
DefaultCachePath = ".cache"
DefaultToolsPath = "tools"
DefaultModuleArchiveTemplate = "{module.resref}.mod"
DefaultHAKManifest = "haks.json"
DefaultHAKArchiveTemplate = "{hak.name}.hak"
DefaultSourceJSONPattern = "{resref}.{extension}.json"
DefaultScriptsSourceDir = "scripts"
DefaultScriptsCache = "{paths.cache}/ncs"
DefaultMusicStageRoot = "{paths.cache}/music"
DefaultCreditsRoot = "{paths.cache}/credits"
DefaultCreditsOverlayFile = "CREDITS.md"
DefaultTopDataBuild = "{paths.build}/topdata"
DefaultTopDataCompiled2DADir = "2da"
DefaultTopDataCompiledTLK = "sow_tlk.tlk"
DefaultTopDataWikiOutputRoot = "wiki"
DefaultTopDataWikiPagesDir = "pages"
DefaultTopDataWikiStateFile = "state.json"
DefaultTopDataPackageHAK = "sow_top.hak"
DefaultTopDataPackageTLK = "sow_tlk.tlk"
DefaultAutogenCacheRoot = "{paths.cache}"
DefaultAutogenCacheMaxAge = time.Hour
DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH"
DefaultBuildPath = "build"
DefaultCachePath = ".cache"
DefaultToolsPath = "tools"
DefaultModuleArchiveTemplate = "{module.resref}.mod"
DefaultHAKManifest = "haks.json"
DefaultHAKArchiveTemplate = "{hak.name}.hak"
DefaultSourceJSONPattern = "{resref}.{extension}.json"
DefaultScriptsSourceDir = "scripts"
DefaultScriptsCache = "{paths.cache}/ncs"
DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER"
DefaultMusicStageRoot = "{paths.cache}/music"
DefaultCreditsRoot = "{paths.cache}/credits"
DefaultCreditsOverlayFile = "CREDITS.md"
DefaultTopDataBuild = "{paths.build}/topdata"
DefaultTopDataCompiled2DADir = "2da"
DefaultTopDataCompiledTLK = "sow_tlk.tlk"
DefaultTopDataWikiOutputRoot = "wiki"
DefaultTopDataWikiPagesDir = "pages"
DefaultTopDataWikiStateFile = "state.json"
DefaultWikiDeployManifest = ".wiki_deploy_manifest.json"
DefaultWikiDeployEditSummary = "Auto-generated from native builder"
DefaultTopDataPackageHAK = "sow_top.hak"
DefaultTopDataPackageTLK = "sow_tlk.tlk"
DefaultAutogenCacheRoot = "{paths.cache}"
DefaultAutogenCacheMaxAge = time.Hour
DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH"
DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH"
DefaultAutogenReleaseProvider = "gitea"
DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL"
DefaultAutogenRepoEnv = "SOW_ASSETS_REPO"
DefaultExtractLayout = "nwn_canonical_json"
DefaultExtractHAKDiscovery = "build_glob"
)
type ConfigProvenance map[string]ConfigValueProvenance
@@ -48,6 +58,7 @@ type EffectiveConfig struct {
Module ModuleConfig `json:"module" yaml:"module"`
Paths EffectivePathConfig `json:"paths" yaml:"paths"`
Outputs EffectiveOutputConfig `json:"outputs" yaml:"outputs"`
Build BuildConfig `json:"build" yaml:"build"`
Inventory InventoryConfig `json:"inventory" yaml:"inventory"`
Scripts EffectiveScriptsConfig `json:"scripts" yaml:"scripts"`
Music EffectiveMusicConfig `json:"music" yaml:"music"`
@@ -56,6 +67,7 @@ type EffectiveConfig struct {
Autogen EffectiveAutogenConfig `json:"autogen" yaml:"autogen"`
HAKs []HAKConfig `json:"haks" yaml:"haks"`
Provenance ConfigProvenance `json:"provenance" yaml:"provenance"`
Overrides []ConfigOverride `json:"overrides,omitempty" yaml:"overrides,omitempty"`
}
type EffectivePathConfig struct {
@@ -73,8 +85,21 @@ type EffectiveOutputConfig struct {
}
type EffectiveScriptsConfig struct {
SourceDir string `json:"source_dir" yaml:"source_dir"`
Cache string `json:"cache" yaml:"cache"`
SourceDir string `json:"source_dir" yaml:"source_dir"`
Cache string `json:"cache" yaml:"cache"`
Compiler EffectiveScriptCompilerConfig `json:"compiler" yaml:"compiler"`
}
type EffectiveScriptCompilerConfig struct {
Path string `json:"path" yaml:"path"`
Search []string `json:"search" yaml:"search"`
Env EffectiveScriptCompilerEnvConfig `json:"env" yaml:"env"`
}
type EffectiveScriptCompilerEnvConfig struct {
Path string `json:"path" yaml:"path"`
NWNRoot []string `json:"nwn_root" yaml:"nwn_root"`
NWNUserDirectory []string `json:"nwn_user_directory" yaml:"nwn_user_directory"`
}
type EffectiveMusicConfig struct {
@@ -98,15 +123,29 @@ type EffectiveTopDataConfig struct {
}
type EffectiveAutogenConfig struct {
Producers []AutogenProducerConfig `json:"producers" yaml:"producers"`
Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"`
Cache EffectiveAutogenCacheConfig `json:"cache" yaml:"cache"`
Producers []AutogenProducerConfig `json:"producers" yaml:"producers"`
Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"`
Cache EffectiveAutogenCacheConfig `json:"cache" yaml:"cache"`
ReleaseSource EffectiveAutogenReleaseSourceConfig `json:"release_source" yaml:"release_source"`
}
type EffectiveAutogenCacheConfig struct {
Root string `json:"root" yaml:"root"`
MaxAge string `json:"max_age" yaml:"max_age"`
RefreshEnv string `json:"refresh_env" yaml:"refresh_env"`
Root string `json:"root" yaml:"root"`
MaxAge string `json:"max_age" yaml:"max_age"`
RefreshEnv string `json:"refresh_env" yaml:"refresh_env"`
PartsManifestRefreshEnv string `json:"parts_manifest_refresh_env" yaml:"parts_manifest_refresh_env"`
}
type EffectiveAutogenReleaseSourceConfig struct {
Provider string `json:"provider" yaml:"provider"`
ServerURLEnv string `json:"server_url_env" yaml:"server_url_env"`
RepoEnv string `json:"repo_env" yaml:"repo_env"`
}
type ConfigOverride struct {
Key string `json:"key" yaml:"key"`
Value string `json:"value" yaml:"value"`
Source string `json:"source" yaml:"source"`
}
func (p *Project) EffectiveConfig() EffectiveConfig {
@@ -133,6 +172,15 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
scripts := EffectiveScriptsConfig{
SourceDir: defaultString(p.Config.Scripts.SourceDir, DefaultScriptsSourceDir),
Cache: expandPathTemplate(defaultString(p.Config.Scripts.Cache, DefaultScriptsCache), paths),
Compiler: EffectiveScriptCompilerConfig{
Path: strings.TrimSpace(p.Config.Scripts.Compiler.Path),
Search: expandPathTemplates(defaultStringSlice(p.Config.Scripts.Compiler.Search, defaultScriptCompilerSearch()), paths),
Env: EffectiveScriptCompilerEnvConfig{
Path: defaultString(p.Config.Scripts.Compiler.Env.Path, DefaultScriptCompilerEnvPath),
NWNRoot: defaultStringSlice(p.Config.Scripts.Compiler.Env.NWNRoot, []string{"SOW_NWN_ROOT", "NWN_ROOT"}),
NWNUserDirectory: defaultStringSlice(p.Config.Scripts.Compiler.Env.NWNUserDirectory, []string{"SOW_NWN_USER_DIRECTORY", "NWN_HOME", "NWN_USER_DIRECTORY"}),
},
},
}
topBuild := expandPathTemplate(defaultString(p.Config.TopData.Build, DefaultTopDataBuild), paths)
top := EffectiveTopDataConfig{
@@ -145,17 +193,28 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
PackageHAK: defaultString(p.Config.TopData.PackageHAK, DefaultTopDataPackageHAK),
PackageTLK: defaultString(p.Config.TopData.PackageTLK, DefaultTopDataPackageTLK),
Wiki: TopDataWikiConfig{
OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot),
PagesDir: defaultString(p.Config.TopData.Wiki.PagesDir, DefaultTopDataWikiPagesDir),
StateFile: defaultString(p.Config.TopData.Wiki.StateFile, DefaultTopDataWikiStateFile),
OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot),
PagesDir: defaultString(p.Config.TopData.Wiki.PagesDir, DefaultTopDataWikiPagesDir),
StateFile: defaultString(p.Config.TopData.Wiki.StateFile, DefaultTopDataWikiStateFile),
ManagedNamespaces: defaultStringSlice(p.Config.TopData.Wiki.ManagedNamespaces, []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}),
DeployManifest: defaultString(p.Config.TopData.Wiki.DeployManifest, DefaultWikiDeployManifest),
DeployEditSummary: defaultString(p.Config.TopData.Wiki.DeployEditSummary, DefaultWikiDeployEditSummary),
},
}
extract := p.Config.Extract
extract.Layout = defaultString(extract.Layout, DefaultExtractLayout)
extract.HAKDiscovery = defaultString(extract.HAKDiscovery, DefaultExtractHAKDiscovery)
if extract.CleanupStale == nil {
defaultCleanup := true
extract.CleanupStale = &defaultCleanup
}
return EffectiveConfig{
effective := EffectiveConfig{
ConfigSource: p.ConfigSource,
Module: p.Config.Module,
Paths: paths,
Outputs: outputs,
Build: p.Config.Build,
Inventory: inventory,
Scripts: scripts,
Music: EffectiveMusicConfig{
@@ -166,19 +225,27 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
MaxStemLength: 16,
},
TopData: top,
Extract: p.Config.Extract,
Extract: extract,
Autogen: EffectiveAutogenConfig{
Producers: slices.Clone(p.Config.Autogen.Producers),
Consumers: slices.Clone(p.Config.Autogen.Consumers),
Cache: EffectiveAutogenCacheConfig{
Root: expandPathTemplate(defaultString(p.Config.Autogen.Cache.Root, DefaultAutogenCacheRoot), paths),
MaxAge: DefaultAutogenCacheMaxAge.String(),
RefreshEnv: DefaultAutogenRefreshEnv,
Root: expandPathTemplate(defaultString(p.Config.Autogen.Cache.Root, DefaultAutogenCacheRoot), paths),
MaxAge: defaultString(p.Config.Autogen.Cache.MaxAge, DefaultAutogenCacheMaxAge.String()),
RefreshEnv: defaultString(p.Config.Autogen.Cache.RefreshEnv, DefaultAutogenRefreshEnv),
PartsManifestRefreshEnv: DefaultPartsManifestRefreshEnv,
},
ReleaseSource: EffectiveAutogenReleaseSourceConfig{
Provider: defaultString(p.Config.Autogen.ReleaseSource.Provider, DefaultAutogenReleaseProvider),
ServerURLEnv: defaultString(p.Config.Autogen.ReleaseSource.ServerURLEnv, DefaultAutogenServerURLEnv),
RepoEnv: defaultString(p.Config.Autogen.ReleaseSource.RepoEnv, DefaultAutogenRepoEnv),
},
},
HAKs: slices.Clone(p.Config.HAKs),
Provenance: provenance,
}
effective.Overrides = activeOverrides(effective)
return effective
}
func (p *Project) EffectiveConfigJSON() ([]byte, error) {
@@ -223,32 +290,44 @@ func lookupEffectiveValue(effective EffectiveConfig, key string) (any, bool) {
func markMissingDefaults(provenance ConfigProvenance) {
defaults := map[string]string{
"paths.build": "build",
"paths.cache": ".cache",
"paths.tools": "tools",
"outputs.module_archive": DefaultModuleArchiveTemplate,
"outputs.hak_manifest": DefaultHAKManifest,
"outputs.hak_archive": DefaultHAKArchiveTemplate,
"inventory.source_extensions": "NWN source resource extensions",
"inventory.asset_extensions": "NWN asset resource extensions",
"inventory.source_json_pattern": DefaultSourceJSONPattern,
"scripts.source_dir": DefaultScriptsSourceDir,
"scripts.cache": DefaultScriptsCache,
"music.stage_root": DefaultMusicStageRoot,
"music.credits_root": DefaultCreditsRoot,
"music.credits_overlay": DefaultCreditsOverlayFile,
"music.max_stem_length": "16",
"topdata.build": DefaultTopDataBuild,
"topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir,
"topdata.compiled_tlk": DefaultTopDataCompiledTLK,
"topdata.package_hak": DefaultTopDataPackageHAK,
"topdata.package_tlk": DefaultTopDataPackageTLK,
"topdata.wiki.output_root": DefaultTopDataWikiOutputRoot,
"topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir,
"topdata.wiki.state_file": DefaultTopDataWikiStateFile,
"autogen.cache.root": DefaultAutogenCacheRoot,
"autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(),
"autogen.cache.refresh_env": DefaultAutogenRefreshEnv,
"paths.build": "build",
"paths.cache": ".cache",
"paths.tools": "tools",
"outputs.module_archive": DefaultModuleArchiveTemplate,
"outputs.hak_manifest": DefaultHAKManifest,
"outputs.hak_archive": DefaultHAKArchiveTemplate,
"inventory.source_extensions": "NWN source resource extensions",
"inventory.asset_extensions": "NWN asset resource extensions",
"inventory.source_json_pattern": DefaultSourceJSONPattern,
"scripts.source_dir": DefaultScriptsSourceDir,
"scripts.cache": DefaultScriptsCache,
"scripts.compiler.env.path": DefaultScriptCompilerEnvPath,
"scripts.compiler.search": "repo tools and PATH:nwn_script_comp",
"music.stage_root": DefaultMusicStageRoot,
"music.credits_root": DefaultCreditsRoot,
"music.credits_overlay": DefaultCreditsOverlayFile,
"music.max_stem_length": "16",
"topdata.build": DefaultTopDataBuild,
"topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir,
"topdata.compiled_tlk": DefaultTopDataCompiledTLK,
"topdata.package_hak": DefaultTopDataPackageHAK,
"topdata.package_tlk": DefaultTopDataPackageTLK,
"topdata.wiki.output_root": DefaultTopDataWikiOutputRoot,
"topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir,
"topdata.wiki.state_file": DefaultTopDataWikiStateFile,
"topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces",
"topdata.wiki.deploy_manifest": DefaultWikiDeployManifest,
"topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary,
"extract.layout": DefaultExtractLayout,
"extract.hak_discovery": DefaultExtractHAKDiscovery,
"extract.cleanup_stale": "true",
"autogen.cache.root": DefaultAutogenCacheRoot,
"autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(),
"autogen.cache.refresh_env": DefaultAutogenRefreshEnv,
"autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv,
"autogen.release_source.provider": DefaultAutogenReleaseProvider,
"autogen.release_source.server_url_env": DefaultAutogenServerURLEnv,
"autogen.release_source.repo_env": DefaultAutogenRepoEnv,
}
for key, detail := range defaults {
if _, ok := provenance[key]; !ok {
@@ -302,6 +381,75 @@ func expandPathTemplate(template string, paths EffectivePathConfig) string {
return filepath.ToSlash(replacer.Replace(template))
}
func expandPathTemplates(templates []string, paths EffectivePathConfig) []string {
out := make([]string, 0, len(templates))
for _, template := range templates {
out = append(out, expandPathTemplate(template, paths))
}
return out
}
func defaultScriptCompilerSearch() []string {
return []string{
"{paths.tools}/script-compiler/nwn_script_comp",
"{paths.tools}/script-compiler/nwn_script_comp.exe",
"{paths.tools}/nwn_script_comp",
"{paths.tools}/nwn_script_comp.exe",
"PATH:nwn_script_comp",
"PATH:nwn_script_comp.exe",
}
}
func (p *Project) ActiveOverrides() []ConfigOverride {
return activeOverrides(p.EffectiveConfig())
}
func activeOverrides(effective EffectiveConfig) []ConfigOverride {
envs := map[string]string{
"build.keep_existing_haks": effectiveBuildKeepExistingEnv,
"scripts.compiler.path": effective.Scripts.Compiler.Env.Path,
"autogen.cache.refresh": effective.Autogen.Cache.RefreshEnv,
"autogen.cache.parts_manifest_refresh": effective.Autogen.Cache.PartsManifestRefreshEnv,
"autogen.release_source.server_url": effective.Autogen.ReleaseSource.ServerURLEnv,
"autogen.release_source.repo": effective.Autogen.ReleaseSource.RepoEnv,
"music.ffmpeg": "SOW_FFMPEG",
"music.ffprobe": "SOW_FFPROBE",
"wiki.endpoint": "NODEBB_API_ENDPOINT",
"wiki.token": "NODEBB_API_TOKEN",
"wiki.categories": "NODEBB_WIKI_CATEGORIES",
"release.version": "GITHUB_REF_NAME",
}
for _, key := range effective.Scripts.Compiler.Env.NWNRoot {
envs["scripts.compiler.env.nwn_root."+key] = key
}
for _, key := range effective.Scripts.Compiler.Env.NWNUserDirectory {
envs["scripts.compiler.env.nwn_user_directory."+key] = key
}
keys := make([]string, 0, len(envs))
for key := range envs {
keys = append(keys, key)
}
slices.Sort(keys)
overrides := make([]ConfigOverride, 0)
for _, key := range keys {
env := envs[key]
if value := strings.TrimSpace(os.Getenv(env)); value != "" {
if isSensitiveOverrideKey(key) || isSensitiveOverrideKey(env) {
value = "<set>"
}
overrides = append(overrides, ConfigOverride{Key: key, Value: value, Source: "env:" + env})
}
}
return overrides
}
const effectiveBuildKeepExistingEnv = "SOW_BUILD_HAKS_KEEP_EXISTING"
func isSensitiveOverrideKey(key string) bool {
lower := strings.ToLower(key)
return strings.Contains(lower, "token") || strings.Contains(lower, "password")
}
func (o EffectiveOutputConfig) ModuleArchiveName(module ModuleConfig) string {
return strings.NewReplacer("{module.resref}", strings.TrimSpace(module.ResRef)).Replace(o.ModuleArchive)
}
internal/project/project.go
+159 -29
View File
@@ -11,6 +11,7 @@ import (
"path/filepath"
"slices"
"strings"
"time"
"gopkg.in/yaml.v3"
)
@@ -51,6 +52,7 @@ type Config struct {
Module ModuleConfig `json:"module" yaml:"module"`
Paths PathConfig `json:"paths" yaml:"paths"`
Outputs OutputConfig `json:"outputs" yaml:"outputs"`
Build BuildConfig `json:"build" yaml:"build"`
HAKs []HAKConfig `json:"haks" yaml:"haks"`
Inventory InventoryConfig `json:"inventory" yaml:"inventory"`
Scripts ScriptsConfig `json:"scripts" yaml:"scripts"`
@@ -81,6 +83,10 @@ type OutputConfig struct {
HAKArchive string `json:"hak_archive" yaml:"hak_archive"`
}
type BuildConfig struct {
KeepExistingHAKs bool `json:"keep_existing_haks,omitempty" yaml:"keep_existing_haks,omitempty"`
}
type InventoryConfig struct {
SourceExtensions []string `json:"source_extensions" yaml:"source_extensions"`
AssetExtensions []string `json:"asset_extensions" yaml:"asset_extensions"`
@@ -88,8 +94,21 @@ type InventoryConfig struct {
}
type ScriptsConfig struct {
SourceDir string `json:"source_dir" yaml:"source_dir"`
Cache string `json:"cache" yaml:"cache"`
SourceDir string `json:"source_dir" yaml:"source_dir"`
Cache string `json:"cache" yaml:"cache"`
Compiler ScriptCompilerConfig `json:"compiler" yaml:"compiler"`
}
type ScriptCompilerConfig struct {
Path string `json:"path" yaml:"path"`
Search []string `json:"search" yaml:"search"`
Env ScriptCompilerEnvConfig `json:"env" yaml:"env"`
}
type ScriptCompilerEnvConfig struct {
Path string `json:"path" yaml:"path"`
NWNRoot []string `json:"nwn_root" yaml:"nwn_root"`
NWNUserDirectory []string `json:"nwn_user_directory" yaml:"nwn_user_directory"`
}
type HAKConfig struct {
@@ -118,24 +137,39 @@ type TopDataConfig struct {
}
type TopDataWikiConfig struct {
OutputRoot string `json:"output_root" yaml:"output_root"`
PagesDir string `json:"pages_dir" yaml:"pages_dir"`
StateFile string `json:"state_file" yaml:"state_file"`
OutputRoot string `json:"output_root" yaml:"output_root"`
PagesDir string `json:"pages_dir" yaml:"pages_dir"`
StateFile string `json:"state_file" yaml:"state_file"`
ManagedNamespaces []string `json:"managed_namespaces" yaml:"managed_namespaces"`
DeployManifest string `json:"deploy_manifest" yaml:"deploy_manifest"`
DeployEditSummary string `json:"deploy_edit_summary" yaml:"deploy_edit_summary"`
}
type ExtractConfig struct {
IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"`
Layout string `json:"layout" yaml:"layout"`
HAKDiscovery string `json:"hak_discovery" yaml:"hak_discovery"`
CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"`
DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"`
}
type AutogenConfig struct {
Producers []AutogenProducerConfig `json:"producers" yaml:"producers"`
Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"`
Cache AutogenCacheConfig `json:"cache" yaml:"cache"`
Producers []AutogenProducerConfig `json:"producers" yaml:"producers"`
Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"`
Cache AutogenCacheConfig `json:"cache" yaml:"cache"`
ReleaseSource AutogenReleaseSourceConfig `json:"release_source" yaml:"release_source"`
}
type AutogenCacheConfig struct {
Root string `json:"root" yaml:"root"`
Root string `json:"root" yaml:"root"`
MaxAge string `json:"max_age" yaml:"max_age"`
RefreshEnv string `json:"refresh_env" yaml:"refresh_env"`
}
type AutogenReleaseSourceConfig struct {
Provider string `json:"provider" yaml:"provider"`
ServerURLEnv string `json:"server_url_env" yaml:"server_url_env"`
RepoEnv string `json:"repo_env" yaml:"repo_env"`
}
type AutogenProducerConfig struct {
@@ -329,26 +363,37 @@ func (p *Project) ValidateLayout() error {
failures = append(failures, errors.New("at least one of paths.source, paths.assets, or topdata.source is required"))
}
for field, value := range map[string]string{
"paths.source": p.Config.Paths.Source,
"paths.assets": p.Config.Paths.Assets,
"paths.build": p.Config.Paths.Build,
"paths.cache": p.Config.Paths.Cache,
"paths.tools": p.Config.Paths.Tools,
"outputs.module_archive": p.Config.Outputs.ModuleArchive,
"outputs.hak_manifest": p.Config.Outputs.HAKManifest,
"outputs.hak_archive": p.Config.Outputs.HAKArchive,
"scripts.source_dir": p.Config.Scripts.SourceDir,
"scripts.cache": p.Config.Scripts.Cache,
"topdata.source": p.Config.TopData.Source,
"topdata.build": p.Config.TopData.Build,
"topdata.reference_builder": p.Config.TopData.ReferenceBuilder,
"topdata.assets": p.Config.TopData.Assets,
"topdata.compiled_2da_dir": p.Config.TopData.Compiled2DADir,
"topdata.compiled_tlk": p.Config.TopData.CompiledTLK,
"topdata.wiki.output_root": p.Config.TopData.Wiki.OutputRoot,
"topdata.wiki.pages_dir": p.Config.TopData.Wiki.PagesDir,
"topdata.wiki.state_file": p.Config.TopData.Wiki.StateFile,
"autogen.cache.root": p.Config.Autogen.Cache.Root,
"paths.source": p.Config.Paths.Source,
"paths.assets": p.Config.Paths.Assets,
"paths.build": p.Config.Paths.Build,
"paths.cache": p.Config.Paths.Cache,
"paths.tools": p.Config.Paths.Tools,
"outputs.module_archive": p.Config.Outputs.ModuleArchive,
"outputs.hak_manifest": p.Config.Outputs.HAKManifest,
"outputs.hak_archive": p.Config.Outputs.HAKArchive,
"scripts.source_dir": p.Config.Scripts.SourceDir,
"scripts.cache": p.Config.Scripts.Cache,
"scripts.compiler.path": p.Config.Scripts.Compiler.Path,
"scripts.compiler.env.path": p.Config.Scripts.Compiler.Env.Path,
"topdata.source": p.Config.TopData.Source,
"topdata.build": p.Config.TopData.Build,
"topdata.reference_builder": p.Config.TopData.ReferenceBuilder,
"topdata.assets": p.Config.TopData.Assets,
"topdata.compiled_2da_dir": p.Config.TopData.Compiled2DADir,
"topdata.compiled_tlk": p.Config.TopData.CompiledTLK,
"topdata.wiki.output_root": p.Config.TopData.Wiki.OutputRoot,
"topdata.wiki.pages_dir": p.Config.TopData.Wiki.PagesDir,
"topdata.wiki.state_file": p.Config.TopData.Wiki.StateFile,
"topdata.wiki.deploy_manifest": p.Config.TopData.Wiki.DeployManifest,
"topdata.wiki.deploy_edit_summary": p.Config.TopData.Wiki.DeployEditSummary,
"extract.layout": p.Config.Extract.Layout,
"extract.hak_discovery": p.Config.Extract.HAKDiscovery,
"autogen.cache.root": p.Config.Autogen.Cache.Root,
"autogen.cache.max_age": p.Config.Autogen.Cache.MaxAge,
"autogen.cache.refresh_env": p.Config.Autogen.Cache.RefreshEnv,
"autogen.release_source.provider": p.Config.Autogen.ReleaseSource.Provider,
"autogen.release_source.server_url_env": p.Config.Autogen.ReleaseSource.ServerURLEnv,
"autogen.release_source.repo_env": p.Config.Autogen.ReleaseSource.RepoEnv,
} {
if strings.Contains(value, "\x00") {
failures = append(failures, fmt.Errorf("%s must not contain NUL bytes", field))
@@ -360,11 +405,13 @@ func (p *Project) ValidateLayout() error {
failures = append(failures, validateRelativePath("outputs.hak_manifest", effective.Outputs.HAKManifest)...)
failures = append(failures, validateRelativePath("outputs.hak_archive", effective.Outputs.HAKArchive)...)
failures = append(failures, validateRelativePath("scripts.cache", effective.Scripts.Cache)...)
failures = append(failures, validateRelativePath("scripts.source_dir", effective.Scripts.SourceDir)...)
failures = append(failures, validateRelativePath("topdata.compiled_2da_dir", effective.TopData.Compiled2DADir)...)
failures = append(failures, validateRelativePath("topdata.compiled_tlk", effective.TopData.CompiledTLK)...)
failures = append(failures, validateRelativePath("topdata.wiki.output_root", effective.TopData.Wiki.OutputRoot)...)
failures = append(failures, validateRelativePath("topdata.wiki.pages_dir", effective.TopData.Wiki.PagesDir)...)
failures = append(failures, validateRelativePath("topdata.wiki.state_file", effective.TopData.Wiki.StateFile)...)
failures = append(failures, validateRelativePath("topdata.wiki.deploy_manifest", effective.TopData.Wiki.DeployManifest)...)
failures = append(failures, validateRelativePath("autogen.cache.root", effective.Autogen.Cache.Root)...)
if err := validateOutputFileName("outputs.module_archive", filepath.Base(effective.Outputs.ModuleArchive), ".mod"); err != nil {
failures = append(failures, err)
@@ -375,6 +422,25 @@ func (p *Project) ValidateLayout() error {
if err := validateOutputFileName("outputs.hak_archive", filepath.Base(effective.Outputs.HAKArchive), ".hak"); err != nil {
failures = append(failures, err)
}
if err := validateOutputFileName("topdata.wiki.deploy_manifest", filepath.Base(effective.TopData.Wiki.DeployManifest), ".json"); err != nil {
failures = append(failures, err)
}
switch effective.Extract.Layout {
case "nwn_canonical_json":
default:
failures = append(failures, fmt.Errorf("extract.layout %q is not supported", effective.Extract.Layout))
}
switch effective.Extract.HAKDiscovery {
case "build_glob", "configured_haks":
default:
failures = append(failures, fmt.Errorf("extract.hak_discovery %q is not supported", effective.Extract.HAKDiscovery))
}
if _, err := time.ParseDuration(effective.Autogen.Cache.MaxAge); err != nil {
failures = append(failures, fmt.Errorf("autogen.cache.max_age must be a duration: %w", err))
}
if effective.Autogen.ReleaseSource.Provider != "gitea" {
failures = append(failures, fmt.Errorf("autogen.release_source.provider %q is not supported", effective.Autogen.ReleaseSource.Provider))
}
if p.HasTopData() {
if err := validateOutputFileName("topdata.package_hak", p.TopDataPackageHAKName(), ".hak"); err != nil {
failures = append(failures, err)
@@ -685,6 +751,46 @@ func (p *Project) ScriptCacheDir() string {
return p.rootPath(p.EffectiveConfig().Scripts.Cache)
}
func (p *Project) ScriptCompilerPath() string {
path := strings.TrimSpace(p.EffectiveConfig().Scripts.Compiler.Path)
if path == "" {
return ""
}
if filepath.IsAbs(path) {
return path
}
return filepath.Join(p.Root, filepath.FromSlash(path))
}
func (p *Project) ScriptCompilerSearchPaths() []string {
search := p.EffectiveConfig().Scripts.Compiler.Search
out := make([]string, 0, len(search))
for _, candidate := range search {
candidate = strings.TrimSpace(candidate)
if candidate == "" {
continue
}
if strings.HasPrefix(candidate, "PATH:") || filepath.IsAbs(candidate) {
out = append(out, candidate)
continue
}
out = append(out, filepath.Join(p.Root, filepath.FromSlash(candidate)))
}
return out
}
func (p *Project) ScriptCompilerPathEnv() string {
return p.EffectiveConfig().Scripts.Compiler.Env.Path
}
func (p *Project) ScriptCompilerNWNRootEnvKeys() []string {
return slices.Clone(p.EffectiveConfig().Scripts.Compiler.Env.NWNRoot)
}
func (p *Project) ScriptCompilerNWNUserDirectoryEnvKeys() []string {
return slices.Clone(p.EffectiveConfig().Scripts.Compiler.Env.NWNUserDirectory)
}
func (p *Project) MusicStageDir() string {
return p.rootPath(p.EffectiveConfig().Music.StageRoot)
}
@@ -701,6 +807,30 @@ func (p *Project) AutogenCachePath(name string) string {
return filepath.Join(p.AutogenCacheDir(), filepath.FromSlash(strings.TrimSpace(name)))
}
func (p *Project) AutogenCacheMaxAge() time.Duration {
duration, err := time.ParseDuration(p.EffectiveConfig().Autogen.Cache.MaxAge)
if err != nil {
return DefaultAutogenCacheMaxAge
}
return duration
}
func (p *Project) AutogenRefreshEnv() string {
return p.EffectiveConfig().Autogen.Cache.RefreshEnv
}
func (p *Project) PartsManifestRefreshEnv() string {
return p.EffectiveConfig().Autogen.Cache.PartsManifestRefreshEnv
}
func (p *Project) AssetsServerURLEnv() string {
return p.EffectiveConfig().Autogen.ReleaseSource.ServerURLEnv
}
func (p *Project) AssetsRepoEnv() string {
return p.EffectiveConfig().Autogen.ReleaseSource.RepoEnv
}
func (p *Project) rootPath(path string) string {
if strings.TrimSpace(path) == "" {
return p.Root
internal/project/project_test.go
+110
View File
@@ -136,6 +136,116 @@ autogen:
}
}
func TestEffectiveConfigHonorsConfiguredCompilerExtractWikiAndAutogenPolicy(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `
module:
name: Test Module
resref: testmod
paths:
source: src
tools: custom-tools
scripts:
source_dir: module-scripts
compiler:
path: bin/compiler
search:
- "{paths.tools}/compiler"
env:
path: TEST_COMPILER
nwn_root:
- TEST_NWN_ROOT
nwn_user_directory:
- TEST_NWN_USER
extract:
layout: nwn_canonical_json
hak_discovery: configured_haks
cleanup_stale: false
topdata:
wiki:
managed_namespaces:
- skills
deploy_manifest: wiki-manifest.json
deploy_edit_summary: Custom summary
autogen:
cache:
root: "{paths.cache}/released"
max_age: 30m
refresh_env: TEST_AUTOGEN_REFRESH
release_source:
provider: gitea
server_url_env: TEST_ASSETS_SERVER
repo_env: TEST_ASSETS_REPO
`)
proj, err := Load(root)
if err != nil {
t.Fatalf("Load returned error: %v", err)
}
effective := proj.EffectiveConfig()
if got, want := proj.ScriptSourceDir(), filepath.Join(root, "src", "module-scripts"); got != want {
t.Fatalf("expected script source dir %s, got %s", want, got)
}
if got, want := proj.ScriptCompilerPath(), filepath.Join(root, "bin", "compiler"); got != want {
t.Fatalf("expected compiler path %s, got %s", want, got)
}
if got, want := effective.Scripts.Compiler.Search[0], "custom-tools/compiler"; got != want {
t.Fatalf("expected compiler search %q, got %q", want, got)
}
if got, want := effective.Scripts.Compiler.Env.Path, "TEST_COMPILER"; got != want {
t.Fatalf("expected compiler env %q, got %q", want, got)
}
if got, want := effective.Extract.HAKDiscovery, "configured_haks"; got != want {
t.Fatalf("expected extract hak discovery %q, got %q", want, got)
}
if effective.Extract.CleanupStale == nil || *effective.Extract.CleanupStale {
t.Fatalf("expected cleanup_stale false, got %#v", effective.Extract.CleanupStale)
}
if got, want := effective.TopData.Wiki.DeployManifest, "wiki-manifest.json"; got != want {
t.Fatalf("expected wiki deploy manifest %q, got %q", want, got)
}
if got, want := effective.Autogen.Cache.MaxAge, "30m"; got != want {
t.Fatalf("expected autogen cache max age %q, got %q", want, got)
}
if got, want := proj.AutogenRefreshEnv(), "TEST_AUTOGEN_REFRESH"; got != want {
t.Fatalf("expected autogen refresh env %q, got %q", want, got)
}
}
func TestActiveOverridesAreVisibleAndSensitiveValuesAreMasked(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `
module:
name: Test Module
resref: testmod
paths:
source: src
`)
t.Setenv("SOW_BUILD_HAKS_KEEP_EXISTING", "1")
t.Setenv("NODEBB_API_TOKEN", "secret-token")
proj, err := Load(root)
if err != nil {
t.Fatalf("Load returned error: %v", err)
}
overrides := proj.ActiveOverrides()
var sawKeepExisting, sawMaskedToken bool
for _, override := range overrides {
if override.Key == "build.keep_existing_haks" && override.Value == "1" {
sawKeepExisting = true
}
if override.Key == "wiki.token" && override.Value == "<set>" {
sawMaskedToken = true
}
}
if !sawKeepExisting {
t.Fatalf("expected build keep existing override in %#v", overrides)
}
if !sawMaskedToken {
t.Fatalf("expected masked wiki token override in %#v", overrides)
}
}
func TestEffectiveConfigJSONIsDeterministic(t *testing.T) {
root := t.TempDir()
writeProjectFile(t, filepath.Join(root, ConfigFile), `
internal/topdata/autogen.go
+5 -5
View File
@@ -329,8 +329,8 @@ func deriveAutogenManifestEntry(rel string, derive project.AutogenDeriveConfig)
func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) {
cachePath := p.AutogenCachePath(consumer.Manifest.CacheName)
if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) == "" {
manifest, fresh, err := readFreshAutogenManifestCache(cachePath, autogenManifestCacheMaxAge)
if strings.TrimSpace(os.Getenv(p.AutogenRefreshEnv())) == "" {
manifest, fresh, err := readFreshAutogenManifestCache(cachePath, p.AutogenCacheMaxAge())
if err != nil {
if progress != nil {
progress(fmt.Sprintf("Ignoring cached autogen manifest %s: %v", consumer.Manifest.CacheName, err))
@@ -352,7 +352,7 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.
}
}
spec, err := deriveSowAssetsRepoSpec(p.Root)
spec, err := deriveSowAssetsRepoSpec(p)
if err != nil {
return nil, err
}
@@ -383,7 +383,7 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.
}
func releasedAutogenManifestCacheCurrent(p *project.Project, consumer project.AutogenConsumerConfig, cachePath string, manifest *autogenManifest) (bool, time.Time, error) {
spec, err := deriveSowAssetsRepoSpec(p.Root)
spec, err := deriveSowAssetsRepoSpec(p)
if err != nil {
return false, time.Time{}, err
}
@@ -409,7 +409,7 @@ func localAutogenManifestComparableTime(cachePath string, manifest *autogenManif
}
func newestReleasedAutogenManifestInput(p *project.Project, consumer project.AutogenConsumerConfig, now time.Time, cachePath string) (time.Time, string, error) {
if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) != "" {
if strings.TrimSpace(os.Getenv(p.AutogenRefreshEnv())) != "" {
return now, cachePath, nil
}
internal/topdata/native.go
+1 -1
View File
@@ -360,7 +360,7 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
}
}
filesTLK, err := compiler.finish(outputTLK)
filesTLK, err := compiler.finish(outputTLK, filepath.Base(p.TopDataCompiledTLKPath()))
if err != nil {
return BuildResult{}, err
}
internal/topdata/parts_manifest.go
+6 -6
View File
@@ -46,7 +46,7 @@ func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (ma
progress = func(string) {}
}
cachePath := p.AutogenCachePath(partsManifestCacheFileName)
if strings.TrimSpace(os.Getenv("SOW_PARTS_MANIFEST_REFRESH")) == "" {
if strings.TrimSpace(os.Getenv(p.PartsManifestRefreshEnv())) == "" {
manifest, fresh, err := readFreshPartsManifestCache(cachePath, partsManifestCacheMaxAge)
if err != nil {
progress(fmt.Sprintf("Ignoring cached parts manifest: %v", err))
@@ -55,7 +55,7 @@ func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (ma
return inventoryFromPartsManifest(manifest), nil
}
}
spec, err := deriveSowAssetsRepoSpec(p.Root)
spec, err := deriveSowAssetsRepoSpec(p)
if err != nil {
return nil, err
}
@@ -74,10 +74,10 @@ func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (ma
return inventoryFromPartsManifest(manifest), nil
}
func deriveSowAssetsRepoSpec(root string) (giteaRepoSpec, error) {
serverOverride := strings.TrimSpace(os.Getenv("SOW_ASSETS_SERVER_URL"))
repoOverride := strings.TrimSpace(os.Getenv("SOW_ASSETS_REPO"))
remote, err := gitOutput(root, "remote", "get-url", "origin")
func deriveSowAssetsRepoSpec(p *project.Project) (giteaRepoSpec, error) {
serverOverride := strings.TrimSpace(os.Getenv(p.AssetsServerURLEnv()))
repoOverride := strings.TrimSpace(os.Getenv(p.AssetsRepoEnv()))
remote, err := gitOutput(p.Root, "remote", "get-url", "origin")
spec := giteaRepoSpec{BaseURL: defaultGiteaBaseURL, Owner: defaultSowAssetsRepoOwner, Repo: defaultSowAssetsRepoName}
if err == nil && strings.TrimSpace(remote) != "" {
if derived, ok := parseRepoSpec(strings.TrimSpace(remote)); ok {
internal/topdata/tlk_native.go
+5 -2
View File
@@ -356,7 +356,7 @@ func (c *tlkCompiler) allocateID() int {
return id
}
func (c *tlkCompiler) finish(outputDir string) (int, error) {
func (c *tlkCompiler) finish(outputDir, tlkName string) (int, error) {
if err := os.MkdirAll(outputDir, 0o755); err != nil {
return 0, fmt.Errorf("create tlk output dir: %w", err)
}
@@ -388,7 +388,10 @@ func (c *tlkCompiler) finish(outputDir string) (int, error) {
entries[id] = c.active[key]
}
if err := writeTLKBinary(filepath.Join(outputDir, defaultTLKName), entries, c.language); err != nil {
if strings.TrimSpace(tlkName) == "" {
tlkName = defaultTLKName
}
if err := writeTLKBinary(filepath.Join(outputDir, tlkName), entries, c.language); err != nil {
return 0, err
}
if maxID < 0 {
internal/topdata/wiki_deploy.go
+7 -11
View File
@@ -20,16 +20,12 @@ import (
)
const (
defaultDeployManifestName = ".wiki_deploy_manifest.json"
defaultEditSummary = "Auto-generated from native builder"
managedStartMarker = "[//]: # (sow-topdata-wiki:managed:start)"
managedEndMarker = "[//]: # (sow-topdata-wiki:managed:end)"
legacyManagedStartMarker = "<!-- sow-topdata-wiki:managed:start -->"
legacyManagedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
managedStartMarker = "[//]: # (sow-topdata-wiki:managed:start)"
managedEndMarker = "[//]: # (sow-topdata-wiki:managed:end)"
legacyManagedStartMarker = "<!-- sow-topdata-wiki:managed:start -->"
legacyManagedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
)
var defaultManagedNamespaces = []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}
type DeployWikiOptions struct {
SourceDir string
Endpoint string
@@ -107,12 +103,12 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
namespaces := opts.Namespaces
if len(namespaces) == 0 {
namespaces = defaultManagedNamespaces
namespaces = p.EffectiveConfig().TopData.Wiki.ManagedNamespaces
}
manifestPath := opts.ManifestPath
if manifestPath == "" {
manifestPath = filepath.Join(filepath.Dir(opts.SourceDir), defaultDeployManifestName)
manifestPath = filepath.Join(filepath.Dir(opts.SourceDir), p.EffectiveConfig().TopData.Wiki.DeployManifest)
}
pages, err := collectLocalPages(opts.SourceDir, namespaces)
@@ -137,7 +133,7 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
return result, errors.New("remote managed wiki content drifted; rerun with --force to overwrite")
}
summary := defaultEditSummary
summary := p.EffectiveConfig().TopData.Wiki.DeployEditSummary
if opts.Version != "" {
summary = fmt.Sprintf("%s (%s)", summary, opts.Version)
}
internal/topdata/wiki_native.go
+11 -12
View File
@@ -30,9 +30,8 @@ const (
)
var (
wikiManagedNamespaces = []string{"classes", "feat", "itemtypes", "races", "skills", "spells"}
wikiStatusPriority = map[string]int{wikiStatusVanilla: 0, wikiStatusModified: 1, wikiStatusNew: 2}
wikiStatusLabels = map[string]string{wikiStatusNew: "New", wikiStatusModified: "Modified", wikiStatusVanilla: "Vanilla"}
wikiStatusPriority = map[string]int{wikiStatusVanilla: 0, wikiStatusModified: 1, wikiStatusNew: 2}
wikiStatusLabels = map[string]string{wikiStatusNew: "New", wikiStatusModified: "Modified", wikiStatusVanilla: "Vanilla"}
)
//go:embed wiki_templates/*.txt
@@ -138,7 +137,7 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
if err := generateEntityPages(outputDir, ctx, writePage); err != nil {
return wikiResult{}, err
}
if err := generateStatusPages(outputDir, pageStatuses, pageTitles); err != nil {
if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces); err != nil {
return wikiResult{}, err
}
@@ -1237,12 +1236,12 @@ func wikiStatusNoun(category string) string {
}
}
func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string) error {
summaries := buildWikiNamespaceSummaries(pageStatuses)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.md"), renderWikiStatusReportPage(summaries)); err != nil {
func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string, namespaces []string) error {
summaries := buildWikiNamespaceSummaries(pageStatuses, namespaces)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.md"), renderWikiStatusReportPage(summaries, namespaces)); err != nil {
return err
}
for _, namespace := range wikiManagedNamespaces {
for _, namespace := range namespaces {
summary := summaries[namespace]
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".md"), renderWikiNamespaceFragment(namespace, summary)); err != nil {
return err
@@ -1271,9 +1270,9 @@ func writeWikiFile(path, content string) error {
return os.WriteFile(path, []byte(ensureTrailingNewline(content)), 0o644)
}
func buildWikiNamespaceSummaries(pageStatuses map[string]string) map[string]map[string]any {
func buildWikiNamespaceSummaries(pageStatuses map[string]string, namespaces []string) map[string]map[string]any {
summaries := map[string]map[string]any{}
for _, namespace := range wikiManagedNamespaces {
for _, namespace := range namespaces {
statuses := []string{}
for pageID, status := range pageStatuses {
if strings.SplitN(pageID, ":", 2)[0] == namespace {
@@ -1325,9 +1324,9 @@ func renderWikiNamespaceFragment(namespace string, summary map[string]any) strin
}, "\n")
}
func renderWikiStatusReportPage(summaries map[string]map[string]any) string {
func renderWikiStatusReportPage(summaries map[string]map[string]any, namespaces []string) string {
lines := []string{"====== Wiki Status ======", "", "This page is auto-generated from builder source provenance.", ""}
for _, namespace := range wikiManagedNamespaces {
for _, namespace := range namespaces {
summary := summaries[namespace]
lines = append(lines, "===== "+namespaceTitle(namespace)+" =====", "")
lines = append(lines, fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]))