From b39bdf13e8d8a0e818790521aaca0c7308f7e9dc Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Thu, 7 May 2026 14:22:47 +0200 Subject: [PATCH] Configuration Hardening 2 What changed: - Added visible runtime override reporting with masking for sensitive values. - Expanded effective config for build cleanup, script compiler discovery/env, extract layout/cleanup/ HAK discovery, wiki deploy defaults, autogen cache policy, and release-source env names. - Wired configured values into build, script compiler resolution, extract, autogen cache refresh/max age, parts manifest release source, wiki generation/deploy, and TLK output naming. - Added config sources override output and validation-rule hints in config explain. - Added CONFIGURATION_HARDENING_AUDIT.md. - Updated README.md with config commands, schema defaults, and JSON artifact boundaries. - Added regression tests for override visibility/masking, compiler/extract/wiki/autogen config, configured HAK discovery, and config sources output. --- CONFIGURATION_HARDENING_AUDIT.md | 70 ++++++ README.md | 89 +++++++- internal/app/app.go | 37 ++- internal/app/app_test.go | 31 +++ internal/pipeline/build.go | 50 ++-- internal/pipeline/extract.go | 36 ++- internal/pipeline/pipeline_test.go | 71 ++++++ internal/pipeline/script_compiler_env.go | 16 +- internal/project/effective.go | 278 +++++++++++++++++------ internal/project/project.go | 188 ++++++++++++--- internal/project/project_test.go | 110 +++++++++ internal/topdata/autogen.go | 10 +- internal/topdata/native.go | 2 +- internal/topdata/parts_manifest.go | 12 +- internal/topdata/tlk_native.go | 7 +- internal/topdata/wiki_deploy.go | 18 +- internal/topdata/wiki_native.go | 23 +- 17 files changed, 877 insertions(+), 171 deletions(-) create mode 100644 CONFIGURATION_HARDENING_AUDIT.md diff --git a/CONFIGURATION_HARDENING_AUDIT.md b/CONFIGURATION_HARDENING_AUDIT.md new file mode 100644 index 0000000..b5c1e99 --- /dev/null +++ b/CONFIGURATION_HARDENING_AUDIT.md @@ -0,0 +1,70 @@ +# Configuration Hardening Audit + +This audit records the current repository-configuration authority model for +`sow-toolkit`. + +## Authority Rules + +- Human-authored root repository configuration is YAML: `nwn-tool.yaml`, then + `nwn-tool.yml`. +- `nwn-tool.json` is legacy compatibility only. It remains readable during the + migration window and is reported as deprecated provenance. +- Generated and machine-readable artifacts remain JSON. This includes HAK + manifests, canonical GFF JSON, topdata datasets, autogen manifests, caches, + wiki state, deploy manifests, and credits inventories. +- Toolkit defaults are centralized in `internal/project/effective.go` and are + visible through `sow-toolkit config effective`. +- Environment variables are treated as runtime overrides and are listed by + `sow-toolkit config sources` and `config effective`. Sensitive values are + masked. + +## Implemented Schema Expansion + +- `paths.cache`, `paths.tools` +- `outputs.module_archive`, `outputs.hak_manifest`, `outputs.hak_archive` +- `build.keep_existing_haks` +- `inventory.source_extensions`, `inventory.asset_extensions`, + `inventory.source_json_pattern` +- `scripts.source_dir`, `scripts.cache`, `scripts.compiler` +- `extract.layout`, `extract.hak_discovery`, `extract.cleanup_stale` +- `topdata.compiled_2da_dir`, `topdata.compiled_tlk`, package names, and wiki + output/deploy defaults +- `autogen.cache.root`, `autogen.cache.max_age`, + `autogen.cache.refresh_env`, and release-source env names + +## Remaining Engine Invariants + +These are intentionally still implemented as toolkit behavior rather than +repository-specific YAML: + +- Canonical GFF JSON shape. +- ERF/MOD/HAK binary serialization. +- Supported autogen algorithms: `parts_rows`, `head_visualeffects`, + `trailing_numeric_suffix`, `model_stem`, and `first_path_segment`. +- Native topdata dataset internals and generated JSON dataset formats. +- NWN validator semantics for required module fields and resource-reference + checks. These are candidates for future validation profiles, but are not + repository-specific defaults. + +## Migration Checklist + +For each consumer repository: + +1. Confirm root config is `nwn-tool.yaml` or `nwn-tool.yml`. +2. Remove root `nwn-tool.json` unless it is a deliberate test fixture. +3. Run `sow-toolkit config effective --json` and record generated output, + cache paths, HAK manifest paths, topdata package names, script compiler + resolution policy, and active overrides. +4. Move wrapper-provided environment behavior into YAML where it is stable + repository policy. +5. Keep credentials and one-off CI values as runtime overrides. +6. Run build/validate workflows and compare generated JSON artifacts. + +## Follow-Up Tickets + +- Remove legacy root JSON loading after all active repositories migrate. +- Add validation-profile schema if repositories need to vary validator rules. +- Complete the separate music dataset refactor described in + `MUSIC_REFACTOR_CONTRACT.md`. +- Add static linting for new repository-specific literals in command and + pipeline code. diff --git a/README.md b/README.md index e17bb89..838e136 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,92 @@ music: envi/music/westgate: mus_wg_ ``` +Resolved configuration can be inspected without scanning or building: + +```bash +sow-toolkit config validate +sow-toolkit config effective --json +sow-toolkit config inspect paths.build +sow-toolkit config explain topdata.package_hak +sow-toolkit config sources +``` + +`config effective` prints the normalized values consumed by commands, including +toolkit defaults, YAML values, legacy status, active environment overrides, and +provenance. Sensitive override values such as tokens are reported as set without +printing the secret. + +Common configurable defaults: + +```yaml +paths: + source: src + assets: assets + build: build + cache: .cache + tools: tools + +outputs: + module_archive: "{module.resref}.mod" + hak_manifest: haks.json + hak_archive: "{hak.name}.hak" + +build: + keep_existing_haks: false + +inventory: + source_extensions: [] # empty means built-in NWN source extensions + asset_extensions: [] # empty means built-in NWN asset extensions + source_json_pattern: "{resref}.{extension}.json" + +scripts: + source_dir: scripts + cache: "{paths.cache}/ncs" + compiler: + path: "" + env: + path: SOW_NWN_SCRIPT_COMPILER + nwn_root: [SOW_NWN_ROOT, NWN_ROOT] + nwn_user_directory: [SOW_NWN_USER_DIRECTORY, NWN_HOME, NWN_USER_DIRECTORY] + search: + - "{paths.tools}/script-compiler/nwn_script_comp" + - "{paths.tools}/script-compiler/nwn_script_comp.exe" + - "{paths.tools}/nwn_script_comp" + - "{paths.tools}/nwn_script_comp.exe" + - PATH:nwn_script_comp + +extract: + layout: nwn_canonical_json + hak_discovery: build_glob # or configured_haks + cleanup_stale: true + ignore_extensions: [] + delete_module_archive_after_success: false + +topdata: + build: "{paths.build}/topdata" + compiled_2da_dir: 2da + compiled_tlk: sow_tlk.tlk + package_hak: sow_top.hak + package_tlk: sow_tlk.tlk + wiki: + output_root: wiki + pages_dir: pages + state_file: state.json + managed_namespaces: [classes, feat, itemtypes, races, skills, spells, meta] + deploy_manifest: .wiki_deploy_manifest.json + deploy_edit_summary: Auto-generated from native builder + +autogen: + cache: + root: "{paths.cache}" + max_age: 1h + refresh_env: SOW_AUTOGEN_MANIFEST_REFRESH + release_source: + provider: gitea + server_url_env: SOW_ASSETS_SERVER_URL + repo_env: SOW_ASSETS_REPO +``` + Generated and machine-readable artifacts remain JSON, including HAK manifests, topdata datasets, caches, credits inventories, build reports, and canonical GFF documents. Nested metadata such as `topdata/templates/config.json` is not root @@ -109,7 +195,8 @@ Consumer wrapper resolution is intentionally aligned: 2. Else prefer a sibling `../sow-tools` checkout for local source development 3. Else install the latest published `sow-tools` release artifact -NWScript compilation behavior is also aligned: +NWScript compilation behavior is also aligned and configurable under +`scripts.compiler`: 1. Prefer `SOW_NWN_SCRIPT_COMPILER` when explicitly set 2. Else prefer a local compiler in `tools/script-compiler/` or `tools/` diff --git a/internal/app/app.go b/internal/app/app.go index d946b60..996ac0c 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -149,7 +149,7 @@ var commands = []command{ }, { name: "extract", - description: "Extract a module archive back into the src/ tree.", + description: "Extract built archives back into the configured source and asset trees.", run: runExtract, }, { @@ -169,7 +169,7 @@ var commands = []command{ }, { name: "apply-hak-manifest", - description: "Apply a generated HAK manifest to src/module/module.ifo.json.", + description: "Apply a generated HAK manifest to the configured module source.", run: runApplyHAKManifest, }, { @@ -749,6 +749,9 @@ func runConfig(ctx context) error { if provenance.Deprecated { fmt.Fprintf(ctx.stdout, "deprecated: true\n") } + if rules := configValidationRules(key); len(rules) > 0 { + fmt.Fprintf(ctx.stdout, "validation: %s\n", strings.Join(rules, "; ")) + } return nil case "sources": if len(ctx.args) > 2 { @@ -759,12 +762,42 @@ func runConfig(ctx context) error { fmt.Fprintf(ctx.stdout, "format: %s\n", p.ConfigSource.Format) fmt.Fprintf(ctx.stdout, "legacy: %t\n", p.ConfigSource.Legacy) fmt.Fprintf(ctx.stdout, "candidates: %s\n", strings.Join(project.ConfigFileCandidates, ", ")) + overrides := p.ActiveOverrides() + if len(overrides) == 0 { + fmt.Fprintln(ctx.stdout, "active overrides: none") + return nil + } + fmt.Fprintln(ctx.stdout, "active overrides:") + for _, override := range overrides { + fmt.Fprintf(ctx.stdout, " %s=%s (%s)\n", override.Key, override.Value, override.Source) + } return nil default: return fmt.Errorf("unknown config subcommand %q", ctx.args[1]) } } +func configValidationRules(key string) []string { + switch { + case strings.HasPrefix(key, "paths."), strings.Contains(key, ".root"), strings.Contains(key, ".dir"), strings.Contains(key, ".cache"): + return []string{"relative paths must not escape the repository root unless explicitly documented"} + case strings.HasPrefix(key, "outputs."): + return []string{"generated output names must use the configured extension and must not escape the repository root"} + case key == "extract.layout": + return []string{"supported value: nwn_canonical_json"} + case key == "extract.hak_discovery": + return []string{"supported values: build_glob, configured_haks"} + case key == "autogen.cache.max_age": + return []string{"Go duration string, for example 1h or 30m"} + case strings.HasPrefix(key, "topdata.package_hak"): + return []string{"must be a .hak file name"} + case strings.HasPrefix(key, "topdata.package_tlk"), strings.HasPrefix(key, "topdata.compiled_tlk"): + return []string{"must be a .tlk file name"} + default: + return nil + } +} + func parseConfigFormatArgs(commandName string, args []string) (string, error) { format := "yaml" for _, arg := range args { diff --git a/internal/app/app_test.go b/internal/app/app_test.go index 4c09a03..4817b7f 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -286,6 +286,37 @@ paths: } } +func TestRunConfigSourcesListsActiveOverrides(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src +`) + t.Setenv("SOW_BUILD_HAKS_KEEP_EXISTING", "1") + + var stdout bytes.Buffer + ctx := context{ + stdout: &stdout, + stderr: &bytes.Buffer{}, + cwd: root, + args: []string{"config", "sources"}, + } + + if err := runConfig(ctx); err != nil { + t.Fatalf("runConfig failed: %v", err) + } + output := stdout.String() + if !strings.Contains(output, "active overrides:") { + t.Fatalf("expected active overrides section, got %q", output) + } + if !strings.Contains(output, "build.keep_existing_haks=1") { + t.Fatalf("expected keep existing override, got %q", output) + } +} + func mkdirAll(t *testing.T, path string) { t.Helper() if err := os.MkdirAll(path, 0o755); err != nil { diff --git a/internal/pipeline/build.go b/internal/pipeline/build.go index b3fa307..dc25205 100644 --- a/internal/pipeline/build.go +++ b/internal/pipeline/build.go @@ -215,7 +215,7 @@ func planHAKs(p *project.Project, progress ProgressFunc) (BuildResult, error) { } func planOrBuildHAKs(p *project.Project, progress ProgressFunc, writeArchives bool, archiveNames []string, sourceManifestPath string) (result BuildResult, err error) { - preserveExistingHAKs := envBool("SOW_BUILD_HAKS_KEEP_EXISTING") + preserveExistingHAKs := p.EffectiveConfig().Build.KeepExistingHAKs || envBool("SOW_BUILD_HAKS_KEEP_EXISTING") progressf(progress, "Validating project...") if err := validateForBuild(p); err != nil { @@ -556,7 +556,7 @@ func compileReferencedScripts(p *project.Project) ([]erf.Resource, error) { if err != nil { return nil, err } - compilerEnv, err := resolveScriptCompilerEnvironment() + compilerEnv, err := resolveScriptCompilerEnvironment(p) if err != nil { return nil, err } @@ -630,40 +630,40 @@ func referencedScriptResrefs(p *project.Project) ([]string, error) { } func resolveScriptCompiler(p *project.Project) (string, error) { - if configured := strings.TrimSpace(os.Getenv("SOW_NWN_SCRIPT_COMPILER")); configured != "" { + if configured := p.ScriptCompilerPath(); configured != "" { + return configured, nil + } + if configured := strings.TrimSpace(os.Getenv(p.ScriptCompilerPathEnv())); configured != "" { return configured, nil } - candidates := []string{ - filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp"), - filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp.exe"), - filepath.Join(p.Root, "tools", "nwn_script_comp"), - filepath.Join(p.Root, "tools", "nwn_script_comp.exe"), - } + candidates := p.ScriptCompilerSearchPaths() if runtime.GOOS == "windows" { - candidates = []string{ - filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp.exe"), - filepath.Join(p.Root, "tools", "script-compiler", "nwn_script_comp"), - filepath.Join(p.Root, "tools", "nwn_script_comp.exe"), - filepath.Join(p.Root, "tools", "nwn_script_comp"), - } + slices.SortStableFunc(candidates, func(a, b string) int { + aExe := strings.HasSuffix(strings.ToLower(a), ".exe") + bExe := strings.HasSuffix(strings.ToLower(b), ".exe") + if aExe == bExe { + return strings.Compare(a, b) + } + if aExe { + return -1 + } + return 1 + }) } for _, candidate := range candidates { + if binary, ok := strings.CutPrefix(candidate, "PATH:"); ok { + if compiler, err := exec.LookPath(binary); err == nil { + return compiler, nil + } + continue + } if info, err := os.Stat(candidate); err == nil && !info.IsDir() { return candidate, nil } } - if compiler, err := exec.LookPath("nwn_script_comp"); err == nil { - return compiler, nil - } - if runtime.GOOS == "windows" { - if compiler, err := exec.LookPath("nwn_script_comp.exe"); err == nil { - return compiler, nil - } - } - - return "", errors.New("script compiler not found; install nwn_script_comp into ./tools or set SOW_NWN_SCRIPT_COMPILER") + return "", fmt.Errorf("script compiler not found; configure scripts.compiler.path or set %s", p.ScriptCompilerPathEnv()) } func validatorLoadDocument(path string) (gff.Document, string, string, error) { diff --git a/internal/pipeline/extract.go b/internal/pipeline/extract.go index 57f99e5..cc0dd0f 100644 --- a/internal/pipeline/extract.go +++ b/internal/pipeline/extract.go @@ -57,7 +57,7 @@ func Extract(p *project.Project, files ...string) (ExtractResult, error) { failures = append(failures, errs...) } - hakPaths, err := filepath.Glob(filepath.Join(p.BuildDir(), "*.hak")) + hakPaths, err := extractHAKPaths(p) if err != nil { return result, fmt.Errorf("scan hak archives: %w", err) } @@ -88,9 +88,11 @@ func Extract(p *project.Project, files ...string) (ExtractResult, error) { failures = append(failures, errs...) } - removed, errs := cleanupStaleFiles(p, desired) - result.Removed = removed - failures = append(failures, errs...) + if p.EffectiveConfig().Extract.CleanupStale == nil || *p.EffectiveConfig().Extract.CleanupStale { + removed, errs := cleanupStaleFiles(p, desired) + result.Removed = removed + failures = append(failures, errs...) + } if len(failures) > 0 { return result, errors.Join(failures...) @@ -157,7 +159,7 @@ func extractedFile(p *project.Project, resource erf.Resource, extension string) switch extension { case "nss": - return filepath.Join(p.SourceDir(), "scripts", resref+".nss"), resource.Data, nil + return filepath.Join(p.ScriptSourceDir(), resref+".nss"), resource.Data, nil case "utc", "utd", "ute", "uti", "utm", "utp", "uts", "utt", "utw", "are", "dlg", "fac", "gic", "git", "ifo", "itp", "jrl": document, err := gff.Read(bytes.NewReader(resource.Data)) @@ -175,6 +177,30 @@ func extractedFile(p *project.Project, resource erf.Resource, extension string) } } +func extractHAKPaths(p *project.Project) ([]string, error) { + switch p.EffectiveConfig().Extract.HAKDiscovery { + case "configured_haks": + paths := make([]string, 0, len(p.Config.HAKs)) + for _, hak := range p.Config.HAKs { + path := p.HAKArchivePath(hak.Name) + if _, err := os.Stat(path); err == nil { + paths = append(paths, path) + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + } + slices.Sort(paths) + return paths, nil + default: + hakPaths, err := filepath.Glob(filepath.Join(p.BuildDir(), "*.hak")) + if err != nil { + return nil, err + } + slices.Sort(hakPaths) + return hakPaths, nil + } +} + type writeState int const ( diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index d1a7f97..1d25a94 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -175,6 +175,77 @@ func TestExtractReadsHAKAssets(t *testing.T) { } } +func TestExtractConfiguredHAKDiscoveryIgnoresUnconfiguredArchives(t *testing.T) { + root := t.TempDir() + mustMkdir(t, filepath.Join(root, "src")) + mustMkdir(t, filepath.Join(root, "assets")) + mustMkdir(t, filepath.Join(root, "build")) + + mustWriteFile(t, filepath.Join(root, "nwn-tool.yaml"), ` +module: + name: Test Module + resref: testmod +paths: + source: src + assets: assets + build: build +extract: + hak_discovery: configured_haks +haks: + - name: wanted + priority: 1 + include: + - wanted/** +`) + + p, err := project.Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := p.ValidateLayout(); err != nil { + t.Fatalf("validate layout: %v", err) + } + + modFile, err := os.Create(p.ModuleArchivePath()) + if err != nil { + t.Fatalf("create mod: %v", err) + } + if err := erf.Write(modFile, erf.New("MOD ", nil)); err != nil { + t.Fatalf("write mod: %v", err) + } + if err := modFile.Close(); err != nil { + t.Fatalf("close mod: %v", err) + } + for _, name := range []string{"wanted", "ignored"} { + hakFile, err := os.Create(filepath.Join(root, "build", name+".hak")) + if err != nil { + t.Fatalf("create hak: %v", err) + } + if err := erf.Write(hakFile, erf.New("HAK ", []erf.Resource{ + {Name: name + "_asset", Type: 0x07D2, Data: []byte("mdl-data")}, + })); err != nil { + t.Fatalf("write hak: %v", err) + } + if err := hakFile.Close(); err != nil { + t.Fatalf("close hak: %v", err) + } + } + + result, err := Extract(p) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(result.HAKPaths) != 1 || filepath.Base(result.HAKPaths[0]) != "wanted.hak" { + t.Fatalf("expected only configured hak, got %#v", result.HAKPaths) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "wanted_asset.mdl")); err != nil { + t.Fatalf("expected wanted asset: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "assets", "mdl", "ignored_asset.mdl")); !os.IsNotExist(err) { + t.Fatalf("expected ignored asset to remain absent, err=%v", err) + } +} + func TestExtractDeletesConsumedModuleArchiveAfterSuccessWhenConfigured(t *testing.T) { root := t.TempDir() mustMkdir(t, filepath.Join(root, "src", "module")) diff --git a/internal/pipeline/script_compiler_env.go b/internal/pipeline/script_compiler_env.go index c884de7..994b3d9 100644 --- a/internal/pipeline/script_compiler_env.go +++ b/internal/pipeline/script_compiler_env.go @@ -9,19 +9,21 @@ import ( "regexp" "runtime" "strings" + + "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" ) -func resolveScriptCompilerEnvironment() ([]string, error) { +func resolveScriptCompilerEnvironment(p *project.Project) ([]string, error) { env := os.Environ() - userDir, err := resolveNWNUserDirectory() + userDir, err := resolveNWNUserDirectory(p.ScriptCompilerNWNUserDirectoryEnvKeys()) if err != nil { return nil, err } env = upsertEnv(env, "NWN_HOME", userDir) env = upsertEnv(env, "NWN_USER_DIRECTORY", userDir) - root, err := resolveNWNRoot() + root, err := resolveNWNRoot(p.ScriptCompilerNWNRootEnvKeys()) if err != nil { return nil, err } @@ -32,8 +34,8 @@ func resolveScriptCompilerEnvironment() ([]string, error) { return env, nil } -func resolveNWNUserDirectory() (string, error) { - for _, key := range []string{"SOW_NWN_USER_DIRECTORY", "NWN_HOME", "NWN_USER_DIRECTORY"} { +func resolveNWNUserDirectory(envKeys []string) (string, error) { + for _, key := range envKeys { if value := strings.TrimSpace(os.Getenv(key)); value != "" { path := filepath.Clean(expandUserPath(value)) if err := os.MkdirAll(path, 0o755); err != nil { @@ -71,8 +73,8 @@ func defaultNWNUserDirectory() (string, error) { } } -func resolveNWNRoot() (string, error) { - for _, key := range []string{"SOW_NWN_ROOT", "NWN_ROOT"} { +func resolveNWNRoot(envKeys []string) (string, error) { + for _, key := range envKeys { if value := strings.TrimSpace(os.Getenv(key)); value != "" { path := filepath.Clean(expandUserPath(value)) if !looksLikeNWNInstall(path) { diff --git a/internal/project/effective.go b/internal/project/effective.go index 3ee23b1..cdba63e 100644 --- a/internal/project/effective.go +++ b/internal/project/effective.go @@ -3,6 +3,7 @@ package project import ( "encoding/json" "fmt" + "os" "path/filepath" "slices" "strings" @@ -10,29 +11,38 @@ import ( ) const ( - DefaultBuildPath = "build" - DefaultCachePath = ".cache" - DefaultToolsPath = "tools" - DefaultModuleArchiveTemplate = "{module.resref}.mod" - DefaultHAKManifest = "haks.json" - DefaultHAKArchiveTemplate = "{hak.name}.hak" - DefaultSourceJSONPattern = "{resref}.{extension}.json" - DefaultScriptsSourceDir = "scripts" - DefaultScriptsCache = "{paths.cache}/ncs" - DefaultMusicStageRoot = "{paths.cache}/music" - DefaultCreditsRoot = "{paths.cache}/credits" - DefaultCreditsOverlayFile = "CREDITS.md" - DefaultTopDataBuild = "{paths.build}/topdata" - DefaultTopDataCompiled2DADir = "2da" - DefaultTopDataCompiledTLK = "sow_tlk.tlk" - DefaultTopDataWikiOutputRoot = "wiki" - DefaultTopDataWikiPagesDir = "pages" - DefaultTopDataWikiStateFile = "state.json" - DefaultTopDataPackageHAK = "sow_top.hak" - DefaultTopDataPackageTLK = "sow_tlk.tlk" - DefaultAutogenCacheRoot = "{paths.cache}" - DefaultAutogenCacheMaxAge = time.Hour - DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH" + DefaultBuildPath = "build" + DefaultCachePath = ".cache" + DefaultToolsPath = "tools" + DefaultModuleArchiveTemplate = "{module.resref}.mod" + DefaultHAKManifest = "haks.json" + DefaultHAKArchiveTemplate = "{hak.name}.hak" + DefaultSourceJSONPattern = "{resref}.{extension}.json" + DefaultScriptsSourceDir = "scripts" + DefaultScriptsCache = "{paths.cache}/ncs" + DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER" + DefaultMusicStageRoot = "{paths.cache}/music" + DefaultCreditsRoot = "{paths.cache}/credits" + DefaultCreditsOverlayFile = "CREDITS.md" + DefaultTopDataBuild = "{paths.build}/topdata" + DefaultTopDataCompiled2DADir = "2da" + DefaultTopDataCompiledTLK = "sow_tlk.tlk" + DefaultTopDataWikiOutputRoot = "wiki" + DefaultTopDataWikiPagesDir = "pages" + DefaultTopDataWikiStateFile = "state.json" + DefaultWikiDeployManifest = ".wiki_deploy_manifest.json" + DefaultWikiDeployEditSummary = "Auto-generated from native builder" + DefaultTopDataPackageHAK = "sow_top.hak" + DefaultTopDataPackageTLK = "sow_tlk.tlk" + DefaultAutogenCacheRoot = "{paths.cache}" + DefaultAutogenCacheMaxAge = time.Hour + DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH" + DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH" + DefaultAutogenReleaseProvider = "gitea" + DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL" + DefaultAutogenRepoEnv = "SOW_ASSETS_REPO" + DefaultExtractLayout = "nwn_canonical_json" + DefaultExtractHAKDiscovery = "build_glob" ) type ConfigProvenance map[string]ConfigValueProvenance @@ -48,6 +58,7 @@ type EffectiveConfig struct { Module ModuleConfig `json:"module" yaml:"module"` Paths EffectivePathConfig `json:"paths" yaml:"paths"` Outputs EffectiveOutputConfig `json:"outputs" yaml:"outputs"` + Build BuildConfig `json:"build" yaml:"build"` Inventory InventoryConfig `json:"inventory" yaml:"inventory"` Scripts EffectiveScriptsConfig `json:"scripts" yaml:"scripts"` Music EffectiveMusicConfig `json:"music" yaml:"music"` @@ -56,6 +67,7 @@ type EffectiveConfig struct { Autogen EffectiveAutogenConfig `json:"autogen" yaml:"autogen"` HAKs []HAKConfig `json:"haks" yaml:"haks"` Provenance ConfigProvenance `json:"provenance" yaml:"provenance"` + Overrides []ConfigOverride `json:"overrides,omitempty" yaml:"overrides,omitempty"` } type EffectivePathConfig struct { @@ -73,8 +85,21 @@ type EffectiveOutputConfig struct { } type EffectiveScriptsConfig struct { - SourceDir string `json:"source_dir" yaml:"source_dir"` - Cache string `json:"cache" yaml:"cache"` + SourceDir string `json:"source_dir" yaml:"source_dir"` + Cache string `json:"cache" yaml:"cache"` + Compiler EffectiveScriptCompilerConfig `json:"compiler" yaml:"compiler"` +} + +type EffectiveScriptCompilerConfig struct { + Path string `json:"path" yaml:"path"` + Search []string `json:"search" yaml:"search"` + Env EffectiveScriptCompilerEnvConfig `json:"env" yaml:"env"` +} + +type EffectiveScriptCompilerEnvConfig struct { + Path string `json:"path" yaml:"path"` + NWNRoot []string `json:"nwn_root" yaml:"nwn_root"` + NWNUserDirectory []string `json:"nwn_user_directory" yaml:"nwn_user_directory"` } type EffectiveMusicConfig struct { @@ -98,15 +123,29 @@ type EffectiveTopDataConfig struct { } type EffectiveAutogenConfig struct { - Producers []AutogenProducerConfig `json:"producers" yaml:"producers"` - Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"` - Cache EffectiveAutogenCacheConfig `json:"cache" yaml:"cache"` + Producers []AutogenProducerConfig `json:"producers" yaml:"producers"` + Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"` + Cache EffectiveAutogenCacheConfig `json:"cache" yaml:"cache"` + ReleaseSource EffectiveAutogenReleaseSourceConfig `json:"release_source" yaml:"release_source"` } type EffectiveAutogenCacheConfig struct { - Root string `json:"root" yaml:"root"` - MaxAge string `json:"max_age" yaml:"max_age"` - RefreshEnv string `json:"refresh_env" yaml:"refresh_env"` + Root string `json:"root" yaml:"root"` + MaxAge string `json:"max_age" yaml:"max_age"` + RefreshEnv string `json:"refresh_env" yaml:"refresh_env"` + PartsManifestRefreshEnv string `json:"parts_manifest_refresh_env" yaml:"parts_manifest_refresh_env"` +} + +type EffectiveAutogenReleaseSourceConfig struct { + Provider string `json:"provider" yaml:"provider"` + ServerURLEnv string `json:"server_url_env" yaml:"server_url_env"` + RepoEnv string `json:"repo_env" yaml:"repo_env"` +} + +type ConfigOverride struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + Source string `json:"source" yaml:"source"` } func (p *Project) EffectiveConfig() EffectiveConfig { @@ -133,6 +172,15 @@ func (p *Project) EffectiveConfig() EffectiveConfig { scripts := EffectiveScriptsConfig{ SourceDir: defaultString(p.Config.Scripts.SourceDir, DefaultScriptsSourceDir), Cache: expandPathTemplate(defaultString(p.Config.Scripts.Cache, DefaultScriptsCache), paths), + Compiler: EffectiveScriptCompilerConfig{ + Path: strings.TrimSpace(p.Config.Scripts.Compiler.Path), + Search: expandPathTemplates(defaultStringSlice(p.Config.Scripts.Compiler.Search, defaultScriptCompilerSearch()), paths), + Env: EffectiveScriptCompilerEnvConfig{ + Path: defaultString(p.Config.Scripts.Compiler.Env.Path, DefaultScriptCompilerEnvPath), + NWNRoot: defaultStringSlice(p.Config.Scripts.Compiler.Env.NWNRoot, []string{"SOW_NWN_ROOT", "NWN_ROOT"}), + NWNUserDirectory: defaultStringSlice(p.Config.Scripts.Compiler.Env.NWNUserDirectory, []string{"SOW_NWN_USER_DIRECTORY", "NWN_HOME", "NWN_USER_DIRECTORY"}), + }, + }, } topBuild := expandPathTemplate(defaultString(p.Config.TopData.Build, DefaultTopDataBuild), paths) top := EffectiveTopDataConfig{ @@ -145,17 +193,28 @@ func (p *Project) EffectiveConfig() EffectiveConfig { PackageHAK: defaultString(p.Config.TopData.PackageHAK, DefaultTopDataPackageHAK), PackageTLK: defaultString(p.Config.TopData.PackageTLK, DefaultTopDataPackageTLK), Wiki: TopDataWikiConfig{ - OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot), - PagesDir: defaultString(p.Config.TopData.Wiki.PagesDir, DefaultTopDataWikiPagesDir), - StateFile: defaultString(p.Config.TopData.Wiki.StateFile, DefaultTopDataWikiStateFile), + OutputRoot: defaultString(p.Config.TopData.Wiki.OutputRoot, DefaultTopDataWikiOutputRoot), + PagesDir: defaultString(p.Config.TopData.Wiki.PagesDir, DefaultTopDataWikiPagesDir), + StateFile: defaultString(p.Config.TopData.Wiki.StateFile, DefaultTopDataWikiStateFile), + ManagedNamespaces: defaultStringSlice(p.Config.TopData.Wiki.ManagedNamespaces, []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}), + DeployManifest: defaultString(p.Config.TopData.Wiki.DeployManifest, DefaultWikiDeployManifest), + DeployEditSummary: defaultString(p.Config.TopData.Wiki.DeployEditSummary, DefaultWikiDeployEditSummary), }, } + extract := p.Config.Extract + extract.Layout = defaultString(extract.Layout, DefaultExtractLayout) + extract.HAKDiscovery = defaultString(extract.HAKDiscovery, DefaultExtractHAKDiscovery) + if extract.CleanupStale == nil { + defaultCleanup := true + extract.CleanupStale = &defaultCleanup + } - return EffectiveConfig{ + effective := EffectiveConfig{ ConfigSource: p.ConfigSource, Module: p.Config.Module, Paths: paths, Outputs: outputs, + Build: p.Config.Build, Inventory: inventory, Scripts: scripts, Music: EffectiveMusicConfig{ @@ -166,19 +225,27 @@ func (p *Project) EffectiveConfig() EffectiveConfig { MaxStemLength: 16, }, TopData: top, - Extract: p.Config.Extract, + Extract: extract, Autogen: EffectiveAutogenConfig{ Producers: slices.Clone(p.Config.Autogen.Producers), Consumers: slices.Clone(p.Config.Autogen.Consumers), Cache: EffectiveAutogenCacheConfig{ - Root: expandPathTemplate(defaultString(p.Config.Autogen.Cache.Root, DefaultAutogenCacheRoot), paths), - MaxAge: DefaultAutogenCacheMaxAge.String(), - RefreshEnv: DefaultAutogenRefreshEnv, + Root: expandPathTemplate(defaultString(p.Config.Autogen.Cache.Root, DefaultAutogenCacheRoot), paths), + MaxAge: defaultString(p.Config.Autogen.Cache.MaxAge, DefaultAutogenCacheMaxAge.String()), + RefreshEnv: defaultString(p.Config.Autogen.Cache.RefreshEnv, DefaultAutogenRefreshEnv), + PartsManifestRefreshEnv: DefaultPartsManifestRefreshEnv, + }, + ReleaseSource: EffectiveAutogenReleaseSourceConfig{ + Provider: defaultString(p.Config.Autogen.ReleaseSource.Provider, DefaultAutogenReleaseProvider), + ServerURLEnv: defaultString(p.Config.Autogen.ReleaseSource.ServerURLEnv, DefaultAutogenServerURLEnv), + RepoEnv: defaultString(p.Config.Autogen.ReleaseSource.RepoEnv, DefaultAutogenRepoEnv), }, }, HAKs: slices.Clone(p.Config.HAKs), Provenance: provenance, } + effective.Overrides = activeOverrides(effective) + return effective } func (p *Project) EffectiveConfigJSON() ([]byte, error) { @@ -223,32 +290,44 @@ func lookupEffectiveValue(effective EffectiveConfig, key string) (any, bool) { func markMissingDefaults(provenance ConfigProvenance) { defaults := map[string]string{ - "paths.build": "build", - "paths.cache": ".cache", - "paths.tools": "tools", - "outputs.module_archive": DefaultModuleArchiveTemplate, - "outputs.hak_manifest": DefaultHAKManifest, - "outputs.hak_archive": DefaultHAKArchiveTemplate, - "inventory.source_extensions": "NWN source resource extensions", - "inventory.asset_extensions": "NWN asset resource extensions", - "inventory.source_json_pattern": DefaultSourceJSONPattern, - "scripts.source_dir": DefaultScriptsSourceDir, - "scripts.cache": DefaultScriptsCache, - "music.stage_root": DefaultMusicStageRoot, - "music.credits_root": DefaultCreditsRoot, - "music.credits_overlay": DefaultCreditsOverlayFile, - "music.max_stem_length": "16", - "topdata.build": DefaultTopDataBuild, - "topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir, - "topdata.compiled_tlk": DefaultTopDataCompiledTLK, - "topdata.package_hak": DefaultTopDataPackageHAK, - "topdata.package_tlk": DefaultTopDataPackageTLK, - "topdata.wiki.output_root": DefaultTopDataWikiOutputRoot, - "topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir, - "topdata.wiki.state_file": DefaultTopDataWikiStateFile, - "autogen.cache.root": DefaultAutogenCacheRoot, - "autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(), - "autogen.cache.refresh_env": DefaultAutogenRefreshEnv, + "paths.build": "build", + "paths.cache": ".cache", + "paths.tools": "tools", + "outputs.module_archive": DefaultModuleArchiveTemplate, + "outputs.hak_manifest": DefaultHAKManifest, + "outputs.hak_archive": DefaultHAKArchiveTemplate, + "inventory.source_extensions": "NWN source resource extensions", + "inventory.asset_extensions": "NWN asset resource extensions", + "inventory.source_json_pattern": DefaultSourceJSONPattern, + "scripts.source_dir": DefaultScriptsSourceDir, + "scripts.cache": DefaultScriptsCache, + "scripts.compiler.env.path": DefaultScriptCompilerEnvPath, + "scripts.compiler.search": "repo tools and PATH:nwn_script_comp", + "music.stage_root": DefaultMusicStageRoot, + "music.credits_root": DefaultCreditsRoot, + "music.credits_overlay": DefaultCreditsOverlayFile, + "music.max_stem_length": "16", + "topdata.build": DefaultTopDataBuild, + "topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir, + "topdata.compiled_tlk": DefaultTopDataCompiledTLK, + "topdata.package_hak": DefaultTopDataPackageHAK, + "topdata.package_tlk": DefaultTopDataPackageTLK, + "topdata.wiki.output_root": DefaultTopDataWikiOutputRoot, + "topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir, + "topdata.wiki.state_file": DefaultTopDataWikiStateFile, + "topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces", + "topdata.wiki.deploy_manifest": DefaultWikiDeployManifest, + "topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary, + "extract.layout": DefaultExtractLayout, + "extract.hak_discovery": DefaultExtractHAKDiscovery, + "extract.cleanup_stale": "true", + "autogen.cache.root": DefaultAutogenCacheRoot, + "autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(), + "autogen.cache.refresh_env": DefaultAutogenRefreshEnv, + "autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv, + "autogen.release_source.provider": DefaultAutogenReleaseProvider, + "autogen.release_source.server_url_env": DefaultAutogenServerURLEnv, + "autogen.release_source.repo_env": DefaultAutogenRepoEnv, } for key, detail := range defaults { if _, ok := provenance[key]; !ok { @@ -302,6 +381,75 @@ func expandPathTemplate(template string, paths EffectivePathConfig) string { return filepath.ToSlash(replacer.Replace(template)) } +func expandPathTemplates(templates []string, paths EffectivePathConfig) []string { + out := make([]string, 0, len(templates)) + for _, template := range templates { + out = append(out, expandPathTemplate(template, paths)) + } + return out +} + +func defaultScriptCompilerSearch() []string { + return []string{ + "{paths.tools}/script-compiler/nwn_script_comp", + "{paths.tools}/script-compiler/nwn_script_comp.exe", + "{paths.tools}/nwn_script_comp", + "{paths.tools}/nwn_script_comp.exe", + "PATH:nwn_script_comp", + "PATH:nwn_script_comp.exe", + } +} + +func (p *Project) ActiveOverrides() []ConfigOverride { + return activeOverrides(p.EffectiveConfig()) +} + +func activeOverrides(effective EffectiveConfig) []ConfigOverride { + envs := map[string]string{ + "build.keep_existing_haks": effectiveBuildKeepExistingEnv, + "scripts.compiler.path": effective.Scripts.Compiler.Env.Path, + "autogen.cache.refresh": effective.Autogen.Cache.RefreshEnv, + "autogen.cache.parts_manifest_refresh": effective.Autogen.Cache.PartsManifestRefreshEnv, + "autogen.release_source.server_url": effective.Autogen.ReleaseSource.ServerURLEnv, + "autogen.release_source.repo": effective.Autogen.ReleaseSource.RepoEnv, + "music.ffmpeg": "SOW_FFMPEG", + "music.ffprobe": "SOW_FFPROBE", + "wiki.endpoint": "NODEBB_API_ENDPOINT", + "wiki.token": "NODEBB_API_TOKEN", + "wiki.categories": "NODEBB_WIKI_CATEGORIES", + "release.version": "GITHUB_REF_NAME", + } + for _, key := range effective.Scripts.Compiler.Env.NWNRoot { + envs["scripts.compiler.env.nwn_root."+key] = key + } + for _, key := range effective.Scripts.Compiler.Env.NWNUserDirectory { + envs["scripts.compiler.env.nwn_user_directory."+key] = key + } + keys := make([]string, 0, len(envs)) + for key := range envs { + keys = append(keys, key) + } + slices.Sort(keys) + overrides := make([]ConfigOverride, 0) + for _, key := range keys { + env := envs[key] + if value := strings.TrimSpace(os.Getenv(env)); value != "" { + if isSensitiveOverrideKey(key) || isSensitiveOverrideKey(env) { + value = "" + } + overrides = append(overrides, ConfigOverride{Key: key, Value: value, Source: "env:" + env}) + } + } + return overrides +} + +const effectiveBuildKeepExistingEnv = "SOW_BUILD_HAKS_KEEP_EXISTING" + +func isSensitiveOverrideKey(key string) bool { + lower := strings.ToLower(key) + return strings.Contains(lower, "token") || strings.Contains(lower, "password") +} + func (o EffectiveOutputConfig) ModuleArchiveName(module ModuleConfig) string { return strings.NewReplacer("{module.resref}", strings.TrimSpace(module.ResRef)).Replace(o.ModuleArchive) } diff --git a/internal/project/project.go b/internal/project/project.go index ef54e21..d7ed92b 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -11,6 +11,7 @@ import ( "path/filepath" "slices" "strings" + "time" "gopkg.in/yaml.v3" ) @@ -51,6 +52,7 @@ type Config struct { Module ModuleConfig `json:"module" yaml:"module"` Paths PathConfig `json:"paths" yaml:"paths"` Outputs OutputConfig `json:"outputs" yaml:"outputs"` + Build BuildConfig `json:"build" yaml:"build"` HAKs []HAKConfig `json:"haks" yaml:"haks"` Inventory InventoryConfig `json:"inventory" yaml:"inventory"` Scripts ScriptsConfig `json:"scripts" yaml:"scripts"` @@ -81,6 +83,10 @@ type OutputConfig struct { HAKArchive string `json:"hak_archive" yaml:"hak_archive"` } +type BuildConfig struct { + KeepExistingHAKs bool `json:"keep_existing_haks,omitempty" yaml:"keep_existing_haks,omitempty"` +} + type InventoryConfig struct { SourceExtensions []string `json:"source_extensions" yaml:"source_extensions"` AssetExtensions []string `json:"asset_extensions" yaml:"asset_extensions"` @@ -88,8 +94,21 @@ type InventoryConfig struct { } type ScriptsConfig struct { - SourceDir string `json:"source_dir" yaml:"source_dir"` - Cache string `json:"cache" yaml:"cache"` + SourceDir string `json:"source_dir" yaml:"source_dir"` + Cache string `json:"cache" yaml:"cache"` + Compiler ScriptCompilerConfig `json:"compiler" yaml:"compiler"` +} + +type ScriptCompilerConfig struct { + Path string `json:"path" yaml:"path"` + Search []string `json:"search" yaml:"search"` + Env ScriptCompilerEnvConfig `json:"env" yaml:"env"` +} + +type ScriptCompilerEnvConfig struct { + Path string `json:"path" yaml:"path"` + NWNRoot []string `json:"nwn_root" yaml:"nwn_root"` + NWNUserDirectory []string `json:"nwn_user_directory" yaml:"nwn_user_directory"` } type HAKConfig struct { @@ -118,24 +137,39 @@ type TopDataConfig struct { } type TopDataWikiConfig struct { - OutputRoot string `json:"output_root" yaml:"output_root"` - PagesDir string `json:"pages_dir" yaml:"pages_dir"` - StateFile string `json:"state_file" yaml:"state_file"` + OutputRoot string `json:"output_root" yaml:"output_root"` + PagesDir string `json:"pages_dir" yaml:"pages_dir"` + StateFile string `json:"state_file" yaml:"state_file"` + ManagedNamespaces []string `json:"managed_namespaces" yaml:"managed_namespaces"` + DeployManifest string `json:"deploy_manifest" yaml:"deploy_manifest"` + DeployEditSummary string `json:"deploy_edit_summary" yaml:"deploy_edit_summary"` } type ExtractConfig struct { IgnoreExtensions []string `json:"ignore_extensions" yaml:"ignore_extensions"` + Layout string `json:"layout" yaml:"layout"` + HAKDiscovery string `json:"hak_discovery" yaml:"hak_discovery"` + CleanupStale *bool `json:"cleanup_stale,omitempty" yaml:"cleanup_stale,omitempty"` DeleteModuleArchiveAfterSuccess bool `json:"delete_module_archive_after_success,omitempty" yaml:"delete_module_archive_after_success,omitempty"` } type AutogenConfig struct { - Producers []AutogenProducerConfig `json:"producers" yaml:"producers"` - Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"` - Cache AutogenCacheConfig `json:"cache" yaml:"cache"` + Producers []AutogenProducerConfig `json:"producers" yaml:"producers"` + Consumers []AutogenConsumerConfig `json:"consumers" yaml:"consumers"` + Cache AutogenCacheConfig `json:"cache" yaml:"cache"` + ReleaseSource AutogenReleaseSourceConfig `json:"release_source" yaml:"release_source"` } type AutogenCacheConfig struct { - Root string `json:"root" yaml:"root"` + Root string `json:"root" yaml:"root"` + MaxAge string `json:"max_age" yaml:"max_age"` + RefreshEnv string `json:"refresh_env" yaml:"refresh_env"` +} + +type AutogenReleaseSourceConfig struct { + Provider string `json:"provider" yaml:"provider"` + ServerURLEnv string `json:"server_url_env" yaml:"server_url_env"` + RepoEnv string `json:"repo_env" yaml:"repo_env"` } type AutogenProducerConfig struct { @@ -329,26 +363,37 @@ func (p *Project) ValidateLayout() error { failures = append(failures, errors.New("at least one of paths.source, paths.assets, or topdata.source is required")) } for field, value := range map[string]string{ - "paths.source": p.Config.Paths.Source, - "paths.assets": p.Config.Paths.Assets, - "paths.build": p.Config.Paths.Build, - "paths.cache": p.Config.Paths.Cache, - "paths.tools": p.Config.Paths.Tools, - "outputs.module_archive": p.Config.Outputs.ModuleArchive, - "outputs.hak_manifest": p.Config.Outputs.HAKManifest, - "outputs.hak_archive": p.Config.Outputs.HAKArchive, - "scripts.source_dir": p.Config.Scripts.SourceDir, - "scripts.cache": p.Config.Scripts.Cache, - "topdata.source": p.Config.TopData.Source, - "topdata.build": p.Config.TopData.Build, - "topdata.reference_builder": p.Config.TopData.ReferenceBuilder, - "topdata.assets": p.Config.TopData.Assets, - "topdata.compiled_2da_dir": p.Config.TopData.Compiled2DADir, - "topdata.compiled_tlk": p.Config.TopData.CompiledTLK, - "topdata.wiki.output_root": p.Config.TopData.Wiki.OutputRoot, - "topdata.wiki.pages_dir": p.Config.TopData.Wiki.PagesDir, - "topdata.wiki.state_file": p.Config.TopData.Wiki.StateFile, - "autogen.cache.root": p.Config.Autogen.Cache.Root, + "paths.source": p.Config.Paths.Source, + "paths.assets": p.Config.Paths.Assets, + "paths.build": p.Config.Paths.Build, + "paths.cache": p.Config.Paths.Cache, + "paths.tools": p.Config.Paths.Tools, + "outputs.module_archive": p.Config.Outputs.ModuleArchive, + "outputs.hak_manifest": p.Config.Outputs.HAKManifest, + "outputs.hak_archive": p.Config.Outputs.HAKArchive, + "scripts.source_dir": p.Config.Scripts.SourceDir, + "scripts.cache": p.Config.Scripts.Cache, + "scripts.compiler.path": p.Config.Scripts.Compiler.Path, + "scripts.compiler.env.path": p.Config.Scripts.Compiler.Env.Path, + "topdata.source": p.Config.TopData.Source, + "topdata.build": p.Config.TopData.Build, + "topdata.reference_builder": p.Config.TopData.ReferenceBuilder, + "topdata.assets": p.Config.TopData.Assets, + "topdata.compiled_2da_dir": p.Config.TopData.Compiled2DADir, + "topdata.compiled_tlk": p.Config.TopData.CompiledTLK, + "topdata.wiki.output_root": p.Config.TopData.Wiki.OutputRoot, + "topdata.wiki.pages_dir": p.Config.TopData.Wiki.PagesDir, + "topdata.wiki.state_file": p.Config.TopData.Wiki.StateFile, + "topdata.wiki.deploy_manifest": p.Config.TopData.Wiki.DeployManifest, + "topdata.wiki.deploy_edit_summary": p.Config.TopData.Wiki.DeployEditSummary, + "extract.layout": p.Config.Extract.Layout, + "extract.hak_discovery": p.Config.Extract.HAKDiscovery, + "autogen.cache.root": p.Config.Autogen.Cache.Root, + "autogen.cache.max_age": p.Config.Autogen.Cache.MaxAge, + "autogen.cache.refresh_env": p.Config.Autogen.Cache.RefreshEnv, + "autogen.release_source.provider": p.Config.Autogen.ReleaseSource.Provider, + "autogen.release_source.server_url_env": p.Config.Autogen.ReleaseSource.ServerURLEnv, + "autogen.release_source.repo_env": p.Config.Autogen.ReleaseSource.RepoEnv, } { if strings.Contains(value, "\x00") { failures = append(failures, fmt.Errorf("%s must not contain NUL bytes", field)) @@ -360,11 +405,13 @@ func (p *Project) ValidateLayout() error { failures = append(failures, validateRelativePath("outputs.hak_manifest", effective.Outputs.HAKManifest)...) failures = append(failures, validateRelativePath("outputs.hak_archive", effective.Outputs.HAKArchive)...) failures = append(failures, validateRelativePath("scripts.cache", effective.Scripts.Cache)...) + failures = append(failures, validateRelativePath("scripts.source_dir", effective.Scripts.SourceDir)...) failures = append(failures, validateRelativePath("topdata.compiled_2da_dir", effective.TopData.Compiled2DADir)...) failures = append(failures, validateRelativePath("topdata.compiled_tlk", effective.TopData.CompiledTLK)...) failures = append(failures, validateRelativePath("topdata.wiki.output_root", effective.TopData.Wiki.OutputRoot)...) failures = append(failures, validateRelativePath("topdata.wiki.pages_dir", effective.TopData.Wiki.PagesDir)...) failures = append(failures, validateRelativePath("topdata.wiki.state_file", effective.TopData.Wiki.StateFile)...) + failures = append(failures, validateRelativePath("topdata.wiki.deploy_manifest", effective.TopData.Wiki.DeployManifest)...) failures = append(failures, validateRelativePath("autogen.cache.root", effective.Autogen.Cache.Root)...) if err := validateOutputFileName("outputs.module_archive", filepath.Base(effective.Outputs.ModuleArchive), ".mod"); err != nil { failures = append(failures, err) @@ -375,6 +422,25 @@ func (p *Project) ValidateLayout() error { if err := validateOutputFileName("outputs.hak_archive", filepath.Base(effective.Outputs.HAKArchive), ".hak"); err != nil { failures = append(failures, err) } + if err := validateOutputFileName("topdata.wiki.deploy_manifest", filepath.Base(effective.TopData.Wiki.DeployManifest), ".json"); err != nil { + failures = append(failures, err) + } + switch effective.Extract.Layout { + case "nwn_canonical_json": + default: + failures = append(failures, fmt.Errorf("extract.layout %q is not supported", effective.Extract.Layout)) + } + switch effective.Extract.HAKDiscovery { + case "build_glob", "configured_haks": + default: + failures = append(failures, fmt.Errorf("extract.hak_discovery %q is not supported", effective.Extract.HAKDiscovery)) + } + if _, err := time.ParseDuration(effective.Autogen.Cache.MaxAge); err != nil { + failures = append(failures, fmt.Errorf("autogen.cache.max_age must be a duration: %w", err)) + } + if effective.Autogen.ReleaseSource.Provider != "gitea" { + failures = append(failures, fmt.Errorf("autogen.release_source.provider %q is not supported", effective.Autogen.ReleaseSource.Provider)) + } if p.HasTopData() { if err := validateOutputFileName("topdata.package_hak", p.TopDataPackageHAKName(), ".hak"); err != nil { failures = append(failures, err) @@ -685,6 +751,46 @@ func (p *Project) ScriptCacheDir() string { return p.rootPath(p.EffectiveConfig().Scripts.Cache) } +func (p *Project) ScriptCompilerPath() string { + path := strings.TrimSpace(p.EffectiveConfig().Scripts.Compiler.Path) + if path == "" { + return "" + } + if filepath.IsAbs(path) { + return path + } + return filepath.Join(p.Root, filepath.FromSlash(path)) +} + +func (p *Project) ScriptCompilerSearchPaths() []string { + search := p.EffectiveConfig().Scripts.Compiler.Search + out := make([]string, 0, len(search)) + for _, candidate := range search { + candidate = strings.TrimSpace(candidate) + if candidate == "" { + continue + } + if strings.HasPrefix(candidate, "PATH:") || filepath.IsAbs(candidate) { + out = append(out, candidate) + continue + } + out = append(out, filepath.Join(p.Root, filepath.FromSlash(candidate))) + } + return out +} + +func (p *Project) ScriptCompilerPathEnv() string { + return p.EffectiveConfig().Scripts.Compiler.Env.Path +} + +func (p *Project) ScriptCompilerNWNRootEnvKeys() []string { + return slices.Clone(p.EffectiveConfig().Scripts.Compiler.Env.NWNRoot) +} + +func (p *Project) ScriptCompilerNWNUserDirectoryEnvKeys() []string { + return slices.Clone(p.EffectiveConfig().Scripts.Compiler.Env.NWNUserDirectory) +} + func (p *Project) MusicStageDir() string { return p.rootPath(p.EffectiveConfig().Music.StageRoot) } @@ -701,6 +807,30 @@ func (p *Project) AutogenCachePath(name string) string { return filepath.Join(p.AutogenCacheDir(), filepath.FromSlash(strings.TrimSpace(name))) } +func (p *Project) AutogenCacheMaxAge() time.Duration { + duration, err := time.ParseDuration(p.EffectiveConfig().Autogen.Cache.MaxAge) + if err != nil { + return DefaultAutogenCacheMaxAge + } + return duration +} + +func (p *Project) AutogenRefreshEnv() string { + return p.EffectiveConfig().Autogen.Cache.RefreshEnv +} + +func (p *Project) PartsManifestRefreshEnv() string { + return p.EffectiveConfig().Autogen.Cache.PartsManifestRefreshEnv +} + +func (p *Project) AssetsServerURLEnv() string { + return p.EffectiveConfig().Autogen.ReleaseSource.ServerURLEnv +} + +func (p *Project) AssetsRepoEnv() string { + return p.EffectiveConfig().Autogen.ReleaseSource.RepoEnv +} + func (p *Project) rootPath(path string) string { if strings.TrimSpace(path) == "" { return p.Root diff --git a/internal/project/project_test.go b/internal/project/project_test.go index fe10d89..3fb6a5f 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -136,6 +136,116 @@ autogen: } } +func TestEffectiveConfigHonorsConfiguredCompilerExtractWikiAndAutogenPolicy(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + tools: custom-tools +scripts: + source_dir: module-scripts + compiler: + path: bin/compiler + search: + - "{paths.tools}/compiler" + env: + path: TEST_COMPILER + nwn_root: + - TEST_NWN_ROOT + nwn_user_directory: + - TEST_NWN_USER +extract: + layout: nwn_canonical_json + hak_discovery: configured_haks + cleanup_stale: false +topdata: + wiki: + managed_namespaces: + - skills + deploy_manifest: wiki-manifest.json + deploy_edit_summary: Custom summary +autogen: + cache: + root: "{paths.cache}/released" + max_age: 30m + refresh_env: TEST_AUTOGEN_REFRESH + release_source: + provider: gitea + server_url_env: TEST_ASSETS_SERVER + repo_env: TEST_ASSETS_REPO +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + effective := proj.EffectiveConfig() + if got, want := proj.ScriptSourceDir(), filepath.Join(root, "src", "module-scripts"); got != want { + t.Fatalf("expected script source dir %s, got %s", want, got) + } + if got, want := proj.ScriptCompilerPath(), filepath.Join(root, "bin", "compiler"); got != want { + t.Fatalf("expected compiler path %s, got %s", want, got) + } + if got, want := effective.Scripts.Compiler.Search[0], "custom-tools/compiler"; got != want { + t.Fatalf("expected compiler search %q, got %q", want, got) + } + if got, want := effective.Scripts.Compiler.Env.Path, "TEST_COMPILER"; got != want { + t.Fatalf("expected compiler env %q, got %q", want, got) + } + if got, want := effective.Extract.HAKDiscovery, "configured_haks"; got != want { + t.Fatalf("expected extract hak discovery %q, got %q", want, got) + } + if effective.Extract.CleanupStale == nil || *effective.Extract.CleanupStale { + t.Fatalf("expected cleanup_stale false, got %#v", effective.Extract.CleanupStale) + } + if got, want := effective.TopData.Wiki.DeployManifest, "wiki-manifest.json"; got != want { + t.Fatalf("expected wiki deploy manifest %q, got %q", want, got) + } + if got, want := effective.Autogen.Cache.MaxAge, "30m"; got != want { + t.Fatalf("expected autogen cache max age %q, got %q", want, got) + } + if got, want := proj.AutogenRefreshEnv(), "TEST_AUTOGEN_REFRESH"; got != want { + t.Fatalf("expected autogen refresh env %q, got %q", want, got) + } +} + +func TestActiveOverridesAreVisibleAndSensitiveValuesAreMasked(t *testing.T) { + root := t.TempDir() + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src +`) + t.Setenv("SOW_BUILD_HAKS_KEEP_EXISTING", "1") + t.Setenv("NODEBB_API_TOKEN", "secret-token") + + proj, err := Load(root) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + overrides := proj.ActiveOverrides() + var sawKeepExisting, sawMaskedToken bool + for _, override := range overrides { + if override.Key == "build.keep_existing_haks" && override.Value == "1" { + sawKeepExisting = true + } + if override.Key == "wiki.token" && override.Value == "" { + sawMaskedToken = true + } + } + if !sawKeepExisting { + t.Fatalf("expected build keep existing override in %#v", overrides) + } + if !sawMaskedToken { + t.Fatalf("expected masked wiki token override in %#v", overrides) + } +} + func TestEffectiveConfigJSONIsDeterministic(t *testing.T) { root := t.TempDir() writeProjectFile(t, filepath.Join(root, ConfigFile), ` diff --git a/internal/topdata/autogen.go b/internal/topdata/autogen.go index 9fed478..d177032 100644 --- a/internal/topdata/autogen.go +++ b/internal/topdata/autogen.go @@ -329,8 +329,8 @@ func deriveAutogenManifestEntry(rel string, derive project.AutogenDeriveConfig) func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project.AutogenConsumerConfig, progress func(string)) ([]autogenManifestEntry, error) { cachePath := p.AutogenCachePath(consumer.Manifest.CacheName) - if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) == "" { - manifest, fresh, err := readFreshAutogenManifestCache(cachePath, autogenManifestCacheMaxAge) + if strings.TrimSpace(os.Getenv(p.AutogenRefreshEnv())) == "" { + manifest, fresh, err := readFreshAutogenManifestCache(cachePath, p.AutogenCacheMaxAge()) if err != nil { if progress != nil { progress(fmt.Sprintf("Ignoring cached autogen manifest %s: %v", consumer.Manifest.CacheName, err)) @@ -352,7 +352,7 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project. } } - spec, err := deriveSowAssetsRepoSpec(p.Root) + spec, err := deriveSowAssetsRepoSpec(p) if err != nil { return nil, err } @@ -383,7 +383,7 @@ func resolveReleasedAutogenManifestEntries(p *project.Project, consumer project. } func releasedAutogenManifestCacheCurrent(p *project.Project, consumer project.AutogenConsumerConfig, cachePath string, manifest *autogenManifest) (bool, time.Time, error) { - spec, err := deriveSowAssetsRepoSpec(p.Root) + spec, err := deriveSowAssetsRepoSpec(p) if err != nil { return false, time.Time{}, err } @@ -409,7 +409,7 @@ func localAutogenManifestComparableTime(cachePath string, manifest *autogenManif } func newestReleasedAutogenManifestInput(p *project.Project, consumer project.AutogenConsumerConfig, now time.Time, cachePath string) (time.Time, string, error) { - if strings.TrimSpace(os.Getenv("SOW_AUTOGEN_MANIFEST_REFRESH")) != "" { + if strings.TrimSpace(os.Getenv(p.AutogenRefreshEnv())) != "" { return now, cachePath, nil } diff --git a/internal/topdata/native.go b/internal/topdata/native.go index 71c4b27..39ded9a 100644 --- a/internal/topdata/native.go +++ b/internal/topdata/native.go @@ -360,7 +360,7 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress } } - filesTLK, err := compiler.finish(outputTLK) + filesTLK, err := compiler.finish(outputTLK, filepath.Base(p.TopDataCompiledTLKPath())) if err != nil { return BuildResult{}, err } diff --git a/internal/topdata/parts_manifest.go b/internal/topdata/parts_manifest.go index f8a1a17..0527a7c 100644 --- a/internal/topdata/parts_manifest.go +++ b/internal/topdata/parts_manifest.go @@ -46,7 +46,7 @@ func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (ma progress = func(string) {} } cachePath := p.AutogenCachePath(partsManifestCacheFileName) - if strings.TrimSpace(os.Getenv("SOW_PARTS_MANIFEST_REFRESH")) == "" { + if strings.TrimSpace(os.Getenv(p.PartsManifestRefreshEnv())) == "" { manifest, fresh, err := readFreshPartsManifestCache(cachePath, partsManifestCacheMaxAge) if err != nil { progress(fmt.Sprintf("Ignoring cached parts manifest: %v", err)) @@ -55,7 +55,7 @@ func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (ma return inventoryFromPartsManifest(manifest), nil } } - spec, err := deriveSowAssetsRepoSpec(p.Root) + spec, err := deriveSowAssetsRepoSpec(p) if err != nil { return nil, err } @@ -74,10 +74,10 @@ func resolveReleasedPartsManifest(p *project.Project, progress func(string)) (ma return inventoryFromPartsManifest(manifest), nil } -func deriveSowAssetsRepoSpec(root string) (giteaRepoSpec, error) { - serverOverride := strings.TrimSpace(os.Getenv("SOW_ASSETS_SERVER_URL")) - repoOverride := strings.TrimSpace(os.Getenv("SOW_ASSETS_REPO")) - remote, err := gitOutput(root, "remote", "get-url", "origin") +func deriveSowAssetsRepoSpec(p *project.Project) (giteaRepoSpec, error) { + serverOverride := strings.TrimSpace(os.Getenv(p.AssetsServerURLEnv())) + repoOverride := strings.TrimSpace(os.Getenv(p.AssetsRepoEnv())) + remote, err := gitOutput(p.Root, "remote", "get-url", "origin") spec := giteaRepoSpec{BaseURL: defaultGiteaBaseURL, Owner: defaultSowAssetsRepoOwner, Repo: defaultSowAssetsRepoName} if err == nil && strings.TrimSpace(remote) != "" { if derived, ok := parseRepoSpec(strings.TrimSpace(remote)); ok { diff --git a/internal/topdata/tlk_native.go b/internal/topdata/tlk_native.go index 4c1f5df..01da36f 100644 --- a/internal/topdata/tlk_native.go +++ b/internal/topdata/tlk_native.go @@ -356,7 +356,7 @@ func (c *tlkCompiler) allocateID() int { return id } -func (c *tlkCompiler) finish(outputDir string) (int, error) { +func (c *tlkCompiler) finish(outputDir, tlkName string) (int, error) { if err := os.MkdirAll(outputDir, 0o755); err != nil { return 0, fmt.Errorf("create tlk output dir: %w", err) } @@ -388,7 +388,10 @@ func (c *tlkCompiler) finish(outputDir string) (int, error) { entries[id] = c.active[key] } - if err := writeTLKBinary(filepath.Join(outputDir, defaultTLKName), entries, c.language); err != nil { + if strings.TrimSpace(tlkName) == "" { + tlkName = defaultTLKName + } + if err := writeTLKBinary(filepath.Join(outputDir, tlkName), entries, c.language); err != nil { return 0, err } if maxID < 0 { diff --git a/internal/topdata/wiki_deploy.go b/internal/topdata/wiki_deploy.go index eb39298..10a7e62 100644 --- a/internal/topdata/wiki_deploy.go +++ b/internal/topdata/wiki_deploy.go @@ -20,16 +20,12 @@ import ( ) const ( - defaultDeployManifestName = ".wiki_deploy_manifest.json" - defaultEditSummary = "Auto-generated from native builder" - managedStartMarker = "[//]: # (sow-topdata-wiki:managed:start)" - managedEndMarker = "[//]: # (sow-topdata-wiki:managed:end)" - legacyManagedStartMarker = "" - legacyManagedEndMarker = "" + managedStartMarker = "[//]: # (sow-topdata-wiki:managed:start)" + managedEndMarker = "[//]: # (sow-topdata-wiki:managed:end)" + legacyManagedStartMarker = "" + legacyManagedEndMarker = "" ) -var defaultManagedNamespaces = []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"} - type DeployWikiOptions struct { SourceDir string Endpoint string @@ -107,12 +103,12 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress namespaces := opts.Namespaces if len(namespaces) == 0 { - namespaces = defaultManagedNamespaces + namespaces = p.EffectiveConfig().TopData.Wiki.ManagedNamespaces } manifestPath := opts.ManifestPath if manifestPath == "" { - manifestPath = filepath.Join(filepath.Dir(opts.SourceDir), defaultDeployManifestName) + manifestPath = filepath.Join(filepath.Dir(opts.SourceDir), p.EffectiveConfig().TopData.Wiki.DeployManifest) } pages, err := collectLocalPages(opts.SourceDir, namespaces) @@ -137,7 +133,7 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress return result, errors.New("remote managed wiki content drifted; rerun with --force to overwrite") } - summary := defaultEditSummary + summary := p.EffectiveConfig().TopData.Wiki.DeployEditSummary if opts.Version != "" { summary = fmt.Sprintf("%s (%s)", summary, opts.Version) } diff --git a/internal/topdata/wiki_native.go b/internal/topdata/wiki_native.go index 379321a..c76e303 100644 --- a/internal/topdata/wiki_native.go +++ b/internal/topdata/wiki_native.go @@ -30,9 +30,8 @@ const ( ) var ( - wikiManagedNamespaces = []string{"classes", "feat", "itemtypes", "races", "skills", "spells"} - wikiStatusPriority = map[string]int{wikiStatusVanilla: 0, wikiStatusModified: 1, wikiStatusNew: 2} - wikiStatusLabels = map[string]string{wikiStatusNew: "New", wikiStatusModified: "Modified", wikiStatusVanilla: "Vanilla"} + wikiStatusPriority = map[string]int{wikiStatusVanilla: 0, wikiStatusModified: 1, wikiStatusNew: 2} + wikiStatusLabels = map[string]string{wikiStatusNew: "New", wikiStatusModified: "Modified", wikiStatusVanilla: "Vanilla"} ) //go:embed wiki_templates/*.txt @@ -138,7 +137,7 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres if err := generateEntityPages(outputDir, ctx, writePage); err != nil { return wikiResult{}, err } - if err := generateStatusPages(outputDir, pageStatuses, pageTitles); err != nil { + if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces); err != nil { return wikiResult{}, err } @@ -1237,12 +1236,12 @@ func wikiStatusNoun(category string) string { } } -func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string) error { - summaries := buildWikiNamespaceSummaries(pageStatuses) - if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.md"), renderWikiStatusReportPage(summaries)); err != nil { +func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string, namespaces []string) error { + summaries := buildWikiNamespaceSummaries(pageStatuses, namespaces) + if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.md"), renderWikiStatusReportPage(summaries, namespaces)); err != nil { return err } - for _, namespace := range wikiManagedNamespaces { + for _, namespace := range namespaces { summary := summaries[namespace] if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".md"), renderWikiNamespaceFragment(namespace, summary)); err != nil { return err @@ -1271,9 +1270,9 @@ func writeWikiFile(path, content string) error { return os.WriteFile(path, []byte(ensureTrailingNewline(content)), 0o644) } -func buildWikiNamespaceSummaries(pageStatuses map[string]string) map[string]map[string]any { +func buildWikiNamespaceSummaries(pageStatuses map[string]string, namespaces []string) map[string]map[string]any { summaries := map[string]map[string]any{} - for _, namespace := range wikiManagedNamespaces { + for _, namespace := range namespaces { statuses := []string{} for pageID, status := range pageStatuses { if strings.SplitN(pageID, ":", 2)[0] == namespace { @@ -1325,9 +1324,9 @@ func renderWikiNamespaceFragment(namespace string, summary map[string]any) strin }, "\n") } -func renderWikiStatusReportPage(summaries map[string]map[string]any) string { +func renderWikiStatusReportPage(summaries map[string]map[string]any, namespaces []string) string { lines := []string{"====== Wiki Status ======", "", "This page is auto-generated from builder source provenance.", ""} - for _, namespace := range wikiManagedNamespaces { + for _, namespace := range namespaces { summary := summaries[namespace] lines = append(lines, "===== "+namespaceTitle(namespace)+" =====", "") lines = append(lines, fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]))