ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Wiki pipeline (#3)

Browse Source 
Implemented the wiki pipeline to a shippable local state across `toolkit/`, `module/`, and the NodeBB wiki plugin.

**Scope Audited**
- Toolkit config/project loading, wiki renderer, deployer, app command wiring.
- Module `nwn-tool.yaml`, wiki source/docs, release script, topdata docs.
- Plugin sanitizer/API storage contract docs and sanitizer tests.

**Changes Made**
- Added topdata-owned wiki declarations under `module/topdata/wiki/`.
- Added `topdata.wiki.*` config fields, validation, effective config output.
- Switched generated wiki pages to `.html` with HTML comment managed/manual markers.
- Added `page-index.json` generation and deterministic metadata.
- Added manual-section merge preservation and stale page `report`/`archive` handling.
- Added namespace `category_env` loading from `topdata/wiki/namespaces.yaml`, while keeping `--category`/`NODEBB_WIKI_CATEGORIES` overrides.
- Updated release deployment to keep dry-run first and make stale cleanup opt-in via `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY`.
- Updated plugin sanitizer to preserve only `sow-topdata-wiki` comments.

**Configuration/Schema Impact**
- `module/nwn-tool.yaml` now declares wiki source, renderer, link strategy, templates, manual sections, managed marker policy, and stale policy.
- Deploy manifest entries now support stale/archive metadata: `stale`, `last_seen_hash`, `archived_hash`, `title`, `namespace`, `tid`, `pid`, `cid`.

**Compatibility Impact**
- Legacy Markdown managed markers are still readable for migration.
- Generated output is now HTML, not Markdown.
- Existing mapped pages update by `pid`; creates still require explicit `--create`.

**Tests Added/Updated**
- Go tests for wiki config validation, HTML rendering, page index, manual merge, stale report/archive, app summary output.
- Plugin sanitizer test for generated topdata bot HTML markers/subset.

**Validation Performed**
- `go test ./internal/project ./internal/topdata ./internal/app` passed.
- `./build-tool.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./validate-topdata.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force` passed, generating 1345 entity pages.
- `deploy-wiki --dry-run --create` with dummy endpoint/token and namespace CID env vars passed locally: 1377 planned creates, 0 stale/drift.
- `node tests/wiki-html-sanitizer.test.js` passed.

**Remaining Risks**
- Full plugin `npm test` is blocked by an existing unrelated drawer CSS contract failure in `tests/wiki-article-drawers.test.js`.
- Controlled live migration was not performed because it requires a non-production NodeBB namespace and credentials.
- Direct `PUT /api/v3/posts/{pid}` save-filter behavior is documented as needing live confirmation against the deployed NodeBB/plugin stack.

Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/3
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
This commit is contained in:
archvillainette
2026-05-15 09:09:37 +02:00
committed by archvillainette
parent d767ec0a25
commit 9b2084344d
9 changed files with 1030 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/project/effective.go
+115 -81
View File
@@ -13,41 +13,51 @@ import (
)
const (
DefaultBuildPath = "build"
DefaultCachePath = ".cache"
DefaultToolsPath = "tools"
DefaultModuleArchiveTemplate = "{module.resref}.mod"
DefaultHAKManifest = "haks.json"
DefaultHAKArchiveTemplate = "{hak.name}.hak"
DefaultSourceJSONPattern = "{resref}.{extension}.json"
DefaultValidationProfile = "nwn_module"
DefaultScriptsSourceDir = "scripts"
DefaultScriptsCache = "{paths.cache}/ncs"
DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER"
DefaultMusicStageRoot = "{paths.cache}/music"
DefaultCreditsRoot = "{paths.cache}/credits"
DefaultCreditsOverlayFile = "CREDITS.md"
DefaultMusicNamingScheme = "nwn_bmu"
DefaultMusicOutputExtension = ".bmu"
DefaultTopDataBuild = "{paths.build}/topdata"
DefaultTopDataCompiled2DADir = "2da"
DefaultTopDataCompiledTLK = "sow_tlk.tlk"
DefaultTopDataWikiOutputRoot = "wiki"
DefaultTopDataWikiPagesDir = "pages"
DefaultTopDataWikiStateFile = "state.json"
DefaultWikiDeployManifest = ".wiki_deploy_manifest.json"
DefaultWikiDeployEditSummary = "Auto-generated from native builder"
DefaultTopDataPackageHAK = "sow_top.hak"
DefaultTopDataPackageTLK = "sow_tlk.tlk"
DefaultAutogenCacheRoot = "{paths.cache}"
DefaultAutogenCacheMaxAge = time.Hour
DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH"
DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH"
DefaultAutogenReleaseProvider = "gitea"
DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL"
DefaultAutogenRepoEnv = "SOW_ASSETS_REPO"
DefaultExtractLayout = "nwn_canonical_json"
DefaultExtractHAKDiscovery = "build_glob"
DefaultBuildPath = "build"
DefaultCachePath = ".cache"
DefaultToolsPath = "tools"
DefaultModuleArchiveTemplate = "{module.resref}.mod"
DefaultHAKManifest = "haks.json"
DefaultHAKArchiveTemplate = "{hak.name}.hak"
DefaultSourceJSONPattern = "{resref}.{extension}.json"
DefaultValidationProfile = "nwn_module"
DefaultScriptsSourceDir = "scripts"
DefaultScriptsCache = "{paths.cache}/ncs"
DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER"
DefaultMusicStageRoot = "{paths.cache}/music"
DefaultCreditsRoot = "{paths.cache}/credits"
DefaultCreditsOverlayFile = "CREDITS.md"
DefaultMusicNamingScheme = "nwn_bmu"
DefaultMusicOutputExtension = ".bmu"
DefaultTopDataBuild = "{paths.build}/topdata"
DefaultTopDataCompiled2DADir = "2da"
DefaultTopDataCompiledTLK = "sow_tlk.tlk"
DefaultTopDataWikiOutputRoot = "wiki"
DefaultTopDataWikiPagesDir = "pages"
DefaultTopDataWikiStateFile = "state.json"
DefaultTopDataWikiSource = "topdata/wiki"
DefaultTopDataWikiRenderer = "nodebb_tiptap_html"
DefaultTopDataWikiLinkStrategy = "preserve_westgate_wiki_links"
DefaultTopDataWikiNamespacesFile = "namespaces.yaml"
DefaultTopDataWikiTemplatesDir = "templates"
DefaultTopDataWikiManualSectionsDir = "manual-sections"
DefaultTopDataWikiStaleDefault = "report"
DefaultTopDataWikiStaleLiveCleanup = "archive"
DefaultTopDataWikiMarkerFormat = "html_comments"
DefaultTopDataWikiPageMarkerPrefix = "sow-topdata-wiki"
DefaultWikiDeployManifest = ".wiki_deploy_manifest.json"
DefaultWikiDeployEditSummary = "Auto-generated from native builder"
DefaultTopDataPackageHAK = "sow_top.hak"
DefaultTopDataPackageTLK = "sow_tlk.tlk"
DefaultAutogenCacheRoot = "{paths.cache}"
DefaultAutogenCacheMaxAge = time.Hour
DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH"
DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH"
DefaultAutogenReleaseProvider = "gitea"
DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL"
DefaultAutogenRepoEnv = "SOW_ASSETS_REPO"
DefaultExtractLayout = "nwn_canonical_json"
DefaultExtractHAKDiscovery = "build_glob"
)
type ConfigProvenance map[string]ConfigValueProvenance
@@ -228,6 +238,20 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
ManagedNamespaces: defaultStringSlice(p.Config.TopData.Wiki.ManagedNamespaces, []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}),
DeployManifest: defaultString(p.Config.TopData.Wiki.DeployManifest, DefaultWikiDeployManifest),
DeployEditSummary: defaultString(p.Config.TopData.Wiki.DeployEditSummary, DefaultWikiDeployEditSummary),
Source: defaultString(p.Config.TopData.Wiki.Source, DefaultTopDataWikiSource),
Renderer: defaultString(p.Config.TopData.Wiki.Renderer, DefaultTopDataWikiRenderer),
LinkStrategy: defaultString(p.Config.TopData.Wiki.LinkStrategy, DefaultTopDataWikiLinkStrategy),
NamespacesFile: defaultString(p.Config.TopData.Wiki.NamespacesFile, DefaultTopDataWikiNamespacesFile),
TemplatesDir: defaultString(p.Config.TopData.Wiki.TemplatesDir, DefaultTopDataWikiTemplatesDir),
ManualSectionsDir: defaultString(p.Config.TopData.Wiki.ManualSectionsDir, DefaultTopDataWikiManualSectionsDir),
StalePages: TopDataWikiStalePagesConfig{
Default: defaultString(p.Config.TopData.Wiki.StalePages.Default, DefaultTopDataWikiStaleDefault),
LiveCleanup: defaultString(p.Config.TopData.Wiki.StalePages.LiveCleanup, DefaultTopDataWikiStaleLiveCleanup),
},
ManagedRegion: TopDataWikiManagedRegionConfig{
MarkerFormat: defaultString(p.Config.TopData.Wiki.ManagedRegion.MarkerFormat, DefaultTopDataWikiMarkerFormat),
PageMarkerPrefix: defaultString(p.Config.TopData.Wiki.ManagedRegion.PageMarkerPrefix, DefaultTopDataWikiPageMarkerPrefix),
},
},
}
extract := p.Config.Extract
@@ -407,52 +431,62 @@ func lookupEffectiveValue(effective EffectiveConfig, key string) (any, bool) {
func markMissingDefaults(provenance ConfigProvenance) {
defaults := map[string]string{
"paths.build": "build",
"paths.cache": ".cache",
"paths.tools": "tools",
"outputs.module_archive": DefaultModuleArchiveTemplate,
"outputs.hak_manifest": DefaultHAKManifest,
"outputs.hak_archive": DefaultHAKArchiveTemplate,
"inventory.source_extensions": "NWN source resource extensions",
"inventory.asset_extensions": "NWN asset resource extensions",
"inventory.source_json_pattern": DefaultSourceJSONPattern,
"validation.profile": DefaultValidationProfile,
"validation.builtin_script_prefixes": "NWN built-in script prefixes",
"validation.required_fields": "NWN required GFF fields",
"scripts.source_dir": DefaultScriptsSourceDir,
"scripts.cache": DefaultScriptsCache,
"scripts.compiler.env.path": DefaultScriptCompilerEnvPath,
"scripts.compiler.search": "repo tools and PATH:nwn_script_comp",
"music.stage_root": DefaultMusicStageRoot,
"music.credits_root": DefaultCreditsRoot,
"music.credits_overlay": DefaultCreditsOverlayFile,
"music.max_stem_length": "16",
"music.naming_scheme": DefaultMusicNamingScheme,
"music.output_extension": DefaultMusicOutputExtension,
"music.convert_extensions": strings.Join(music.DefaultConvertExtensions(), ", "),
"topdata.build": DefaultTopDataBuild,
"topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir,
"topdata.compiled_tlk": DefaultTopDataCompiledTLK,
"topdata.package_hak": DefaultTopDataPackageHAK,
"topdata.package_tlk": DefaultTopDataPackageTLK,
"topdata.wiki.output_root": DefaultTopDataWikiOutputRoot,
"topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir,
"topdata.wiki.state_file": DefaultTopDataWikiStateFile,
"topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces",
"topdata.wiki.deploy_manifest": DefaultWikiDeployManifest,
"topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary,
"extract.layout": DefaultExtractLayout,
"extract.hak_discovery": DefaultExtractHAKDiscovery,
"extract.archives": DefaultModuleArchiveTemplate,
"extract.cleanup_stale": "true",
"extract.consume_archives": "false",
"autogen.cache.root": DefaultAutogenCacheRoot,
"autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(),
"autogen.cache.refresh_env": DefaultAutogenRefreshEnv,
"autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv,
"autogen.release_source.provider": DefaultAutogenReleaseProvider,
"autogen.release_source.server_url_env": DefaultAutogenServerURLEnv,
"autogen.release_source.repo_env": DefaultAutogenRepoEnv,
"paths.build": "build",
"paths.cache": ".cache",
"paths.tools": "tools",
"outputs.module_archive": DefaultModuleArchiveTemplate,
"outputs.hak_manifest": DefaultHAKManifest,
"outputs.hak_archive": DefaultHAKArchiveTemplate,
"inventory.source_extensions": "NWN source resource extensions",
"inventory.asset_extensions": "NWN asset resource extensions",
"inventory.source_json_pattern": DefaultSourceJSONPattern,
"validation.profile": DefaultValidationProfile,
"validation.builtin_script_prefixes": "NWN built-in script prefixes",
"validation.required_fields": "NWN required GFF fields",
"scripts.source_dir": DefaultScriptsSourceDir,
"scripts.cache": DefaultScriptsCache,
"scripts.compiler.env.path": DefaultScriptCompilerEnvPath,
"scripts.compiler.search": "repo tools and PATH:nwn_script_comp",
"music.stage_root": DefaultMusicStageRoot,
"music.credits_root": DefaultCreditsRoot,
"music.credits_overlay": DefaultCreditsOverlayFile,
"music.max_stem_length": "16",
"music.naming_scheme": DefaultMusicNamingScheme,
"music.output_extension": DefaultMusicOutputExtension,
"music.convert_extensions": strings.Join(music.DefaultConvertExtensions(), ", "),
"topdata.build": DefaultTopDataBuild,
"topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir,
"topdata.compiled_tlk": DefaultTopDataCompiledTLK,
"topdata.package_hak": DefaultTopDataPackageHAK,
"topdata.package_tlk": DefaultTopDataPackageTLK,
"topdata.wiki.output_root": DefaultTopDataWikiOutputRoot,
"topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir,
"topdata.wiki.state_file": DefaultTopDataWikiStateFile,
"topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces",
"topdata.wiki.deploy_manifest": DefaultWikiDeployManifest,
"topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary,
"topdata.wiki.source": DefaultTopDataWikiSource,
"topdata.wiki.renderer": DefaultTopDataWikiRenderer,
"topdata.wiki.link_strategy": DefaultTopDataWikiLinkStrategy,
"topdata.wiki.namespaces_file": DefaultTopDataWikiNamespacesFile,
"topdata.wiki.templates_dir": DefaultTopDataWikiTemplatesDir,
"topdata.wiki.manual_sections_dir": DefaultTopDataWikiManualSectionsDir,
"topdata.wiki.stale_pages.default": DefaultTopDataWikiStaleDefault,
"topdata.wiki.stale_pages.live_cleanup": DefaultTopDataWikiStaleLiveCleanup,
"topdata.wiki.managed_region.marker_format": DefaultTopDataWikiMarkerFormat,
"topdata.wiki.managed_region.page_marker_prefix": DefaultTopDataWikiPageMarkerPrefix,
"extract.layout": DefaultExtractLayout,
"extract.hak_discovery": DefaultExtractHAKDiscovery,
"extract.archives": DefaultModuleArchiveTemplate,
"extract.cleanup_stale": "true",
"extract.consume_archives": "false",
"autogen.cache.root": DefaultAutogenCacheRoot,
"autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(),
"autogen.cache.refresh_env": DefaultAutogenRefreshEnv,
"autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv,
"autogen.release_source.provider": DefaultAutogenReleaseProvider,
"autogen.release_source.server_url_env": DefaultAutogenServerURLEnv,
"autogen.release_source.repo_env": DefaultAutogenRepoEnv,
}
for key, detail := range defaults {
if _, ok := provenance[key]; !ok {