From 9b2084344d5a023b78af98ed7400fc30c764318d Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Fri, 15 May 2026 09:09:37 +0200 Subject: [PATCH] Wiki pipeline (#3) Implemented the wiki pipeline to a shippable local state across `toolkit/`, `module/`, and the NodeBB wiki plugin. **Scope Audited** - Toolkit config/project loading, wiki renderer, deployer, app command wiring. - Module `nwn-tool.yaml`, wiki source/docs, release script, topdata docs. - Plugin sanitizer/API storage contract docs and sanitizer tests. **Changes Made** - Added topdata-owned wiki declarations under `module/topdata/wiki/`. - Added `topdata.wiki.*` config fields, validation, effective config output. - Switched generated wiki pages to `.html` with HTML comment managed/manual markers. - Added `page-index.json` generation and deterministic metadata. - Added manual-section merge preservation and stale page `report`/`archive` handling. - Added namespace `category_env` loading from `topdata/wiki/namespaces.yaml`, while keeping `--category`/`NODEBB_WIKI_CATEGORIES` overrides. - Updated release deployment to keep dry-run first and make stale cleanup opt-in via `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY`. - Updated plugin sanitizer to preserve only `sow-topdata-wiki` comments. **Configuration/Schema Impact** - `module/nwn-tool.yaml` now declares wiki source, renderer, link strategy, templates, manual sections, managed marker policy, and stale policy. - Deploy manifest entries now support stale/archive metadata: `stale`, `last_seen_hash`, `archived_hash`, `title`, `namespace`, `tid`, `pid`, `cid`. **Compatibility Impact** - Legacy Markdown managed markers are still readable for migration. - Generated output is now HTML, not Markdown. - Existing mapped pages update by `pid`; creates still require explicit `--create`. **Tests Added/Updated** - Go tests for wiki config validation, HTML rendering, page index, manual merge, stale report/archive, app summary output. - Plugin sanitizer test for generated topdata bot HTML markers/subset. **Validation Performed** - `go test ./internal/project ./internal/topdata ./internal/app` passed. - `./build-tool.sh` passed. - `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./validate-topdata.sh` passed. - `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force` passed, generating 1345 entity pages. - `deploy-wiki --dry-run --create` with dummy endpoint/token and namespace CID env vars passed locally: 1377 planned creates, 0 stale/drift. - `node tests/wiki-html-sanitizer.test.js` passed. **Remaining Risks** - Full plugin `npm test` is blocked by an existing unrelated drawer CSS contract failure in `tests/wiki-article-drawers.test.js`. - Controlled live migration was not performed because it requires a non-production NodeBB namespace and credentials. - Direct `PUT /api/v3/posts/{pid}` save-filter behavior is documented as needing live confirmation against the deployed NodeBB/plugin stack. Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/3 Co-authored-by: vickydotbat Co-committed-by: vickydotbat --- internal/app/app.go | 18 +- internal/app/app_test.go | 9 +- internal/project/effective.go | 196 +++++++++------ internal/project/project.go | 62 ++++- internal/project/project_test.go | 81 ++++++ internal/topdata/wiki_deploy.go | 283 ++++++++++++++++++++- internal/topdata/wiki_deploy_test.go | 163 ++++++++++++ internal/topdata/wiki_native.go | 354 +++++++++++++++++++++++---- internal/topdata/wiki_native_test.go | 30 ++- 9 files changed, 1030 insertions(+), 166 deletions(-) diff --git a/internal/app/app.go b/internal/app/app.go index e4235f2..fd7362e 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1089,7 +1089,7 @@ func (c *topdataConsole) emitWikiBuildResult(outputDir string, pageCount int, st fmt.Fprintf(c.stdout, "wiki status: %s\n", status) } -func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skipped, stale, drifted int, manifest string) { +func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skipped, stale, archived, drifted int, manifest string) { spin.linebreak() fmt.Fprintln(c.stdout, "Deploy Wiki ----------") fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) @@ -1098,6 +1098,7 @@ func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skip fmt.Fprintf(c.stdout, "updated: %d\n", updated) fmt.Fprintf(c.stdout, "skipped: %d\n", skipped) fmt.Fprintf(c.stdout, "stale: %d\n", stale) + fmt.Fprintf(c.stdout, "archived: %d\n", archived) fmt.Fprintf(c.stdout, "drifted: %d\n", drifted) fmt.Fprintf(c.stdout, "manifest: %s\n", c.relPath(manifest)) } @@ -1868,7 +1869,7 @@ func runDeployWiki(ctx context) error { return err } - console.emitWikiDeployResult(result.LocalPages, result.Created, result.Updated, result.Skipped, result.Stale, result.Drifted, result.Manifest) + console.emitWikiDeployResult(result.LocalPages, result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Drifted, result.Manifest) return nil } @@ -1936,6 +1937,12 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO return opts, fmt.Errorf("--manifest requires a value") } opts.ManifestPath = args[i] + case "--stale-policy": + i++ + if i >= len(args) { + return opts, fmt.Errorf("--stale-policy requires a value") + } + opts.StalePolicy = args[i] case "--dry-run": opts.DryRun = true case "--force": @@ -1943,7 +1950,7 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO case "--create": opts.AllowCreates = true case "-h", "--help": - return opts, fmt.Errorf("usage: %s [--source-dir ] [--endpoint ] [--token ] [--version ] [--namespace ...] [--category ...] [--manifest ] [--dry-run] [--create] [--force]", commandName) + return opts, fmt.Errorf("usage: %s [--source-dir ] [--endpoint ] [--token ] [--version ] [--namespace ...] [--category ...] [--manifest ] [--stale-policy ] [--dry-run] [--create] [--force]", commandName) default: if value, ok, err := requireInlineFlagValue(arg, "--source-dir"); ok || err != nil { if err != nil { @@ -1995,6 +2002,11 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO return opts, err } opts.ManifestPath = value + } else if value, ok, err := requireInlineFlagValue(arg, "--stale-policy"); ok || err != nil { + if err != nil { + return opts, err + } + opts.StalePolicy = value } else if arg == "--dry-run" { opts.DryRun = true } else if arg == "--force" { diff --git a/internal/app/app_test.go b/internal/app/app_test.go index f42d6ef..5eba59e 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -242,13 +242,16 @@ func TestTopdataConsoleDebugProgressAndRelativePaths(t *testing.T) { level: logLevelDebug, } - console.progress("NodeBB wiki plan: create 1, update 2, skip 3, drift 0") - console.emitWikiDeployResult(10, 1, 2, 3, 4, 0, "/workspace/project/build/wiki/deploy-manifest.json") + console.progress("NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, drift 0") + console.emitWikiDeployResult(10, 1, 2, 3, 4, 5, 0, "/workspace/project/build/wiki/deploy-manifest.json") output := stdout.String() - if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, drift 0") { + if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, drift 0") { t.Fatalf("expected debug progress line, got %q", output) } + if !strings.Contains(output, "archived: 5") { + t.Fatalf("expected archived deploy count, got %q", output) + } if !strings.Contains(output, "manifest: build/wiki/deploy-manifest.json") { t.Fatalf("expected relative deploy manifest path, got %q", output) } diff --git a/internal/project/effective.go b/internal/project/effective.go index 2aea4cf..bb2406b 100644 --- a/internal/project/effective.go +++ b/internal/project/effective.go @@ -13,41 +13,51 @@ import ( ) const ( - DefaultBuildPath = "build" - DefaultCachePath = ".cache" - DefaultToolsPath = "tools" - DefaultModuleArchiveTemplate = "{module.resref}.mod" - DefaultHAKManifest = "haks.json" - DefaultHAKArchiveTemplate = "{hak.name}.hak" - DefaultSourceJSONPattern = "{resref}.{extension}.json" - DefaultValidationProfile = "nwn_module" - DefaultScriptsSourceDir = "scripts" - DefaultScriptsCache = "{paths.cache}/ncs" - DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER" - DefaultMusicStageRoot = "{paths.cache}/music" - DefaultCreditsRoot = "{paths.cache}/credits" - DefaultCreditsOverlayFile = "CREDITS.md" - DefaultMusicNamingScheme = "nwn_bmu" - DefaultMusicOutputExtension = ".bmu" - DefaultTopDataBuild = "{paths.build}/topdata" - DefaultTopDataCompiled2DADir = "2da" - DefaultTopDataCompiledTLK = "sow_tlk.tlk" - DefaultTopDataWikiOutputRoot = "wiki" - DefaultTopDataWikiPagesDir = "pages" - DefaultTopDataWikiStateFile = "state.json" - DefaultWikiDeployManifest = ".wiki_deploy_manifest.json" - DefaultWikiDeployEditSummary = "Auto-generated from native builder" - DefaultTopDataPackageHAK = "sow_top.hak" - DefaultTopDataPackageTLK = "sow_tlk.tlk" - DefaultAutogenCacheRoot = "{paths.cache}" - DefaultAutogenCacheMaxAge = time.Hour - DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH" - DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH" - DefaultAutogenReleaseProvider = "gitea" - DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL" - DefaultAutogenRepoEnv = "SOW_ASSETS_REPO" - DefaultExtractLayout = "nwn_canonical_json" - DefaultExtractHAKDiscovery = "build_glob" + DefaultBuildPath = "build" + DefaultCachePath = ".cache" + DefaultToolsPath = "tools" + DefaultModuleArchiveTemplate = "{module.resref}.mod" + DefaultHAKManifest = "haks.json" + DefaultHAKArchiveTemplate = "{hak.name}.hak" + DefaultSourceJSONPattern = "{resref}.{extension}.json" + DefaultValidationProfile = "nwn_module" + DefaultScriptsSourceDir = "scripts" + DefaultScriptsCache = "{paths.cache}/ncs" + DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER" + DefaultMusicStageRoot = "{paths.cache}/music" + DefaultCreditsRoot = "{paths.cache}/credits" + DefaultCreditsOverlayFile = "CREDITS.md" + DefaultMusicNamingScheme = "nwn_bmu" + DefaultMusicOutputExtension = ".bmu" + DefaultTopDataBuild = "{paths.build}/topdata" + DefaultTopDataCompiled2DADir = "2da" + DefaultTopDataCompiledTLK = "sow_tlk.tlk" + DefaultTopDataWikiOutputRoot = "wiki" + DefaultTopDataWikiPagesDir = "pages" + DefaultTopDataWikiStateFile = "state.json" + DefaultTopDataWikiSource = "topdata/wiki" + DefaultTopDataWikiRenderer = "nodebb_tiptap_html" + DefaultTopDataWikiLinkStrategy = "preserve_westgate_wiki_links" + DefaultTopDataWikiNamespacesFile = "namespaces.yaml" + DefaultTopDataWikiTemplatesDir = "templates" + DefaultTopDataWikiManualSectionsDir = "manual-sections" + DefaultTopDataWikiStaleDefault = "report" + DefaultTopDataWikiStaleLiveCleanup = "archive" + DefaultTopDataWikiMarkerFormat = "html_comments" + DefaultTopDataWikiPageMarkerPrefix = "sow-topdata-wiki" + DefaultWikiDeployManifest = ".wiki_deploy_manifest.json" + DefaultWikiDeployEditSummary = "Auto-generated from native builder" + DefaultTopDataPackageHAK = "sow_top.hak" + DefaultTopDataPackageTLK = "sow_tlk.tlk" + DefaultAutogenCacheRoot = "{paths.cache}" + DefaultAutogenCacheMaxAge = time.Hour + DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH" + DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH" + DefaultAutogenReleaseProvider = "gitea" + DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL" + DefaultAutogenRepoEnv = "SOW_ASSETS_REPO" + DefaultExtractLayout = "nwn_canonical_json" + DefaultExtractHAKDiscovery = "build_glob" ) type ConfigProvenance map[string]ConfigValueProvenance @@ -228,6 +238,20 @@ func (p *Project) EffectiveConfig() EffectiveConfig { ManagedNamespaces: defaultStringSlice(p.Config.TopData.Wiki.ManagedNamespaces, []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}), DeployManifest: defaultString(p.Config.TopData.Wiki.DeployManifest, DefaultWikiDeployManifest), DeployEditSummary: defaultString(p.Config.TopData.Wiki.DeployEditSummary, DefaultWikiDeployEditSummary), + Source: defaultString(p.Config.TopData.Wiki.Source, DefaultTopDataWikiSource), + Renderer: defaultString(p.Config.TopData.Wiki.Renderer, DefaultTopDataWikiRenderer), + LinkStrategy: defaultString(p.Config.TopData.Wiki.LinkStrategy, DefaultTopDataWikiLinkStrategy), + NamespacesFile: defaultString(p.Config.TopData.Wiki.NamespacesFile, DefaultTopDataWikiNamespacesFile), + TemplatesDir: defaultString(p.Config.TopData.Wiki.TemplatesDir, DefaultTopDataWikiTemplatesDir), + ManualSectionsDir: defaultString(p.Config.TopData.Wiki.ManualSectionsDir, DefaultTopDataWikiManualSectionsDir), + StalePages: TopDataWikiStalePagesConfig{ + Default: defaultString(p.Config.TopData.Wiki.StalePages.Default, DefaultTopDataWikiStaleDefault), + LiveCleanup: defaultString(p.Config.TopData.Wiki.StalePages.LiveCleanup, DefaultTopDataWikiStaleLiveCleanup), + }, + ManagedRegion: TopDataWikiManagedRegionConfig{ + MarkerFormat: defaultString(p.Config.TopData.Wiki.ManagedRegion.MarkerFormat, DefaultTopDataWikiMarkerFormat), + PageMarkerPrefix: defaultString(p.Config.TopData.Wiki.ManagedRegion.PageMarkerPrefix, DefaultTopDataWikiPageMarkerPrefix), + }, }, } extract := p.Config.Extract @@ -407,52 +431,62 @@ func lookupEffectiveValue(effective EffectiveConfig, key string) (any, bool) { func markMissingDefaults(provenance ConfigProvenance) { defaults := map[string]string{ - "paths.build": "build", - "paths.cache": ".cache", - "paths.tools": "tools", - "outputs.module_archive": DefaultModuleArchiveTemplate, - "outputs.hak_manifest": DefaultHAKManifest, - "outputs.hak_archive": DefaultHAKArchiveTemplate, - "inventory.source_extensions": "NWN source resource extensions", - "inventory.asset_extensions": "NWN asset resource extensions", - "inventory.source_json_pattern": DefaultSourceJSONPattern, - "validation.profile": DefaultValidationProfile, - "validation.builtin_script_prefixes": "NWN built-in script prefixes", - "validation.required_fields": "NWN required GFF fields", - "scripts.source_dir": DefaultScriptsSourceDir, - "scripts.cache": DefaultScriptsCache, - "scripts.compiler.env.path": DefaultScriptCompilerEnvPath, - "scripts.compiler.search": "repo tools and PATH:nwn_script_comp", - "music.stage_root": DefaultMusicStageRoot, - "music.credits_root": DefaultCreditsRoot, - "music.credits_overlay": DefaultCreditsOverlayFile, - "music.max_stem_length": "16", - "music.naming_scheme": DefaultMusicNamingScheme, - "music.output_extension": DefaultMusicOutputExtension, - "music.convert_extensions": strings.Join(music.DefaultConvertExtensions(), ", "), - "topdata.build": DefaultTopDataBuild, - "topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir, - "topdata.compiled_tlk": DefaultTopDataCompiledTLK, - "topdata.package_hak": DefaultTopDataPackageHAK, - "topdata.package_tlk": DefaultTopDataPackageTLK, - "topdata.wiki.output_root": DefaultTopDataWikiOutputRoot, - "topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir, - "topdata.wiki.state_file": DefaultTopDataWikiStateFile, - "topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces", - "topdata.wiki.deploy_manifest": DefaultWikiDeployManifest, - "topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary, - "extract.layout": DefaultExtractLayout, - "extract.hak_discovery": DefaultExtractHAKDiscovery, - "extract.archives": DefaultModuleArchiveTemplate, - "extract.cleanup_stale": "true", - "extract.consume_archives": "false", - "autogen.cache.root": DefaultAutogenCacheRoot, - "autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(), - "autogen.cache.refresh_env": DefaultAutogenRefreshEnv, - "autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv, - "autogen.release_source.provider": DefaultAutogenReleaseProvider, - "autogen.release_source.server_url_env": DefaultAutogenServerURLEnv, - "autogen.release_source.repo_env": DefaultAutogenRepoEnv, + "paths.build": "build", + "paths.cache": ".cache", + "paths.tools": "tools", + "outputs.module_archive": DefaultModuleArchiveTemplate, + "outputs.hak_manifest": DefaultHAKManifest, + "outputs.hak_archive": DefaultHAKArchiveTemplate, + "inventory.source_extensions": "NWN source resource extensions", + "inventory.asset_extensions": "NWN asset resource extensions", + "inventory.source_json_pattern": DefaultSourceJSONPattern, + "validation.profile": DefaultValidationProfile, + "validation.builtin_script_prefixes": "NWN built-in script prefixes", + "validation.required_fields": "NWN required GFF fields", + "scripts.source_dir": DefaultScriptsSourceDir, + "scripts.cache": DefaultScriptsCache, + "scripts.compiler.env.path": DefaultScriptCompilerEnvPath, + "scripts.compiler.search": "repo tools and PATH:nwn_script_comp", + "music.stage_root": DefaultMusicStageRoot, + "music.credits_root": DefaultCreditsRoot, + "music.credits_overlay": DefaultCreditsOverlayFile, + "music.max_stem_length": "16", + "music.naming_scheme": DefaultMusicNamingScheme, + "music.output_extension": DefaultMusicOutputExtension, + "music.convert_extensions": strings.Join(music.DefaultConvertExtensions(), ", "), + "topdata.build": DefaultTopDataBuild, + "topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir, + "topdata.compiled_tlk": DefaultTopDataCompiledTLK, + "topdata.package_hak": DefaultTopDataPackageHAK, + "topdata.package_tlk": DefaultTopDataPackageTLK, + "topdata.wiki.output_root": DefaultTopDataWikiOutputRoot, + "topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir, + "topdata.wiki.state_file": DefaultTopDataWikiStateFile, + "topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces", + "topdata.wiki.deploy_manifest": DefaultWikiDeployManifest, + "topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary, + "topdata.wiki.source": DefaultTopDataWikiSource, + "topdata.wiki.renderer": DefaultTopDataWikiRenderer, + "topdata.wiki.link_strategy": DefaultTopDataWikiLinkStrategy, + "topdata.wiki.namespaces_file": DefaultTopDataWikiNamespacesFile, + "topdata.wiki.templates_dir": DefaultTopDataWikiTemplatesDir, + "topdata.wiki.manual_sections_dir": DefaultTopDataWikiManualSectionsDir, + "topdata.wiki.stale_pages.default": DefaultTopDataWikiStaleDefault, + "topdata.wiki.stale_pages.live_cleanup": DefaultTopDataWikiStaleLiveCleanup, + "topdata.wiki.managed_region.marker_format": DefaultTopDataWikiMarkerFormat, + "topdata.wiki.managed_region.page_marker_prefix": DefaultTopDataWikiPageMarkerPrefix, + "extract.layout": DefaultExtractLayout, + "extract.hak_discovery": DefaultExtractHAKDiscovery, + "extract.archives": DefaultModuleArchiveTemplate, + "extract.cleanup_stale": "true", + "extract.consume_archives": "false", + "autogen.cache.root": DefaultAutogenCacheRoot, + "autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(), + "autogen.cache.refresh_env": DefaultAutogenRefreshEnv, + "autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv, + "autogen.release_source.provider": DefaultAutogenReleaseProvider, + "autogen.release_source.server_url_env": DefaultAutogenServerURLEnv, + "autogen.release_source.repo_env": DefaultAutogenRepoEnv, } for key, detail := range defaults { if _, ok := provenance[key]; !ok { diff --git a/internal/project/project.go b/internal/project/project.go index 050cfc8..6ad8cf4 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -204,12 +204,30 @@ type TopDataConfig struct { } type TopDataWikiConfig struct { - OutputRoot string `json:"output_root" yaml:"output_root"` - PagesDir string `json:"pages_dir" yaml:"pages_dir"` - StateFile string `json:"state_file" yaml:"state_file"` - ManagedNamespaces []string `json:"managed_namespaces" yaml:"managed_namespaces"` - DeployManifest string `json:"deploy_manifest" yaml:"deploy_manifest"` - DeployEditSummary string `json:"deploy_edit_summary" yaml:"deploy_edit_summary"` + OutputRoot string `json:"output_root" yaml:"output_root"` + PagesDir string `json:"pages_dir" yaml:"pages_dir"` + StateFile string `json:"state_file" yaml:"state_file"` + ManagedNamespaces []string `json:"managed_namespaces" yaml:"managed_namespaces"` + DeployManifest string `json:"deploy_manifest" yaml:"deploy_manifest"` + DeployEditSummary string `json:"deploy_edit_summary" yaml:"deploy_edit_summary"` + Source string `json:"source" yaml:"source"` + Renderer string `json:"renderer" yaml:"renderer"` + LinkStrategy string `json:"link_strategy" yaml:"link_strategy"` + NamespacesFile string `json:"namespaces_file" yaml:"namespaces_file"` + TemplatesDir string `json:"templates_dir" yaml:"templates_dir"` + ManualSectionsDir string `json:"manual_sections_dir" yaml:"manual_sections_dir"` + StalePages TopDataWikiStalePagesConfig `json:"stale_pages" yaml:"stale_pages"` + ManagedRegion TopDataWikiManagedRegionConfig `json:"managed_region" yaml:"managed_region"` +} + +type TopDataWikiStalePagesConfig struct { + Default string `json:"default" yaml:"default"` + LiveCleanup string `json:"live_cleanup" yaml:"live_cleanup"` +} + +type TopDataWikiManagedRegionConfig struct { + MarkerFormat string `json:"marker_format" yaml:"marker_format"` + PageMarkerPrefix string `json:"page_marker_prefix" yaml:"page_marker_prefix"` } type ExtractConfig struct { @@ -485,6 +503,10 @@ func (p *Project) ValidateLayout() error { failures = append(failures, validateRelativePath("topdata.wiki.pages_dir", effective.TopData.Wiki.PagesDir)...) failures = append(failures, validateRelativePath("topdata.wiki.state_file", effective.TopData.Wiki.StateFile)...) failures = append(failures, validateRelativePath("topdata.wiki.deploy_manifest", effective.TopData.Wiki.DeployManifest)...) + failures = append(failures, validateTreeRootPath("topdata.wiki.source", effective.TopData.Wiki.Source)...) + failures = append(failures, validateRelativePath("topdata.wiki.namespaces_file", effective.TopData.Wiki.NamespacesFile)...) + failures = append(failures, validateTreeRootPath("topdata.wiki.templates_dir", effective.TopData.Wiki.TemplatesDir)...) + failures = append(failures, validateTreeRootPath("topdata.wiki.manual_sections_dir", effective.TopData.Wiki.ManualSectionsDir)...) failures = append(failures, validateRelativePath("autogen.cache.root", effective.Autogen.Cache.Root)...) if err := validateOutputFileName("outputs.module_archive", filepath.Base(effective.Outputs.ModuleArchive), ".mod"); err != nil { failures = append(failures, err) @@ -498,6 +520,26 @@ func (p *Project) ValidateLayout() error { if err := validateOutputFileName("topdata.wiki.deploy_manifest", filepath.Base(effective.TopData.Wiki.DeployManifest), ".json"); err != nil { failures = append(failures, err) } + switch effective.TopData.Wiki.Renderer { + case "nodebb_tiptap_html": + default: + failures = append(failures, fmt.Errorf("topdata.wiki.renderer %q is not supported", effective.TopData.Wiki.Renderer)) + } + switch effective.TopData.Wiki.LinkStrategy { + case "preserve_westgate_wiki_links": + default: + failures = append(failures, fmt.Errorf("topdata.wiki.link_strategy %q is not supported", effective.TopData.Wiki.LinkStrategy)) + } + for key, policy := range map[string]string{ + "topdata.wiki.stale_pages.default": effective.TopData.Wiki.StalePages.Default, + "topdata.wiki.stale_pages.live_cleanup": effective.TopData.Wiki.StalePages.LiveCleanup, + } { + switch policy { + case "report", "archive", "unpublish": + default: + failures = append(failures, fmt.Errorf("%s %q is not supported", key, policy)) + } + } switch effective.Extract.Layout { case "nwn_canonical_json": default: @@ -834,6 +876,14 @@ func (p *Project) TopDataWikiRootDir() string { return filepath.Join(p.TopDataBuildDir(), wikiRoot) } +func (p *Project) TopDataWikiSourceDir() string { + source := filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.Source) + if filepath.IsAbs(source) { + return source + } + return filepath.Join(p.Root, source) +} + func (p *Project) TopDataWikiPagesDir() string { return filepath.Join(p.TopDataWikiRootDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.PagesDir)) } diff --git a/internal/project/project_test.go b/internal/project/project_test.go index e038c15..7679169 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -256,10 +256,19 @@ extract: cleanup_stale: false topdata: wiki: + source: topdata/wiki managed_namespaces: - skills + renderer: nodebb_tiptap_html + link_strategy: preserve_westgate_wiki_links + namespaces_file: namespaces.yaml + templates_dir: templates + manual_sections_dir: manual-sections deploy_manifest: wiki-manifest.json deploy_edit_summary: Custom summary + stale_pages: + default: report + live_cleanup: archive autogen: cache: root: "{paths.cache}/released" @@ -297,6 +306,30 @@ autogen: if got, want := effective.TopData.Wiki.DeployManifest, "wiki-manifest.json"; got != want { t.Fatalf("expected wiki deploy manifest %q, got %q", want, got) } + if got, want := effective.TopData.Wiki.Source, "topdata/wiki"; got != want { + t.Fatalf("expected wiki source %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.Renderer, "nodebb_tiptap_html"; got != want { + t.Fatalf("expected wiki renderer %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.LinkStrategy, "preserve_westgate_wiki_links"; got != want { + t.Fatalf("expected wiki link strategy %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.NamespacesFile, "namespaces.yaml"; got != want { + t.Fatalf("expected wiki namespaces file %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.TemplatesDir, "templates"; got != want { + t.Fatalf("expected wiki templates dir %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.ManualSectionsDir, "manual-sections"; got != want { + t.Fatalf("expected wiki manual sections dir %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.StalePages.Default, "report"; got != want { + t.Fatalf("expected wiki stale default %q, got %q", want, got) + } + if got, want := effective.TopData.Wiki.StalePages.LiveCleanup, "archive"; got != want { + t.Fatalf("expected wiki stale live cleanup %q, got %q", want, got) + } if got, want := effective.Autogen.Cache.MaxAge, "30m"; got != want { t.Fatalf("expected autogen cache max age %q, got %q", want, got) } @@ -704,6 +737,54 @@ func TestValidateLayoutRejectsUnsafeGeneratedOutputPaths(t *testing.T) { } } +func TestValidateLayoutRejectsInvalidTopDataWikiConfig(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + + proj := &Project{ + Root: root, + Config: Config{ + Module: ModuleConfig{Name: "Test", ResRef: "test"}, + Paths: PathConfig{Source: "src", Build: "build"}, + TopData: TopDataConfig{ + Source: "topdata", + Wiki: TopDataWikiConfig{ + Source: "../wiki", + Renderer: "dokuwiki", + LinkStrategy: "rewrite_links", + NamespacesFile: "../namespaces.yaml", + TemplatesDir: ".", + ManualSectionsDir: "../manual", + StalePages: TopDataWikiStalePagesConfig{ + Default: "delete", + LiveCleanup: "delete", + }, + }, + }, + }, + } + + err := proj.ValidateLayout() + if err == nil { + t.Fatal("expected invalid wiki config validation error") + } + for _, needle := range []string{ + "topdata.wiki.source", + "topdata.wiki.renderer", + "topdata.wiki.link_strategy", + "topdata.wiki.namespaces_file", + "topdata.wiki.templates_dir", + "topdata.wiki.manual_sections_dir", + "topdata.wiki.stale_pages.default", + "topdata.wiki.stale_pages.live_cleanup", + } { + if !strings.Contains(err.Error(), needle) { + t.Fatalf("expected validation error to mention %s, got %v", needle, err) + } + } +} + func TestValidateLayoutRejectsRootSourceAndAssetPaths(t *testing.T) { root := t.TempDir() mkdirAll(t, filepath.Join(root, "build")) diff --git a/internal/topdata/wiki_deploy.go b/internal/topdata/wiki_deploy.go index 10a7e62..d15a18b 100644 --- a/internal/topdata/wiki_deploy.go +++ b/internal/topdata/wiki_deploy.go @@ -17,13 +17,18 @@ import ( "time" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" + "gopkg.in/yaml.v3" ) const ( - managedStartMarker = "[//]: # (sow-topdata-wiki:managed:start)" - managedEndMarker = "[//]: # (sow-topdata-wiki:managed:end)" - legacyManagedStartMarker = "" - legacyManagedEndMarker = "" + managedStartMarker = "" + legacyMarkdownStartMarker = "[//]: # (sow-topdata-wiki:managed:start)" + legacyMarkdownEndMarker = "[//]: # (sow-topdata-wiki:managed:end)" + legacyManagedStartMarker = "" + legacyManagedEndMarker = "" + manualStartMarkerPrefix = "") + end := strings.Index(content, managedEndMarker) + if startEnd >= 0 && end >= 0 { + startMarker := content[start : start+startEnd+len("-->")] + return startMarker, managedEndMarker, start, end + } + } for _, markers := range [][2]string{ - {managedStartMarker, managedEndMarker}, + {legacyMarkdownStartMarker, legacyMarkdownEndMarker}, {legacyManagedStartMarker, legacyManagedEndMarker}, } { start := strings.Index(content, markers[0]) @@ -344,6 +498,109 @@ func managedRegionBounds(content string) (string, string, int, int) { return "", "", -1, -1 } +func extractManagedMarkers(content string) (string, string, bool) { + startMarker, endMarker, start, end := managedRegionBounds(content) + return startMarker, endMarker, start >= 0 && end >= 0 +} + +func mergeManualRegions(merged, existing string) string { + existingManual := extractManualRegions(existing) + if len(existingManual) == 0 { + return merged + } + return replaceManualRegions(merged, existingManual) +} + +func extractManualRegions(content string) map[string]string { + regions := map[string]string{} + offset := 0 + for { + startRel := strings.Index(content[offset:], manualStartMarkerPrefix) + if startRel < 0 { + break + } + start := offset + startRel + startEndRel := strings.Index(content[start:], "-->") + if startEndRel < 0 { + break + } + startMarker := content[start : start+startEndRel+len("-->")] + id := markerID(startMarker) + if id == "" { + offset = start + len(startMarker) + continue + } + endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\"" + end := strings.Index(content[start+len(startMarker):], endPrefix) + if end < 0 { + offset = start + len(startMarker) + continue + } + end += start + len(startMarker) + endEndRel := strings.Index(content[end:], "-->") + if endEndRel < 0 { + break + } + endMarker := content[end : end+endEndRel+len("-->")] + regions[id] = startMarker + strings.TrimRight(content[start+len(startMarker):end], "\n") + "\n" + endMarker + offset = end + len(endMarker) + } + return regions +} + +func replaceManualRegions(content string, replacements map[string]string) string { + for id, replacement := range replacements { + start := strings.Index(content, manualStartMarkerPrefix+" id=\""+id+"\"") + if start < 0 { + continue + } + endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\"" + end := strings.Index(content[start:], endPrefix) + if end < 0 { + continue + } + end += start + endEndRel := strings.Index(content[end:], "-->") + if endEndRel < 0 { + continue + } + endEnd := end + endEndRel + len("-->") + content = content[:start] + replacement + content[endEnd:] + } + return content +} + +func markerID(marker string) string { + const prefix = `id="` + start := strings.Index(marker, prefix) + if start < 0 { + return "" + } + start += len(prefix) + end := strings.Index(marker[start:], `"`) + if end < 0 { + return "" + } + return marker[start : start+end] +} + +func renderArchivedWikiPage(pageID, title string) string { + if title == "" { + title = extractHTMLTitle("", pageID) + } + body := strings.Join([]string{ + "

" + html.EscapeString(title) + "

", + "

This generated page is no longer present in the current Shadows Over Westgate topdata wiki source.

", + }, "\n") + hash := computeManagedHash(body) + return ensureTrailingNewline(strings.Join([]string{ + "", + "", + body, + "", + }, "\n")) +} + func extractPageTitle(content, fallback string) string { if title := extractMarkdownTitle(content); title != "" { return title diff --git a/internal/topdata/wiki_deploy_test.go b/internal/topdata/wiki_deploy_test.go index e214563..f1d9f1d 100644 --- a/internal/topdata/wiki_deploy_test.go +++ b/internal/topdata/wiki_deploy_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strings" "testing" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" @@ -194,3 +195,165 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) { t.Fatalf("DeployWikiWithOptions create failed: %v", err) } } + +func TestDeployWikiMergesHTMLManagedAndManualRegions(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Athletics

+

Generated athletics page

+ + +

Notes

+

+ + +

See Also

+

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + old := ` + +

Athletics

+

Old generated text

+ + +

Notes

+

Keep this human note.

+ +` + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:athletics": {Hash: computeManagedHash(old), TID: 7, PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + var updated string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7, "content": old}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42": + var req struct { + Content string `json:"content"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode update request: %v", err) + } + updated = req.Content + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions update failed: %v", err) + } + if result.Updated != 1 { + t.Fatalf("expected one update, got %#v", result) + } + if !containsAll(updated, "

Generated athletics page

", "

Keep this human note.

", `manual:start id="see_also"`) { + t.Fatalf("expected update to replace managed HTML and preserve/bootstrap manual regions:\n%s", updated) + } +} + +func TestDeployWikiReportsAndArchivesStalePages(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + + var archived string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut || r.URL.Path != "/api/v3/posts/42" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + var req struct { + Content string `json:"content"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode archive request: %v", err) + } + archived = req.Content + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}}) + })) + defer server.Close() + + report, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + DryRun: true, + StalePolicy: "report", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions stale report failed: %v", err) + } + if report.Stale != 1 { + t.Fatalf("expected one stale report, got %#v", report) + } + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + StalePolicy: "archive", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions stale archive failed: %v", err) + } + if result.Stale != 1 || result.Archived != 1 { + t.Fatalf("expected one stale archive, got %#v", result) + } + if !containsAll(archived, "

Retired Skill

", "no longer present", `", + "", + strings.TrimSpace(body), + "", + } + for _, section := range manualSections { + parts = append(parts, + "", + strings.TrimSpace(section.InitialHTML), + "", + ) + } + return ensureTrailingNewline(strings.Join(parts, "\n")) +} + +func dokuWikiToNodeBBHTML(text string) string { + text = strings.ReplaceAll(strings.TrimSpace(text), "\r\n", "\n") + if text == "" { + return "" + } + lines := strings.Split(text, "\n") + var out []string + var paragraph []string + var list []string + var tableRows []string + + flushParagraph := func() { + if len(paragraph) == 0 { + return + } + out = append(out, "

"+renderInlineHTML(strings.Join(paragraph, " "))+"

") + paragraph = nil + } + flushList := func() { + if len(list) == 0 { + return + } + out = append(out, "
    ") + for _, item := range list { + out = append(out, "
  • "+renderInlineHTML(item)+"
    • ") + } + out = append(out, "
") + list = nil + } + flushTable := func() { + if len(tableRows) == 0 { + return + } + out = append(out, "") + out = append(out, tableRows...) + out = append(out, "
") + tableRows = nil + } + flushBlocks := func() { + flushParagraph() + flushList() + flushTable() + } + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + flushBlocks() + continue + } + if strings.HasPrefix(trimmed, "

%s", level, renderInlineHTML(heading), level)) + continue + } + if strings.HasPrefix(trimmed, " * ") || strings.HasPrefix(trimmed, "* ") || strings.HasPrefix(trimmed, "- ") { + flushParagraph() + flushTable() + item := strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(trimmed, " * "), "* "), "- ")) + list = append(list, item) + continue + } + if label, value, ok := parseFactLine(trimmed); ok { + flushParagraph() + flushList() + tableRows = append(tableRows, ""+html.EscapeString(label)+""+renderInlineHTML(value)+"") + continue + } + flushList() + flushTable() + for _, part := range strings.Split(trimmed, `\\`) { + part = strings.TrimSpace(part) + if part != "" { + paragraph = append(paragraph, part) + } + } + } + flushBlocks() + return strings.TrimSpace(strings.Join(out, "\n")) +} + +func parseFactLine(line string) (string, string, bool) { + if !strings.HasPrefix(line, "**") { + return "", "", false + } + rest := strings.TrimPrefix(line, "**") + label, value, ok := strings.Cut(rest, ":**") + if !ok { + return "", "", false + } + return strings.TrimSpace(label), strings.Trim(strings.TrimSpace(value), `\`), true +} + +func renderInlineHTML(text string) string { + var out strings.Builder + for { + start := strings.Index(text, "[[") + if start < 0 { + out.WriteString(html.EscapeString(text)) + break + } + end := strings.Index(text[start+2:], "]]") + if end < 0 { + out.WriteString(html.EscapeString(text)) + break + } + end += start + 2 + out.WriteString(html.EscapeString(text[:start])) + out.WriteString(text[start : end+2]) + text = text[end+2:] + } + return out.String() +} + func splitDokuWikiNotes(text string) (string, string) { const delimiter = "===== Notes =====" before, after, ok := strings.Cut(text, delimiter) diff --git a/internal/topdata/wiki_native_test.go b/internal/topdata/wiki_native_test.go index 0e915fe..e0dea1b 100644 --- a/internal/topdata/wiki_native_test.go +++ b/internal/topdata/wiki_native_test.go @@ -42,26 +42,40 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) { if result.WikiPages != 1 { t.Fatalf("expected one generated entity page, got %d", result.WikiPages) } - pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md") + pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html") got, err := os.ReadFile(pagePath) if err != nil { t.Fatalf("read generated page: %v", err) } text := string(got) - if !strings.Contains(text, "# Athletics") || !strings.Contains(text, "This skill is unchanged from vanilla.") { + if !strings.Contains(text, "

Athletics

") || !strings.Contains(text, `class="wiki-callout wiki-callout--status"`) || !strings.Contains(text, "This skill is unchanged from vanilla.") { t.Fatalf("unexpected generated page:\n%s", text) } - if !strings.Contains(text, managedStartMarker) || !strings.Contains(text, "Ability: Strength.") || !strings.Contains(text, "- Climb") { - t.Fatalf("expected description to be rendered into wiki page:\n%s", text) + if !strings.Contains(text, ``) || !strings.Contains(text, ``) || !strings.Contains(text, "

See Also

") { + t.Fatalf("expected generated page to include manual section placeholders:\n%s", text) + } + statusPath := filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus.html") statusRaw, err := os.ReadFile(statusPath) if err != nil { t.Fatalf("read status page: %v", err) } - if !strings.Contains(string(statusRaw), "**Vanilla:** 1") { + if !strings.Contains(string(statusRaw), "Vanilla1") { t.Fatalf("expected wiki status summary to count vanilla page:\n%s", string(statusRaw)) } + indexRaw, err := os.ReadFile(filepath.Join(root, ".cache", "wiki", "page-index.json")) + if err != nil { + t.Fatalf("read page index: %v", err) + } + indexText := string(indexRaw) + if !strings.Contains(indexText, `"page_id": "skills:athletics"`) || !strings.Contains(indexText, `"output_path": "pages/skills/athletics.html"`) || !strings.Contains(indexText, `"edit_policy": "preserve_manual_sections"`) { + t.Fatalf("expected deterministic page-index metadata, got:\n%s", indexText) + } if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil { t.Fatalf("expected state file after first build: %v", err) } @@ -201,7 +215,7 @@ func TestBuildNativeRegeneratesWikiWhenTLKTextChanges(t *testing.T) { if _, err := BuildNative(proj, nil); err != nil { t.Fatalf("BuildNative after text change failed: %v", err) } - pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md") + pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html") got, err := os.ReadFile(pagePath) if err != nil { t.Fatalf("read regenerated page: %v", err) @@ -243,7 +257,7 @@ func TestBuildPackageIgnoresWikiSourcesForFreshness(t *testing.T) { t.Fatalf("BuildNative failed: %v", err) } - pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md") + pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html") newTime := time.Now().Add(2 * time.Second) if err := os.Chtimes(pagePath, newTime, newTime); err != nil { t.Fatalf("touch wiki page: %v", err)