ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Wiki pipeline (#3)

Browse Source 
Implemented the wiki pipeline to a shippable local state across `toolkit/`, `module/`, and the NodeBB wiki plugin.

**Scope Audited**
- Toolkit config/project loading, wiki renderer, deployer, app command wiring.
- Module `nwn-tool.yaml`, wiki source/docs, release script, topdata docs.
- Plugin sanitizer/API storage contract docs and sanitizer tests.

**Changes Made**
- Added topdata-owned wiki declarations under `module/topdata/wiki/`.
- Added `topdata.wiki.*` config fields, validation, effective config output.
- Switched generated wiki pages to `.html` with HTML comment managed/manual markers.
- Added `page-index.json` generation and deterministic metadata.
- Added manual-section merge preservation and stale page `report`/`archive` handling.
- Added namespace `category_env` loading from `topdata/wiki/namespaces.yaml`, while keeping `--category`/`NODEBB_WIKI_CATEGORIES` overrides.
- Updated release deployment to keep dry-run first and make stale cleanup opt-in via `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY`.
- Updated plugin sanitizer to preserve only `sow-topdata-wiki` comments.

**Configuration/Schema Impact**
- `module/nwn-tool.yaml` now declares wiki source, renderer, link strategy, templates, manual sections, managed marker policy, and stale policy.
- Deploy manifest entries now support stale/archive metadata: `stale`, `last_seen_hash`, `archived_hash`, `title`, `namespace`, `tid`, `pid`, `cid`.

**Compatibility Impact**
- Legacy Markdown managed markers are still readable for migration.
- Generated output is now HTML, not Markdown.
- Existing mapped pages update by `pid`; creates still require explicit `--create`.

**Tests Added/Updated**
- Go tests for wiki config validation, HTML rendering, page index, manual merge, stale report/archive, app summary output.
- Plugin sanitizer test for generated topdata bot HTML markers/subset.

**Validation Performed**
- `go test ./internal/project ./internal/topdata ./internal/app` passed.
- `./build-tool.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./validate-topdata.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force` passed, generating 1345 entity pages.
- `deploy-wiki --dry-run --create` with dummy endpoint/token and namespace CID env vars passed locally: 1377 planned creates, 0 stale/drift.
- `node tests/wiki-html-sanitizer.test.js` passed.

**Remaining Risks**
- Full plugin `npm test` is blocked by an existing unrelated drawer CSS contract failure in `tests/wiki-article-drawers.test.js`.
- Controlled live migration was not performed because it requires a non-production NodeBB namespace and credentials.
- Direct `PUT /api/v3/posts/{pid}` save-filter behavior is documented as needing live confirmation against the deployed NodeBB/plugin stack.

Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/3
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
This commit is contained in:
archvillainette
2026-05-15 09:09:37 +02:00
committed by archvillainette
parent d767ec0a25
commit 9b2084344d
9 changed files with 1030 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/app/app.go
+15 -3
View File
@@ -1089,7 +1089,7 @@ func (c *topdataConsole) emitWikiBuildResult(outputDir string, pageCount int, st
fmt.Fprintf(c.stdout, "wiki status: %s\n", status) fmt.Fprintf(c.stdout, "wiki status: %s\n", status)
} }
func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skipped, stale, drifted int, manifest string) { func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skipped, stale, archived, drifted int, manifest string) {
spin.linebreak() spin.linebreak()
fmt.Fprintln(c.stdout, "Deploy Wiki ----------") fmt.Fprintln(c.stdout, "Deploy Wiki ----------")
fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) fmt.Fprintf(c.stdout, "project: %s\n", c.projectName)
@@ -1098,6 +1098,7 @@ func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skip
fmt.Fprintf(c.stdout, "updated: %d\n", updated) fmt.Fprintf(c.stdout, "updated: %d\n", updated)
fmt.Fprintf(c.stdout, "skipped: %d\n", skipped) fmt.Fprintf(c.stdout, "skipped: %d\n", skipped)
fmt.Fprintf(c.stdout, "stale: %d\n", stale) fmt.Fprintf(c.stdout, "stale: %d\n", stale)
fmt.Fprintf(c.stdout, "archived: %d\n", archived)
fmt.Fprintf(c.stdout, "drifted: %d\n", drifted) fmt.Fprintf(c.stdout, "drifted: %d\n", drifted)
fmt.Fprintf(c.stdout, "manifest: %s\n", c.relPath(manifest)) fmt.Fprintf(c.stdout, "manifest: %s\n", c.relPath(manifest))
} }
@@ -1868,7 +1869,7 @@ func runDeployWiki(ctx context) error {
return err return err
} }
console.emitWikiDeployResult(result.LocalPages, result.Created, result.Updated, result.Skipped, result.Stale, result.Drifted, result.Manifest) console.emitWikiDeployResult(result.LocalPages, result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Drifted, result.Manifest)
return nil return nil
} }
@@ -1936,6 +1937,12 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO
return opts, fmt.Errorf("--manifest requires a value") return opts, fmt.Errorf("--manifest requires a value")
} }
opts.ManifestPath = args[i] opts.ManifestPath = args[i]
case "--stale-policy":
i++
if i >= len(args) {
return opts, fmt.Errorf("--stale-policy requires a value")
}
opts.StalePolicy = args[i]
case "--dry-run": case "--dry-run":
opts.DryRun = true opts.DryRun = true
case "--force": case "--force":
@@ -1943,7 +1950,7 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO
case "--create": case "--create":
opts.AllowCreates = true opts.AllowCreates = true
case "-h", "--help": case "-h", "--help":
return opts, fmt.Errorf("usage: %s [--source-dir <path>] [--endpoint <url>] [--token <token>] [--version <ver>] [--namespace <ns> ...] [--category <namespace=cid> ...] [--manifest <path>] [--dry-run] [--create] [--force]", commandName) return opts, fmt.Errorf("usage: %s [--source-dir <path>] [--endpoint <url>] [--token <token>] [--version <ver>] [--namespace <ns> ...] [--category <namespace=cid> ...] [--manifest <path>] [--stale-policy <report|archive>] [--dry-run] [--create] [--force]", commandName)
default: default:
if value, ok, err := requireInlineFlagValue(arg, "--source-dir"); ok || err != nil { if value, ok, err := requireInlineFlagValue(arg, "--source-dir"); ok || err != nil {
if err != nil { if err != nil {
@@ -1995,6 +2002,11 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO
return opts, err return opts, err
} }
opts.ManifestPath = value opts.ManifestPath = value
} else if value, ok, err := requireInlineFlagValue(arg, "--stale-policy"); ok || err != nil {
if err != nil {
return opts, err
}
opts.StalePolicy = value
} else if arg == "--dry-run" { } else if arg == "--dry-run" {
opts.DryRun = true opts.DryRun = true
} else if arg == "--force" { } else if arg == "--force" {
internal/app/app_test.go
+6 -3
View File
@@ -242,13 +242,16 @@ func TestTopdataConsoleDebugProgressAndRelativePaths(t *testing.T) {
level: logLevelDebug, level: logLevelDebug,
} }
console.progress("NodeBB wiki plan: create 1, update 2, skip 3, drift 0") console.progress("NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, drift 0")
console.emitWikiDeployResult(10, 1, 2, 3, 4, 0, "/workspace/project/build/wiki/deploy-manifest.json") console.emitWikiDeployResult(10, 1, 2, 3, 4, 5, 0, "/workspace/project/build/wiki/deploy-manifest.json")
output := stdout.String() output := stdout.String()
if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, drift 0") { if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, drift 0") {
t.Fatalf("expected debug progress line, got %q", output) t.Fatalf("expected debug progress line, got %q", output)
} }
if !strings.Contains(output, "archived: 5") {
t.Fatalf("expected archived deploy count, got %q", output)
}
if !strings.Contains(output, "manifest: build/wiki/deploy-manifest.json") { if !strings.Contains(output, "manifest: build/wiki/deploy-manifest.json") {
t.Fatalf("expected relative deploy manifest path, got %q", output) t.Fatalf("expected relative deploy manifest path, got %q", output)
} }
internal/project/effective.go
+115 -81
View File
@@ -13,41 +13,51 @@ import (
) )
const ( const (
DefaultBuildPath = "build" DefaultBuildPath = "build"
DefaultCachePath = ".cache" DefaultCachePath = ".cache"
DefaultToolsPath = "tools" DefaultToolsPath = "tools"
DefaultModuleArchiveTemplate = "{module.resref}.mod" DefaultModuleArchiveTemplate = "{module.resref}.mod"
DefaultHAKManifest = "haks.json" DefaultHAKManifest = "haks.json"
DefaultHAKArchiveTemplate = "{hak.name}.hak" DefaultHAKArchiveTemplate = "{hak.name}.hak"
DefaultSourceJSONPattern = "{resref}.{extension}.json" DefaultSourceJSONPattern = "{resref}.{extension}.json"
DefaultValidationProfile = "nwn_module" DefaultValidationProfile = "nwn_module"
DefaultScriptsSourceDir = "scripts" DefaultScriptsSourceDir = "scripts"
DefaultScriptsCache = "{paths.cache}/ncs" DefaultScriptsCache = "{paths.cache}/ncs"
DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER" DefaultScriptCompilerEnvPath = "SOW_NWN_SCRIPT_COMPILER"
DefaultMusicStageRoot = "{paths.cache}/music" DefaultMusicStageRoot = "{paths.cache}/music"
DefaultCreditsRoot = "{paths.cache}/credits" DefaultCreditsRoot = "{paths.cache}/credits"
DefaultCreditsOverlayFile = "CREDITS.md" DefaultCreditsOverlayFile = "CREDITS.md"
DefaultMusicNamingScheme = "nwn_bmu" DefaultMusicNamingScheme = "nwn_bmu"
DefaultMusicOutputExtension = ".bmu" DefaultMusicOutputExtension = ".bmu"
DefaultTopDataBuild = "{paths.build}/topdata" DefaultTopDataBuild = "{paths.build}/topdata"
DefaultTopDataCompiled2DADir = "2da" DefaultTopDataCompiled2DADir = "2da"
DefaultTopDataCompiledTLK = "sow_tlk.tlk" DefaultTopDataCompiledTLK = "sow_tlk.tlk"
DefaultTopDataWikiOutputRoot = "wiki" DefaultTopDataWikiOutputRoot = "wiki"
DefaultTopDataWikiPagesDir = "pages" DefaultTopDataWikiPagesDir = "pages"
DefaultTopDataWikiStateFile = "state.json" DefaultTopDataWikiStateFile = "state.json"
DefaultWikiDeployManifest = ".wiki_deploy_manifest.json" DefaultTopDataWikiSource = "topdata/wiki"
DefaultWikiDeployEditSummary = "Auto-generated from native builder" DefaultTopDataWikiRenderer = "nodebb_tiptap_html"
DefaultTopDataPackageHAK = "sow_top.hak" DefaultTopDataWikiLinkStrategy = "preserve_westgate_wiki_links"
DefaultTopDataPackageTLK = "sow_tlk.tlk" DefaultTopDataWikiNamespacesFile = "namespaces.yaml"
DefaultAutogenCacheRoot = "{paths.cache}" DefaultTopDataWikiTemplatesDir = "templates"
DefaultAutogenCacheMaxAge = time.Hour DefaultTopDataWikiManualSectionsDir = "manual-sections"
DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH" DefaultTopDataWikiStaleDefault = "report"
DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH" DefaultTopDataWikiStaleLiveCleanup = "archive"
DefaultAutogenReleaseProvider = "gitea" DefaultTopDataWikiMarkerFormat = "html_comments"
DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL" DefaultTopDataWikiPageMarkerPrefix = "sow-topdata-wiki"
DefaultAutogenRepoEnv = "SOW_ASSETS_REPO" DefaultWikiDeployManifest = ".wiki_deploy_manifest.json"
DefaultExtractLayout = "nwn_canonical_json" DefaultWikiDeployEditSummary = "Auto-generated from native builder"
DefaultExtractHAKDiscovery = "build_glob" DefaultTopDataPackageHAK = "sow_top.hak"
DefaultTopDataPackageTLK = "sow_tlk.tlk"
DefaultAutogenCacheRoot = "{paths.cache}"
DefaultAutogenCacheMaxAge = time.Hour
DefaultAutogenRefreshEnv = "SOW_AUTOGEN_MANIFEST_REFRESH"
DefaultPartsManifestRefreshEnv = "SOW_PARTS_MANIFEST_REFRESH"
DefaultAutogenReleaseProvider = "gitea"
DefaultAutogenServerURLEnv = "SOW_ASSETS_SERVER_URL"
DefaultAutogenRepoEnv = "SOW_ASSETS_REPO"
DefaultExtractLayout = "nwn_canonical_json"
DefaultExtractHAKDiscovery = "build_glob"
) )
type ConfigProvenance map[string]ConfigValueProvenance type ConfigProvenance map[string]ConfigValueProvenance
@@ -228,6 +238,20 @@ func (p *Project) EffectiveConfig() EffectiveConfig {
ManagedNamespaces: defaultStringSlice(p.Config.TopData.Wiki.ManagedNamespaces, []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}), ManagedNamespaces: defaultStringSlice(p.Config.TopData.Wiki.ManagedNamespaces, []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}),
DeployManifest: defaultString(p.Config.TopData.Wiki.DeployManifest, DefaultWikiDeployManifest), DeployManifest: defaultString(p.Config.TopData.Wiki.DeployManifest, DefaultWikiDeployManifest),
DeployEditSummary: defaultString(p.Config.TopData.Wiki.DeployEditSummary, DefaultWikiDeployEditSummary), DeployEditSummary: defaultString(p.Config.TopData.Wiki.DeployEditSummary, DefaultWikiDeployEditSummary),
Source: defaultString(p.Config.TopData.Wiki.Source, DefaultTopDataWikiSource),
Renderer: defaultString(p.Config.TopData.Wiki.Renderer, DefaultTopDataWikiRenderer),
LinkStrategy: defaultString(p.Config.TopData.Wiki.LinkStrategy, DefaultTopDataWikiLinkStrategy),
NamespacesFile: defaultString(p.Config.TopData.Wiki.NamespacesFile, DefaultTopDataWikiNamespacesFile),
TemplatesDir: defaultString(p.Config.TopData.Wiki.TemplatesDir, DefaultTopDataWikiTemplatesDir),
ManualSectionsDir: defaultString(p.Config.TopData.Wiki.ManualSectionsDir, DefaultTopDataWikiManualSectionsDir),
StalePages: TopDataWikiStalePagesConfig{
Default: defaultString(p.Config.TopData.Wiki.StalePages.Default, DefaultTopDataWikiStaleDefault),
LiveCleanup: defaultString(p.Config.TopData.Wiki.StalePages.LiveCleanup, DefaultTopDataWikiStaleLiveCleanup),
},
ManagedRegion: TopDataWikiManagedRegionConfig{
MarkerFormat: defaultString(p.Config.TopData.Wiki.ManagedRegion.MarkerFormat, DefaultTopDataWikiMarkerFormat),
PageMarkerPrefix: defaultString(p.Config.TopData.Wiki.ManagedRegion.PageMarkerPrefix, DefaultTopDataWikiPageMarkerPrefix),
},
}, },
} }
extract := p.Config.Extract extract := p.Config.Extract
@@ -407,52 +431,62 @@ func lookupEffectiveValue(effective EffectiveConfig, key string) (any, bool) {
func markMissingDefaults(provenance ConfigProvenance) { func markMissingDefaults(provenance ConfigProvenance) {
defaults := map[string]string{ defaults := map[string]string{
"paths.build": "build", "paths.build": "build",
"paths.cache": ".cache", "paths.cache": ".cache",
"paths.tools": "tools", "paths.tools": "tools",
"outputs.module_archive": DefaultModuleArchiveTemplate, "outputs.module_archive": DefaultModuleArchiveTemplate,
"outputs.hak_manifest": DefaultHAKManifest, "outputs.hak_manifest": DefaultHAKManifest,
"outputs.hak_archive": DefaultHAKArchiveTemplate, "outputs.hak_archive": DefaultHAKArchiveTemplate,
"inventory.source_extensions": "NWN source resource extensions", "inventory.source_extensions": "NWN source resource extensions",
"inventory.asset_extensions": "NWN asset resource extensions", "inventory.asset_extensions": "NWN asset resource extensions",
"inventory.source_json_pattern": DefaultSourceJSONPattern, "inventory.source_json_pattern": DefaultSourceJSONPattern,
"validation.profile": DefaultValidationProfile, "validation.profile": DefaultValidationProfile,
"validation.builtin_script_prefixes": "NWN built-in script prefixes", "validation.builtin_script_prefixes": "NWN built-in script prefixes",
"validation.required_fields": "NWN required GFF fields", "validation.required_fields": "NWN required GFF fields",
"scripts.source_dir": DefaultScriptsSourceDir, "scripts.source_dir": DefaultScriptsSourceDir,
"scripts.cache": DefaultScriptsCache, "scripts.cache": DefaultScriptsCache,
"scripts.compiler.env.path": DefaultScriptCompilerEnvPath, "scripts.compiler.env.path": DefaultScriptCompilerEnvPath,
"scripts.compiler.search": "repo tools and PATH:nwn_script_comp", "scripts.compiler.search": "repo tools and PATH:nwn_script_comp",
"music.stage_root": DefaultMusicStageRoot, "music.stage_root": DefaultMusicStageRoot,
"music.credits_root": DefaultCreditsRoot, "music.credits_root": DefaultCreditsRoot,
"music.credits_overlay": DefaultCreditsOverlayFile, "music.credits_overlay": DefaultCreditsOverlayFile,
"music.max_stem_length": "16", "music.max_stem_length": "16",
"music.naming_scheme": DefaultMusicNamingScheme, "music.naming_scheme": DefaultMusicNamingScheme,
"music.output_extension": DefaultMusicOutputExtension, "music.output_extension": DefaultMusicOutputExtension,
"music.convert_extensions": strings.Join(music.DefaultConvertExtensions(), ", "), "music.convert_extensions": strings.Join(music.DefaultConvertExtensions(), ", "),
"topdata.build": DefaultTopDataBuild, "topdata.build": DefaultTopDataBuild,
"topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir, "topdata.compiled_2da_dir": DefaultTopDataCompiled2DADir,
"topdata.compiled_tlk": DefaultTopDataCompiledTLK, "topdata.compiled_tlk": DefaultTopDataCompiledTLK,
"topdata.package_hak": DefaultTopDataPackageHAK, "topdata.package_hak": DefaultTopDataPackageHAK,
"topdata.package_tlk": DefaultTopDataPackageTLK, "topdata.package_tlk": DefaultTopDataPackageTLK,
"topdata.wiki.output_root": DefaultTopDataWikiOutputRoot, "topdata.wiki.output_root": DefaultTopDataWikiOutputRoot,
"topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir, "topdata.wiki.pages_dir": DefaultTopDataWikiPagesDir,
"topdata.wiki.state_file": DefaultTopDataWikiStateFile, "topdata.wiki.state_file": DefaultTopDataWikiStateFile,
"topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces", "topdata.wiki.managed_namespaces": "NWN topdata wiki namespaces",
"topdata.wiki.deploy_manifest": DefaultWikiDeployManifest, "topdata.wiki.deploy_manifest": DefaultWikiDeployManifest,
"topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary, "topdata.wiki.deploy_edit_summary": DefaultWikiDeployEditSummary,
"extract.layout": DefaultExtractLayout, "topdata.wiki.source": DefaultTopDataWikiSource,
"extract.hak_discovery": DefaultExtractHAKDiscovery, "topdata.wiki.renderer": DefaultTopDataWikiRenderer,
"extract.archives": DefaultModuleArchiveTemplate, "topdata.wiki.link_strategy": DefaultTopDataWikiLinkStrategy,
"extract.cleanup_stale": "true", "topdata.wiki.namespaces_file": DefaultTopDataWikiNamespacesFile,
"extract.consume_archives": "false", "topdata.wiki.templates_dir": DefaultTopDataWikiTemplatesDir,
"autogen.cache.root": DefaultAutogenCacheRoot, "topdata.wiki.manual_sections_dir": DefaultTopDataWikiManualSectionsDir,
"autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(), "topdata.wiki.stale_pages.default": DefaultTopDataWikiStaleDefault,
"autogen.cache.refresh_env": DefaultAutogenRefreshEnv, "topdata.wiki.stale_pages.live_cleanup": DefaultTopDataWikiStaleLiveCleanup,
"autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv, "topdata.wiki.managed_region.marker_format": DefaultTopDataWikiMarkerFormat,
"autogen.release_source.provider": DefaultAutogenReleaseProvider, "topdata.wiki.managed_region.page_marker_prefix": DefaultTopDataWikiPageMarkerPrefix,
"autogen.release_source.server_url_env": DefaultAutogenServerURLEnv, "extract.layout": DefaultExtractLayout,
"autogen.release_source.repo_env": DefaultAutogenRepoEnv, "extract.hak_discovery": DefaultExtractHAKDiscovery,
"extract.archives": DefaultModuleArchiveTemplate,
"extract.cleanup_stale": "true",
"extract.consume_archives": "false",
"autogen.cache.root": DefaultAutogenCacheRoot,
"autogen.cache.max_age": DefaultAutogenCacheMaxAge.String(),
"autogen.cache.refresh_env": DefaultAutogenRefreshEnv,
"autogen.cache.parts_manifest_refresh_env": DefaultPartsManifestRefreshEnv,
"autogen.release_source.provider": DefaultAutogenReleaseProvider,
"autogen.release_source.server_url_env": DefaultAutogenServerURLEnv,
"autogen.release_source.repo_env": DefaultAutogenRepoEnv,
} }
for key, detail := range defaults { for key, detail := range defaults {
if _, ok := provenance[key]; !ok { if _, ok := provenance[key]; !ok {
internal/project/project.go
+56 -6
View File
@@ -204,12 +204,30 @@ type TopDataConfig struct {
} }
type TopDataWikiConfig struct { type TopDataWikiConfig struct {
OutputRoot string `json:"output_root" yaml:"output_root"` OutputRoot string `json:"output_root" yaml:"output_root"`
PagesDir string `json:"pages_dir" yaml:"pages_dir"` PagesDir string `json:"pages_dir" yaml:"pages_dir"`
StateFile string `json:"state_file" yaml:"state_file"` StateFile string `json:"state_file" yaml:"state_file"`
ManagedNamespaces []string `json:"managed_namespaces" yaml:"managed_namespaces"` ManagedNamespaces []string `json:"managed_namespaces" yaml:"managed_namespaces"`
DeployManifest string `json:"deploy_manifest" yaml:"deploy_manifest"` DeployManifest string `json:"deploy_manifest" yaml:"deploy_manifest"`
DeployEditSummary string `json:"deploy_edit_summary" yaml:"deploy_edit_summary"` DeployEditSummary string `json:"deploy_edit_summary" yaml:"deploy_edit_summary"`
Source string `json:"source" yaml:"source"`
Renderer string `json:"renderer" yaml:"renderer"`
LinkStrategy string `json:"link_strategy" yaml:"link_strategy"`
NamespacesFile string `json:"namespaces_file" yaml:"namespaces_file"`
TemplatesDir string `json:"templates_dir" yaml:"templates_dir"`
ManualSectionsDir string `json:"manual_sections_dir" yaml:"manual_sections_dir"`
StalePages TopDataWikiStalePagesConfig `json:"stale_pages" yaml:"stale_pages"`
ManagedRegion TopDataWikiManagedRegionConfig `json:"managed_region" yaml:"managed_region"`
}
type TopDataWikiStalePagesConfig struct {
Default string `json:"default" yaml:"default"`
LiveCleanup string `json:"live_cleanup" yaml:"live_cleanup"`
}
type TopDataWikiManagedRegionConfig struct {
MarkerFormat string `json:"marker_format" yaml:"marker_format"`
PageMarkerPrefix string `json:"page_marker_prefix" yaml:"page_marker_prefix"`
} }
type ExtractConfig struct { type ExtractConfig struct {
@@ -485,6 +503,10 @@ func (p *Project) ValidateLayout() error {
failures = append(failures, validateRelativePath("topdata.wiki.pages_dir", effective.TopData.Wiki.PagesDir)...) failures = append(failures, validateRelativePath("topdata.wiki.pages_dir", effective.TopData.Wiki.PagesDir)...)
failures = append(failures, validateRelativePath("topdata.wiki.state_file", effective.TopData.Wiki.StateFile)...) failures = append(failures, validateRelativePath("topdata.wiki.state_file", effective.TopData.Wiki.StateFile)...)
failures = append(failures, validateRelativePath("topdata.wiki.deploy_manifest", effective.TopData.Wiki.DeployManifest)...) failures = append(failures, validateRelativePath("topdata.wiki.deploy_manifest", effective.TopData.Wiki.DeployManifest)...)
failures = append(failures, validateTreeRootPath("topdata.wiki.source", effective.TopData.Wiki.Source)...)
failures = append(failures, validateRelativePath("topdata.wiki.namespaces_file", effective.TopData.Wiki.NamespacesFile)...)
failures = append(failures, validateTreeRootPath("topdata.wiki.templates_dir", effective.TopData.Wiki.TemplatesDir)...)
failures = append(failures, validateTreeRootPath("topdata.wiki.manual_sections_dir", effective.TopData.Wiki.ManualSectionsDir)...)
failures = append(failures, validateRelativePath("autogen.cache.root", effective.Autogen.Cache.Root)...) failures = append(failures, validateRelativePath("autogen.cache.root", effective.Autogen.Cache.Root)...)
if err := validateOutputFileName("outputs.module_archive", filepath.Base(effective.Outputs.ModuleArchive), ".mod"); err != nil { if err := validateOutputFileName("outputs.module_archive", filepath.Base(effective.Outputs.ModuleArchive), ".mod"); err != nil {
failures = append(failures, err) failures = append(failures, err)
@@ -498,6 +520,26 @@ func (p *Project) ValidateLayout() error {
if err := validateOutputFileName("topdata.wiki.deploy_manifest", filepath.Base(effective.TopData.Wiki.DeployManifest), ".json"); err != nil { if err := validateOutputFileName("topdata.wiki.deploy_manifest", filepath.Base(effective.TopData.Wiki.DeployManifest), ".json"); err != nil {
failures = append(failures, err) failures = append(failures, err)
} }
switch effective.TopData.Wiki.Renderer {
case "nodebb_tiptap_html":
default:
failures = append(failures, fmt.Errorf("topdata.wiki.renderer %q is not supported", effective.TopData.Wiki.Renderer))
}
switch effective.TopData.Wiki.LinkStrategy {
case "preserve_westgate_wiki_links":
default:
failures = append(failures, fmt.Errorf("topdata.wiki.link_strategy %q is not supported", effective.TopData.Wiki.LinkStrategy))
}
for key, policy := range map[string]string{
"topdata.wiki.stale_pages.default": effective.TopData.Wiki.StalePages.Default,
"topdata.wiki.stale_pages.live_cleanup": effective.TopData.Wiki.StalePages.LiveCleanup,
} {
switch policy {
case "report", "archive", "unpublish":
default:
failures = append(failures, fmt.Errorf("%s %q is not supported", key, policy))
}
}
switch effective.Extract.Layout { switch effective.Extract.Layout {
case "nwn_canonical_json": case "nwn_canonical_json":
default: default:
@@ -834,6 +876,14 @@ func (p *Project) TopDataWikiRootDir() string {
return filepath.Join(p.TopDataBuildDir(), wikiRoot) return filepath.Join(p.TopDataBuildDir(), wikiRoot)
} }
func (p *Project) TopDataWikiSourceDir() string {
source := filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.Source)
if filepath.IsAbs(source) {
return source
}
return filepath.Join(p.Root, source)
}
func (p *Project) TopDataWikiPagesDir() string { func (p *Project) TopDataWikiPagesDir() string {
return filepath.Join(p.TopDataWikiRootDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.PagesDir)) return filepath.Join(p.TopDataWikiRootDir(), filepath.FromSlash(p.EffectiveConfig().TopData.Wiki.PagesDir))
} }
internal/project/project_test.go
+81
View File
@@ -256,10 +256,19 @@ extract:
cleanup_stale: false cleanup_stale: false
topdata: topdata:
wiki: wiki:
source: topdata/wiki
managed_namespaces: managed_namespaces:
- skills - skills
renderer: nodebb_tiptap_html
link_strategy: preserve_westgate_wiki_links
namespaces_file: namespaces.yaml
templates_dir: templates
manual_sections_dir: manual-sections
deploy_manifest: wiki-manifest.json deploy_manifest: wiki-manifest.json
deploy_edit_summary: Custom summary deploy_edit_summary: Custom summary
stale_pages:
default: report
live_cleanup: archive
autogen: autogen:
cache: cache:
root: "{paths.cache}/released" root: "{paths.cache}/released"
@@ -297,6 +306,30 @@ autogen:
if got, want := effective.TopData.Wiki.DeployManifest, "wiki-manifest.json"; got != want { if got, want := effective.TopData.Wiki.DeployManifest, "wiki-manifest.json"; got != want {
t.Fatalf("expected wiki deploy manifest %q, got %q", want, got) t.Fatalf("expected wiki deploy manifest %q, got %q", want, got)
} }
if got, want := effective.TopData.Wiki.Source, "topdata/wiki"; got != want {
t.Fatalf("expected wiki source %q, got %q", want, got)
}
if got, want := effective.TopData.Wiki.Renderer, "nodebb_tiptap_html"; got != want {
t.Fatalf("expected wiki renderer %q, got %q", want, got)
}
if got, want := effective.TopData.Wiki.LinkStrategy, "preserve_westgate_wiki_links"; got != want {
t.Fatalf("expected wiki link strategy %q, got %q", want, got)
}
if got, want := effective.TopData.Wiki.NamespacesFile, "namespaces.yaml"; got != want {
t.Fatalf("expected wiki namespaces file %q, got %q", want, got)
}
if got, want := effective.TopData.Wiki.TemplatesDir, "templates"; got != want {
t.Fatalf("expected wiki templates dir %q, got %q", want, got)
}
if got, want := effective.TopData.Wiki.ManualSectionsDir, "manual-sections"; got != want {
t.Fatalf("expected wiki manual sections dir %q, got %q", want, got)
}
if got, want := effective.TopData.Wiki.StalePages.Default, "report"; got != want {
t.Fatalf("expected wiki stale default %q, got %q", want, got)
}
if got, want := effective.TopData.Wiki.StalePages.LiveCleanup, "archive"; got != want {
t.Fatalf("expected wiki stale live cleanup %q, got %q", want, got)
}
if got, want := effective.Autogen.Cache.MaxAge, "30m"; got != want { if got, want := effective.Autogen.Cache.MaxAge, "30m"; got != want {
t.Fatalf("expected autogen cache max age %q, got %q", want, got) t.Fatalf("expected autogen cache max age %q, got %q", want, got)
} }
@@ -704,6 +737,54 @@ func TestValidateLayoutRejectsUnsafeGeneratedOutputPaths(t *testing.T) {
} }
} }
func TestValidateLayoutRejectsInvalidTopDataWikiConfig(t *testing.T) {
root := t.TempDir()
mkdirAll(t, filepath.Join(root, "src"))
mkdirAll(t, filepath.Join(root, "build"))
proj := &Project{
Root: root,
Config: Config{
Module: ModuleConfig{Name: "Test", ResRef: "test"},
Paths: PathConfig{Source: "src", Build: "build"},
TopData: TopDataConfig{
Source: "topdata",
Wiki: TopDataWikiConfig{
Source: "../wiki",
Renderer: "dokuwiki",
LinkStrategy: "rewrite_links",
NamespacesFile: "../namespaces.yaml",
TemplatesDir: ".",
ManualSectionsDir: "../manual",
StalePages: TopDataWikiStalePagesConfig{
Default: "delete",
LiveCleanup: "delete",
},
},
},
},
}
err := proj.ValidateLayout()
if err == nil {
t.Fatal("expected invalid wiki config validation error")
}
for _, needle := range []string{
"topdata.wiki.source",
"topdata.wiki.renderer",
"topdata.wiki.link_strategy",
"topdata.wiki.namespaces_file",
"topdata.wiki.templates_dir",
"topdata.wiki.manual_sections_dir",
"topdata.wiki.stale_pages.default",
"topdata.wiki.stale_pages.live_cleanup",
} {
if !strings.Contains(err.Error(), needle) {
t.Fatalf("expected validation error to mention %s, got %v", needle, err)
}
}
}
func TestValidateLayoutRejectsRootSourceAndAssetPaths(t *testing.T) { func TestValidateLayoutRejectsRootSourceAndAssetPaths(t *testing.T) {
root := t.TempDir() root := t.TempDir()
mkdirAll(t, filepath.Join(root, "build")) mkdirAll(t, filepath.Join(root, "build"))
internal/topdata/wiki_deploy.go
+270 -13
View File
@@ -17,13 +17,18 @@ import (
"time" "time"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
"gopkg.in/yaml.v3"
) )
const ( const (
managedStartMarker = "[//]: # (sow-topdata-wiki:managed:start)" managedStartMarker = "<!-- sow-topdata-wiki:managed:start"
managedEndMarker = "[//]: # (sow-topdata-wiki:managed:end)" managedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
legacyManagedStartMarker = "<!-- sow-topdata-wiki:managed:start -->" legacyMarkdownStartMarker = "[//]: # (sow-topdata-wiki:managed:start)"
legacyManagedEndMarker = "<!-- sow-topdata-wiki:managed:end -->" legacyMarkdownEndMarker = "[//]: # (sow-topdata-wiki:managed:end)"
legacyManagedStartMarker = "<!-- sow-topdata-wiki:managed:start -->"
legacyManagedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
manualStartMarkerPrefix = "<!-- sow-topdata-wiki:manual:start"
manualEndMarkerPrefix = "<!-- sow-topdata-wiki:manual:end"
) )
type DeployWikiOptions struct { type DeployWikiOptions struct {
@@ -48,6 +53,7 @@ type deployResult struct {
Created int Created int
Updated int Updated int
Stale int Stale int
Archived int
Skipped int Skipped int
Drifted int Drifted int
Manifest string Manifest string
@@ -67,10 +73,15 @@ type wikiDeployManifest struct {
} }
type wikiDeployManifestPage struct { type wikiDeployManifestPage struct {
Hash string `json:"hash"` Hash string `json:"hash"`
TID int `json:"tid,omitempty"` LastSeenHash string `json:"last_seen_hash,omitempty"`
PID int `json:"pid,omitempty"` ArchivedHash string `json:"archived_hash,omitempty"`
CID int `json:"cid,omitempty"` Title string `json:"title,omitempty"`
Namespace string `json:"namespace,omitempty"`
Stale bool `json:"stale,omitempty"`
TID int `json:"tid,omitempty"`
PID int `json:"pid,omitempty"`
CID int `json:"cid,omitempty"`
} }
type wikiDeployPlan struct { type wikiDeployPlan struct {
@@ -81,6 +92,19 @@ type wikiDeployPlan struct {
RemoteHash string RemoteHash string
} }
type wikiNamespacesDocument struct {
Namespaces []wikiNamespaceDeclaration `json:"namespaces" yaml:"namespaces"`
}
type wikiNamespaceDeclaration struct {
ID string `json:"id" yaml:"id"`
Title string `json:"title" yaml:"title"`
CategoryEnv string `json:"category_env" yaml:"category_env"`
PageTitle string `json:"page_title" yaml:"page_title"`
Tags []string `json:"tags" yaml:"tags"`
EditPolicy string `json:"edit_policy" yaml:"edit_policy"`
}
func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress func(string)) (deployResult, error) { func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress func(string)) (deployResult, error) {
if progress == nil { if progress == nil {
progress = func(string) {} progress = func(string) {}
@@ -100,10 +124,33 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
if opts.Username != "" || opts.Password != "" || opts.NotesDelimiter != "" { if opts.Username != "" || opts.Password != "" || opts.NotesDelimiter != "" {
return deployResult{}, errors.New("DokuWiki deployment options are no longer supported; use NodeBB endpoint, token, and category mappings") return deployResult{}, errors.New("DokuWiki deployment options are no longer supported; use NodeBB endpoint, token, and category mappings")
} }
if opts.StalePolicy != "" && opts.StalePolicy != "report" && opts.StalePolicy != "archive" {
return deployResult{}, fmt.Errorf("wiki stale policy %q is not supported", opts.StalePolicy)
}
namespaces := opts.Namespaces namespaces := opts.Namespaces
if len(namespaces) == 0 { if len(namespaces) == 0 {
namespaces = p.EffectiveConfig().TopData.Wiki.ManagedNamespaces declarations, err := loadWikiNamespaceDeclarations(p)
if err != nil {
return deployResult{}, err
}
for _, declaration := range declarations {
namespaces = append(namespaces, declaration.ID)
}
if len(namespaces) == 0 {
namespaces = p.EffectiveConfig().TopData.Wiki.ManagedNamespaces
}
envCategories, err := categoryIDsFromNamespaceEnv(declarations)
if err != nil {
return deployResult{}, err
}
if len(envCategories) > 0 {
merged := envCategories
for namespace, cid := range opts.CategoryIDs {
merged[namespace] = cid
}
opts.CategoryIDs = merged
}
} }
manifestPath := opts.ManifestPath manifestPath := opts.ManifestPath
@@ -125,7 +172,7 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
result.LocalPages = len(pages) result.LocalPages = len(pages)
result.Manifest = manifestPath result.Manifest = manifestPath
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, skip %d, drift %d", result.Created, result.Updated, result.Skipped, result.Drifted)) progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, skip %d, stale %d, archive %d, drift %d", result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Drifted))
if opts.DryRun { if opts.DryRun {
return result, nil return result, nil
} }
@@ -152,6 +199,10 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil { if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil {
return result, err return result, err
} }
case "archive":
if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil {
return result, err
}
} }
} }
if err := saveDeployManifest(manifestPath, nextManifest); err != nil { if err := saveDeployManifest(manifestPath, nextManifest); err != nil {
@@ -160,6 +211,62 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
return result, nil return result, nil
} }
func loadWikiNamespaceDeclarations(p *project.Project) ([]wikiNamespaceDeclaration, error) {
cfg := p.EffectiveConfig().TopData.Wiki
path := filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(cfg.NamespacesFile))
raw, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("read wiki namespaces: %w", err)
}
var doc wikiNamespacesDocument
if err := yaml.Unmarshal(raw, &doc); err != nil {
return nil, fmt.Errorf("parse wiki namespaces: %w", err)
}
seen := map[string]struct{}{}
for _, declaration := range doc.Namespaces {
id := strings.TrimSpace(declaration.ID)
if id == "" {
return nil, errors.New("topdata wiki namespace id is required")
}
if _, ok := seen[id]; ok {
return nil, fmt.Errorf("duplicate topdata wiki namespace id %q", id)
}
seen[id] = struct{}{}
if strings.TrimSpace(declaration.CategoryEnv) == "" {
return nil, fmt.Errorf("topdata wiki namespace %q category_env is required", id)
}
switch strings.TrimSpace(declaration.EditPolicy) {
case "", "preserve_manual_sections", "generated_only":
default:
return nil, fmt.Errorf("topdata wiki namespace %q edit_policy %q is not supported", id, declaration.EditPolicy)
}
}
return doc.Namespaces, nil
}
func categoryIDsFromNamespaceEnv(declarations []wikiNamespaceDeclaration) (map[string]int, error) {
categories := map[string]int{}
for _, declaration := range declarations {
envName := strings.TrimSpace(declaration.CategoryEnv)
if envName == "" {
continue
}
raw := strings.TrimSpace(os.Getenv(envName))
if raw == "" {
continue
}
cid, err := strconv.Atoi(raw)
if err != nil || cid <= 0 {
return nil, fmt.Errorf("topdata wiki namespace %q env %s has invalid cid %q", declaration.ID, envName, raw)
}
categories[declaration.ID] = cid
}
return categories, nil
}
func collectLocalPages(sourceDir string, namespaces []string) (map[string]wikiDeployPage, error) { func collectLocalPages(sourceDir string, namespaces []string) (map[string]wikiDeployPage, error) {
namespaceSet := map[string]struct{}{} namespaceSet := map[string]struct{}{}
for _, namespace := range namespaces { for _, namespace := range namespaces {
@@ -224,6 +331,10 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
page := pages[pageID] page := pages[pageID]
entry := manifest.Pages[pageID] entry := manifest.Pages[pageID]
entry.Hash = page.Hash entry.Hash = page.Hash
entry.LastSeenHash = page.Hash
entry.Title = page.Title
entry.Namespace = page.Namespace
entry.Stale = false
if entry.CID == 0 { if entry.CID == 0 {
entry.CID = opts.CategoryIDs[page.Namespace] entry.CID = opts.CategoryIDs[page.Namespace]
} }
@@ -267,7 +378,33 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
} }
for pageID := range manifest.Pages { for pageID := range manifest.Pages {
if _, ok := pages[pageID]; !ok { if _, ok := pages[pageID]; !ok {
entry := manifest.Pages[pageID]
entry.Stale = true
if entry.Namespace == "" {
entry.Namespace = strings.SplitN(pageID, ":", 2)[0]
}
if entry.Title == "" {
entry.Title = extractHTMLTitle("", pageID)
}
next.Pages[pageID] = entry
result.Stale++ result.Stale++
policy := strings.TrimSpace(opts.StalePolicy)
if policy == "" {
policy = "report"
}
if policy == "archive" {
body := renderArchivedWikiPage(pageID, entry.Title)
entry.Hash = computeManagedHash(body)
entry.ArchivedHash = entry.Hash
next.Pages[pageID] = entry
result.Archived++
plans = append(plans, wikiDeployPlan{
Page: wikiDeployPage{PageID: pageID, Title: entry.Title, Namespace: entry.Namespace, Content: body, Hash: entry.Hash},
Entry: entry,
Action: "archive",
Content: body,
})
}
} }
} }
return plans, result, next, nil return plans, result, next, nil
@@ -322,17 +459,34 @@ func mergeManagedContent(existing, generated string) string {
if !ok { if !ok {
return generated return generated
} }
_, endMarker, start, end := managedRegionBounds(existing) startMarker, endMarker, start, end := managedRegionBounds(existing)
if start < 0 || end < 0 || end < start { if start < 0 || end < 0 || end < start {
return generated return generated
} }
if strings.Contains(generated, manualStartMarkerPrefix) {
return mergeManualRegions(generated, existing)
}
end += len(endMarker) end += len(endMarker)
return existing[:start] + managedStartMarker + "\n" + generatedRegion + "\n" + managedEndMarker + existing[end:] generatedStart, generatedEnd, ok := extractManagedMarkers(generated)
if !ok {
generatedStart = startMarker
generatedEnd = endMarker
}
merged := existing[:start] + generatedStart + "\n" + generatedRegion + "\n" + generatedEnd + existing[end:]
return mergeManualRegions(merged, existing)
} }
func managedRegionBounds(content string) (string, string, int, int) { func managedRegionBounds(content string) (string, string, int, int) {
if start := strings.Index(content, managedStartMarker); start >= 0 {
startEnd := strings.Index(content[start:], "-->")
end := strings.Index(content, managedEndMarker)
if startEnd >= 0 && end >= 0 {
startMarker := content[start : start+startEnd+len("-->")]
return startMarker, managedEndMarker, start, end
}
}
for _, markers := range [][2]string{ for _, markers := range [][2]string{
{managedStartMarker, managedEndMarker}, {legacyMarkdownStartMarker, legacyMarkdownEndMarker},
{legacyManagedStartMarker, legacyManagedEndMarker}, {legacyManagedStartMarker, legacyManagedEndMarker},
} { } {
start := strings.Index(content, markers[0]) start := strings.Index(content, markers[0])
@@ -344,6 +498,109 @@ func managedRegionBounds(content string) (string, string, int, int) {
return "", "", -1, -1 return "", "", -1, -1
} }
func extractManagedMarkers(content string) (string, string, bool) {
startMarker, endMarker, start, end := managedRegionBounds(content)
return startMarker, endMarker, start >= 0 && end >= 0
}
func mergeManualRegions(merged, existing string) string {
existingManual := extractManualRegions(existing)
if len(existingManual) == 0 {
return merged
}
return replaceManualRegions(merged, existingManual)
}
func extractManualRegions(content string) map[string]string {
regions := map[string]string{}
offset := 0
for {
startRel := strings.Index(content[offset:], manualStartMarkerPrefix)
if startRel < 0 {
break
}
start := offset + startRel
startEndRel := strings.Index(content[start:], "-->")
if startEndRel < 0 {
break
}
startMarker := content[start : start+startEndRel+len("-->")]
id := markerID(startMarker)
if id == "" {
offset = start + len(startMarker)
continue
}
endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\""
end := strings.Index(content[start+len(startMarker):], endPrefix)
if end < 0 {
offset = start + len(startMarker)
continue
}
end += start + len(startMarker)
endEndRel := strings.Index(content[end:], "-->")
if endEndRel < 0 {
break
}
endMarker := content[end : end+endEndRel+len("-->")]
regions[id] = startMarker + strings.TrimRight(content[start+len(startMarker):end], "\n") + "\n" + endMarker
offset = end + len(endMarker)
}
return regions
}
func replaceManualRegions(content string, replacements map[string]string) string {
for id, replacement := range replacements {
start := strings.Index(content, manualStartMarkerPrefix+" id=\""+id+"\"")
if start < 0 {
continue
}
endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\""
end := strings.Index(content[start:], endPrefix)
if end < 0 {
continue
}
end += start
endEndRel := strings.Index(content[end:], "-->")
if endEndRel < 0 {
continue
}
endEnd := end + endEndRel + len("-->")
content = content[:start] + replacement + content[endEnd:]
}
return content
}
func markerID(marker string) string {
const prefix = `id="`
start := strings.Index(marker, prefix)
if start < 0 {
return ""
}
start += len(prefix)
end := strings.Index(marker[start:], `"`)
if end < 0 {
return ""
}
return marker[start : start+end]
}
func renderArchivedWikiPage(pageID, title string) string {
if title == "" {
title = extractHTMLTitle("", pageID)
}
body := strings.Join([]string{
"<h1>" + html.EscapeString(title) + "</h1>",
"<p>This generated page is no longer present in the current Shadows Over Westgate topdata wiki source.</p>",
}, "\n")
hash := computeManagedHash(body)
return ensureTrailingNewline(strings.Join([]string{
"<!-- sow-topdata-wiki:page=" + pageID + " -->",
"<!-- sow-topdata-wiki:managed:start hash=\"sha256:" + hash + "\" -->",
body,
"<!-- sow-topdata-wiki:managed:end -->",
}, "\n"))
}
func extractPageTitle(content, fallback string) string { func extractPageTitle(content, fallback string) string {
if title := extractMarkdownTitle(content); title != "" { if title := extractMarkdownTitle(content); title != "" {
return title return title
internal/topdata/wiki_deploy_test.go
+163
View File
@@ -6,6 +6,7 @@ import (
"net/http/httptest" "net/http/httptest"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
@@ -194,3 +195,165 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) {
t.Fatalf("DeployWikiWithOptions create failed: %v", err) t.Fatalf("DeployWikiWithOptions create failed: %v", err)
} }
} }
func TestDeployWikiMergesHTMLManagedAndManualRegions(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
generated := `<!-- sow-topdata-wiki:page=skills:athletics -->
<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
<h1>Athletics</h1>
<p>Generated athletics page</p>
<!-- sow-topdata-wiki:managed:end -->
<!-- sow-topdata-wiki:manual:start id="notes" -->
<h2>Notes</h2>
<p></p>
<!-- sow-topdata-wiki:manual:end id="notes" -->
<!-- sow-topdata-wiki:manual:start id="see_also" -->
<h2>See Also</h2>
<p></p>
<!-- sow-topdata-wiki:manual:end id="see_also" -->
`
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil {
t.Fatalf("write source page: %v", err)
}
old := `<!-- sow-topdata-wiki:page=skills:athletics -->
<!-- sow-topdata-wiki:managed:start hash="sha256:old" -->
<h1>Athletics</h1>
<p>Old generated text</p>
<!-- sow-topdata-wiki:managed:end -->
<!-- sow-topdata-wiki:manual:start id="notes" -->
<h2>Notes</h2>
<p>Keep this human note.</p>
<!-- sow-topdata-wiki:manual:end id="notes" -->
`
manifestPath := filepath.Join(root, "deploy-manifest.json")
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
Version: "nodebb-v1",
Pages: map[string]wikiDeployManifestPage{
"skills:athletics": {Hash: computeManagedHash(old), TID: 7, PID: 42, CID: 3},
},
}); err != nil {
t.Fatalf("write deploy manifest: %v", err)
}
var updated string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42":
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7, "content": old}})
case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42":
var req struct {
Content string `json:"content"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatalf("decode update request: %v", err)
}
updated = req.Content
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}})
default:
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
}
}))
defer server.Close()
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions update failed: %v", err)
}
if result.Updated != 1 {
t.Fatalf("expected one update, got %#v", result)
}
if !containsAll(updated, "<p>Generated athletics page</p>", "<p>Keep this human note.</p>", `manual:start id="see_also"`) {
t.Fatalf("expected update to replace managed HTML and preserve/bootstrap manual regions:\n%s", updated)
}
}
func TestDeployWikiReportsAndArchivesStalePages(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
manifestPath := filepath.Join(root, "deploy-manifest.json")
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
Version: "nodebb-v1",
Pages: map[string]wikiDeployManifestPage{
"skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3},
},
}); err != nil {
t.Fatalf("write deploy manifest: %v", err)
}
var archived string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut || r.URL.Path != "/api/v3/posts/42" {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
}
var req struct {
Content string `json:"content"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatalf("decode archive request: %v", err)
}
archived = req.Content
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}})
}))
defer server.Close()
report, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
DryRun: true,
StalePolicy: "report",
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions stale report failed: %v", err)
}
if report.Stale != 1 {
t.Fatalf("expected one stale report, got %#v", report)
}
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
StalePolicy: "archive",
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions stale archive failed: %v", err)
}
if result.Stale != 1 || result.Archived != 1 {
t.Fatalf("expected one stale archive, got %#v", result)
}
if !containsAll(archived, "<h1>Retired Skill</h1>", "no longer present", `<!-- sow-topdata-wiki:managed:start hash="sha256:`) {
t.Fatalf("unexpected archived body:\n%s", archived)
}
manifest := loadDeployManifest(manifestPath)
entry := manifest.Pages["skills:retired"]
if !entry.Stale || entry.ArchivedHash == "" || entry.PID != 42 {
t.Fatalf("expected archived stale manifest entry, got %#v", entry)
}
}
func containsAll(haystack string, needles ...string) bool {
for _, needle := range needles {
if !strings.Contains(haystack, needle) {
return false
}
}
return true
}
internal/topdata/wiki_native.go
+302 -52
View File
@@ -5,6 +5,7 @@ import (
"embed" "embed"
"encoding/json" "encoding/json"
"fmt" "fmt"
"html"
"io/fs" "io/fs"
"os" "os"
"path/filepath" "path/filepath"
@@ -14,6 +15,7 @@ import (
"time" "time"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
"gopkg.in/yaml.v3"
) )
const ( const (
@@ -24,7 +26,7 @@ const (
legacyWikiRootDirName = ".wiki" legacyWikiRootDirName = ".wiki"
wikiPagesDirName = "pages" wikiPagesDirName = "pages"
wikiStateFileName = "state.json" wikiStateFileName = "state.json"
wikiGeneratorVersion = "nodebb-markdown-v1" wikiGeneratorVersion = "nodebb-tiptap-html-v1"
wikiGeneratedStatus = "generated" wikiGeneratedStatus = "generated"
wikiSkippedStatus = "skipped" wikiSkippedStatus = "skipped"
) )
@@ -43,6 +45,34 @@ type wikiStateDocument struct {
Pages int `json:"pages"` Pages int `json:"pages"`
} }
type wikiPageIndex struct {
Version string `json:"version"`
Pages []wikiPageIndexEntry `json:"pages"`
}
type wikiPageIndexEntry struct {
PageID string `json:"page_id"`
Namespace string `json:"namespace"`
SourceDataset string `json:"source_dataset"`
SourceKey string `json:"source_key"`
Title string `json:"title"`
Hash string `json:"hash"`
OutputPath string `json:"output_path"`
EditPolicy string `json:"edit_policy"`
StalePolicy string `json:"stale_policy"`
}
type wikiManualSectionsDocument struct {
ManualSections []wikiManualSection `json:"manual_sections" yaml:"manual_sections"`
}
type wikiManualSection struct {
ID string `json:"id" yaml:"id"`
Heading string `json:"heading" yaml:"heading"`
Placement string `json:"placement" yaml:"placement"`
InitialHTML string `json:"initial_html" yaml:"initial_html"`
}
type wikiResult struct { type wikiResult struct {
OutputDir string OutputDir string
PageCount int PageCount int
@@ -117,6 +147,7 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
pageTitles := map[string]string{} pageTitles := map[string]string{}
pageStatuses := map[string]string{} pageStatuses := map[string]string{}
pageIndex := wikiPageIndex{Version: wikiGeneratorVersion}
pageCount := 0 pageCount := 0
writePage := func(pageID, content, title, status string) error { writePage := func(pageID, content, title, status string) error {
@@ -124,12 +155,28 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err return err
} }
content = renderNodeBBManagedMarkdown(pageID, content) content = renderNodeBBManagedHTML(pageID, content, loadWikiManualSections(p))
if err := os.WriteFile(path, []byte(content), 0o644); err != nil { if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
return err return err
} }
pageTitles[pageID] = title pageTitles[pageID] = title
pageStatuses[pageID] = status pageStatuses[pageID] = status
namespace := strings.SplitN(pageID, ":", 2)[0]
rel, err := filepath.Rel(rootDir, path)
if err != nil {
rel = filepath.Base(path)
}
pageIndex.Pages = append(pageIndex.Pages, wikiPageIndexEntry{
PageID: pageID,
Namespace: namespace,
SourceDataset: categoryFromWikiNamespace(namespace),
SourceKey: pageID,
Title: title,
Hash: computeManagedHash(content),
OutputPath: filepath.ToSlash(rel),
EditPolicy: wikiEditPolicy(namespace),
StalePolicy: p.EffectiveConfig().TopData.Wiki.StalePages.Default,
})
pageCount++ pageCount++
return nil return nil
} }
@@ -137,7 +184,10 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
if err := generateEntityPages(outputDir, ctx, writePage); err != nil { if err := generateEntityPages(outputDir, ctx, writePage); err != nil {
return wikiResult{}, err return wikiResult{}, err
} }
if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces); err != nil { if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces, loadWikiManualSections(p)); err != nil {
return wikiResult{}, err
}
if err := saveWikiPageIndex(filepath.Join(rootDir, "page-index.json"), pageIndex); err != nil {
return wikiResult{}, err return wikiResult{}, err
} }
@@ -156,6 +206,56 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
}, nil }, nil
} }
func loadWikiManualSections(p *project.Project) []wikiManualSection {
defaults := []wikiManualSection{
{ID: "notes", Heading: "Notes", InitialHTML: "<h2>Notes</h2><p></p>"},
{ID: "see_also", Heading: "See Also", InitialHTML: "<h2>See Also</h2><p></p>"},
}
cfg := p.EffectiveConfig().TopData.Wiki
path := filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(cfg.ManualSectionsDir), "default.yaml")
raw, err := os.ReadFile(path)
if err != nil {
return defaults
}
var doc wikiManualSectionsDocument
if err := yaml.Unmarshal(raw, &doc); err != nil || len(doc.ManualSections) == 0 {
return defaults
}
out := []wikiManualSection{}
for _, section := range doc.ManualSections {
section.ID = strings.TrimSpace(section.ID)
if section.ID == "" {
continue
}
if strings.TrimSpace(section.InitialHTML) == "" {
heading := strings.TrimSpace(section.Heading)
if heading == "" {
heading = section.ID
}
section.InitialHTML = "<h2>" + html.EscapeString(heading) + "</h2><p></p>"
}
out = append(out, section)
}
if len(out) == 0 {
return defaults
}
return out
}
func saveWikiPageIndex(path string, index wikiPageIndex) error {
slices.SortFunc(index.Pages, func(a, b wikiPageIndexEntry) int {
return strings.Compare(a.PageID, b.PageID)
})
raw, err := json.MarshalIndent(index, "", " ")
if err != nil {
return fmt.Errorf("marshal wiki page index: %w", err)
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, append(raw, '\n'), 0o644)
}
func loadWikiState(path string) (wikiStateDocument, error) { func loadWikiState(path string) (wikiStateDocument, error) {
raw, err := os.ReadFile(path) raw, err := os.ReadFile(path)
if err != nil { if err != nil {
@@ -593,24 +693,20 @@ func (ctx *wikiContext) shouldGeneratePage(category, key string, row map[string]
} }
func (ctx *wikiContext) renderPage(category, key string, row map[string]any, title string) (string, error) { func (ctx *wikiContext) renderPage(category, key string, row map[string]any, title string) (string, error) {
templateName := category sections := []string{
if category == "baseitems" { "====== " + title + " ======",
templateName = "items" "",
} else if category == "racialtypes" { buildWikiStatusBlock(ctx.pageStatus(category, key, row), category),
templateName = "races" "",
toDokuWiki(ctx.resolveRowDescription(category, row)),
"",
ctx.renderFacts(category, key, row),
"",
ctx.renderProgression(category, row),
"",
ctx.renderRaceNameForms(row),
} }
templateBytes, err := wikiTemplateFS.ReadFile("wiki_templates/" + templateName + ".txt") return ensureTrailingNewline(strings.Join(sections, "\n")), nil
if err != nil {
return "", err
}
rendered := string(templateBytes)
rendered = strings.ReplaceAll(rendered, "{{name}}", title)
rendered = strings.ReplaceAll(rendered, "{{status}}", buildWikiStatusBlock(ctx.pageStatus(category, key, row), category))
rendered = strings.ReplaceAll(rendered, "{{description}}", toDokuWiki(ctx.resolveRowDescription(category, row)))
rendered = strings.ReplaceAll(rendered, "{{facts}}", ctx.renderFacts(category, key, row))
rendered = strings.ReplaceAll(rendered, "{{progression}}", ctx.renderProgression(category, row))
rendered = strings.ReplaceAll(rendered, "{{nameforms}}", ctx.renderRaceNameForms(row))
return ensureTrailingNewline(rendered), nil
} }
func (ctx *wikiContext) renderFacts(category, key string, row map[string]any) string { func (ctx *wikiContext) renderFacts(category, key string, row map[string]any) string {
@@ -1205,16 +1301,13 @@ func mergeRowWikiStatus(key string, row map[string]any, statuses map[string]stri
func buildWikiStatusBlock(status, category string) string { func buildWikiStatusBlock(status, category string) string {
message := "This " + wikiStatusNoun(category) + " is unchanged from vanilla." message := "This " + wikiStatusNoun(category) + " is unchanged from vanilla."
wrap := "info"
switch status { switch status {
case wikiStatusNew: case wikiStatusNew:
message = "This is a new " + wikiStatusNoun(category) + "!" message = "This is a new " + wikiStatusNoun(category) + "!"
wrap = "tip"
case wikiStatusModified: case wikiStatusModified:
message = "This " + wikiStatusNoun(category) + " has been altered from vanilla." message = "This " + wikiStatusNoun(category) + " has been altered from vanilla."
wrap = "help"
} }
return "<WRAP round " + wrap + ">\n" + message + "\n</WRAP>" return "<p class=\"wiki-callout wiki-callout--status\">" + html.EscapeString(message) + "</p>"
} }
func wikiStatusNoun(category string) string { func wikiStatusNoun(category string) string {
@@ -1236,37 +1329,37 @@ func wikiStatusNoun(category string) string {
} }
} }
func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string, namespaces []string) error { func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string, namespaces []string, manualSections []wikiManualSection) error {
summaries := buildWikiNamespaceSummaries(pageStatuses, namespaces) summaries := buildWikiNamespaceSummaries(pageStatuses, namespaces)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.md"), renderWikiStatusReportPage(summaries, namespaces)); err != nil { if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.html"), renderWikiStatusReportPage(summaries, namespaces), manualSections); err != nil {
return err return err
} }
for _, namespace := range namespaces { for _, namespace := range namespaces {
summary := summaries[namespace] summary := summaries[namespace]
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".md"), renderWikiNamespaceFragment(namespace, summary)); err != nil { if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".html"), renderWikiNamespaceFragment(namespace, summary), manualSections); err != nil {
return err return err
} }
for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} { for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} {
pageIDs := filterWikiPageIDs(pageStatuses, namespace, status) pageIDs := filterWikiPageIDs(pageStatuses, namespace, status)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".md"), renderWikiStatusListingPage(namespaceTitle(namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil { if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".html"), renderWikiStatusListingPage(namespaceTitle(namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles), manualSections); err != nil {
return err return err
} }
} }
} }
for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} { for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} {
pageIDs := filterWikiPageIDs(pageStatuses, "", status) pageIDs := filterWikiPageIDs(pageStatuses, "", status)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".md"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil { if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".html"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles), manualSections); err != nil {
return err return err
} }
} }
return nil return nil
} }
func writeWikiFile(path, content string) error { func writeWikiFile(path, content string, manualSections []wikiManualSection) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err return err
} }
content = renderNodeBBManagedMarkdown(wikiRelPathToPageID(path), content) content = renderNodeBBManagedHTML(wikiRelPathToPageID(path), content, manualSections)
return os.WriteFile(path, []byte(ensureTrailingNewline(content)), 0o644) return os.WriteFile(path, []byte(ensureTrailingNewline(content)), 0o644)
} }
@@ -1311,32 +1404,15 @@ func rollupWikiStatuses(statuses []string) string {
} }
func renderWikiNamespaceFragment(namespace string, summary map[string]any) string { func renderWikiNamespaceFragment(namespace string, summary map[string]any) string {
return strings.Join([]string{ return renderStatusTable(namespaceTitle(namespace), summary, "")
"====== " + namespaceTitle(namespace) + " ======",
"",
fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]),
fmt.Sprintf("**Generated Pages:** %d\\\\", summary["total"].(int)),
fmt.Sprintf("**New:** %d\\\\", summary["new"].(int)),
fmt.Sprintf("**Modified:** %d\\\\", summary["modified"].(int)),
fmt.Sprintf("**Vanilla:** %d\\\\", summary["vanilla"].(int)),
"",
fmt.Sprintf(" * [[%s:index|%s Index]]", namespace, namespaceTitle(namespace)),
}, "\n")
} }
func renderWikiStatusReportPage(summaries map[string]map[string]any, namespaces []string) string { func renderWikiStatusReportPage(summaries map[string]map[string]any, namespaces []string) string {
lines := []string{"====== Wiki Status ======", "", "This page is auto-generated from builder source provenance.", ""} lines := []string{"====== Wiki Status ======", "", "This page is auto-generated from builder source provenance.", ""}
for _, namespace := range namespaces { for _, namespace := range namespaces {
summary := summaries[namespace] summary := summaries[namespace]
lines = append(lines, "===== "+namespaceTitle(namespace)+" =====", "") lines = append(lines, renderStatusTable(namespaceTitle(namespace), summary, fmt.Sprintf("[[%s:index|%s]]", namespace, namespaceTitle(namespace))))
lines = append(lines, fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]))
lines = append(lines, fmt.Sprintf("**Generated Pages:** %d\\\\", summary["total"].(int)))
lines = append(lines, fmt.Sprintf("**New:** %d\\\\", summary["new"].(int)))
lines = append(lines, fmt.Sprintf("**Modified:** %d\\\\", summary["modified"].(int)))
lines = append(lines, fmt.Sprintf("**Vanilla:** %d\\\\", summary["vanilla"].(int)))
lines = append(lines, fmt.Sprintf("**Namespace:** [[%s:index|%s]]", namespace, namespaceTitle(namespace)), "")
} }
lines = append(lines, "<WRAP center round important>", "**Auto-generated content.** This section is rebuilt from game data on each release.", "</WRAP>", "", "===== Notes =====")
return strings.Join(lines, "\n") return strings.Join(lines, "\n")
} }
@@ -1353,10 +1429,25 @@ func renderWikiStatusListingPage(title string, pageIDs []string, pageTitles map[
lines = append(lines, " * [["+pageID+"|"+title+"]]") lines = append(lines, " * [["+pageID+"|"+title+"]]")
} }
} }
lines = append(lines, "", "<WRAP center round important>", "**Auto-generated content.** This section is rebuilt from game data on each release.", "</WRAP>", "", "===== Notes =====")
return strings.Join(lines, "\n") return strings.Join(lines, "\n")
} }
func renderStatusTable(title string, summary map[string]any, namespaceLink string) string {
rows := []string{
"===== " + title + " =====",
"",
fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]),
fmt.Sprintf("**Generated Pages:** %d\\\\", summary["total"].(int)),
fmt.Sprintf("**New:** %d\\\\", summary["new"].(int)),
fmt.Sprintf("**Modified:** %d\\\\", summary["modified"].(int)),
fmt.Sprintf("**Vanilla:** %d\\\\", summary["vanilla"].(int)),
}
if namespaceLink != "" {
rows = append(rows, fmt.Sprintf("**Namespace:** %s", namespaceLink))
}
return strings.Join(rows, "\n")
}
func filterWikiPageIDs(pageStatuses map[string]string, namespace, status string) []string { func filterWikiPageIDs(pageStatuses map[string]string, namespace, status string) []string {
pageIDs := []string{} pageIDs := []string{}
for pageID, candidate := range pageStatuses { for pageID, candidate := range pageStatuses {
@@ -1395,7 +1486,7 @@ func wikiPageIDForKey(key string) string {
} }
func wikiPageIDToRelPath(pageID string) string { func wikiPageIDToRelPath(pageID string) string {
return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".md") return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".html")
} }
func wikiRelPathToPageID(path string) string { func wikiRelPathToPageID(path string) string {
@@ -1417,6 +1508,24 @@ func namespaceTitle(namespace string) string {
} }
} }
func categoryFromWikiNamespace(namespace string) string {
switch namespace {
case "itemtypes":
return "baseitems"
case "races":
return "racialtypes"
default:
return namespace
}
}
func wikiEditPolicy(namespace string) string {
if namespace == "meta" {
return "generated_only"
}
return "preserve_manual_sections"
}
func normalizeWikiScalar(value any) string { func normalizeWikiScalar(value any) string {
switch typed := value.(type) { switch typed := value.(type) {
case string: case string:
@@ -1598,6 +1707,147 @@ func renderNodeBBManagedMarkdown(pageID, dokuText string) string {
return ensureTrailingNewline(strings.Join(parts, "\n")) return ensureTrailingNewline(strings.Join(parts, "\n"))
} }
func renderNodeBBManagedHTML(pageID, sourceText string, manualSections []wikiManualSection) string {
body := sourceText
if !strings.Contains(sourceText, "<h1") {
body = dokuWikiToNodeBBHTML(sourceText)
}
hash := computeManagedHash(body)
parts := []string{
"<!-- sow-topdata-wiki:page=" + pageID + " -->",
"<!-- sow-topdata-wiki:managed:start hash=\"sha256:" + hash + "\" -->",
strings.TrimSpace(body),
"<!-- sow-topdata-wiki:managed:end -->",
}
for _, section := range manualSections {
parts = append(parts,
"<!-- sow-topdata-wiki:manual:start id=\""+html.EscapeString(section.ID)+"\" -->",
strings.TrimSpace(section.InitialHTML),
"<!-- sow-topdata-wiki:manual:end id=\""+html.EscapeString(section.ID)+"\" -->",
)
}
return ensureTrailingNewline(strings.Join(parts, "\n"))
}
func dokuWikiToNodeBBHTML(text string) string {
text = strings.ReplaceAll(strings.TrimSpace(text), "\r\n", "\n")
if text == "" {
return ""
}
lines := strings.Split(text, "\n")
var out []string
var paragraph []string
var list []string
var tableRows []string
flushParagraph := func() {
if len(paragraph) == 0 {
return
}
out = append(out, "<p>"+renderInlineHTML(strings.Join(paragraph, " "))+"</p>")
paragraph = nil
}
flushList := func() {
if len(list) == 0 {
return
}
out = append(out, "<ul>")
for _, item := range list {
out = append(out, "<li>"+renderInlineHTML(item)+"</li>")
}
out = append(out, "</ul>")
list = nil
}
flushTable := func() {
if len(tableRows) == 0 {
return
}
out = append(out, "<table><tbody>")
out = append(out, tableRows...)
out = append(out, "</tbody></table>")
tableRows = nil
}
flushBlocks := func() {
flushParagraph()
flushList()
flushTable()
}
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if trimmed == "" {
flushBlocks()
continue
}
if strings.HasPrefix(trimmed, "<p class=\"wiki-callout") {
flushBlocks()
out = append(out, trimmed)
continue
}
if heading, level, ok := parseDokuHeading(trimmed); ok {
flushBlocks()
out = append(out, fmt.Sprintf("<h%d>%s</h%d>", level, renderInlineHTML(heading), level))
continue
}
if strings.HasPrefix(trimmed, " * ") || strings.HasPrefix(trimmed, "* ") || strings.HasPrefix(trimmed, "- ") {
flushParagraph()
flushTable()
item := strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(trimmed, " * "), "* "), "- "))
list = append(list, item)
continue
}
if label, value, ok := parseFactLine(trimmed); ok {
flushParagraph()
flushList()
tableRows = append(tableRows, "<tr><th>"+html.EscapeString(label)+"</th><td>"+renderInlineHTML(value)+"</td></tr>")
continue
}
flushList()
flushTable()
for _, part := range strings.Split(trimmed, `\\`) {
part = strings.TrimSpace(part)
if part != "" {
paragraph = append(paragraph, part)
}
}
}
flushBlocks()
return strings.TrimSpace(strings.Join(out, "\n"))
}
func parseFactLine(line string) (string, string, bool) {
if !strings.HasPrefix(line, "**") {
return "", "", false
}
rest := strings.TrimPrefix(line, "**")
label, value, ok := strings.Cut(rest, ":**")
if !ok {
return "", "", false
}
return strings.TrimSpace(label), strings.Trim(strings.TrimSpace(value), `\`), true
}
func renderInlineHTML(text string) string {
var out strings.Builder
for {
start := strings.Index(text, "[[")
if start < 0 {
out.WriteString(html.EscapeString(text))
break
}
end := strings.Index(text[start+2:], "]]")
if end < 0 {
out.WriteString(html.EscapeString(text))
break
}
end += start + 2
out.WriteString(html.EscapeString(text[:start]))
out.WriteString(text[start : end+2])
text = text[end+2:]
}
return out.String()
}
func splitDokuWikiNotes(text string) (string, string) { func splitDokuWikiNotes(text string) (string, string) {
const delimiter = "===== Notes =====" const delimiter = "===== Notes ====="
before, after, ok := strings.Cut(text, delimiter) before, after, ok := strings.Cut(text, delimiter)
internal/topdata/wiki_native_test.go
+22 -8
View File
@@ -42,26 +42,40 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) {
if result.WikiPages != 1 { if result.WikiPages != 1 {
t.Fatalf("expected one generated entity page, got %d", result.WikiPages) t.Fatalf("expected one generated entity page, got %d", result.WikiPages)
} }
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md") pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
got, err := os.ReadFile(pagePath) got, err := os.ReadFile(pagePath)
if err != nil { if err != nil {
t.Fatalf("read generated page: %v", err) t.Fatalf("read generated page: %v", err)
} }
text := string(got) text := string(got)
if !strings.Contains(text, "# Athletics") || !strings.Contains(text, "This skill is unchanged from vanilla.") { if !strings.Contains(text, "<h1>Athletics</h1>") || !strings.Contains(text, `class="wiki-callout wiki-callout--status"`) || !strings.Contains(text, "This skill is unchanged from vanilla.") {
t.Fatalf("unexpected generated page:\n%s", text) t.Fatalf("unexpected generated page:\n%s", text)
} }
if !strings.Contains(text, managedStartMarker) || !strings.Contains(text, "Ability: Strength.") || !strings.Contains(text, "- Climb") { if !strings.Contains(text, `<!-- sow-topdata-wiki:page=skills:athletics -->`) || !strings.Contains(text, `<!-- sow-topdata-wiki:managed:start hash="sha256:`) || !strings.Contains(text, "Ability: Strength.") || !strings.Contains(text, "<li>Climb</li>") {
t.Fatalf("expected description to be rendered into wiki page:\n%s", text) t.Fatalf("expected HTML managed markers and description to be rendered into wiki page:\n%s", text)
} }
statusPath := filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus.md") if strings.Contains(text, "[//]: #") || strings.Contains(text, "<WRAP>") {
t.Fatalf("expected generated HTML to omit DokuWiki-era syntax:\n%s", text)
}
if !strings.Contains(text, `<!-- sow-topdata-wiki:manual:start id="notes" -->`) || !strings.Contains(text, "<h2>See Also</h2>") {
t.Fatalf("expected generated page to include manual section placeholders:\n%s", text)
}
statusPath := filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus.html")
statusRaw, err := os.ReadFile(statusPath) statusRaw, err := os.ReadFile(statusPath)
if err != nil { if err != nil {
t.Fatalf("read status page: %v", err) t.Fatalf("read status page: %v", err)
} }
if !strings.Contains(string(statusRaw), "**Vanilla:** 1") { if !strings.Contains(string(statusRaw), "<th>Vanilla</th><td>1</td>") {
t.Fatalf("expected wiki status summary to count vanilla page:\n%s", string(statusRaw)) t.Fatalf("expected wiki status summary to count vanilla page:\n%s", string(statusRaw))
} }
indexRaw, err := os.ReadFile(filepath.Join(root, ".cache", "wiki", "page-index.json"))
if err != nil {
t.Fatalf("read page index: %v", err)
}
indexText := string(indexRaw)
if !strings.Contains(indexText, `"page_id": "skills:athletics"`) || !strings.Contains(indexText, `"output_path": "pages/skills/athletics.html"`) || !strings.Contains(indexText, `"edit_policy": "preserve_manual_sections"`) {
t.Fatalf("expected deterministic page-index metadata, got:\n%s", indexText)
}
if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil { if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil {
t.Fatalf("expected state file after first build: %v", err) t.Fatalf("expected state file after first build: %v", err)
} }
@@ -201,7 +215,7 @@ func TestBuildNativeRegeneratesWikiWhenTLKTextChanges(t *testing.T) {
if _, err := BuildNative(proj, nil); err != nil { if _, err := BuildNative(proj, nil); err != nil {
t.Fatalf("BuildNative after text change failed: %v", err) t.Fatalf("BuildNative after text change failed: %v", err)
} }
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md") pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
got, err := os.ReadFile(pagePath) got, err := os.ReadFile(pagePath)
if err != nil { if err != nil {
t.Fatalf("read regenerated page: %v", err) t.Fatalf("read regenerated page: %v", err)
@@ -243,7 +257,7 @@ func TestBuildPackageIgnoresWikiSourcesForFreshness(t *testing.T) {
t.Fatalf("BuildNative failed: %v", err) t.Fatalf("BuildNative failed: %v", err)
} }
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md") pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
newTime := time.Now().Add(2 * time.Second) newTime := time.Now().Add(2 * time.Second)
if err := os.Chtimes(pagePath, newTime, newTime); err != nil { if err := os.Chtimes(pagePath, newTime, newTime); err != nil {
t.Fatalf("touch wiki page: %v", err) t.Fatalf("touch wiki page: %v", err)