Wiki pipeline (#3)

Implemented the wiki pipeline to a shippable local state across `toolkit/`, `module/`, and the NodeBB wiki plugin.

**Scope Audited**
- Toolkit config/project loading, wiki renderer, deployer, app command wiring.
- Module `nwn-tool.yaml`, wiki source/docs, release script, topdata docs.
- Plugin sanitizer/API storage contract docs and sanitizer tests.

**Changes Made**
- Added topdata-owned wiki declarations under `module/topdata/wiki/`.
- Added `topdata.wiki.*` config fields, validation, effective config output.
- Switched generated wiki pages to `.html` with HTML comment managed/manual markers.
- Added `page-index.json` generation and deterministic metadata.
- Added manual-section merge preservation and stale page `report`/`archive` handling.
- Added namespace `category_env` loading from `topdata/wiki/namespaces.yaml`, while keeping `--category`/`NODEBB_WIKI_CATEGORIES` overrides.
- Updated release deployment to keep dry-run first and make stale cleanup opt-in via `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY`.
- Updated plugin sanitizer to preserve only `sow-topdata-wiki` comments.

**Configuration/Schema Impact**
- `module/nwn-tool.yaml` now declares wiki source, renderer, link strategy, templates, manual sections, managed marker policy, and stale policy.
- Deploy manifest entries now support stale/archive metadata: `stale`, `last_seen_hash`, `archived_hash`, `title`, `namespace`, `tid`, `pid`, `cid`.

**Compatibility Impact**
- Legacy Markdown managed markers are still readable for migration.
- Generated output is now HTML, not Markdown.
- Existing mapped pages update by `pid`; creates still require explicit `--create`.

**Tests Added/Updated**
- Go tests for wiki config validation, HTML rendering, page index, manual merge, stale report/archive, app summary output.
- Plugin sanitizer test for generated topdata bot HTML markers/subset.

**Validation Performed**
- `go test ./internal/project ./internal/topdata ./internal/app` passed.
- `./build-tool.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./validate-topdata.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force` passed, generating 1345 entity pages.
- `deploy-wiki --dry-run --create` with dummy endpoint/token and namespace CID env vars passed locally: 1377 planned creates, 0 stale/drift.
- `node tests/wiki-html-sanitizer.test.js` passed.

**Remaining Risks**
- Full plugin `npm test` is blocked by an existing unrelated drawer CSS contract failure in `tests/wiki-article-drawers.test.js`.
- Controlled live migration was not performed because it requires a non-production NodeBB namespace and credentials.
- Direct `PUT /api/v3/posts/{pid}` save-filter behavior is documented as needing live confirmation against the deployed NodeBB/plugin stack.

Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/3
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
This commit is contained in:
2026-05-15 09:09:37 +02:00
committed by archvillainette
parent d767ec0a25
commit 9b2084344d
9 changed files with 1030 additions and 166 deletions
+270 -13
View File
@@ -17,13 +17,18 @@ import (
"time"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
"gopkg.in/yaml.v3"
)
const (
managedStartMarker = "[//]: # (sow-topdata-wiki:managed:start)"
managedEndMarker = "[//]: # (sow-topdata-wiki:managed:end)"
legacyManagedStartMarker = "<!-- sow-topdata-wiki:managed:start -->"
legacyManagedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
managedStartMarker = "<!-- sow-topdata-wiki:managed:start"
managedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
legacyMarkdownStartMarker = "[//]: # (sow-topdata-wiki:managed:start)"
legacyMarkdownEndMarker = "[//]: # (sow-topdata-wiki:managed:end)"
legacyManagedStartMarker = "<!-- sow-topdata-wiki:managed:start -->"
legacyManagedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
manualStartMarkerPrefix = "<!-- sow-topdata-wiki:manual:start"
manualEndMarkerPrefix = "<!-- sow-topdata-wiki:manual:end"
)
type DeployWikiOptions struct {
@@ -48,6 +53,7 @@ type deployResult struct {
Created int
Updated int
Stale int
Archived int
Skipped int
Drifted int
Manifest string
@@ -67,10 +73,15 @@ type wikiDeployManifest struct {
}
type wikiDeployManifestPage struct {
Hash string `json:"hash"`
TID int `json:"tid,omitempty"`
PID int `json:"pid,omitempty"`
CID int `json:"cid,omitempty"`
Hash string `json:"hash"`
LastSeenHash string `json:"last_seen_hash,omitempty"`
ArchivedHash string `json:"archived_hash,omitempty"`
Title string `json:"title,omitempty"`
Namespace string `json:"namespace,omitempty"`
Stale bool `json:"stale,omitempty"`
TID int `json:"tid,omitempty"`
PID int `json:"pid,omitempty"`
CID int `json:"cid,omitempty"`
}
type wikiDeployPlan struct {
@@ -81,6 +92,19 @@ type wikiDeployPlan struct {
RemoteHash string
}
type wikiNamespacesDocument struct {
Namespaces []wikiNamespaceDeclaration `json:"namespaces" yaml:"namespaces"`
}
type wikiNamespaceDeclaration struct {
ID string `json:"id" yaml:"id"`
Title string `json:"title" yaml:"title"`
CategoryEnv string `json:"category_env" yaml:"category_env"`
PageTitle string `json:"page_title" yaml:"page_title"`
Tags []string `json:"tags" yaml:"tags"`
EditPolicy string `json:"edit_policy" yaml:"edit_policy"`
}
func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress func(string)) (deployResult, error) {
if progress == nil {
progress = func(string) {}
@@ -100,10 +124,33 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
if opts.Username != "" || opts.Password != "" || opts.NotesDelimiter != "" {
return deployResult{}, errors.New("DokuWiki deployment options are no longer supported; use NodeBB endpoint, token, and category mappings")
}
if opts.StalePolicy != "" && opts.StalePolicy != "report" && opts.StalePolicy != "archive" {
return deployResult{}, fmt.Errorf("wiki stale policy %q is not supported", opts.StalePolicy)
}
namespaces := opts.Namespaces
if len(namespaces) == 0 {
namespaces = p.EffectiveConfig().TopData.Wiki.ManagedNamespaces
declarations, err := loadWikiNamespaceDeclarations(p)
if err != nil {
return deployResult{}, err
}
for _, declaration := range declarations {
namespaces = append(namespaces, declaration.ID)
}
if len(namespaces) == 0 {
namespaces = p.EffectiveConfig().TopData.Wiki.ManagedNamespaces
}
envCategories, err := categoryIDsFromNamespaceEnv(declarations)
if err != nil {
return deployResult{}, err
}
if len(envCategories) > 0 {
merged := envCategories
for namespace, cid := range opts.CategoryIDs {
merged[namespace] = cid
}
opts.CategoryIDs = merged
}
}
manifestPath := opts.ManifestPath
@@ -125,7 +172,7 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
result.LocalPages = len(pages)
result.Manifest = manifestPath
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, skip %d, drift %d", result.Created, result.Updated, result.Skipped, result.Drifted))
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, skip %d, stale %d, archive %d, drift %d", result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Drifted))
if opts.DryRun {
return result, nil
}
@@ -152,6 +199,10 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil {
return result, err
}
case "archive":
if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil {
return result, err
}
}
}
if err := saveDeployManifest(manifestPath, nextManifest); err != nil {
@@ -160,6 +211,62 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
return result, nil
}
func loadWikiNamespaceDeclarations(p *project.Project) ([]wikiNamespaceDeclaration, error) {
cfg := p.EffectiveConfig().TopData.Wiki
path := filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(cfg.NamespacesFile))
raw, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("read wiki namespaces: %w", err)
}
var doc wikiNamespacesDocument
if err := yaml.Unmarshal(raw, &doc); err != nil {
return nil, fmt.Errorf("parse wiki namespaces: %w", err)
}
seen := map[string]struct{}{}
for _, declaration := range doc.Namespaces {
id := strings.TrimSpace(declaration.ID)
if id == "" {
return nil, errors.New("topdata wiki namespace id is required")
}
if _, ok := seen[id]; ok {
return nil, fmt.Errorf("duplicate topdata wiki namespace id %q", id)
}
seen[id] = struct{}{}
if strings.TrimSpace(declaration.CategoryEnv) == "" {
return nil, fmt.Errorf("topdata wiki namespace %q category_env is required", id)
}
switch strings.TrimSpace(declaration.EditPolicy) {
case "", "preserve_manual_sections", "generated_only":
default:
return nil, fmt.Errorf("topdata wiki namespace %q edit_policy %q is not supported", id, declaration.EditPolicy)
}
}
return doc.Namespaces, nil
}
func categoryIDsFromNamespaceEnv(declarations []wikiNamespaceDeclaration) (map[string]int, error) {
categories := map[string]int{}
for _, declaration := range declarations {
envName := strings.TrimSpace(declaration.CategoryEnv)
if envName == "" {
continue
}
raw := strings.TrimSpace(os.Getenv(envName))
if raw == "" {
continue
}
cid, err := strconv.Atoi(raw)
if err != nil || cid <= 0 {
return nil, fmt.Errorf("topdata wiki namespace %q env %s has invalid cid %q", declaration.ID, envName, raw)
}
categories[declaration.ID] = cid
}
return categories, nil
}
func collectLocalPages(sourceDir string, namespaces []string) (map[string]wikiDeployPage, error) {
namespaceSet := map[string]struct{}{}
for _, namespace := range namespaces {
@@ -224,6 +331,10 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
page := pages[pageID]
entry := manifest.Pages[pageID]
entry.Hash = page.Hash
entry.LastSeenHash = page.Hash
entry.Title = page.Title
entry.Namespace = page.Namespace
entry.Stale = false
if entry.CID == 0 {
entry.CID = opts.CategoryIDs[page.Namespace]
}
@@ -267,7 +378,33 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
}
for pageID := range manifest.Pages {
if _, ok := pages[pageID]; !ok {
entry := manifest.Pages[pageID]
entry.Stale = true
if entry.Namespace == "" {
entry.Namespace = strings.SplitN(pageID, ":", 2)[0]
}
if entry.Title == "" {
entry.Title = extractHTMLTitle("", pageID)
}
next.Pages[pageID] = entry
result.Stale++
policy := strings.TrimSpace(opts.StalePolicy)
if policy == "" {
policy = "report"
}
if policy == "archive" {
body := renderArchivedWikiPage(pageID, entry.Title)
entry.Hash = computeManagedHash(body)
entry.ArchivedHash = entry.Hash
next.Pages[pageID] = entry
result.Archived++
plans = append(plans, wikiDeployPlan{
Page: wikiDeployPage{PageID: pageID, Title: entry.Title, Namespace: entry.Namespace, Content: body, Hash: entry.Hash},
Entry: entry,
Action: "archive",
Content: body,
})
}
}
}
return plans, result, next, nil
@@ -322,17 +459,34 @@ func mergeManagedContent(existing, generated string) string {
if !ok {
return generated
}
_, endMarker, start, end := managedRegionBounds(existing)
startMarker, endMarker, start, end := managedRegionBounds(existing)
if start < 0 || end < 0 || end < start {
return generated
}
if strings.Contains(generated, manualStartMarkerPrefix) {
return mergeManualRegions(generated, existing)
}
end += len(endMarker)
return existing[:start] + managedStartMarker + "\n" + generatedRegion + "\n" + managedEndMarker + existing[end:]
generatedStart, generatedEnd, ok := extractManagedMarkers(generated)
if !ok {
generatedStart = startMarker
generatedEnd = endMarker
}
merged := existing[:start] + generatedStart + "\n" + generatedRegion + "\n" + generatedEnd + existing[end:]
return mergeManualRegions(merged, existing)
}
func managedRegionBounds(content string) (string, string, int, int) {
if start := strings.Index(content, managedStartMarker); start >= 0 {
startEnd := strings.Index(content[start:], "-->")
end := strings.Index(content, managedEndMarker)
if startEnd >= 0 && end >= 0 {
startMarker := content[start : start+startEnd+len("-->")]
return startMarker, managedEndMarker, start, end
}
}
for _, markers := range [][2]string{
{managedStartMarker, managedEndMarker},
{legacyMarkdownStartMarker, legacyMarkdownEndMarker},
{legacyManagedStartMarker, legacyManagedEndMarker},
} {
start := strings.Index(content, markers[0])
@@ -344,6 +498,109 @@ func managedRegionBounds(content string) (string, string, int, int) {
return "", "", -1, -1
}
func extractManagedMarkers(content string) (string, string, bool) {
startMarker, endMarker, start, end := managedRegionBounds(content)
return startMarker, endMarker, start >= 0 && end >= 0
}
func mergeManualRegions(merged, existing string) string {
existingManual := extractManualRegions(existing)
if len(existingManual) == 0 {
return merged
}
return replaceManualRegions(merged, existingManual)
}
func extractManualRegions(content string) map[string]string {
regions := map[string]string{}
offset := 0
for {
startRel := strings.Index(content[offset:], manualStartMarkerPrefix)
if startRel < 0 {
break
}
start := offset + startRel
startEndRel := strings.Index(content[start:], "-->")
if startEndRel < 0 {
break
}
startMarker := content[start : start+startEndRel+len("-->")]
id := markerID(startMarker)
if id == "" {
offset = start + len(startMarker)
continue
}
endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\""
end := strings.Index(content[start+len(startMarker):], endPrefix)
if end < 0 {
offset = start + len(startMarker)
continue
}
end += start + len(startMarker)
endEndRel := strings.Index(content[end:], "-->")
if endEndRel < 0 {
break
}
endMarker := content[end : end+endEndRel+len("-->")]
regions[id] = startMarker + strings.TrimRight(content[start+len(startMarker):end], "\n") + "\n" + endMarker
offset = end + len(endMarker)
}
return regions
}
func replaceManualRegions(content string, replacements map[string]string) string {
for id, replacement := range replacements {
start := strings.Index(content, manualStartMarkerPrefix+" id=\""+id+"\"")
if start < 0 {
continue
}
endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\""
end := strings.Index(content[start:], endPrefix)
if end < 0 {
continue
}
end += start
endEndRel := strings.Index(content[end:], "-->")
if endEndRel < 0 {
continue
}
endEnd := end + endEndRel + len("-->")
content = content[:start] + replacement + content[endEnd:]
}
return content
}
func markerID(marker string) string {
const prefix = `id="`
start := strings.Index(marker, prefix)
if start < 0 {
return ""
}
start += len(prefix)
end := strings.Index(marker[start:], `"`)
if end < 0 {
return ""
}
return marker[start : start+end]
}
func renderArchivedWikiPage(pageID, title string) string {
if title == "" {
title = extractHTMLTitle("", pageID)
}
body := strings.Join([]string{
"<h1>" + html.EscapeString(title) + "</h1>",
"<p>This generated page is no longer present in the current Shadows Over Westgate topdata wiki source.</p>",
}, "\n")
hash := computeManagedHash(body)
return ensureTrailingNewline(strings.Join([]string{
"<!-- sow-topdata-wiki:page=" + pageID + " -->",
"<!-- sow-topdata-wiki:managed:start hash=\"sha256:" + hash + "\" -->",
body,
"<!-- sow-topdata-wiki:managed:end -->",
}, "\n"))
}
func extractPageTitle(content, fallback string) string {
if title := extractMarkdownTitle(content); title != "" {
return title
+163
View File
@@ -6,6 +6,7 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
@@ -194,3 +195,165 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) {
t.Fatalf("DeployWikiWithOptions create failed: %v", err)
}
}
func TestDeployWikiMergesHTMLManagedAndManualRegions(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
generated := `<!-- sow-topdata-wiki:page=skills:athletics -->
<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
<h1>Athletics</h1>
<p>Generated athletics page</p>
<!-- sow-topdata-wiki:managed:end -->
<!-- sow-topdata-wiki:manual:start id="notes" -->
<h2>Notes</h2>
<p></p>
<!-- sow-topdata-wiki:manual:end id="notes" -->
<!-- sow-topdata-wiki:manual:start id="see_also" -->
<h2>See Also</h2>
<p></p>
<!-- sow-topdata-wiki:manual:end id="see_also" -->
`
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil {
t.Fatalf("write source page: %v", err)
}
old := `<!-- sow-topdata-wiki:page=skills:athletics -->
<!-- sow-topdata-wiki:managed:start hash="sha256:old" -->
<h1>Athletics</h1>
<p>Old generated text</p>
<!-- sow-topdata-wiki:managed:end -->
<!-- sow-topdata-wiki:manual:start id="notes" -->
<h2>Notes</h2>
<p>Keep this human note.</p>
<!-- sow-topdata-wiki:manual:end id="notes" -->
`
manifestPath := filepath.Join(root, "deploy-manifest.json")
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
Version: "nodebb-v1",
Pages: map[string]wikiDeployManifestPage{
"skills:athletics": {Hash: computeManagedHash(old), TID: 7, PID: 42, CID: 3},
},
}); err != nil {
t.Fatalf("write deploy manifest: %v", err)
}
var updated string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42":
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7, "content": old}})
case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42":
var req struct {
Content string `json:"content"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatalf("decode update request: %v", err)
}
updated = req.Content
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}})
default:
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
}
}))
defer server.Close()
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions update failed: %v", err)
}
if result.Updated != 1 {
t.Fatalf("expected one update, got %#v", result)
}
if !containsAll(updated, "<p>Generated athletics page</p>", "<p>Keep this human note.</p>", `manual:start id="see_also"`) {
t.Fatalf("expected update to replace managed HTML and preserve/bootstrap manual regions:\n%s", updated)
}
}
func TestDeployWikiReportsAndArchivesStalePages(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
manifestPath := filepath.Join(root, "deploy-manifest.json")
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
Version: "nodebb-v1",
Pages: map[string]wikiDeployManifestPage{
"skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3},
},
}); err != nil {
t.Fatalf("write deploy manifest: %v", err)
}
var archived string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPut || r.URL.Path != "/api/v3/posts/42" {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
}
var req struct {
Content string `json:"content"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatalf("decode archive request: %v", err)
}
archived = req.Content
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}})
}))
defer server.Close()
report, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
DryRun: true,
StalePolicy: "report",
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions stale report failed: %v", err)
}
if report.Stale != 1 {
t.Fatalf("expected one stale report, got %#v", report)
}
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
StalePolicy: "archive",
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions stale archive failed: %v", err)
}
if result.Stale != 1 || result.Archived != 1 {
t.Fatalf("expected one stale archive, got %#v", result)
}
if !containsAll(archived, "<h1>Retired Skill</h1>", "no longer present", `<!-- sow-topdata-wiki:managed:start hash="sha256:`) {
t.Fatalf("unexpected archived body:\n%s", archived)
}
manifest := loadDeployManifest(manifestPath)
entry := manifest.Pages["skills:retired"]
if !entry.Stale || entry.ArchivedHash == "" || entry.PID != 42 {
t.Fatalf("expected archived stale manifest entry, got %#v", entry)
}
}
func containsAll(haystack string, needles ...string) bool {
for _, needle := range needles {
if !strings.Contains(haystack, needle) {
return false
}
}
return true
}
+302 -52
View File
@@ -5,6 +5,7 @@ import (
"embed"
"encoding/json"
"fmt"
"html"
"io/fs"
"os"
"path/filepath"
@@ -14,6 +15,7 @@ import (
"time"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
"gopkg.in/yaml.v3"
)
const (
@@ -24,7 +26,7 @@ const (
legacyWikiRootDirName = ".wiki"
wikiPagesDirName = "pages"
wikiStateFileName = "state.json"
wikiGeneratorVersion = "nodebb-markdown-v1"
wikiGeneratorVersion = "nodebb-tiptap-html-v1"
wikiGeneratedStatus = "generated"
wikiSkippedStatus = "skipped"
)
@@ -43,6 +45,34 @@ type wikiStateDocument struct {
Pages int `json:"pages"`
}
type wikiPageIndex struct {
Version string `json:"version"`
Pages []wikiPageIndexEntry `json:"pages"`
}
type wikiPageIndexEntry struct {
PageID string `json:"page_id"`
Namespace string `json:"namespace"`
SourceDataset string `json:"source_dataset"`
SourceKey string `json:"source_key"`
Title string `json:"title"`
Hash string `json:"hash"`
OutputPath string `json:"output_path"`
EditPolicy string `json:"edit_policy"`
StalePolicy string `json:"stale_policy"`
}
type wikiManualSectionsDocument struct {
ManualSections []wikiManualSection `json:"manual_sections" yaml:"manual_sections"`
}
type wikiManualSection struct {
ID string `json:"id" yaml:"id"`
Heading string `json:"heading" yaml:"heading"`
Placement string `json:"placement" yaml:"placement"`
InitialHTML string `json:"initial_html" yaml:"initial_html"`
}
type wikiResult struct {
OutputDir string
PageCount int
@@ -117,6 +147,7 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
pageTitles := map[string]string{}
pageStatuses := map[string]string{}
pageIndex := wikiPageIndex{Version: wikiGeneratorVersion}
pageCount := 0
writePage := func(pageID, content, title, status string) error {
@@ -124,12 +155,28 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
content = renderNodeBBManagedMarkdown(pageID, content)
content = renderNodeBBManagedHTML(pageID, content, loadWikiManualSections(p))
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
return err
}
pageTitles[pageID] = title
pageStatuses[pageID] = status
namespace := strings.SplitN(pageID, ":", 2)[0]
rel, err := filepath.Rel(rootDir, path)
if err != nil {
rel = filepath.Base(path)
}
pageIndex.Pages = append(pageIndex.Pages, wikiPageIndexEntry{
PageID: pageID,
Namespace: namespace,
SourceDataset: categoryFromWikiNamespace(namespace),
SourceKey: pageID,
Title: title,
Hash: computeManagedHash(content),
OutputPath: filepath.ToSlash(rel),
EditPolicy: wikiEditPolicy(namespace),
StalePolicy: p.EffectiveConfig().TopData.Wiki.StalePages.Default,
})
pageCount++
return nil
}
@@ -137,7 +184,10 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
if err := generateEntityPages(outputDir, ctx, writePage); err != nil {
return wikiResult{}, err
}
if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces); err != nil {
if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces, loadWikiManualSections(p)); err != nil {
return wikiResult{}, err
}
if err := saveWikiPageIndex(filepath.Join(rootDir, "page-index.json"), pageIndex); err != nil {
return wikiResult{}, err
}
@@ -156,6 +206,56 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
}, nil
}
func loadWikiManualSections(p *project.Project) []wikiManualSection {
defaults := []wikiManualSection{
{ID: "notes", Heading: "Notes", InitialHTML: "<h2>Notes</h2><p></p>"},
{ID: "see_also", Heading: "See Also", InitialHTML: "<h2>See Also</h2><p></p>"},
}
cfg := p.EffectiveConfig().TopData.Wiki
path := filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(cfg.ManualSectionsDir), "default.yaml")
raw, err := os.ReadFile(path)
if err != nil {
return defaults
}
var doc wikiManualSectionsDocument
if err := yaml.Unmarshal(raw, &doc); err != nil || len(doc.ManualSections) == 0 {
return defaults
}
out := []wikiManualSection{}
for _, section := range doc.ManualSections {
section.ID = strings.TrimSpace(section.ID)
if section.ID == "" {
continue
}
if strings.TrimSpace(section.InitialHTML) == "" {
heading := strings.TrimSpace(section.Heading)
if heading == "" {
heading = section.ID
}
section.InitialHTML = "<h2>" + html.EscapeString(heading) + "</h2><p></p>"
}
out = append(out, section)
}
if len(out) == 0 {
return defaults
}
return out
}
func saveWikiPageIndex(path string, index wikiPageIndex) error {
slices.SortFunc(index.Pages, func(a, b wikiPageIndexEntry) int {
return strings.Compare(a.PageID, b.PageID)
})
raw, err := json.MarshalIndent(index, "", " ")
if err != nil {
return fmt.Errorf("marshal wiki page index: %w", err)
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, append(raw, '\n'), 0o644)
}
func loadWikiState(path string) (wikiStateDocument, error) {
raw, err := os.ReadFile(path)
if err != nil {
@@ -593,24 +693,20 @@ func (ctx *wikiContext) shouldGeneratePage(category, key string, row map[string]
}
func (ctx *wikiContext) renderPage(category, key string, row map[string]any, title string) (string, error) {
templateName := category
if category == "baseitems" {
templateName = "items"
} else if category == "racialtypes" {
templateName = "races"
sections := []string{
"====== " + title + " ======",
"",
buildWikiStatusBlock(ctx.pageStatus(category, key, row), category),
"",
toDokuWiki(ctx.resolveRowDescription(category, row)),
"",
ctx.renderFacts(category, key, row),
"",
ctx.renderProgression(category, row),
"",
ctx.renderRaceNameForms(row),
}
templateBytes, err := wikiTemplateFS.ReadFile("wiki_templates/" + templateName + ".txt")
if err != nil {
return "", err
}
rendered := string(templateBytes)
rendered = strings.ReplaceAll(rendered, "{{name}}", title)
rendered = strings.ReplaceAll(rendered, "{{status}}", buildWikiStatusBlock(ctx.pageStatus(category, key, row), category))
rendered = strings.ReplaceAll(rendered, "{{description}}", toDokuWiki(ctx.resolveRowDescription(category, row)))
rendered = strings.ReplaceAll(rendered, "{{facts}}", ctx.renderFacts(category, key, row))
rendered = strings.ReplaceAll(rendered, "{{progression}}", ctx.renderProgression(category, row))
rendered = strings.ReplaceAll(rendered, "{{nameforms}}", ctx.renderRaceNameForms(row))
return ensureTrailingNewline(rendered), nil
return ensureTrailingNewline(strings.Join(sections, "\n")), nil
}
func (ctx *wikiContext) renderFacts(category, key string, row map[string]any) string {
@@ -1205,16 +1301,13 @@ func mergeRowWikiStatus(key string, row map[string]any, statuses map[string]stri
func buildWikiStatusBlock(status, category string) string {
message := "This " + wikiStatusNoun(category) + " is unchanged from vanilla."
wrap := "info"
switch status {
case wikiStatusNew:
message = "This is a new " + wikiStatusNoun(category) + "!"
wrap = "tip"
case wikiStatusModified:
message = "This " + wikiStatusNoun(category) + " has been altered from vanilla."
wrap = "help"
}
return "<WRAP round " + wrap + ">\n" + message + "\n</WRAP>"
return "<p class=\"wiki-callout wiki-callout--status\">" + html.EscapeString(message) + "</p>"
}
func wikiStatusNoun(category string) string {
@@ -1236,37 +1329,37 @@ func wikiStatusNoun(category string) string {
}
}
func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string, namespaces []string) error {
func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string, namespaces []string, manualSections []wikiManualSection) error {
summaries := buildWikiNamespaceSummaries(pageStatuses, namespaces)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.md"), renderWikiStatusReportPage(summaries, namespaces)); err != nil {
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.html"), renderWikiStatusReportPage(summaries, namespaces), manualSections); err != nil {
return err
}
for _, namespace := range namespaces {
summary := summaries[namespace]
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".md"), renderWikiNamespaceFragment(namespace, summary)); err != nil {
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".html"), renderWikiNamespaceFragment(namespace, summary), manualSections); err != nil {
return err
}
for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} {
pageIDs := filterWikiPageIDs(pageStatuses, namespace, status)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".md"), renderWikiStatusListingPage(namespaceTitle(namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".html"), renderWikiStatusListingPage(namespaceTitle(namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles), manualSections); err != nil {
return err
}
}
}
for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} {
pageIDs := filterWikiPageIDs(pageStatuses, "", status)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".md"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".html"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles), manualSections); err != nil {
return err
}
}
return nil
}
func writeWikiFile(path, content string) error {
func writeWikiFile(path, content string, manualSections []wikiManualSection) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
content = renderNodeBBManagedMarkdown(wikiRelPathToPageID(path), content)
content = renderNodeBBManagedHTML(wikiRelPathToPageID(path), content, manualSections)
return os.WriteFile(path, []byte(ensureTrailingNewline(content)), 0o644)
}
@@ -1311,32 +1404,15 @@ func rollupWikiStatuses(statuses []string) string {
}
func renderWikiNamespaceFragment(namespace string, summary map[string]any) string {
return strings.Join([]string{
"====== " + namespaceTitle(namespace) + " ======",
"",
fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]),
fmt.Sprintf("**Generated Pages:** %d\\\\", summary["total"].(int)),
fmt.Sprintf("**New:** %d\\\\", summary["new"].(int)),
fmt.Sprintf("**Modified:** %d\\\\", summary["modified"].(int)),
fmt.Sprintf("**Vanilla:** %d\\\\", summary["vanilla"].(int)),
"",
fmt.Sprintf(" * [[%s:index|%s Index]]", namespace, namespaceTitle(namespace)),
}, "\n")
return renderStatusTable(namespaceTitle(namespace), summary, "")
}
func renderWikiStatusReportPage(summaries map[string]map[string]any, namespaces []string) string {
lines := []string{"====== Wiki Status ======", "", "This page is auto-generated from builder source provenance.", ""}
for _, namespace := range namespaces {
summary := summaries[namespace]
lines = append(lines, "===== "+namespaceTitle(namespace)+" =====", "")
lines = append(lines, fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]))
lines = append(lines, fmt.Sprintf("**Generated Pages:** %d\\\\", summary["total"].(int)))
lines = append(lines, fmt.Sprintf("**New:** %d\\\\", summary["new"].(int)))
lines = append(lines, fmt.Sprintf("**Modified:** %d\\\\", summary["modified"].(int)))
lines = append(lines, fmt.Sprintf("**Vanilla:** %d\\\\", summary["vanilla"].(int)))
lines = append(lines, fmt.Sprintf("**Namespace:** [[%s:index|%s]]", namespace, namespaceTitle(namespace)), "")
lines = append(lines, renderStatusTable(namespaceTitle(namespace), summary, fmt.Sprintf("[[%s:index|%s]]", namespace, namespaceTitle(namespace))))
}
lines = append(lines, "<WRAP center round important>", "**Auto-generated content.** This section is rebuilt from game data on each release.", "</WRAP>", "", "===== Notes =====")
return strings.Join(lines, "\n")
}
@@ -1353,10 +1429,25 @@ func renderWikiStatusListingPage(title string, pageIDs []string, pageTitles map[
lines = append(lines, " * [["+pageID+"|"+title+"]]")
}
}
lines = append(lines, "", "<WRAP center round important>", "**Auto-generated content.** This section is rebuilt from game data on each release.", "</WRAP>", "", "===== Notes =====")
return strings.Join(lines, "\n")
}
func renderStatusTable(title string, summary map[string]any, namespaceLink string) string {
rows := []string{
"===== " + title + " =====",
"",
fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]),
fmt.Sprintf("**Generated Pages:** %d\\\\", summary["total"].(int)),
fmt.Sprintf("**New:** %d\\\\", summary["new"].(int)),
fmt.Sprintf("**Modified:** %d\\\\", summary["modified"].(int)),
fmt.Sprintf("**Vanilla:** %d\\\\", summary["vanilla"].(int)),
}
if namespaceLink != "" {
rows = append(rows, fmt.Sprintf("**Namespace:** %s", namespaceLink))
}
return strings.Join(rows, "\n")
}
func filterWikiPageIDs(pageStatuses map[string]string, namespace, status string) []string {
pageIDs := []string{}
for pageID, candidate := range pageStatuses {
@@ -1395,7 +1486,7 @@ func wikiPageIDForKey(key string) string {
}
func wikiPageIDToRelPath(pageID string) string {
return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".md")
return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".html")
}
func wikiRelPathToPageID(path string) string {
@@ -1417,6 +1508,24 @@ func namespaceTitle(namespace string) string {
}
}
func categoryFromWikiNamespace(namespace string) string {
switch namespace {
case "itemtypes":
return "baseitems"
case "races":
return "racialtypes"
default:
return namespace
}
}
func wikiEditPolicy(namespace string) string {
if namespace == "meta" {
return "generated_only"
}
return "preserve_manual_sections"
}
func normalizeWikiScalar(value any) string {
switch typed := value.(type) {
case string:
@@ -1598,6 +1707,147 @@ func renderNodeBBManagedMarkdown(pageID, dokuText string) string {
return ensureTrailingNewline(strings.Join(parts, "\n"))
}
func renderNodeBBManagedHTML(pageID, sourceText string, manualSections []wikiManualSection) string {
body := sourceText
if !strings.Contains(sourceText, "<h1") {
body = dokuWikiToNodeBBHTML(sourceText)
}
hash := computeManagedHash(body)
parts := []string{
"<!-- sow-topdata-wiki:page=" + pageID + " -->",
"<!-- sow-topdata-wiki:managed:start hash=\"sha256:" + hash + "\" -->",
strings.TrimSpace(body),
"<!-- sow-topdata-wiki:managed:end -->",
}
for _, section := range manualSections {
parts = append(parts,
"<!-- sow-topdata-wiki:manual:start id=\""+html.EscapeString(section.ID)+"\" -->",
strings.TrimSpace(section.InitialHTML),
"<!-- sow-topdata-wiki:manual:end id=\""+html.EscapeString(section.ID)+"\" -->",
)
}
return ensureTrailingNewline(strings.Join(parts, "\n"))
}
func dokuWikiToNodeBBHTML(text string) string {
text = strings.ReplaceAll(strings.TrimSpace(text), "\r\n", "\n")
if text == "" {
return ""
}
lines := strings.Split(text, "\n")
var out []string
var paragraph []string
var list []string
var tableRows []string
flushParagraph := func() {
if len(paragraph) == 0 {
return
}
out = append(out, "<p>"+renderInlineHTML(strings.Join(paragraph, " "))+"</p>")
paragraph = nil
}
flushList := func() {
if len(list) == 0 {
return
}
out = append(out, "<ul>")
for _, item := range list {
out = append(out, "<li>"+renderInlineHTML(item)+"</li>")
}
out = append(out, "</ul>")
list = nil
}
flushTable := func() {
if len(tableRows) == 0 {
return
}
out = append(out, "<table><tbody>")
out = append(out, tableRows...)
out = append(out, "</tbody></table>")
tableRows = nil
}
flushBlocks := func() {
flushParagraph()
flushList()
flushTable()
}
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if trimmed == "" {
flushBlocks()
continue
}
if strings.HasPrefix(trimmed, "<p class=\"wiki-callout") {
flushBlocks()
out = append(out, trimmed)
continue
}
if heading, level, ok := parseDokuHeading(trimmed); ok {
flushBlocks()
out = append(out, fmt.Sprintf("<h%d>%s</h%d>", level, renderInlineHTML(heading), level))
continue
}
if strings.HasPrefix(trimmed, " * ") || strings.HasPrefix(trimmed, "* ") || strings.HasPrefix(trimmed, "- ") {
flushParagraph()
flushTable()
item := strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(trimmed, " * "), "* "), "- "))
list = append(list, item)
continue
}
if label, value, ok := parseFactLine(trimmed); ok {
flushParagraph()
flushList()
tableRows = append(tableRows, "<tr><th>"+html.EscapeString(label)+"</th><td>"+renderInlineHTML(value)+"</td></tr>")
continue
}
flushList()
flushTable()
for _, part := range strings.Split(trimmed, `\\`) {
part = strings.TrimSpace(part)
if part != "" {
paragraph = append(paragraph, part)
}
}
}
flushBlocks()
return strings.TrimSpace(strings.Join(out, "\n"))
}
func parseFactLine(line string) (string, string, bool) {
if !strings.HasPrefix(line, "**") {
return "", "", false
}
rest := strings.TrimPrefix(line, "**")
label, value, ok := strings.Cut(rest, ":**")
if !ok {
return "", "", false
}
return strings.TrimSpace(label), strings.Trim(strings.TrimSpace(value), `\`), true
}
func renderInlineHTML(text string) string {
var out strings.Builder
for {
start := strings.Index(text, "[[")
if start < 0 {
out.WriteString(html.EscapeString(text))
break
}
end := strings.Index(text[start+2:], "]]")
if end < 0 {
out.WriteString(html.EscapeString(text))
break
}
end += start + 2
out.WriteString(html.EscapeString(text[:start]))
out.WriteString(text[start : end+2])
text = text[end+2:]
}
return out.String()
}
func splitDokuWikiNotes(text string) (string, string) {
const delimiter = "===== Notes ====="
before, after, ok := strings.Cut(text, delimiter)
+22 -8
View File
@@ -42,26 +42,40 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) {
if result.WikiPages != 1 {
t.Fatalf("expected one generated entity page, got %d", result.WikiPages)
}
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md")
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
got, err := os.ReadFile(pagePath)
if err != nil {
t.Fatalf("read generated page: %v", err)
}
text := string(got)
if !strings.Contains(text, "# Athletics") || !strings.Contains(text, "This skill is unchanged from vanilla.") {
if !strings.Contains(text, "<h1>Athletics</h1>") || !strings.Contains(text, `class="wiki-callout wiki-callout--status"`) || !strings.Contains(text, "This skill is unchanged from vanilla.") {
t.Fatalf("unexpected generated page:\n%s", text)
}
if !strings.Contains(text, managedStartMarker) || !strings.Contains(text, "Ability: Strength.") || !strings.Contains(text, "- Climb") {
t.Fatalf("expected description to be rendered into wiki page:\n%s", text)
if !strings.Contains(text, `<!-- sow-topdata-wiki:page=skills:athletics -->`) || !strings.Contains(text, `<!-- sow-topdata-wiki:managed:start hash="sha256:`) || !strings.Contains(text, "Ability: Strength.") || !strings.Contains(text, "<li>Climb</li>") {
t.Fatalf("expected HTML managed markers and description to be rendered into wiki page:\n%s", text)
}
statusPath := filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus.md")
if strings.Contains(text, "[//]: #") || strings.Contains(text, "<WRAP>") {
t.Fatalf("expected generated HTML to omit DokuWiki-era syntax:\n%s", text)
}
if !strings.Contains(text, `<!-- sow-topdata-wiki:manual:start id="notes" -->`) || !strings.Contains(text, "<h2>See Also</h2>") {
t.Fatalf("expected generated page to include manual section placeholders:\n%s", text)
}
statusPath := filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus.html")
statusRaw, err := os.ReadFile(statusPath)
if err != nil {
t.Fatalf("read status page: %v", err)
}
if !strings.Contains(string(statusRaw), "**Vanilla:** 1") {
if !strings.Contains(string(statusRaw), "<th>Vanilla</th><td>1</td>") {
t.Fatalf("expected wiki status summary to count vanilla page:\n%s", string(statusRaw))
}
indexRaw, err := os.ReadFile(filepath.Join(root, ".cache", "wiki", "page-index.json"))
if err != nil {
t.Fatalf("read page index: %v", err)
}
indexText := string(indexRaw)
if !strings.Contains(indexText, `"page_id": "skills:athletics"`) || !strings.Contains(indexText, `"output_path": "pages/skills/athletics.html"`) || !strings.Contains(indexText, `"edit_policy": "preserve_manual_sections"`) {
t.Fatalf("expected deterministic page-index metadata, got:\n%s", indexText)
}
if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil {
t.Fatalf("expected state file after first build: %v", err)
}
@@ -201,7 +215,7 @@ func TestBuildNativeRegeneratesWikiWhenTLKTextChanges(t *testing.T) {
if _, err := BuildNative(proj, nil); err != nil {
t.Fatalf("BuildNative after text change failed: %v", err)
}
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md")
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
got, err := os.ReadFile(pagePath)
if err != nil {
t.Fatalf("read regenerated page: %v", err)
@@ -243,7 +257,7 @@ func TestBuildPackageIgnoresWikiSourcesForFreshness(t *testing.T) {
t.Fatalf("BuildNative failed: %v", err)
}
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md")
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
newTime := time.Now().Add(2 * time.Second)
if err := os.Chtimes(pagePath, newTime, newTime); err != nil {
t.Fatalf("touch wiki page: %v", err)