ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Wiki pipeline (#3)

Browse Source 
Implemented the wiki pipeline to a shippable local state across `toolkit/`, `module/`, and the NodeBB wiki plugin.

**Scope Audited**
- Toolkit config/project loading, wiki renderer, deployer, app command wiring.
- Module `nwn-tool.yaml`, wiki source/docs, release script, topdata docs.
- Plugin sanitizer/API storage contract docs and sanitizer tests.

**Changes Made**
- Added topdata-owned wiki declarations under `module/topdata/wiki/`.
- Added `topdata.wiki.*` config fields, validation, effective config output.
- Switched generated wiki pages to `.html` with HTML comment managed/manual markers.
- Added `page-index.json` generation and deterministic metadata.
- Added manual-section merge preservation and stale page `report`/`archive` handling.
- Added namespace `category_env` loading from `topdata/wiki/namespaces.yaml`, while keeping `--category`/`NODEBB_WIKI_CATEGORIES` overrides.
- Updated release deployment to keep dry-run first and make stale cleanup opt-in via `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY`.
- Updated plugin sanitizer to preserve only `sow-topdata-wiki` comments.

**Configuration/Schema Impact**
- `module/nwn-tool.yaml` now declares wiki source, renderer, link strategy, templates, manual sections, managed marker policy, and stale policy.
- Deploy manifest entries now support stale/archive metadata: `stale`, `last_seen_hash`, `archived_hash`, `title`, `namespace`, `tid`, `pid`, `cid`.

**Compatibility Impact**
- Legacy Markdown managed markers are still readable for migration.
- Generated output is now HTML, not Markdown.
- Existing mapped pages update by `pid`; creates still require explicit `--create`.

**Tests Added/Updated**
- Go tests for wiki config validation, HTML rendering, page index, manual merge, stale report/archive, app summary output.
- Plugin sanitizer test for generated topdata bot HTML markers/subset.

**Validation Performed**
- `go test ./internal/project ./internal/topdata ./internal/app` passed.
- `./build-tool.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./validate-topdata.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force` passed, generating 1345 entity pages.
- `deploy-wiki --dry-run --create` with dummy endpoint/token and namespace CID env vars passed locally: 1377 planned creates, 0 stale/drift.
- `node tests/wiki-html-sanitizer.test.js` passed.

**Remaining Risks**
- Full plugin `npm test` is blocked by an existing unrelated drawer CSS contract failure in `tests/wiki-article-drawers.test.js`.
- Controlled live migration was not performed because it requires a non-production NodeBB namespace and credentials.
- Direct `PUT /api/v3/posts/{pid}` save-filter behavior is documented as needing live confirmation against the deployed NodeBB/plugin stack.

Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/3
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
This commit is contained in:
archvillainette
2026-05-15 09:09:37 +02:00
committed by archvillainette
parent d767ec0a25
commit 9b2084344d
9 changed files with 1030 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/topdata/wiki_native.go
+302 -52
View File
@@ -5,6 +5,7 @@ import (
"embed"
"encoding/json"
"fmt"
"html"
"io/fs"
"os"
"path/filepath"
@@ -14,6 +15,7 @@ import (
"time"
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
"gopkg.in/yaml.v3"
)
const (
@@ -24,7 +26,7 @@ const (
legacyWikiRootDirName = ".wiki"
wikiPagesDirName = "pages"
wikiStateFileName = "state.json"
wikiGeneratorVersion = "nodebb-markdown-v1"
wikiGeneratorVersion = "nodebb-tiptap-html-v1"
wikiGeneratedStatus = "generated"
wikiSkippedStatus = "skipped"
)
@@ -43,6 +45,34 @@ type wikiStateDocument struct {
Pages int `json:"pages"`
}
type wikiPageIndex struct {
Version string `json:"version"`
Pages []wikiPageIndexEntry `json:"pages"`
}
type wikiPageIndexEntry struct {
PageID string `json:"page_id"`
Namespace string `json:"namespace"`
SourceDataset string `json:"source_dataset"`
SourceKey string `json:"source_key"`
Title string `json:"title"`
Hash string `json:"hash"`
OutputPath string `json:"output_path"`
EditPolicy string `json:"edit_policy"`
StalePolicy string `json:"stale_policy"`
}
type wikiManualSectionsDocument struct {
ManualSections []wikiManualSection `json:"manual_sections" yaml:"manual_sections"`
}
type wikiManualSection struct {
ID string `json:"id" yaml:"id"`
Heading string `json:"heading" yaml:"heading"`
Placement string `json:"placement" yaml:"placement"`
InitialHTML string `json:"initial_html" yaml:"initial_html"`
}
type wikiResult struct {
OutputDir string
PageCount int
@@ -117,6 +147,7 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
pageTitles := map[string]string{}
pageStatuses := map[string]string{}
pageIndex := wikiPageIndex{Version: wikiGeneratorVersion}
pageCount := 0
writePage := func(pageID, content, title, status string) error {
@@ -124,12 +155,28 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
content = renderNodeBBManagedMarkdown(pageID, content)
content = renderNodeBBManagedHTML(pageID, content, loadWikiManualSections(p))
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
return err
}
pageTitles[pageID] = title
pageStatuses[pageID] = status
namespace := strings.SplitN(pageID, ":", 2)[0]
rel, err := filepath.Rel(rootDir, path)
if err != nil {
rel = filepath.Base(path)
}
pageIndex.Pages = append(pageIndex.Pages, wikiPageIndexEntry{
PageID: pageID,
Namespace: namespace,
SourceDataset: categoryFromWikiNamespace(namespace),
SourceKey: pageID,
Title: title,
Hash: computeManagedHash(content),
OutputPath: filepath.ToSlash(rel),
EditPolicy: wikiEditPolicy(namespace),
StalePolicy: p.EffectiveConfig().TopData.Wiki.StalePages.Default,
})
pageCount++
return nil
}
@@ -137,7 +184,10 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
if err := generateEntityPages(outputDir, ctx, writePage); err != nil {
return wikiResult{}, err
}
if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces); err != nil {
if err := generateStatusPages(outputDir, pageStatuses, pageTitles, p.EffectiveConfig().TopData.Wiki.ManagedNamespaces, loadWikiManualSections(p)); err != nil {
return wikiResult{}, err
}
if err := saveWikiPageIndex(filepath.Join(rootDir, "page-index.json"), pageIndex); err != nil {
return wikiResult{}, err
}
@@ -156,6 +206,56 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
}, nil
}
func loadWikiManualSections(p *project.Project) []wikiManualSection {
defaults := []wikiManualSection{
{ID: "notes", Heading: "Notes", InitialHTML: "<h2>Notes</h2><p></p>"},
{ID: "see_also", Heading: "See Also", InitialHTML: "<h2>See Also</h2><p></p>"},
}
cfg := p.EffectiveConfig().TopData.Wiki
path := filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(cfg.ManualSectionsDir), "default.yaml")
raw, err := os.ReadFile(path)
if err != nil {
return defaults
}
var doc wikiManualSectionsDocument
if err := yaml.Unmarshal(raw, &doc); err != nil || len(doc.ManualSections) == 0 {
return defaults
}
out := []wikiManualSection{}
for _, section := range doc.ManualSections {
section.ID = strings.TrimSpace(section.ID)
if section.ID == "" {
continue
}
if strings.TrimSpace(section.InitialHTML) == "" {
heading := strings.TrimSpace(section.Heading)
if heading == "" {
heading = section.ID
}
section.InitialHTML = "<h2>" + html.EscapeString(heading) + "</h2><p></p>"
}
out = append(out, section)
}
if len(out) == 0 {
return defaults
}
return out
}
func saveWikiPageIndex(path string, index wikiPageIndex) error {
slices.SortFunc(index.Pages, func(a, b wikiPageIndexEntry) int {
return strings.Compare(a.PageID, b.PageID)
})
raw, err := json.MarshalIndent(index, "", " ")
if err != nil {
return fmt.Errorf("marshal wiki page index: %w", err)
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, append(raw, '\n'), 0o644)
}
func loadWikiState(path string) (wikiStateDocument, error) {
raw, err := os.ReadFile(path)
if err != nil {
@@ -593,24 +693,20 @@ func (ctx *wikiContext) shouldGeneratePage(category, key string, row map[string]
}
func (ctx *wikiContext) renderPage(category, key string, row map[string]any, title string) (string, error) {
templateName := category
if category == "baseitems" {
templateName = "items"
} else if category == "racialtypes" {
templateName = "races"
sections := []string{
"====== " + title + " ======",
"",
buildWikiStatusBlock(ctx.pageStatus(category, key, row), category),
"",
toDokuWiki(ctx.resolveRowDescription(category, row)),
"",
ctx.renderFacts(category, key, row),
"",
ctx.renderProgression(category, row),
"",
ctx.renderRaceNameForms(row),
}
templateBytes, err := wikiTemplateFS.ReadFile("wiki_templates/" + templateName + ".txt")
if err != nil {
return "", err
}
rendered := string(templateBytes)
rendered = strings.ReplaceAll(rendered, "{{name}}", title)
rendered = strings.ReplaceAll(rendered, "{{status}}", buildWikiStatusBlock(ctx.pageStatus(category, key, row), category))
rendered = strings.ReplaceAll(rendered, "{{description}}", toDokuWiki(ctx.resolveRowDescription(category, row)))
rendered = strings.ReplaceAll(rendered, "{{facts}}", ctx.renderFacts(category, key, row))
rendered = strings.ReplaceAll(rendered, "{{progression}}", ctx.renderProgression(category, row))
rendered = strings.ReplaceAll(rendered, "{{nameforms}}", ctx.renderRaceNameForms(row))
return ensureTrailingNewline(rendered), nil
return ensureTrailingNewline(strings.Join(sections, "\n")), nil
}
func (ctx *wikiContext) renderFacts(category, key string, row map[string]any) string {
@@ -1205,16 +1301,13 @@ func mergeRowWikiStatus(key string, row map[string]any, statuses map[string]stri
func buildWikiStatusBlock(status, category string) string {
message := "This " + wikiStatusNoun(category) + " is unchanged from vanilla."
wrap := "info"
switch status {
case wikiStatusNew:
message = "This is a new " + wikiStatusNoun(category) + "!"
wrap = "tip"
case wikiStatusModified:
message = "This " + wikiStatusNoun(category) + " has been altered from vanilla."
wrap = "help"
}
return "<WRAP round " + wrap + ">\n" + message + "\n</WRAP>"
return "<p class=\"wiki-callout wiki-callout--status\">" + html.EscapeString(message) + "</p>"
}
func wikiStatusNoun(category string) string {
@@ -1236,37 +1329,37 @@ func wikiStatusNoun(category string) string {
}
}
func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string, namespaces []string) error {
func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string, namespaces []string, manualSections []wikiManualSection) error {
summaries := buildWikiNamespaceSummaries(pageStatuses, namespaces)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.md"), renderWikiStatusReportPage(summaries, namespaces)); err != nil {
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.html"), renderWikiStatusReportPage(summaries, namespaces), manualSections); err != nil {
return err
}
for _, namespace := range namespaces {
summary := summaries[namespace]
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".md"), renderWikiNamespaceFragment(namespace, summary)); err != nil {
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".html"), renderWikiNamespaceFragment(namespace, summary), manualSections); err != nil {
return err
}
for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} {
pageIDs := filterWikiPageIDs(pageStatuses, namespace, status)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".md"), renderWikiStatusListingPage(namespaceTitle(namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".html"), renderWikiStatusListingPage(namespaceTitle(namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles), manualSections); err != nil {
return err
}
}
}
for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} {
pageIDs := filterWikiPageIDs(pageStatuses, "", status)
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".md"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".html"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles), manualSections); err != nil {
return err
}
}
return nil
}
func writeWikiFile(path, content string) error {
func writeWikiFile(path, content string, manualSections []wikiManualSection) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
content = renderNodeBBManagedMarkdown(wikiRelPathToPageID(path), content)
content = renderNodeBBManagedHTML(wikiRelPathToPageID(path), content, manualSections)
return os.WriteFile(path, []byte(ensureTrailingNewline(content)), 0o644)
}
@@ -1311,32 +1404,15 @@ func rollupWikiStatuses(statuses []string) string {
}
func renderWikiNamespaceFragment(namespace string, summary map[string]any) string {
return strings.Join([]string{
"====== " + namespaceTitle(namespace) + " ======",
"",
fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]),
fmt.Sprintf("**Generated Pages:** %d\\\\", summary["total"].(int)),
fmt.Sprintf("**New:** %d\\\\", summary["new"].(int)),
fmt.Sprintf("**Modified:** %d\\\\", summary["modified"].(int)),
fmt.Sprintf("**Vanilla:** %d\\\\", summary["vanilla"].(int)),
"",
fmt.Sprintf(" * [[%s:index|%s Index]]", namespace, namespaceTitle(namespace)),
}, "\n")
return renderStatusTable(namespaceTitle(namespace), summary, "")
}
func renderWikiStatusReportPage(summaries map[string]map[string]any, namespaces []string) string {
lines := []string{"====== Wiki Status ======", "", "This page is auto-generated from builder source provenance.", ""}
for _, namespace := range namespaces {
summary := summaries[namespace]
lines = append(lines, "===== "+namespaceTitle(namespace)+" =====", "")
lines = append(lines, fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]))
lines = append(lines, fmt.Sprintf("**Generated Pages:** %d\\\\", summary["total"].(int)))
lines = append(lines, fmt.Sprintf("**New:** %d\\\\", summary["new"].(int)))
lines = append(lines, fmt.Sprintf("**Modified:** %d\\\\", summary["modified"].(int)))
lines = append(lines, fmt.Sprintf("**Vanilla:** %d\\\\", summary["vanilla"].(int)))
lines = append(lines, fmt.Sprintf("**Namespace:** [[%s:index|%s]]", namespace, namespaceTitle(namespace)), "")
lines = append(lines, renderStatusTable(namespaceTitle(namespace), summary, fmt.Sprintf("[[%s:index|%s]]", namespace, namespaceTitle(namespace))))
}
lines = append(lines, "<WRAP center round important>", "**Auto-generated content.** This section is rebuilt from game data on each release.", "</WRAP>", "", "===== Notes =====")
return strings.Join(lines, "\n")
}
@@ -1353,10 +1429,25 @@ func renderWikiStatusListingPage(title string, pageIDs []string, pageTitles map[
lines = append(lines, " * [["+pageID+"|"+title+"]]")
}
}
lines = append(lines, "", "<WRAP center round important>", "**Auto-generated content.** This section is rebuilt from game data on each release.", "</WRAP>", "", "===== Notes =====")
return strings.Join(lines, "\n")
}
func renderStatusTable(title string, summary map[string]any, namespaceLink string) string {
rows := []string{
"===== " + title + " =====",
"",
fmt.Sprintf("**Namespace Status:** %s\\\\", wikiStatusLabels[summary["status"].(string)]),
fmt.Sprintf("**Generated Pages:** %d\\\\", summary["total"].(int)),
fmt.Sprintf("**New:** %d\\\\", summary["new"].(int)),
fmt.Sprintf("**Modified:** %d\\\\", summary["modified"].(int)),
fmt.Sprintf("**Vanilla:** %d\\\\", summary["vanilla"].(int)),
}
if namespaceLink != "" {
rows = append(rows, fmt.Sprintf("**Namespace:** %s", namespaceLink))
}
return strings.Join(rows, "\n")
}
func filterWikiPageIDs(pageStatuses map[string]string, namespace, status string) []string {
pageIDs := []string{}
for pageID, candidate := range pageStatuses {
@@ -1395,7 +1486,7 @@ func wikiPageIDForKey(key string) string {
}
func wikiPageIDToRelPath(pageID string) string {
return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".md")
return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".html")
}
func wikiRelPathToPageID(path string) string {
@@ -1417,6 +1508,24 @@ func namespaceTitle(namespace string) string {
}
}
func categoryFromWikiNamespace(namespace string) string {
switch namespace {
case "itemtypes":
return "baseitems"
case "races":
return "racialtypes"
default:
return namespace
}
}
func wikiEditPolicy(namespace string) string {
if namespace == "meta" {
return "generated_only"
}
return "preserve_manual_sections"
}
func normalizeWikiScalar(value any) string {
switch typed := value.(type) {
case string:
@@ -1598,6 +1707,147 @@ func renderNodeBBManagedMarkdown(pageID, dokuText string) string {
return ensureTrailingNewline(strings.Join(parts, "\n"))
}
func renderNodeBBManagedHTML(pageID, sourceText string, manualSections []wikiManualSection) string {
body := sourceText
if !strings.Contains(sourceText, "<h1") {
body = dokuWikiToNodeBBHTML(sourceText)
}
hash := computeManagedHash(body)
parts := []string{
"<!-- sow-topdata-wiki:page=" + pageID + " -->",
"<!-- sow-topdata-wiki:managed:start hash=\"sha256:" + hash + "\" -->",
strings.TrimSpace(body),
"<!-- sow-topdata-wiki:managed:end -->",
}
for _, section := range manualSections {
parts = append(parts,
"<!-- sow-topdata-wiki:manual:start id=\""+html.EscapeString(section.ID)+"\" -->",
strings.TrimSpace(section.InitialHTML),
"<!-- sow-topdata-wiki:manual:end id=\""+html.EscapeString(section.ID)+"\" -->",
)
}
return ensureTrailingNewline(strings.Join(parts, "\n"))
}
func dokuWikiToNodeBBHTML(text string) string {
text = strings.ReplaceAll(strings.TrimSpace(text), "\r\n", "\n")
if text == "" {
return ""
}
lines := strings.Split(text, "\n")
var out []string
var paragraph []string
var list []string
var tableRows []string
flushParagraph := func() {
if len(paragraph) == 0 {
return
}
out = append(out, "<p>"+renderInlineHTML(strings.Join(paragraph, " "))+"</p>")
paragraph = nil
}
flushList := func() {
if len(list) == 0 {
return
}
out = append(out, "<ul>")
for _, item := range list {
out = append(out, "<li>"+renderInlineHTML(item)+"</li>")
}
out = append(out, "</ul>")
list = nil
}
flushTable := func() {
if len(tableRows) == 0 {
return
}
out = append(out, "<table><tbody>")
out = append(out, tableRows...)
out = append(out, "</tbody></table>")
tableRows = nil
}
flushBlocks := func() {
flushParagraph()
flushList()
flushTable()
}
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if trimmed == "" {
flushBlocks()
continue
}
if strings.HasPrefix(trimmed, "<p class=\"wiki-callout") {
flushBlocks()
out = append(out, trimmed)
continue
}
if heading, level, ok := parseDokuHeading(trimmed); ok {
flushBlocks()
out = append(out, fmt.Sprintf("<h%d>%s</h%d>", level, renderInlineHTML(heading), level))
continue
}
if strings.HasPrefix(trimmed, " * ") || strings.HasPrefix(trimmed, "* ") || strings.HasPrefix(trimmed, "- ") {
flushParagraph()
flushTable()
item := strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(trimmed, " * "), "* "), "- "))
list = append(list, item)
continue
}
if label, value, ok := parseFactLine(trimmed); ok {
flushParagraph()
flushList()
tableRows = append(tableRows, "<tr><th>"+html.EscapeString(label)+"</th><td>"+renderInlineHTML(value)+"</td></tr>")
continue
}
flushList()
flushTable()
for _, part := range strings.Split(trimmed, `\\`) {
part = strings.TrimSpace(part)
if part != "" {
paragraph = append(paragraph, part)
}
}
}
flushBlocks()
return strings.TrimSpace(strings.Join(out, "\n"))
}
func parseFactLine(line string) (string, string, bool) {
if !strings.HasPrefix(line, "**") {
return "", "", false
}
rest := strings.TrimPrefix(line, "**")
label, value, ok := strings.Cut(rest, ":**")
if !ok {
return "", "", false
}
return strings.TrimSpace(label), strings.Trim(strings.TrimSpace(value), `\`), true
}
func renderInlineHTML(text string) string {
var out strings.Builder
for {
start := strings.Index(text, "[[")
if start < 0 {
out.WriteString(html.EscapeString(text))
break
}
end := strings.Index(text[start+2:], "]]")
if end < 0 {
out.WriteString(html.EscapeString(text))
break
}
end += start + 2
out.WriteString(html.EscapeString(text[:start]))
out.WriteString(text[start : end+2])
text = text[end+2:]
}
return out.String()
}
func splitDokuWikiNotes(text string) (string, string) {
const delimiter = "===== Notes ====="
before, after, ok := strings.Cut(text, delimiter)