Move Wiki Pages to MD

internal/topdata/wiki_deploy.go

@@ -22,8 +22,10 @@ import (
 const (
 	defaultDeployManifestName = ".wiki_deploy_manifest.json"
 	defaultEditSummary        = "Auto-generated from native builder"
-	managedStartMarker        = "<!-- sow-topdata-wiki:managed:start -->"
+	managedStartMarker        = "[//]: # (sow-topdata-wiki:managed:start)"
-	managedEndMarker          = "<!-- sow-topdata-wiki:managed:end -->"
+	managedEndMarker          = "[//]: # (sow-topdata-wiki:managed:end)"
+	legacyManagedStartMarker  = "<!-- sow-topdata-wiki:managed:start -->"
+	legacyManagedEndMarker    = "<!-- sow-topdata-wiki:managed:end -->"
 )
 
 var defaultManagedNamespaces = []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}
@@ -177,14 +179,15 @@ func collectLocalPages(sourceDir string, namespaces []string) (map[string]wikiDe
 			if err != nil {
 				return err
 			}
-			if d.IsDir() || filepath.Ext(path) != ".html" {
+			ext := filepath.Ext(path)
+			if d.IsDir() || (ext != ".md" && ext != ".html") {
 				return nil
 			}
 			rel, err := filepath.Rel(sourceDir, path)
 			if err != nil {
 				return err
 			}
-			pageID := strings.TrimSuffix(filepath.ToSlash(rel), ".html")
+			pageID := strings.TrimSuffix(filepath.ToSlash(rel), ext)
 			pageID = strings.ReplaceAll(pageID, "/", ":")
 			namespace := strings.SplitN(pageID, ":", 2)[0]
 			if _, ok := namespaceSet[namespace]; !ok {
@@ -197,7 +200,7 @@ func collectLocalPages(sourceDir string, namespaces []string) (map[string]wikiDe
 			content := string(raw)
 			pages[pageID] = wikiDeployPage{
 				PageID:    pageID,
-				Title:     extractHTMLTitle(content, pageID),
+				Title:     extractPageTitle(content, pageID),
 				Namespace: namespace,
 				Content:   content,
 				Hash:      computeManagedHash(content),
@@ -310,12 +313,11 @@ func computeManagedHash(content string) string {
 }
 
 func extractManagedRegion(content string) (string, bool) {
-	start := strings.Index(content, managedStartMarker)
+	startMarker, _, start, end := managedRegionBounds(content)
-	end := strings.Index(content, managedEndMarker)
 	if start < 0 || end < 0 || end < start {
 		return "", false
 	}
-	start += len(managedStartMarker)
+	start += len(startMarker)
 	return strings.TrimSpace(content[start:end]), true
 }
 
@@ -324,13 +326,47 @@ func mergeManagedContent(existing, generated string) string {
 	if !ok {
 		return generated
 	}
-	start := strings.Index(existing, managedStartMarker)
+	_, endMarker, start, end := managedRegionBounds(existing)
-	end := strings.Index(existing, managedEndMarker)
 	if start < 0 || end < 0 || end < start {
 		return generated
 	}
-	end += len(managedEndMarker)
+	end += len(endMarker)
-	return existing[:start+len(managedStartMarker)] + "\n" + generatedRegion + "\n" + existing[end:]
+	return existing[:start] + managedStartMarker + "\n" + generatedRegion + "\n" + managedEndMarker + existing[end:]
+}
+
+func managedRegionBounds(content string) (string, string, int, int) {
+	for _, markers := range [][2]string{
+		{managedStartMarker, managedEndMarker},
+		{legacyManagedStartMarker, legacyManagedEndMarker},
+	} {
+		start := strings.Index(content, markers[0])
+		end := strings.Index(content, markers[1])
+		if start >= 0 && end >= 0 {
+			return markers[0], markers[1], start, end
+		}
+	}
+	return "", "", -1, -1
+}
+
+func extractPageTitle(content, fallback string) string {
+	if title := extractMarkdownTitle(content); title != "" {
+		return title
+	}
+	return extractHTMLTitle(content, fallback)
+}
+
+func extractMarkdownTitle(content string) string {
+	for _, line := range strings.Split(content, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if !strings.HasPrefix(trimmed, "# ") {
+			continue
+		}
+		title := strings.TrimSpace(strings.TrimPrefix(trimmed, "# "))
+		if title != "" {
+			return title
+		}
+	}
+	return ""
 }
 
 func extractHTMLTitle(content, fallback string) string {

internal/topdata/wiki_deploy_test.go

@@ -17,8 +17,8 @@ func TestDeployWikiDryRunDoesNotWriteRemoteOrManifest(t *testing.T) {
 	if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
 		t.Fatalf("create source dir: %v", err)
 	}
-	generated := "<!-- sow-topdata-wiki:managed:start -->\n<h1>Athletics</h1>\n<p>Generated athletics page</p>\n<!-- sow-topdata-wiki:managed:end -->\n"
+	generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Athletics\n\nGenerated athletics page\n[//]: # (sow-topdata-wiki:managed:end)\n"
-	if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil {
+	if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.md"), []byte(generated), 0644); err != nil {
 		t.Fatalf("write source page: %v", err)
 	}
 	old := "<!-- sow-topdata-wiki:managed:start -->\n<h1>Athletics</h1>\n<p>Existing athletics page</p>\n<!-- sow-topdata-wiki:managed:end -->\n"
@@ -81,8 +81,8 @@ func TestDeployWikiCreatesNodeBBTopicAndWritesManifest(t *testing.T) {
 	if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
 		t.Fatalf("create source dir: %v", err)
 	}
-	generated := "<!-- sow-topdata-wiki:managed:start -->\n<h1>Athletics</h1>\n<p>Generated athletics page</p>\n<!-- sow-topdata-wiki:managed:end -->\n"
+	generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Athletics\n\nGenerated athletics page\n[//]: # (sow-topdata-wiki:managed:end)\n"
-	if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil {
+	if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.md"), []byte(generated), 0644); err != nil {
 		t.Fatalf("write source page: %v", err)
 	}
 
@@ -147,8 +147,8 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) {
 	if err := os.MkdirAll(filepath.Join(sourceDir, "spells"), 0755); err != nil {
 		t.Fatalf("create source dir: %v", err)
 	}
-	generated := "<!-- sow-topdata-wiki:managed:start -->\n<h1>Aid</h1>\n<p>Generated spell page</p>\n<!-- sow-topdata-wiki:managed:end -->\n"
+	generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Aid\n\nGenerated spell page\n[//]: # (sow-topdata-wiki:managed:end)\n"
-	if err := os.WriteFile(filepath.Join(sourceDir, "spells", "aid.html"), []byte(generated), 0644); err != nil {
+	if err := os.WriteFile(filepath.Join(sourceDir, "spells", "aid.md"), []byte(generated), 0644); err != nil {
 		t.Fatalf("write source page: %v", err)
 	}
 

internal/topdata/wiki_native.go

@@ -5,7 +5,6 @@ import (
 	"embed"
 	"encoding/json"
 	"fmt"
-	"html"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -25,7 +24,7 @@ const (
 	legacyWikiRootDirName = ".wiki"
 	wikiPagesDirName      = "pages"
 	wikiStateFileName     = "state.json"
-	wikiGeneratorVersion  = "nodebb-html-v1"
+	wikiGeneratorVersion  = "nodebb-markdown-v1"
 	wikiGeneratedStatus   = "generated"
 	wikiSkippedStatus     = "skipped"
 )
@@ -126,7 +125,7 @@ func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progres
 		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
 			return err
 		}
-		content = renderNodeBBManagedHTML(pageID, content)
+		content = renderNodeBBManagedMarkdown(pageID, content)
 		if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
 			return err
 		}
@@ -1240,24 +1239,24 @@ func wikiStatusNoun(category string) string {
 
 func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string) error {
 	summaries := buildWikiNamespaceSummaries(pageStatuses)
-	if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.html"), renderWikiStatusReportPage(summaries)); err != nil {
+	if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.md"), renderWikiStatusReportPage(summaries)); err != nil {
 		return err
 	}
 	for _, namespace := range wikiManagedNamespaces {
 		summary := summaries[namespace]
-		if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".html"), renderWikiNamespaceFragment(namespace, summary)); err != nil {
+		if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".md"), renderWikiNamespaceFragment(namespace, summary)); err != nil {
 			return err
 		}
 		for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} {
 			pageIDs := filterWikiPageIDs(pageStatuses, namespace, status)
-			if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".html"), renderWikiStatusListingPage(namespaceTitle(namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
+			if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".md"), renderWikiStatusListingPage(namespaceTitle(namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
 				return err
 			}
 		}
 	}
 	for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} {
 		pageIDs := filterWikiPageIDs(pageStatuses, "", status)
-		if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".html"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
+		if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".md"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
 			return err
 		}
 	}
@@ -1268,7 +1267,7 @@ func writeWikiFile(path, content string) error {
 	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
 		return err
 	}
-	content = renderNodeBBManagedHTML(wikiRelPathToPageID(path), content)
+	content = renderNodeBBManagedMarkdown(wikiRelPathToPageID(path), content)
 	return os.WriteFile(path, []byte(ensureTrailingNewline(content)), 0o644)
 }
 
@@ -1397,11 +1396,12 @@ func wikiPageIDForKey(key string) string {
 }
 
 func wikiPageIDToRelPath(pageID string) string {
-	return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".html")
+	return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".md")
 }
 
 func wikiRelPathToPageID(path string) string {
 	path = filepath.ToSlash(path)
+	path = strings.TrimSuffix(path, ".md")
 	path = strings.TrimSuffix(path, ".html")
 	if index := strings.LastIndex(path, "/pages/"); index >= 0 {
 		path = path[index+len("/pages/"):]
@@ -1583,18 +1583,18 @@ func toDokuWiki(text string) string {
 	return strings.Join(out, "\n")
 }
 
-func renderNodeBBManagedHTML(pageID, dokuText string) string {
+func renderNodeBBManagedMarkdown(pageID, dokuText string) string {
 	generated, notes := splitDokuWikiNotes(dokuText)
-	body := dokuWikiToNodeBBHTML(generated)
+	body := dokuWikiToNodeBBMarkdown(generated)
 	parts := []string{
-		"<!-- sow-topdata-wiki:page=" + html.EscapeString(pageID) + " -->",
+		"[//]: # (sow-topdata-wiki:page=" + pageID + ")",
-		"<!-- sow-topdata-wiki:managed:start -->",
+		"[//]: # (sow-topdata-wiki:managed:start)",
 		body,
-		"<!-- sow-topdata-wiki:managed:end -->",
+		"[//]: # (sow-topdata-wiki:managed:end)",
 	}
-	notesHTML := dokuWikiToNodeBBHTML("===== Notes =====\n" + notes)
+	notesMarkdown := dokuWikiToNodeBBMarkdown("===== Notes =====\n" + notes)
-	if strings.TrimSpace(notesHTML) != "" {
+	if strings.TrimSpace(notesMarkdown) != "" {
-		parts = append(parts, notesHTML)
+		parts = append(parts, notesMarkdown)
 	}
 	return ensureTrailingNewline(strings.Join(parts, "\n"))
 }
@@ -1608,72 +1608,59 @@ func splitDokuWikiNotes(text string) (string, string) {
 	return before, after
 }
 
-func dokuWikiToNodeBBHTML(text string) string {
+func dokuWikiToNodeBBMarkdown(text string) string {
 	text = strings.ReplaceAll(strings.TrimSpace(text), "\r\n", "\n")
 	if text == "" {
 		return ""
 	}
 
 	var out []string
-	var paragraph []string
-	inList := false
 	inNotice := false
 
-	flushParagraph := func() {
-		if len(paragraph) == 0 {
-			return
-		}
-		out = append(out, "<p>"+strings.Join(paragraph, "<br>")+"</p>")
-		paragraph = nil
-	}
-	closeList := func() {
-		if inList {
-			out = append(out, "</ul>")
-			inList = false
-		}
-	}
 	closeNotice := func() {
 		if inNotice {
-			out = append(out, "</aside>")
 			inNotice = false
+			if len(out) > 0 && strings.TrimSpace(out[len(out)-1]) != "" {
+				out = append(out, "")
+			}
 		}
 	}
+	appendBlank := func() {
+		if len(out) == 0 || strings.TrimSpace(out[len(out)-1]) == "" {
+			return
+		}
+		out = append(out, "")
+	}
 
 	for _, line := range strings.Split(text, "\n") {
 		trimmed := strings.TrimSpace(line)
 		if trimmed == "" {
-			flushParagraph()
+			appendBlank()
-			closeList()
 			continue
 		}
 		if strings.HasPrefix(trimmed, "<WRAP") {
-			flushParagraph()
+			appendBlank()
-			closeList()
-			out = append(out, `<aside class="sow-generated-note">`)
 			inNotice = true
 			continue
 		}
 		if trimmed == "</WRAP>" {
-			flushParagraph()
-			closeList()
 			closeNotice()
 			continue
 		}
 		if heading, level, ok := parseDokuHeading(trimmed); ok {
-			flushParagraph()
-			closeList()
 			closeNotice()
-			out = append(out, fmt.Sprintf("<h%d>%s</h%d>", level, renderInlineNodeBBHTML(heading), level))
+			appendBlank()
+			out = append(out, strings.Repeat("#", level)+" "+renderInlineNodeBBMarkdown(heading))
+			out = append(out, "")
 			continue
 		}
 		if strings.HasPrefix(trimmed, "  * ") || strings.HasPrefix(trimmed, "* ") || strings.HasPrefix(trimmed, "- ") {
-			flushParagraph()
-			if !inList {
-				out = append(out, "<ul>")
-				inList = true
-			}
 			item := strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(trimmed, "  * "), "* "), "- "))
-			out = append(out, "<li>"+renderInlineNodeBBHTML(item)+"</li>")
+			prefix := "- "
+			if inNotice {
+				prefix = "> - "
+			}
+			out = append(out, prefix+renderInlineNodeBBMarkdown(item))
 			continue
 		}
 
@@ -1681,14 +1668,16 @@ func dokuWikiToNodeBBHTML(text string) string {
 		for _, part := range parts {
 			part = strings.TrimSpace(part)
 			if part != "" {
-				paragraph = append(paragraph, renderInlineNodeBBHTML(part))
+				prefix := ""
+				if inNotice {
+					prefix = "> "
+				}
+				out = append(out, prefix+renderInlineNodeBBMarkdown(part))
 			}
 		}
 	}
-	flushParagraph()
-	closeList()
 	closeNotice()
-	return strings.Join(out, "\n")
+	return strings.TrimSpace(strings.Join(out, "\n"))
 }
 
 func parseDokuHeading(line string) (string, int, bool) {
@@ -1714,29 +1703,8 @@ func parseDokuHeading(line string) (string, int, bool) {
 	return title, level, true
 }
 
-func renderInlineNodeBBHTML(text string) string {
+func renderInlineNodeBBMarkdown(text string) string {
-	escaped := html.EscapeString(text)
+	return renderWikiLinks(text)
-	var out strings.Builder
-	boldOpen := false
-	for {
-		index := strings.Index(escaped, "**")
-		if index < 0 {
-			out.WriteString(escaped)
-			break
-		}
-		out.WriteString(escaped[:index])
-		if boldOpen {
-			out.WriteString("</strong>")
-		} else {
-			out.WriteString("<strong>")
-		}
-		boldOpen = !boldOpen
-		escaped = escaped[index+2:]
-	}
-	if boldOpen {
-		out.WriteString("</strong>")
-	}
-	return renderWikiLinks(out.String())
 }
 
 func renderWikiLinks(text string) string {

internal/topdata/wiki_native_test.go

@@ -42,24 +42,24 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) {
 	if result.WikiPages != 1 {
 		t.Fatalf("expected one generated entity page, got %d", result.WikiPages)
 	}
-	pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
+	pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md")
 	got, err := os.ReadFile(pagePath)
 	if err != nil {
 		t.Fatalf("read generated page: %v", err)
 	}
 	text := string(got)
-	if !strings.Contains(text, "<h1>Athletics</h1>") || !strings.Contains(text, "This skill is unchanged from vanilla.") {
+	if !strings.Contains(text, "# Athletics") || !strings.Contains(text, "This skill is unchanged from vanilla.") {
 		t.Fatalf("unexpected generated page:\n%s", text)
 	}
-	if !strings.Contains(text, managedStartMarker) || !strings.Contains(text, "Ability: Strength.") || !strings.Contains(text, "<li>Climb</li>") {
+	if !strings.Contains(text, managedStartMarker) || !strings.Contains(text, "Ability: Strength.") || !strings.Contains(text, "- Climb") {
 		t.Fatalf("expected description to be rendered into wiki page:\n%s", text)
 	}
-	statusPath := filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus.html")
+	statusPath := filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus.md")
 	statusRaw, err := os.ReadFile(statusPath)
 	if err != nil {
 		t.Fatalf("read status page: %v", err)
 	}
-	if !strings.Contains(string(statusRaw), "<strong>Vanilla:</strong> 1") {
+	if !strings.Contains(string(statusRaw), "**Vanilla:** 1") {
 		t.Fatalf("expected wiki status summary to count vanilla page:\n%s", string(statusRaw))
 	}
 	if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil {
@@ -201,7 +201,7 @@ func TestBuildNativeRegeneratesWikiWhenTLKTextChanges(t *testing.T) {
 	if _, err := BuildNative(proj, nil); err != nil {
 		t.Fatalf("BuildNative after text change failed: %v", err)
 	}
-	pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
+	pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md")
 	got, err := os.ReadFile(pagePath)
 	if err != nil {
 		t.Fatalf("read regenerated page: %v", err)
@@ -243,7 +243,7 @@ func TestBuildPackageIgnoresWikiSourcesForFreshness(t *testing.T) {
 		t.Fatalf("BuildNative failed: %v", err)
 	}
 
-	pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
+	pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.md")
 	newTime := time.Now().Add(2 * time.Second)
 	if err := os.Chtimes(pagePath, newTime, newTime); err != nil {
 		t.Fatalf("touch wiki page: %v", err)