Implemented the wiki pipeline to a shippable local state across `toolkit/`, `module/`, and the NodeBB wiki plugin.
**Scope Audited**
- Toolkit config/project loading, wiki renderer, deployer, app command wiring.
- Module `nwn-tool.yaml`, wiki source/docs, release script, topdata docs.
- Plugin sanitizer/API storage contract docs and sanitizer tests.
**Changes Made**
- Added topdata-owned wiki declarations under `module/topdata/wiki/`.
- Added `topdata.wiki.*` config fields, validation, effective config output.
- Switched generated wiki pages to `.html` with HTML comment managed/manual markers.
- Added `page-index.json` generation and deterministic metadata.
- Added manual-section merge preservation and stale page `report`/`archive` handling.
- Added namespace `category_env` loading from `topdata/wiki/namespaces.yaml`, while keeping `--category`/`NODEBB_WIKI_CATEGORIES` overrides.
- Updated release deployment to keep dry-run first and make stale cleanup opt-in via `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY`.
- Updated plugin sanitizer to preserve only `sow-topdata-wiki` comments.
**Configuration/Schema Impact**
- `module/nwn-tool.yaml` now declares wiki source, renderer, link strategy, templates, manual sections, managed marker policy, and stale policy.
- Deploy manifest entries now support stale/archive metadata: `stale`, `last_seen_hash`, `archived_hash`, `title`, `namespace`, `tid`, `pid`, `cid`.
**Compatibility Impact**
- Legacy Markdown managed markers are still readable for migration.
- Generated output is now HTML, not Markdown.
- Existing mapped pages update by `pid`; creates still require explicit `--create`.
**Tests Added/Updated**
- Go tests for wiki config validation, HTML rendering, page index, manual merge, stale report/archive, app summary output.
- Plugin sanitizer test for generated topdata bot HTML markers/subset.
**Validation Performed**
- `go test ./internal/project ./internal/topdata ./internal/app` passed.
- `./build-tool.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./validate-topdata.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force` passed, generating 1345 entity pages.
- `deploy-wiki --dry-run --create` with dummy endpoint/token and namespace CID env vars passed locally: 1377 planned creates, 0 stale/drift.
- `node tests/wiki-html-sanitizer.test.js` passed.
**Remaining Risks**
- Full plugin `npm test` is blocked by an existing unrelated drawer CSS contract failure in `tests/wiki-article-drawers.test.js`.
- Controlled live migration was not performed because it requires a non-production NodeBB namespace and credentials.
- Direct `PUT /api/v3/posts/{pid}` save-filter behavior is documented as needing live confirmation against the deployed NodeBB/plugin stack.
Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/3
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
838 lines
23 KiB
Go
838 lines
23 KiB
Go
package topdata
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
const (
|
|
managedStartMarker = "<!-- sow-topdata-wiki:managed:start"
|
|
managedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
|
|
legacyMarkdownStartMarker = "[//]: # (sow-topdata-wiki:managed:start)"
|
|
legacyMarkdownEndMarker = "[//]: # (sow-topdata-wiki:managed:end)"
|
|
legacyManagedStartMarker = "<!-- sow-topdata-wiki:managed:start -->"
|
|
legacyManagedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
|
|
manualStartMarkerPrefix = "<!-- sow-topdata-wiki:manual:start"
|
|
manualEndMarkerPrefix = "<!-- sow-topdata-wiki:manual:end"
|
|
)
|
|
|
|
type DeployWikiOptions struct {
|
|
SourceDir string
|
|
Endpoint string
|
|
Token string
|
|
Version string
|
|
Namespaces []string
|
|
CategoryIDs map[string]int
|
|
ManifestPath string
|
|
DryRun bool
|
|
Force bool
|
|
AllowCreates bool
|
|
StalePolicy string
|
|
Username string
|
|
Password string
|
|
NotesDelimiter string
|
|
}
|
|
|
|
type deployResult struct {
|
|
LocalPages int
|
|
Created int
|
|
Updated int
|
|
Stale int
|
|
Archived int
|
|
Skipped int
|
|
Drifted int
|
|
Manifest string
|
|
}
|
|
|
|
type wikiDeployPage struct {
|
|
PageID string
|
|
Title string
|
|
Namespace string
|
|
Content string
|
|
Hash string
|
|
}
|
|
|
|
type wikiDeployManifest struct {
|
|
Version string `json:"version"`
|
|
Pages map[string]wikiDeployManifestPage `json:"pages"`
|
|
}
|
|
|
|
type wikiDeployManifestPage struct {
|
|
Hash string `json:"hash"`
|
|
LastSeenHash string `json:"last_seen_hash,omitempty"`
|
|
ArchivedHash string `json:"archived_hash,omitempty"`
|
|
Title string `json:"title,omitempty"`
|
|
Namespace string `json:"namespace,omitempty"`
|
|
Stale bool `json:"stale,omitempty"`
|
|
TID int `json:"tid,omitempty"`
|
|
PID int `json:"pid,omitempty"`
|
|
CID int `json:"cid,omitempty"`
|
|
}
|
|
|
|
type wikiDeployPlan struct {
|
|
Page wikiDeployPage
|
|
Entry wikiDeployManifestPage
|
|
Action string
|
|
Content string
|
|
RemoteHash string
|
|
}
|
|
|
|
type wikiNamespacesDocument struct {
|
|
Namespaces []wikiNamespaceDeclaration `json:"namespaces" yaml:"namespaces"`
|
|
}
|
|
|
|
type wikiNamespaceDeclaration struct {
|
|
ID string `json:"id" yaml:"id"`
|
|
Title string `json:"title" yaml:"title"`
|
|
CategoryEnv string `json:"category_env" yaml:"category_env"`
|
|
PageTitle string `json:"page_title" yaml:"page_title"`
|
|
Tags []string `json:"tags" yaml:"tags"`
|
|
EditPolicy string `json:"edit_policy" yaml:"edit_policy"`
|
|
}
|
|
|
|
func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress func(string)) (deployResult, error) {
|
|
if progress == nil {
|
|
progress = func(string) {}
|
|
}
|
|
if opts.SourceDir == "" {
|
|
opts.SourceDir = wikiOutputPagesDir(p)
|
|
}
|
|
if _, err := os.Stat(opts.SourceDir); err != nil {
|
|
return deployResult{}, fmt.Errorf("wiki source directory not found: %w", err)
|
|
}
|
|
if opts.Endpoint == "" {
|
|
return deployResult{}, errors.New("NODEBB_API_ENDPOINT is required")
|
|
}
|
|
if opts.Token == "" {
|
|
return deployResult{}, errors.New("NODEBB_API_TOKEN is required")
|
|
}
|
|
if opts.Username != "" || opts.Password != "" || opts.NotesDelimiter != "" {
|
|
return deployResult{}, errors.New("DokuWiki deployment options are no longer supported; use NodeBB endpoint, token, and category mappings")
|
|
}
|
|
if opts.StalePolicy != "" && opts.StalePolicy != "report" && opts.StalePolicy != "archive" {
|
|
return deployResult{}, fmt.Errorf("wiki stale policy %q is not supported", opts.StalePolicy)
|
|
}
|
|
|
|
namespaces := opts.Namespaces
|
|
if len(namespaces) == 0 {
|
|
declarations, err := loadWikiNamespaceDeclarations(p)
|
|
if err != nil {
|
|
return deployResult{}, err
|
|
}
|
|
for _, declaration := range declarations {
|
|
namespaces = append(namespaces, declaration.ID)
|
|
}
|
|
if len(namespaces) == 0 {
|
|
namespaces = p.EffectiveConfig().TopData.Wiki.ManagedNamespaces
|
|
}
|
|
envCategories, err := categoryIDsFromNamespaceEnv(declarations)
|
|
if err != nil {
|
|
return deployResult{}, err
|
|
}
|
|
if len(envCategories) > 0 {
|
|
merged := envCategories
|
|
for namespace, cid := range opts.CategoryIDs {
|
|
merged[namespace] = cid
|
|
}
|
|
opts.CategoryIDs = merged
|
|
}
|
|
}
|
|
|
|
manifestPath := opts.ManifestPath
|
|
if manifestPath == "" {
|
|
manifestPath = filepath.Join(filepath.Dir(opts.SourceDir), p.EffectiveConfig().TopData.Wiki.DeployManifest)
|
|
}
|
|
|
|
pages, err := collectLocalPages(opts.SourceDir, namespaces)
|
|
if err != nil {
|
|
return deployResult{}, err
|
|
}
|
|
manifest := loadDeployManifest(manifestPath)
|
|
client := newNodeBBClient(opts.Endpoint, opts.Token)
|
|
|
|
plans, result, nextManifest, err := planNodeBBDeploy(pages, manifest, opts, client)
|
|
if err != nil {
|
|
return deployResult{}, err
|
|
}
|
|
result.LocalPages = len(pages)
|
|
result.Manifest = manifestPath
|
|
|
|
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, skip %d, stale %d, archive %d, drift %d", result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Drifted))
|
|
if opts.DryRun {
|
|
return result, nil
|
|
}
|
|
if result.Drifted > 0 && !opts.Force {
|
|
return result, errors.New("remote managed wiki content drifted; rerun with --force to overwrite")
|
|
}
|
|
|
|
summary := p.EffectiveConfig().TopData.Wiki.DeployEditSummary
|
|
if opts.Version != "" {
|
|
summary = fmt.Sprintf("%s (%s)", summary, opts.Version)
|
|
}
|
|
for _, plan := range plans {
|
|
switch plan.Action {
|
|
case "create":
|
|
created, err := client.createTopic(plan.Entry.CID, plan.Page.Title, plan.Content, summary)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
entry := nextManifest.Pages[plan.Page.PageID]
|
|
entry.TID = created.TID
|
|
entry.PID = created.PID
|
|
nextManifest.Pages[plan.Page.PageID] = entry
|
|
case "update":
|
|
if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil {
|
|
return result, err
|
|
}
|
|
case "archive":
|
|
if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil {
|
|
return result, err
|
|
}
|
|
}
|
|
}
|
|
if err := saveDeployManifest(manifestPath, nextManifest); err != nil {
|
|
return result, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func loadWikiNamespaceDeclarations(p *project.Project) ([]wikiNamespaceDeclaration, error) {
|
|
cfg := p.EffectiveConfig().TopData.Wiki
|
|
path := filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(cfg.NamespacesFile))
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, nil
|
|
}
|
|
return nil, fmt.Errorf("read wiki namespaces: %w", err)
|
|
}
|
|
var doc wikiNamespacesDocument
|
|
if err := yaml.Unmarshal(raw, &doc); err != nil {
|
|
return nil, fmt.Errorf("parse wiki namespaces: %w", err)
|
|
}
|
|
seen := map[string]struct{}{}
|
|
for _, declaration := range doc.Namespaces {
|
|
id := strings.TrimSpace(declaration.ID)
|
|
if id == "" {
|
|
return nil, errors.New("topdata wiki namespace id is required")
|
|
}
|
|
if _, ok := seen[id]; ok {
|
|
return nil, fmt.Errorf("duplicate topdata wiki namespace id %q", id)
|
|
}
|
|
seen[id] = struct{}{}
|
|
if strings.TrimSpace(declaration.CategoryEnv) == "" {
|
|
return nil, fmt.Errorf("topdata wiki namespace %q category_env is required", id)
|
|
}
|
|
switch strings.TrimSpace(declaration.EditPolicy) {
|
|
case "", "preserve_manual_sections", "generated_only":
|
|
default:
|
|
return nil, fmt.Errorf("topdata wiki namespace %q edit_policy %q is not supported", id, declaration.EditPolicy)
|
|
}
|
|
}
|
|
return doc.Namespaces, nil
|
|
}
|
|
|
|
func categoryIDsFromNamespaceEnv(declarations []wikiNamespaceDeclaration) (map[string]int, error) {
|
|
categories := map[string]int{}
|
|
for _, declaration := range declarations {
|
|
envName := strings.TrimSpace(declaration.CategoryEnv)
|
|
if envName == "" {
|
|
continue
|
|
}
|
|
raw := strings.TrimSpace(os.Getenv(envName))
|
|
if raw == "" {
|
|
continue
|
|
}
|
|
cid, err := strconv.Atoi(raw)
|
|
if err != nil || cid <= 0 {
|
|
return nil, fmt.Errorf("topdata wiki namespace %q env %s has invalid cid %q", declaration.ID, envName, raw)
|
|
}
|
|
categories[declaration.ID] = cid
|
|
}
|
|
return categories, nil
|
|
}
|
|
|
|
func collectLocalPages(sourceDir string, namespaces []string) (map[string]wikiDeployPage, error) {
|
|
namespaceSet := map[string]struct{}{}
|
|
for _, namespace := range namespaces {
|
|
namespaceSet[namespace] = struct{}{}
|
|
}
|
|
pages := map[string]wikiDeployPage{}
|
|
for _, ns := range namespaces {
|
|
nsDir := filepath.Join(sourceDir, ns)
|
|
if !isDir(nsDir) {
|
|
continue
|
|
}
|
|
if err := filepath.WalkDir(nsDir, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ext := filepath.Ext(path)
|
|
if d.IsDir() || (ext != ".md" && ext != ".html") {
|
|
return nil
|
|
}
|
|
rel, err := filepath.Rel(sourceDir, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pageID := strings.TrimSuffix(filepath.ToSlash(rel), ext)
|
|
pageID = strings.ReplaceAll(pageID, "/", ":")
|
|
namespace := strings.SplitN(pageID, ":", 2)[0]
|
|
if _, ok := namespaceSet[namespace]; !ok {
|
|
return nil
|
|
}
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
content := string(raw)
|
|
pages[pageID] = wikiDeployPage{
|
|
PageID: pageID,
|
|
Title: extractPageTitle(content, pageID),
|
|
Namespace: namespace,
|
|
Content: content,
|
|
Hash: computeManagedHash(content),
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return pages, nil
|
|
}
|
|
|
|
func isDir(path string) bool {
|
|
info, err := os.Stat(path)
|
|
return err == nil && info.IsDir()
|
|
}
|
|
|
|
func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) {
|
|
pageIDs := sortedKeysFromMap(pages)
|
|
next := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
|
|
var plans []wikiDeployPlan
|
|
var result deployResult
|
|
|
|
for _, pageID := range pageIDs {
|
|
page := pages[pageID]
|
|
entry := manifest.Pages[pageID]
|
|
entry.Hash = page.Hash
|
|
entry.LastSeenHash = page.Hash
|
|
entry.Title = page.Title
|
|
entry.Namespace = page.Namespace
|
|
entry.Stale = false
|
|
if entry.CID == 0 {
|
|
entry.CID = opts.CategoryIDs[page.Namespace]
|
|
}
|
|
next.Pages[pageID] = entry
|
|
|
|
if entry.PID == 0 {
|
|
if !opts.AllowCreates {
|
|
return nil, result, next, fmt.Errorf("wiki page %q has no manifest post mapping; pass --create and --category %s=<cid> to create it", pageID, page.Namespace)
|
|
}
|
|
if entry.CID == 0 {
|
|
return nil, result, next, fmt.Errorf("wiki page %q requires --category %s=<cid> for creation", pageID, page.Namespace)
|
|
}
|
|
result.Created++
|
|
page.Title = nodeBBTopicTitle(page)
|
|
plans = append(plans, wikiDeployPlan{Page: page, Entry: entry, Action: "create", Content: page.Content})
|
|
continue
|
|
}
|
|
|
|
if manifest.Pages[pageID].Hash == page.Hash {
|
|
result.Skipped++
|
|
continue
|
|
}
|
|
remote, err := client.getPost(entry.PID)
|
|
if err != nil {
|
|
return nil, result, next, err
|
|
}
|
|
remoteHash := computeManagedHash(remote.Content)
|
|
if manifest.Pages[pageID].Hash != "" && remoteHash != manifest.Pages[pageID].Hash {
|
|
result.Drifted++
|
|
if !opts.Force {
|
|
continue
|
|
}
|
|
}
|
|
merged := mergeManagedContent(remote.Content, page.Content)
|
|
if merged == remote.Content {
|
|
result.Skipped++
|
|
continue
|
|
}
|
|
result.Updated++
|
|
plans = append(plans, wikiDeployPlan{Page: page, Entry: entry, Action: "update", Content: merged, RemoteHash: remoteHash})
|
|
}
|
|
for pageID := range manifest.Pages {
|
|
if _, ok := pages[pageID]; !ok {
|
|
entry := manifest.Pages[pageID]
|
|
entry.Stale = true
|
|
if entry.Namespace == "" {
|
|
entry.Namespace = strings.SplitN(pageID, ":", 2)[0]
|
|
}
|
|
if entry.Title == "" {
|
|
entry.Title = extractHTMLTitle("", pageID)
|
|
}
|
|
next.Pages[pageID] = entry
|
|
result.Stale++
|
|
policy := strings.TrimSpace(opts.StalePolicy)
|
|
if policy == "" {
|
|
policy = "report"
|
|
}
|
|
if policy == "archive" {
|
|
body := renderArchivedWikiPage(pageID, entry.Title)
|
|
entry.Hash = computeManagedHash(body)
|
|
entry.ArchivedHash = entry.Hash
|
|
next.Pages[pageID] = entry
|
|
result.Archived++
|
|
plans = append(plans, wikiDeployPlan{
|
|
Page: wikiDeployPage{PageID: pageID, Title: entry.Title, Namespace: entry.Namespace, Content: body, Hash: entry.Hash},
|
|
Entry: entry,
|
|
Action: "archive",
|
|
Content: body,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
return plans, result, next, nil
|
|
}
|
|
|
|
func loadDeployManifest(path string) wikiDeployManifest {
|
|
doc := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return doc
|
|
}
|
|
if err := json.Unmarshal(raw, &doc); err != nil {
|
|
return wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
|
|
}
|
|
if doc.Pages == nil {
|
|
doc.Pages = map[string]wikiDeployManifestPage{}
|
|
}
|
|
return doc
|
|
}
|
|
|
|
func saveDeployManifest(path string, manifest wikiDeployManifest) error {
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
return err
|
|
}
|
|
raw, err := json.MarshalIndent(manifest, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, append(raw, '\n'), 0o644)
|
|
}
|
|
|
|
func computeManagedHash(content string) string {
|
|
managed, ok := extractManagedRegion(content)
|
|
if !ok {
|
|
managed = content
|
|
}
|
|
h := sha256.Sum256([]byte(strings.TrimSpace(managed)))
|
|
return fmt.Sprintf("%x", h[:])
|
|
}
|
|
|
|
func extractManagedRegion(content string) (string, bool) {
|
|
startMarker, _, start, end := managedRegionBounds(content)
|
|
if start < 0 || end < 0 || end < start {
|
|
return "", false
|
|
}
|
|
start += len(startMarker)
|
|
return strings.TrimSpace(content[start:end]), true
|
|
}
|
|
|
|
func mergeManagedContent(existing, generated string) string {
|
|
generatedRegion, ok := extractManagedRegion(generated)
|
|
if !ok {
|
|
return generated
|
|
}
|
|
startMarker, endMarker, start, end := managedRegionBounds(existing)
|
|
if start < 0 || end < 0 || end < start {
|
|
return generated
|
|
}
|
|
if strings.Contains(generated, manualStartMarkerPrefix) {
|
|
return mergeManualRegions(generated, existing)
|
|
}
|
|
end += len(endMarker)
|
|
generatedStart, generatedEnd, ok := extractManagedMarkers(generated)
|
|
if !ok {
|
|
generatedStart = startMarker
|
|
generatedEnd = endMarker
|
|
}
|
|
merged := existing[:start] + generatedStart + "\n" + generatedRegion + "\n" + generatedEnd + existing[end:]
|
|
return mergeManualRegions(merged, existing)
|
|
}
|
|
|
|
func managedRegionBounds(content string) (string, string, int, int) {
|
|
if start := strings.Index(content, managedStartMarker); start >= 0 {
|
|
startEnd := strings.Index(content[start:], "-->")
|
|
end := strings.Index(content, managedEndMarker)
|
|
if startEnd >= 0 && end >= 0 {
|
|
startMarker := content[start : start+startEnd+len("-->")]
|
|
return startMarker, managedEndMarker, start, end
|
|
}
|
|
}
|
|
for _, markers := range [][2]string{
|
|
{legacyMarkdownStartMarker, legacyMarkdownEndMarker},
|
|
{legacyManagedStartMarker, legacyManagedEndMarker},
|
|
} {
|
|
start := strings.Index(content, markers[0])
|
|
end := strings.Index(content, markers[1])
|
|
if start >= 0 && end >= 0 {
|
|
return markers[0], markers[1], start, end
|
|
}
|
|
}
|
|
return "", "", -1, -1
|
|
}
|
|
|
|
func extractManagedMarkers(content string) (string, string, bool) {
|
|
startMarker, endMarker, start, end := managedRegionBounds(content)
|
|
return startMarker, endMarker, start >= 0 && end >= 0
|
|
}
|
|
|
|
func mergeManualRegions(merged, existing string) string {
|
|
existingManual := extractManualRegions(existing)
|
|
if len(existingManual) == 0 {
|
|
return merged
|
|
}
|
|
return replaceManualRegions(merged, existingManual)
|
|
}
|
|
|
|
func extractManualRegions(content string) map[string]string {
|
|
regions := map[string]string{}
|
|
offset := 0
|
|
for {
|
|
startRel := strings.Index(content[offset:], manualStartMarkerPrefix)
|
|
if startRel < 0 {
|
|
break
|
|
}
|
|
start := offset + startRel
|
|
startEndRel := strings.Index(content[start:], "-->")
|
|
if startEndRel < 0 {
|
|
break
|
|
}
|
|
startMarker := content[start : start+startEndRel+len("-->")]
|
|
id := markerID(startMarker)
|
|
if id == "" {
|
|
offset = start + len(startMarker)
|
|
continue
|
|
}
|
|
endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\""
|
|
end := strings.Index(content[start+len(startMarker):], endPrefix)
|
|
if end < 0 {
|
|
offset = start + len(startMarker)
|
|
continue
|
|
}
|
|
end += start + len(startMarker)
|
|
endEndRel := strings.Index(content[end:], "-->")
|
|
if endEndRel < 0 {
|
|
break
|
|
}
|
|
endMarker := content[end : end+endEndRel+len("-->")]
|
|
regions[id] = startMarker + strings.TrimRight(content[start+len(startMarker):end], "\n") + "\n" + endMarker
|
|
offset = end + len(endMarker)
|
|
}
|
|
return regions
|
|
}
|
|
|
|
func replaceManualRegions(content string, replacements map[string]string) string {
|
|
for id, replacement := range replacements {
|
|
start := strings.Index(content, manualStartMarkerPrefix+" id=\""+id+"\"")
|
|
if start < 0 {
|
|
continue
|
|
}
|
|
endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\""
|
|
end := strings.Index(content[start:], endPrefix)
|
|
if end < 0 {
|
|
continue
|
|
}
|
|
end += start
|
|
endEndRel := strings.Index(content[end:], "-->")
|
|
if endEndRel < 0 {
|
|
continue
|
|
}
|
|
endEnd := end + endEndRel + len("-->")
|
|
content = content[:start] + replacement + content[endEnd:]
|
|
}
|
|
return content
|
|
}
|
|
|
|
func markerID(marker string) string {
|
|
const prefix = `id="`
|
|
start := strings.Index(marker, prefix)
|
|
if start < 0 {
|
|
return ""
|
|
}
|
|
start += len(prefix)
|
|
end := strings.Index(marker[start:], `"`)
|
|
if end < 0 {
|
|
return ""
|
|
}
|
|
return marker[start : start+end]
|
|
}
|
|
|
|
func renderArchivedWikiPage(pageID, title string) string {
|
|
if title == "" {
|
|
title = extractHTMLTitle("", pageID)
|
|
}
|
|
body := strings.Join([]string{
|
|
"<h1>" + html.EscapeString(title) + "</h1>",
|
|
"<p>This generated page is no longer present in the current Shadows Over Westgate topdata wiki source.</p>",
|
|
}, "\n")
|
|
hash := computeManagedHash(body)
|
|
return ensureTrailingNewline(strings.Join([]string{
|
|
"<!-- sow-topdata-wiki:page=" + pageID + " -->",
|
|
"<!-- sow-topdata-wiki:managed:start hash=\"sha256:" + hash + "\" -->",
|
|
body,
|
|
"<!-- sow-topdata-wiki:managed:end -->",
|
|
}, "\n"))
|
|
}
|
|
|
|
func extractPageTitle(content, fallback string) string {
|
|
if title := extractMarkdownTitle(content); title != "" {
|
|
return title
|
|
}
|
|
return extractHTMLTitle(content, fallback)
|
|
}
|
|
|
|
func extractMarkdownTitle(content string) string {
|
|
for _, line := range strings.Split(content, "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if !strings.HasPrefix(trimmed, "# ") {
|
|
continue
|
|
}
|
|
title := strings.TrimSpace(strings.TrimPrefix(trimmed, "# "))
|
|
if title != "" {
|
|
return title
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func extractHTMLTitle(content, fallback string) string {
|
|
lower := strings.ToLower(content)
|
|
start := strings.Index(lower, "<h1>")
|
|
end := strings.Index(lower, "</h1>")
|
|
if start >= 0 && end > start {
|
|
title := strings.TrimSpace(content[start+len("<h1>") : end])
|
|
title = strings.ReplaceAll(title, "\n", " ")
|
|
if title != "" {
|
|
return html.UnescapeString(stripSimpleTags(title))
|
|
}
|
|
}
|
|
parts := strings.Split(fallback, ":")
|
|
return namespaceTitle(parts[len(parts)-1])
|
|
}
|
|
|
|
func nodeBBTopicTitle(page wikiDeployPage) string {
|
|
title := strings.TrimSpace(page.Title)
|
|
if len([]rune(title)) >= 5 {
|
|
return title
|
|
}
|
|
namespace := strings.TrimSpace(namespaceTitle(page.Namespace))
|
|
if namespace == "" {
|
|
namespace = "Wiki"
|
|
}
|
|
if title == "" {
|
|
parts := strings.Split(page.PageID, ":")
|
|
title = namespaceTitle(parts[len(parts)-1])
|
|
}
|
|
return namespace + ": " + title
|
|
}
|
|
|
|
func stripSimpleTags(text string) string {
|
|
var out strings.Builder
|
|
inTag := false
|
|
for _, r := range text {
|
|
switch r {
|
|
case '<':
|
|
inTag = true
|
|
case '>':
|
|
inTag = false
|
|
default:
|
|
if !inTag {
|
|
out.WriteRune(r)
|
|
}
|
|
}
|
|
}
|
|
return out.String()
|
|
}
|
|
|
|
func sortedKeysFromMap[V any](m map[string]V) []string {
|
|
keys := make([]string, 0, len(m))
|
|
for key := range m {
|
|
keys = append(keys, key)
|
|
}
|
|
slices.Sort(keys)
|
|
return keys
|
|
}
|
|
|
|
type nodeBBClient struct {
|
|
endpoint string
|
|
token string
|
|
timeout time.Duration
|
|
}
|
|
|
|
type nodeBBPost struct {
|
|
PID int
|
|
TID int
|
|
Content string
|
|
}
|
|
|
|
func newNodeBBClient(endpoint, token string) *nodeBBClient {
|
|
return &nodeBBClient{
|
|
endpoint: strings.TrimRight(endpoint, "/"),
|
|
token: token,
|
|
timeout: 30 * time.Second,
|
|
}
|
|
}
|
|
|
|
func (c *nodeBBClient) getPost(pid int) (nodeBBPost, error) {
|
|
var doc any
|
|
if err := c.request("GET", fmt.Sprintf("/api/v3/posts/%d", pid), nil, &doc); err != nil {
|
|
return nodeBBPost{}, err
|
|
}
|
|
return parseNodeBBPost(doc), nil
|
|
}
|
|
|
|
func (c *nodeBBClient) createTopic(cid int, title, content, summary string) (nodeBBPost, error) {
|
|
body := map[string]any{"cid": cid, "title": title, "content": content}
|
|
if summary != "" {
|
|
body["_uid_note"] = summary
|
|
}
|
|
var doc any
|
|
if err := c.request("POST", "/api/v3/topics", body, &doc); err != nil {
|
|
return nodeBBPost{}, err
|
|
}
|
|
post := parseNodeBBPost(doc)
|
|
if post.TID == 0 || post.PID == 0 {
|
|
return nodeBBPost{}, fmt.Errorf("NodeBB topic create response did not include tid and pid")
|
|
}
|
|
return post, nil
|
|
}
|
|
|
|
func (c *nodeBBClient) updatePost(pid int, content, summary string) error {
|
|
body := map[string]any{"content": content}
|
|
if summary != "" {
|
|
body["_uid_note"] = summary
|
|
}
|
|
return c.request("PUT", fmt.Sprintf("/api/v3/posts/%d", pid), body, nil)
|
|
}
|
|
|
|
func (c *nodeBBClient) request(method, path string, payload any, out any) error {
|
|
var body io.Reader
|
|
if payload != nil {
|
|
raw, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
body = bytes.NewReader(raw)
|
|
}
|
|
req, err := http.NewRequest(method, c.endpoint+path, body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+c.token)
|
|
req.Header.Set("Accept", "application/json")
|
|
if payload != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
client := http.Client{Timeout: c.timeout}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
raw, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if resp.StatusCode >= 400 {
|
|
return fmt.Errorf("NodeBB %s %s failed: HTTP %d: %s", method, path, resp.StatusCode, strings.TrimSpace(string(raw)))
|
|
}
|
|
if out == nil || len(bytes.TrimSpace(raw)) == 0 {
|
|
return nil
|
|
}
|
|
if err := json.Unmarshal(raw, out); err != nil {
|
|
return fmt.Errorf("parse NodeBB response: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func parseNodeBBPost(doc any) nodeBBPost {
|
|
for {
|
|
m, ok := doc.(map[string]any)
|
|
if !ok {
|
|
return nodeBBPost{}
|
|
}
|
|
if response, ok := m["response"]; ok {
|
|
doc = response
|
|
continue
|
|
}
|
|
if postData, ok := m["postData"]; ok {
|
|
doc = postData
|
|
continue
|
|
}
|
|
if mainPost, ok := m["mainPost"]; ok {
|
|
post := parseNodeBBPost(mainPost)
|
|
if post.TID == 0 {
|
|
post.TID = intFromAny(m["tid"])
|
|
}
|
|
return post
|
|
}
|
|
return nodeBBPost{
|
|
PID: firstInt(m, "pid", "mainPid"),
|
|
TID: firstInt(m, "tid"),
|
|
Content: firstString(m, "content"),
|
|
}
|
|
}
|
|
}
|
|
|
|
func firstInt(m map[string]any, keys ...string) int {
|
|
for _, key := range keys {
|
|
if value := intFromAny(m[key]); value != 0 {
|
|
return value
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func intFromAny(value any) int {
|
|
switch typed := value.(type) {
|
|
case int:
|
|
return typed
|
|
case float64:
|
|
return int(typed)
|
|
case json.Number:
|
|
i, _ := typed.Int64()
|
|
return int(i)
|
|
case string:
|
|
i, _ := strconv.Atoi(typed)
|
|
return i
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
func firstString(m map[string]any, keys ...string) string {
|
|
for _, key := range keys {
|
|
if value, ok := m[key].(string); ok {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|