585 lines
15 KiB
Go
585 lines
15 KiB
Go
package topdata
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
|
)
|
|
|
|
const (
|
|
defaultDeployManifestName = ".wiki_deploy_manifest.json"
|
|
defaultEditSummary = "Auto-generated from native builder"
|
|
managedStartMarker = "[//]: # (sow-topdata-wiki:managed:start)"
|
|
managedEndMarker = "[//]: # (sow-topdata-wiki:managed:end)"
|
|
legacyManagedStartMarker = "<!-- sow-topdata-wiki:managed:start -->"
|
|
legacyManagedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
|
|
)
|
|
|
|
var defaultManagedNamespaces = []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}
|
|
|
|
type DeployWikiOptions struct {
|
|
SourceDir string
|
|
Endpoint string
|
|
Token string
|
|
Version string
|
|
Namespaces []string
|
|
CategoryIDs map[string]int
|
|
ManifestPath string
|
|
DryRun bool
|
|
Force bool
|
|
AllowCreates bool
|
|
StalePolicy string
|
|
Username string
|
|
Password string
|
|
NotesDelimiter string
|
|
}
|
|
|
|
type deployResult struct {
|
|
LocalPages int
|
|
Created int
|
|
Updated int
|
|
Stale int
|
|
Skipped int
|
|
Drifted int
|
|
Manifest string
|
|
}
|
|
|
|
type wikiDeployPage struct {
|
|
PageID string
|
|
Title string
|
|
Namespace string
|
|
Content string
|
|
Hash string
|
|
}
|
|
|
|
type wikiDeployManifest struct {
|
|
Version string `json:"version"`
|
|
Pages map[string]wikiDeployManifestPage `json:"pages"`
|
|
}
|
|
|
|
type wikiDeployManifestPage struct {
|
|
Hash string `json:"hash"`
|
|
TID int `json:"tid,omitempty"`
|
|
PID int `json:"pid,omitempty"`
|
|
CID int `json:"cid,omitempty"`
|
|
}
|
|
|
|
type wikiDeployPlan struct {
|
|
Page wikiDeployPage
|
|
Entry wikiDeployManifestPage
|
|
Action string
|
|
Content string
|
|
RemoteHash string
|
|
}
|
|
|
|
func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress func(string)) (deployResult, error) {
|
|
if progress == nil {
|
|
progress = func(string) {}
|
|
}
|
|
if opts.SourceDir == "" {
|
|
opts.SourceDir = wikiOutputPagesDir(p)
|
|
}
|
|
if _, err := os.Stat(opts.SourceDir); err != nil {
|
|
return deployResult{}, fmt.Errorf("wiki source directory not found: %w", err)
|
|
}
|
|
if opts.Endpoint == "" {
|
|
return deployResult{}, errors.New("NODEBB_API_ENDPOINT is required")
|
|
}
|
|
if opts.Token == "" {
|
|
return deployResult{}, errors.New("NODEBB_API_TOKEN is required")
|
|
}
|
|
if opts.Username != "" || opts.Password != "" || opts.NotesDelimiter != "" {
|
|
return deployResult{}, errors.New("DokuWiki deployment options are no longer supported; use NodeBB endpoint, token, and category mappings")
|
|
}
|
|
|
|
namespaces := opts.Namespaces
|
|
if len(namespaces) == 0 {
|
|
namespaces = defaultManagedNamespaces
|
|
}
|
|
|
|
manifestPath := opts.ManifestPath
|
|
if manifestPath == "" {
|
|
manifestPath = filepath.Join(filepath.Dir(opts.SourceDir), defaultDeployManifestName)
|
|
}
|
|
|
|
pages, err := collectLocalPages(opts.SourceDir, namespaces)
|
|
if err != nil {
|
|
return deployResult{}, err
|
|
}
|
|
manifest := loadDeployManifest(manifestPath)
|
|
client := newNodeBBClient(opts.Endpoint, opts.Token)
|
|
|
|
plans, result, nextManifest, err := planNodeBBDeploy(pages, manifest, opts, client)
|
|
if err != nil {
|
|
return deployResult{}, err
|
|
}
|
|
result.LocalPages = len(pages)
|
|
result.Manifest = manifestPath
|
|
|
|
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, skip %d, drift %d", result.Created, result.Updated, result.Skipped, result.Drifted))
|
|
if opts.DryRun {
|
|
return result, nil
|
|
}
|
|
if result.Drifted > 0 && !opts.Force {
|
|
return result, errors.New("remote managed wiki content drifted; rerun with --force to overwrite")
|
|
}
|
|
|
|
summary := defaultEditSummary
|
|
if opts.Version != "" {
|
|
summary = fmt.Sprintf("%s (%s)", summary, opts.Version)
|
|
}
|
|
for _, plan := range plans {
|
|
switch plan.Action {
|
|
case "create":
|
|
created, err := client.createTopic(plan.Entry.CID, plan.Page.Title, plan.Content, summary)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
entry := nextManifest.Pages[plan.Page.PageID]
|
|
entry.TID = created.TID
|
|
entry.PID = created.PID
|
|
nextManifest.Pages[plan.Page.PageID] = entry
|
|
case "update":
|
|
if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil {
|
|
return result, err
|
|
}
|
|
}
|
|
}
|
|
if err := saveDeployManifest(manifestPath, nextManifest); err != nil {
|
|
return result, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func collectLocalPages(sourceDir string, namespaces []string) (map[string]wikiDeployPage, error) {
|
|
namespaceSet := map[string]struct{}{}
|
|
for _, namespace := range namespaces {
|
|
namespaceSet[namespace] = struct{}{}
|
|
}
|
|
pages := map[string]wikiDeployPage{}
|
|
for _, ns := range namespaces {
|
|
nsDir := filepath.Join(sourceDir, ns)
|
|
if !isDir(nsDir) {
|
|
continue
|
|
}
|
|
if err := filepath.WalkDir(nsDir, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ext := filepath.Ext(path)
|
|
if d.IsDir() || (ext != ".md" && ext != ".html") {
|
|
return nil
|
|
}
|
|
rel, err := filepath.Rel(sourceDir, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pageID := strings.TrimSuffix(filepath.ToSlash(rel), ext)
|
|
pageID = strings.ReplaceAll(pageID, "/", ":")
|
|
namespace := strings.SplitN(pageID, ":", 2)[0]
|
|
if _, ok := namespaceSet[namespace]; !ok {
|
|
return nil
|
|
}
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
content := string(raw)
|
|
pages[pageID] = wikiDeployPage{
|
|
PageID: pageID,
|
|
Title: extractPageTitle(content, pageID),
|
|
Namespace: namespace,
|
|
Content: content,
|
|
Hash: computeManagedHash(content),
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return pages, nil
|
|
}
|
|
|
|
func isDir(path string) bool {
|
|
info, err := os.Stat(path)
|
|
return err == nil && info.IsDir()
|
|
}
|
|
|
|
func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) {
|
|
pageIDs := sortedKeysFromMap(pages)
|
|
next := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
|
|
var plans []wikiDeployPlan
|
|
var result deployResult
|
|
|
|
for _, pageID := range pageIDs {
|
|
page := pages[pageID]
|
|
entry := manifest.Pages[pageID]
|
|
entry.Hash = page.Hash
|
|
if entry.CID == 0 {
|
|
entry.CID = opts.CategoryIDs[page.Namespace]
|
|
}
|
|
next.Pages[pageID] = entry
|
|
|
|
if entry.PID == 0 {
|
|
if !opts.AllowCreates {
|
|
return nil, result, next, fmt.Errorf("wiki page %q has no manifest post mapping; pass --create and --category %s=<cid> to create it", pageID, page.Namespace)
|
|
}
|
|
if entry.CID == 0 {
|
|
return nil, result, next, fmt.Errorf("wiki page %q requires --category %s=<cid> for creation", pageID, page.Namespace)
|
|
}
|
|
result.Created++
|
|
page.Title = nodeBBTopicTitle(page)
|
|
plans = append(plans, wikiDeployPlan{Page: page, Entry: entry, Action: "create", Content: page.Content})
|
|
continue
|
|
}
|
|
|
|
if manifest.Pages[pageID].Hash == page.Hash {
|
|
result.Skipped++
|
|
continue
|
|
}
|
|
remote, err := client.getPost(entry.PID)
|
|
if err != nil {
|
|
return nil, result, next, err
|
|
}
|
|
remoteHash := computeManagedHash(remote.Content)
|
|
if manifest.Pages[pageID].Hash != "" && remoteHash != manifest.Pages[pageID].Hash {
|
|
result.Drifted++
|
|
if !opts.Force {
|
|
continue
|
|
}
|
|
}
|
|
merged := mergeManagedContent(remote.Content, page.Content)
|
|
if merged == remote.Content {
|
|
result.Skipped++
|
|
continue
|
|
}
|
|
result.Updated++
|
|
plans = append(plans, wikiDeployPlan{Page: page, Entry: entry, Action: "update", Content: merged, RemoteHash: remoteHash})
|
|
}
|
|
for pageID := range manifest.Pages {
|
|
if _, ok := pages[pageID]; !ok {
|
|
result.Stale++
|
|
}
|
|
}
|
|
return plans, result, next, nil
|
|
}
|
|
|
|
func loadDeployManifest(path string) wikiDeployManifest {
|
|
doc := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return doc
|
|
}
|
|
if err := json.Unmarshal(raw, &doc); err != nil {
|
|
return wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
|
|
}
|
|
if doc.Pages == nil {
|
|
doc.Pages = map[string]wikiDeployManifestPage{}
|
|
}
|
|
return doc
|
|
}
|
|
|
|
func saveDeployManifest(path string, manifest wikiDeployManifest) error {
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
return err
|
|
}
|
|
raw, err := json.MarshalIndent(manifest, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, append(raw, '\n'), 0o644)
|
|
}
|
|
|
|
func computeManagedHash(content string) string {
|
|
managed, ok := extractManagedRegion(content)
|
|
if !ok {
|
|
managed = content
|
|
}
|
|
h := sha256.Sum256([]byte(strings.TrimSpace(managed)))
|
|
return fmt.Sprintf("%x", h[:])
|
|
}
|
|
|
|
func extractManagedRegion(content string) (string, bool) {
|
|
startMarker, _, start, end := managedRegionBounds(content)
|
|
if start < 0 || end < 0 || end < start {
|
|
return "", false
|
|
}
|
|
start += len(startMarker)
|
|
return strings.TrimSpace(content[start:end]), true
|
|
}
|
|
|
|
func mergeManagedContent(existing, generated string) string {
|
|
generatedRegion, ok := extractManagedRegion(generated)
|
|
if !ok {
|
|
return generated
|
|
}
|
|
_, endMarker, start, end := managedRegionBounds(existing)
|
|
if start < 0 || end < 0 || end < start {
|
|
return generated
|
|
}
|
|
end += len(endMarker)
|
|
return existing[:start] + managedStartMarker + "\n" + generatedRegion + "\n" + managedEndMarker + existing[end:]
|
|
}
|
|
|
|
func managedRegionBounds(content string) (string, string, int, int) {
|
|
for _, markers := range [][2]string{
|
|
{managedStartMarker, managedEndMarker},
|
|
{legacyManagedStartMarker, legacyManagedEndMarker},
|
|
} {
|
|
start := strings.Index(content, markers[0])
|
|
end := strings.Index(content, markers[1])
|
|
if start >= 0 && end >= 0 {
|
|
return markers[0], markers[1], start, end
|
|
}
|
|
}
|
|
return "", "", -1, -1
|
|
}
|
|
|
|
func extractPageTitle(content, fallback string) string {
|
|
if title := extractMarkdownTitle(content); title != "" {
|
|
return title
|
|
}
|
|
return extractHTMLTitle(content, fallback)
|
|
}
|
|
|
|
func extractMarkdownTitle(content string) string {
|
|
for _, line := range strings.Split(content, "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if !strings.HasPrefix(trimmed, "# ") {
|
|
continue
|
|
}
|
|
title := strings.TrimSpace(strings.TrimPrefix(trimmed, "# "))
|
|
if title != "" {
|
|
return title
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func extractHTMLTitle(content, fallback string) string {
|
|
lower := strings.ToLower(content)
|
|
start := strings.Index(lower, "<h1>")
|
|
end := strings.Index(lower, "</h1>")
|
|
if start >= 0 && end > start {
|
|
title := strings.TrimSpace(content[start+len("<h1>") : end])
|
|
title = strings.ReplaceAll(title, "\n", " ")
|
|
if title != "" {
|
|
return html.UnescapeString(stripSimpleTags(title))
|
|
}
|
|
}
|
|
parts := strings.Split(fallback, ":")
|
|
return namespaceTitle(parts[len(parts)-1])
|
|
}
|
|
|
|
func nodeBBTopicTitle(page wikiDeployPage) string {
|
|
title := strings.TrimSpace(page.Title)
|
|
if len([]rune(title)) >= 5 {
|
|
return title
|
|
}
|
|
namespace := strings.TrimSpace(namespaceTitle(page.Namespace))
|
|
if namespace == "" {
|
|
namespace = "Wiki"
|
|
}
|
|
if title == "" {
|
|
parts := strings.Split(page.PageID, ":")
|
|
title = namespaceTitle(parts[len(parts)-1])
|
|
}
|
|
return namespace + ": " + title
|
|
}
|
|
|
|
func stripSimpleTags(text string) string {
|
|
var out strings.Builder
|
|
inTag := false
|
|
for _, r := range text {
|
|
switch r {
|
|
case '<':
|
|
inTag = true
|
|
case '>':
|
|
inTag = false
|
|
default:
|
|
if !inTag {
|
|
out.WriteRune(r)
|
|
}
|
|
}
|
|
}
|
|
return out.String()
|
|
}
|
|
|
|
func sortedKeysFromMap[V any](m map[string]V) []string {
|
|
keys := make([]string, 0, len(m))
|
|
for key := range m {
|
|
keys = append(keys, key)
|
|
}
|
|
slices.Sort(keys)
|
|
return keys
|
|
}
|
|
|
|
type nodeBBClient struct {
|
|
endpoint string
|
|
token string
|
|
timeout time.Duration
|
|
}
|
|
|
|
type nodeBBPost struct {
|
|
PID int
|
|
TID int
|
|
Content string
|
|
}
|
|
|
|
func newNodeBBClient(endpoint, token string) *nodeBBClient {
|
|
return &nodeBBClient{
|
|
endpoint: strings.TrimRight(endpoint, "/"),
|
|
token: token,
|
|
timeout: 30 * time.Second,
|
|
}
|
|
}
|
|
|
|
func (c *nodeBBClient) getPost(pid int) (nodeBBPost, error) {
|
|
var doc any
|
|
if err := c.request("GET", fmt.Sprintf("/api/v3/posts/%d", pid), nil, &doc); err != nil {
|
|
return nodeBBPost{}, err
|
|
}
|
|
return parseNodeBBPost(doc), nil
|
|
}
|
|
|
|
func (c *nodeBBClient) createTopic(cid int, title, content, summary string) (nodeBBPost, error) {
|
|
body := map[string]any{"cid": cid, "title": title, "content": content}
|
|
if summary != "" {
|
|
body["_uid_note"] = summary
|
|
}
|
|
var doc any
|
|
if err := c.request("POST", "/api/v3/topics", body, &doc); err != nil {
|
|
return nodeBBPost{}, err
|
|
}
|
|
post := parseNodeBBPost(doc)
|
|
if post.TID == 0 || post.PID == 0 {
|
|
return nodeBBPost{}, fmt.Errorf("NodeBB topic create response did not include tid and pid")
|
|
}
|
|
return post, nil
|
|
}
|
|
|
|
func (c *nodeBBClient) updatePost(pid int, content, summary string) error {
|
|
body := map[string]any{"content": content}
|
|
if summary != "" {
|
|
body["_uid_note"] = summary
|
|
}
|
|
return c.request("PUT", fmt.Sprintf("/api/v3/posts/%d", pid), body, nil)
|
|
}
|
|
|
|
func (c *nodeBBClient) request(method, path string, payload any, out any) error {
|
|
var body io.Reader
|
|
if payload != nil {
|
|
raw, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
body = bytes.NewReader(raw)
|
|
}
|
|
req, err := http.NewRequest(method, c.endpoint+path, body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+c.token)
|
|
req.Header.Set("Accept", "application/json")
|
|
if payload != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
client := http.Client{Timeout: c.timeout}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
raw, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if resp.StatusCode >= 400 {
|
|
return fmt.Errorf("NodeBB %s %s failed: HTTP %d: %s", method, path, resp.StatusCode, strings.TrimSpace(string(raw)))
|
|
}
|
|
if out == nil || len(bytes.TrimSpace(raw)) == 0 {
|
|
return nil
|
|
}
|
|
if err := json.Unmarshal(raw, out); err != nil {
|
|
return fmt.Errorf("parse NodeBB response: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func parseNodeBBPost(doc any) nodeBBPost {
|
|
for {
|
|
m, ok := doc.(map[string]any)
|
|
if !ok {
|
|
return nodeBBPost{}
|
|
}
|
|
if response, ok := m["response"]; ok {
|
|
doc = response
|
|
continue
|
|
}
|
|
if postData, ok := m["postData"]; ok {
|
|
doc = postData
|
|
continue
|
|
}
|
|
if mainPost, ok := m["mainPost"]; ok {
|
|
post := parseNodeBBPost(mainPost)
|
|
if post.TID == 0 {
|
|
post.TID = intFromAny(m["tid"])
|
|
}
|
|
return post
|
|
}
|
|
return nodeBBPost{
|
|
PID: firstInt(m, "pid", "mainPid"),
|
|
TID: firstInt(m, "tid"),
|
|
Content: firstString(m, "content"),
|
|
}
|
|
}
|
|
}
|
|
|
|
func firstInt(m map[string]any, keys ...string) int {
|
|
for _, key := range keys {
|
|
if value := intFromAny(m[key]); value != 0 {
|
|
return value
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func intFromAny(value any) int {
|
|
switch typed := value.(type) {
|
|
case int:
|
|
return typed
|
|
case float64:
|
|
return int(typed)
|
|
case json.Number:
|
|
i, _ := typed.Int64()
|
|
return int(i)
|
|
case string:
|
|
i, _ := strconv.Atoi(typed)
|
|
return i
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
func firstString(m map[string]any, keys ...string) string {
|
|
for _, key := range keys {
|
|
if value, ok := m[key].(string); ok {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|