Wiki Deploy Code
This commit is contained in:
+82
-9
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -139,7 +140,7 @@ var commands = []command{
|
||||
},
|
||||
{
|
||||
name: "deploy-wiki",
|
||||
description: "Deploy wiki pages to DokuWiki.",
|
||||
description: "Deploy generated wiki pages to NodeBB.",
|
||||
run: runDeployWiki,
|
||||
},
|
||||
}
|
||||
@@ -674,9 +675,11 @@ func runDeployWiki(ctx context) error {
|
||||
|
||||
fmt.Fprintf(ctx.stdout, "project: %s\n", p.Config.Module.Name)
|
||||
fmt.Fprintf(ctx.stdout, "local pages: %d\n", result.LocalPages)
|
||||
fmt.Fprintf(ctx.stdout, "created: %d\n", result.Created)
|
||||
fmt.Fprintf(ctx.stdout, "updated: %d\n", result.Updated)
|
||||
fmt.Fprintf(ctx.stdout, "skipped: %d\n", result.Skipped)
|
||||
fmt.Fprintf(ctx.stdout, "deleted: %d\n", result.Deleted)
|
||||
fmt.Fprintf(ctx.stdout, "stale: %d\n", result.Stale)
|
||||
fmt.Fprintf(ctx.stdout, "drifted: %d\n", result.Drifted)
|
||||
fmt.Fprintf(ctx.stdout, "manifest: %s\n", result.Manifest)
|
||||
return nil
|
||||
}
|
||||
@@ -710,6 +713,12 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO
|
||||
return opts, fmt.Errorf("--password requires a value")
|
||||
}
|
||||
opts.Password = args[i]
|
||||
case "--token":
|
||||
i++
|
||||
if i >= len(args) {
|
||||
return opts, fmt.Errorf("--token requires a value")
|
||||
}
|
||||
opts.Token = args[i]
|
||||
case "--version":
|
||||
i++
|
||||
if i >= len(args) {
|
||||
@@ -722,6 +731,17 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO
|
||||
return opts, fmt.Errorf("--namespace requires a value")
|
||||
}
|
||||
opts.Namespaces = append(opts.Namespaces, args[i])
|
||||
case "--category":
|
||||
i++
|
||||
if i >= len(args) {
|
||||
return opts, fmt.Errorf("--category requires a namespace=cid value")
|
||||
}
|
||||
if opts.CategoryIDs == nil {
|
||||
opts.CategoryIDs = map[string]int{}
|
||||
}
|
||||
if err := parseWikiCategoryArg(args[i], opts.CategoryIDs); err != nil {
|
||||
return opts, err
|
||||
}
|
||||
case "--manifest":
|
||||
i++
|
||||
if i >= len(args) {
|
||||
@@ -730,8 +750,12 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO
|
||||
opts.ManifestPath = args[i]
|
||||
case "--dry-run":
|
||||
opts.DryRun = true
|
||||
case "--force":
|
||||
opts.Force = true
|
||||
case "--create":
|
||||
opts.AllowCreates = true
|
||||
case "-h", "--help":
|
||||
return opts, fmt.Errorf("usage: %s [--source-dir <path>] [--endpoint <url>] [--username <user>] [--password <pass>] [--version <ver>] [--namespace <ns> ...] [--manifest <path>] [--dry-run]", commandName)
|
||||
return opts, fmt.Errorf("usage: %s [--source-dir <path>] [--endpoint <url>] [--token <token>] [--version <ver>] [--namespace <ns> ...] [--category <namespace=cid> ...] [--manifest <path>] [--dry-run] [--create] [--force]", commandName)
|
||||
default:
|
||||
if strings.HasPrefix(arg, "--source-dir=") {
|
||||
opts.SourceDir = strings.TrimPrefix(arg, "--source-dir=")
|
||||
@@ -741,14 +765,27 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO
|
||||
opts.Username = strings.TrimPrefix(arg, "--username=")
|
||||
} else if strings.HasPrefix(arg, "--password=") {
|
||||
opts.Password = strings.TrimPrefix(arg, "--password=")
|
||||
} else if strings.HasPrefix(arg, "--token=") {
|
||||
opts.Token = strings.TrimPrefix(arg, "--token=")
|
||||
} else if strings.HasPrefix(arg, "--version=") {
|
||||
opts.Version = strings.TrimPrefix(arg, "--version=")
|
||||
} else if strings.HasPrefix(arg, "--namespace=") {
|
||||
opts.Namespaces = append(opts.Namespaces, strings.TrimPrefix(arg, "--namespace="))
|
||||
} else if strings.HasPrefix(arg, "--category=") {
|
||||
if opts.CategoryIDs == nil {
|
||||
opts.CategoryIDs = map[string]int{}
|
||||
}
|
||||
if err := parseWikiCategoryArg(strings.TrimPrefix(arg, "--category="), opts.CategoryIDs); err != nil {
|
||||
return opts, err
|
||||
}
|
||||
} else if strings.HasPrefix(arg, "--manifest=") {
|
||||
opts.ManifestPath = strings.TrimPrefix(arg, "--manifest=")
|
||||
} else if arg == "--dry-run" {
|
||||
opts.DryRun = true
|
||||
} else if arg == "--force" {
|
||||
opts.Force = true
|
||||
} else if arg == "--create" {
|
||||
opts.AllowCreates = true
|
||||
} else {
|
||||
return opts, fmt.Errorf("unknown %s argument %q", commandName, arg)
|
||||
}
|
||||
@@ -756,21 +793,57 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO
|
||||
}
|
||||
|
||||
if opts.Endpoint == "" {
|
||||
opts.Endpoint = os.Getenv("DOKUWIKI_RPC_ENDPOINT")
|
||||
opts.Endpoint = os.Getenv("NODEBB_API_ENDPOINT")
|
||||
}
|
||||
if opts.Username == "" {
|
||||
opts.Username = os.Getenv("DOKUWIKI_RPC_USERNAME")
|
||||
}
|
||||
if opts.Password == "" {
|
||||
opts.Password = os.Getenv("DOKUWIKI_RPC_PASSWORD")
|
||||
if opts.Token == "" {
|
||||
opts.Token = os.Getenv("NODEBB_API_TOKEN")
|
||||
}
|
||||
if opts.Version == "" {
|
||||
opts.Version = os.Getenv("GITHUB_REF_NAME")
|
||||
}
|
||||
if opts.CategoryIDs == nil {
|
||||
opts.CategoryIDs = map[string]int{}
|
||||
}
|
||||
if raw := os.Getenv("NODEBB_WIKI_CATEGORIES"); raw != "" {
|
||||
if err := parseWikiCategoryList(raw, opts.CategoryIDs); err != nil {
|
||||
return opts, err
|
||||
}
|
||||
}
|
||||
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
func parseWikiCategoryList(raw string, target map[string]int) error {
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
if err := parseWikiCategoryArg(part, target); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseWikiCategoryArg(raw string, target map[string]int) error {
|
||||
namespace, rawCID, ok := strings.Cut(raw, "=")
|
||||
if !ok {
|
||||
return fmt.Errorf("wiki category mapping %q must be namespace=cid", raw)
|
||||
}
|
||||
namespace = strings.TrimSpace(namespace)
|
||||
rawCID = strings.TrimSpace(rawCID)
|
||||
if namespace == "" || rawCID == "" {
|
||||
return fmt.Errorf("wiki category mapping %q must be namespace=cid", raw)
|
||||
}
|
||||
cid, err := strconv.Atoi(rawCID)
|
||||
if err != nil || cid <= 0 {
|
||||
return fmt.Errorf("wiki category mapping %q has invalid cid", raw)
|
||||
}
|
||||
target[namespace] = cid
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadProject(cwd string) (*project.Project, error) {
|
||||
root, err := project.FindRoot(cwd)
|
||||
if err != nil {
|
||||
|
||||
@@ -137,6 +137,7 @@ func Build(p *project.Project, progress func(string)) (BuildResult, error) {
|
||||
|
||||
type NativeBuildOptions struct {
|
||||
BuildWiki bool
|
||||
Force bool
|
||||
}
|
||||
|
||||
type BuildWikiOptions struct {
|
||||
@@ -166,7 +167,7 @@ func BuildWikiWithOptions(p *project.Project, opts BuildWikiOptions, progress fu
|
||||
OutputTLKDir: outputTLK,
|
||||
}
|
||||
|
||||
wikiBuild, err := buildWiki(p, nativeResult, progress)
|
||||
wikiBuild, err := buildWiki(p, nativeResult, opts.Force, progress)
|
||||
if err != nil {
|
||||
return wikiResult{}, err
|
||||
}
|
||||
@@ -378,7 +379,7 @@ func buildNativeUnchecked(p *project.Project, opts NativeBuildOptions, progress
|
||||
OutputTLKDir: outputTLK,
|
||||
Files2DA: files2DA,
|
||||
FilesTLK: filesTLK,
|
||||
}, progress)
|
||||
}, opts.Force, progress)
|
||||
if err != nil {
|
||||
return BuildResult{}, err
|
||||
}
|
||||
|
||||
@@ -105,7 +105,7 @@ func BuildAndPackageWithOptions(p *project.Project, opts BuildAndPackageOptions,
|
||||
}
|
||||
|
||||
func buildAndPackageFull(p *project.Project, opts BuildAndPackageOptions, progress func(string)) (PackageResult, error) {
|
||||
nativeResult, err := BuildNativeWithOptions(p, NativeBuildOptions{BuildWiki: opts.BuildWiki}, progress)
|
||||
nativeResult, err := BuildNativeWithOptions(p, NativeBuildOptions{BuildWiki: opts.BuildWiki, Force: opts.Force}, progress)
|
||||
if err != nil {
|
||||
return PackageResult{}, err
|
||||
}
|
||||
|
||||
+413
-346
@@ -1,16 +1,18 @@
|
||||
package topdata
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -20,61 +22,85 @@ import (
|
||||
const (
|
||||
defaultDeployManifestName = ".wiki_deploy_manifest.json"
|
||||
defaultEditSummary = "Auto-generated from native builder"
|
||||
defaultNotesDelimiter = "===== Notes ====="
|
||||
managedStartMarker = "<!-- sow-topdata-wiki:managed:start -->"
|
||||
managedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
|
||||
)
|
||||
|
||||
var defaultManagedNamespaces = []string{"classes", "feat", "itemtypes", "races", "skills", "spells"}
|
||||
var defaultManagedNamespaces = []string{"classes", "feat", "itemtypes", "races", "skills", "spells", "meta"}
|
||||
|
||||
type DeployWikiOptions struct {
|
||||
SourceDir string
|
||||
Endpoint string
|
||||
Username string
|
||||
Password string
|
||||
Token string
|
||||
Version string
|
||||
Namespaces []string
|
||||
NotesDelimiter string
|
||||
CategoryIDs map[string]int
|
||||
ManifestPath string
|
||||
DryRun bool
|
||||
Force bool
|
||||
AllowCreates bool
|
||||
StalePolicy string
|
||||
Username string
|
||||
Password string
|
||||
NotesDelimiter string
|
||||
}
|
||||
|
||||
type deployResult struct {
|
||||
LocalPages int
|
||||
Created int
|
||||
Updated int
|
||||
Deleted int
|
||||
Stale int
|
||||
Skipped int
|
||||
Drifted int
|
||||
Manifest string
|
||||
}
|
||||
|
||||
type deployPlan struct {
|
||||
PageID string
|
||||
Content string
|
||||
OldContent string
|
||||
type wikiDeployPage struct {
|
||||
PageID string
|
||||
Title string
|
||||
Namespace string
|
||||
Content string
|
||||
Hash string
|
||||
}
|
||||
|
||||
type manifestEntry struct {
|
||||
type wikiDeployManifest struct {
|
||||
Version string `json:"version"`
|
||||
Pages map[string]wikiDeployManifestPage `json:"pages"`
|
||||
}
|
||||
|
||||
type wikiDeployManifestPage struct {
|
||||
Hash string `json:"hash"`
|
||||
TID int `json:"tid,omitempty"`
|
||||
PID int `json:"pid,omitempty"`
|
||||
CID int `json:"cid,omitempty"`
|
||||
}
|
||||
|
||||
type wikiDeployPlan struct {
|
||||
Page wikiDeployPage
|
||||
Entry wikiDeployManifestPage
|
||||
Action string
|
||||
Content string
|
||||
RemoteHash string
|
||||
}
|
||||
|
||||
func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress func(string)) (deployResult, error) {
|
||||
if progress == nil {
|
||||
progress = func(string) {}
|
||||
}
|
||||
|
||||
if opts.SourceDir == "" {
|
||||
opts.SourceDir = wikiOutputPagesDir(p)
|
||||
}
|
||||
if _, err := os.Stat(opts.SourceDir); err != nil {
|
||||
return deployResult{}, fmt.Errorf("wiki source directory not found: %w", err)
|
||||
}
|
||||
|
||||
if opts.Endpoint == "" {
|
||||
return deployResult{}, errors.New("DOKUWIKI_RPC_ENDPOINT is required")
|
||||
return deployResult{}, errors.New("NODEBB_API_ENDPOINT is required")
|
||||
}
|
||||
if opts.Username == "" {
|
||||
return deployResult{}, errors.New("DOKUWIKI_RPC_USERNAME is required")
|
||||
if opts.Token == "" {
|
||||
return deployResult{}, errors.New("NODEBB_API_TOKEN is required")
|
||||
}
|
||||
if opts.Password == "" {
|
||||
return deployResult{}, errors.New("DOKUWIKI_RPC_PASSWORD is required")
|
||||
if opts.Username != "" || opts.Password != "" || opts.NotesDelimiter != "" {
|
||||
return deployResult{}, errors.New("DokuWiki deployment options are no longer supported; use NodeBB endpoint, token, and category mappings")
|
||||
}
|
||||
|
||||
namespaces := opts.Namespaces
|
||||
@@ -82,71 +108,66 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
|
||||
namespaces = defaultManagedNamespaces
|
||||
}
|
||||
|
||||
notesDelimiter := opts.NotesDelimiter
|
||||
if notesDelimiter == "" {
|
||||
notesDelimiter = defaultNotesDelimiter
|
||||
}
|
||||
|
||||
manifestPath := opts.ManifestPath
|
||||
if manifestPath == "" {
|
||||
manifestPath = filepath.Join(filepath.Dir(opts.SourceDir), defaultDeployManifestName)
|
||||
}
|
||||
|
||||
localPages, err := collectLocalPages(opts.SourceDir, namespaces)
|
||||
pages, err := collectLocalPages(opts.SourceDir, namespaces)
|
||||
if err != nil {
|
||||
return deployResult{}, err
|
||||
}
|
||||
|
||||
manifest := loadDeployManifest(manifestPath)
|
||||
client := newDokuWikiClient(opts.Endpoint, opts.Username, opts.Password)
|
||||
client := newNodeBBClient(opts.Endpoint, opts.Token)
|
||||
|
||||
deployments, stalePages, err := planDeploy(localPages, client, namespaces, notesDelimiter, manifest)
|
||||
plans, result, nextManifest, err := planNodeBBDeploy(pages, manifest, opts, client)
|
||||
if err != nil {
|
||||
return deployResult{}, err
|
||||
}
|
||||
result.LocalPages = len(pages)
|
||||
result.Manifest = manifestPath
|
||||
|
||||
progress(fmt.Sprintf("Deploying %d pages...", len(deployments)))
|
||||
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, skip %d, drift %d", result.Created, result.Updated, result.Skipped, result.Drifted))
|
||||
if opts.DryRun {
|
||||
return result, nil
|
||||
}
|
||||
if result.Drifted > 0 && !opts.Force {
|
||||
return result, errors.New("remote managed wiki content drifted; rerun with --force to overwrite")
|
||||
}
|
||||
|
||||
editSummary := defaultEditSummary
|
||||
summary := defaultEditSummary
|
||||
if opts.Version != "" {
|
||||
editSummary = fmt.Sprintf("%s (%s)", defaultEditSummary, opts.Version)
|
||||
summary = fmt.Sprintf("%s (%s)", summary, opts.Version)
|
||||
}
|
||||
|
||||
if !opts.DryRun {
|
||||
for _, plan := range deployments {
|
||||
if err := client.putPage(plan.PageID, plan.Content, editSummary); err != nil {
|
||||
return deployResult{}, err
|
||||
for _, plan := range plans {
|
||||
switch plan.Action {
|
||||
case "create":
|
||||
created, err := client.createTopic(plan.Entry.CID, plan.Page.Title, plan.Content, summary)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
entry := nextManifest.Pages[plan.Page.PageID]
|
||||
entry.TID = created.TID
|
||||
entry.PID = created.PID
|
||||
nextManifest.Pages[plan.Page.PageID] = entry
|
||||
case "update":
|
||||
if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
for _, pageID := range stalePages {
|
||||
if err := client.deletePage(pageID, editSummary); err != nil {
|
||||
return deployResult{}, err
|
||||
}
|
||||
}
|
||||
|
||||
currentHashes := computePageHashes(localPages, notesDelimiter)
|
||||
saveDeployManifest(manifestPath, currentHashes)
|
||||
}
|
||||
|
||||
updated := len(deployments)
|
||||
deleted := len(stalePages)
|
||||
skipped := len(localPages) - updated
|
||||
|
||||
result := deployResult{
|
||||
LocalPages: len(localPages),
|
||||
Updated: updated,
|
||||
Deleted: deleted,
|
||||
Skipped: skipped,
|
||||
Manifest: manifestPath,
|
||||
if err := saveDeployManifest(manifestPath, nextManifest); err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
progress(fmt.Sprintf("Updated: %d, Skipped: %d, Deleted: %d", updated, skipped, deleted))
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func collectLocalPages(sourceDir string, namespaces []string) (map[string]string, error) {
|
||||
pages := map[string]string{}
|
||||
func collectLocalPages(sourceDir string, namespaces []string) (map[string]wikiDeployPage, error) {
|
||||
namespaceSet := map[string]struct{}{}
|
||||
for _, namespace := range namespaces {
|
||||
namespaceSet[namespace] = struct{}{}
|
||||
}
|
||||
pages := map[string]wikiDeployPage{}
|
||||
for _, ns := range namespaces {
|
||||
nsDir := filepath.Join(sourceDir, ns)
|
||||
if !isDir(nsDir) {
|
||||
@@ -156,24 +177,31 @@ func collectLocalPages(sourceDir string, namespaces []string) (map[string]string
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if filepath.Ext(path) != ".txt" {
|
||||
if d.IsDir() || filepath.Ext(path) != ".html" {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(sourceDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pageID := strings.TrimSuffix(rel, ".txt")
|
||||
pageID = strings.ReplaceAll(pageID, string(filepath.Separator), ":")
|
||||
pageID = strings.ToLower(pageID)
|
||||
content, err := os.ReadFile(path)
|
||||
pageID := strings.TrimSuffix(filepath.ToSlash(rel), ".html")
|
||||
pageID = strings.ReplaceAll(pageID, "/", ":")
|
||||
namespace := strings.SplitN(pageID, ":", 2)[0]
|
||||
if _, ok := namespaceSet[namespace]; !ok {
|
||||
return nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pages[pageID] = string(content)
|
||||
content := string(raw)
|
||||
pages[pageID] = wikiDeployPage{
|
||||
PageID: pageID,
|
||||
Title: extractHTMLTitle(content, pageID),
|
||||
Namespace: namespace,
|
||||
Content: content,
|
||||
Hash: computeManagedHash(content),
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
@@ -182,283 +210,322 @@ func collectLocalPages(sourceDir string, namespaces []string) (map[string]string
|
||||
return pages, nil
|
||||
}
|
||||
|
||||
func computePageHash(content, notesDelimiter string) string {
|
||||
generatedBody, _ := splitNotesSection(content, notesDelimiter)
|
||||
h := sha1.New()
|
||||
h.Write([]byte(generatedBody))
|
||||
return fmt.Sprintf("%x", h.Sum(nil))
|
||||
}
|
||||
|
||||
func computePageHashes(pages map[string]string, notesDelimiter string) map[string]string {
|
||||
hashes := map[string]string{}
|
||||
for pageID, content := range pages {
|
||||
hashes[pageID] = computePageHash(content, notesDelimiter)
|
||||
}
|
||||
return hashes
|
||||
}
|
||||
|
||||
func splitNotesSection(content, notesDelimiter string) (string, string) {
|
||||
lines := strings.Split(content, notesDelimiter)
|
||||
if len(lines) <= 1 {
|
||||
return strings.TrimSpace(content), ""
|
||||
}
|
||||
body := strings.TrimSpace(strings.Join(lines[:len(lines)-1], notesDelimiter))
|
||||
notes := strings.TrimSpace(strings.Join(lines[1:], notesDelimiter))
|
||||
return body, notes
|
||||
}
|
||||
|
||||
func loadDeployManifest(path string) map[string]string {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
var doc struct {
|
||||
Pages map[string]string `json:"pages"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &doc); err != nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
result := map[string]string{}
|
||||
for k, v := range doc.Pages {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func saveDeployManifest(path string, hashes map[string]string) {
|
||||
os.MkdirAll(filepath.Dir(path), 0755)
|
||||
doc := struct {
|
||||
Pages map[string]string `json:"pages"`
|
||||
}{
|
||||
Pages: hashes,
|
||||
}
|
||||
data, _ := json.MarshalIndent(doc, "", " ")
|
||||
os.WriteFile(path, append(data, '\n'), 0644)
|
||||
}
|
||||
|
||||
func planDeploy(
|
||||
localPages map[string]string,
|
||||
client *dokuWikiClient,
|
||||
namespaces []string,
|
||||
notesDelimiter string,
|
||||
previousHashes map[string]string,
|
||||
) ([]deployPlan, []string, error) {
|
||||
var deployments []deployPlan
|
||||
remotePages := map[string]string{}
|
||||
|
||||
for pageID, content := range localPages {
|
||||
hash := computePageHash(content, notesDelimiter)
|
||||
if previousHashes[pageID] == hash {
|
||||
continue
|
||||
}
|
||||
|
||||
oldContent, err := client.getPage(pageID)
|
||||
if err != nil {
|
||||
if !errors.Is(err, errPageNotFound) {
|
||||
return nil, nil, err
|
||||
}
|
||||
oldContent = ""
|
||||
}
|
||||
remotePages[pageID] = oldContent
|
||||
|
||||
merged := mergePageContent(content, oldContent, notesDelimiter)
|
||||
if merged != oldContent {
|
||||
deployments = append(deployments, deployPlan{
|
||||
PageID: pageID,
|
||||
Content: merged,
|
||||
OldContent: oldContent,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
remoteIDs := map[string]bool{}
|
||||
for pageID := range remotePages {
|
||||
remoteIDs[pageID] = true
|
||||
}
|
||||
|
||||
for _, ns := range namespaces {
|
||||
pages, err := client.listNamespacePages(ns)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, pageID := range pages {
|
||||
remoteIDs[pageID] = true
|
||||
}
|
||||
}
|
||||
|
||||
var stale []string
|
||||
for pageID := range remoteIDs {
|
||||
if _, exists := localPages[pageID]; !exists {
|
||||
stale = append(stale, pageID)
|
||||
}
|
||||
}
|
||||
slices.Sort(stale)
|
||||
|
||||
return deployments, stale, nil
|
||||
}
|
||||
|
||||
func mergePageContent(generated, existing, notesDelimiter string) string {
|
||||
_, existingNotes := splitNotesSection(existing, notesDelimiter)
|
||||
generatedBody, _ := splitNotesSection(generated, notesDelimiter)
|
||||
|
||||
if existingNotes == "" {
|
||||
return generated
|
||||
}
|
||||
|
||||
return strings.TrimSpace(generatedBody) + "\n\n" + notesDelimiter + "\n\n" + existingNotes + "\n"
|
||||
}
|
||||
|
||||
type dokuWikiClient struct {
|
||||
endpoint string
|
||||
username string
|
||||
password string
|
||||
timeout time.Duration
|
||||
retryAttempts int
|
||||
retryBackoff time.Duration
|
||||
requestDelay time.Duration
|
||||
}
|
||||
|
||||
var errPageNotFound = errors.New("page not found")
|
||||
|
||||
func newDokuWikiClient(endpoint, username, password string) *dokuWikiClient {
|
||||
endpoint = strings.TrimRight(endpoint, "/")
|
||||
if strings.HasSuffix(endpoint, "/doku.php") {
|
||||
endpoint = strings.TrimSuffix(endpoint, "/doku.php") + "/lib/exe/jsonrpc.php"
|
||||
}
|
||||
return &dokuWikiClient{
|
||||
endpoint: endpoint,
|
||||
username: username,
|
||||
password: password,
|
||||
timeout: 30 * time.Second,
|
||||
retryAttempts: 3,
|
||||
retryBackoff: 500 * time.Millisecond,
|
||||
requestDelay: 50 * time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *dokuWikiClient) getPage(pageID string) (string, error) {
|
||||
result, err := c.rpc("wiki.getPage", []any{pageID})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if result == nil {
|
||||
return "", errPageNotFound
|
||||
}
|
||||
s, ok := result.(string)
|
||||
if !ok {
|
||||
return "", nil
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (c *dokuWikiClient) putPage(pageID, content, summary string) error {
|
||||
_, err := c.rpc("wiki.putPage", []any{pageID, content, map[string]any{"sum": summary}})
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *dokuWikiClient) deletePage(pageID, summary string) error {
|
||||
_, err := c.rpc("wiki.putPage", []any{pageID, "", map[string]any{"sum": summary, "minor": false}})
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *dokuWikiClient) listNamespacePages(namespace string) ([]string, error) {
|
||||
result, err := c.rpc("dokuwiki.getPagelist", []any{namespace, map[string]any{}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, ok := result.([]any)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
var pages []string
|
||||
for _, item := range items {
|
||||
switch v := item.(type) {
|
||||
case string:
|
||||
pages = append(pages, v)
|
||||
case map[string]any:
|
||||
if id, ok := v["id"].(string); ok {
|
||||
pages = append(pages, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
return pages, nil
|
||||
}
|
||||
|
||||
func (c *dokuWikiClient) rpc(method string, params []any) (any, error) {
|
||||
var last error
|
||||
for attempt := 1; attempt <= c.retryAttempts; attempt++ {
|
||||
result, err := c.rpcOnce(method, params)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
last = err
|
||||
if attempt < c.retryAttempts {
|
||||
time.Sleep(c.retryBackoff * time.Duration(attempt))
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("%s failed after %d attempts: %w", method, c.retryAttempts, last)
|
||||
}
|
||||
|
||||
func (c *dokuWikiClient) rpcOnce(method string, params []any) (any, error) {
|
||||
if c.requestDelay > 0 {
|
||||
time.Sleep(c.requestDelay)
|
||||
}
|
||||
|
||||
payload := map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"method": method,
|
||||
"params": params,
|
||||
"id": 1,
|
||||
}
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
token := base64.StdEncoding.EncodeToString([]byte(c.username + ":" + c.password))
|
||||
req, err := http.NewRequest("POST", c.endpoint, strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Basic "+token)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var result map[string]any
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
preview := strings.TrimSpace(string(data))
|
||||
if len(preview) > 200 {
|
||||
preview = preview[:200] + "..."
|
||||
}
|
||||
return nil, fmt.Errorf("non-JSON response: %s", preview)
|
||||
}
|
||||
|
||||
if errMsg, ok := result["error"].(any); ok && errMsg != nil {
|
||||
return nil, fmt.Errorf("RPC error: %v", errMsg)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
return result["result"], nil
|
||||
}
|
||||
|
||||
func isDir(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && info.IsDir()
|
||||
}
|
||||
|
||||
func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) {
|
||||
pageIDs := sortedKeysFromMap(pages)
|
||||
next := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
|
||||
var plans []wikiDeployPlan
|
||||
var result deployResult
|
||||
|
||||
for _, pageID := range pageIDs {
|
||||
page := pages[pageID]
|
||||
entry := manifest.Pages[pageID]
|
||||
entry.Hash = page.Hash
|
||||
if entry.CID == 0 {
|
||||
entry.CID = opts.CategoryIDs[page.Namespace]
|
||||
}
|
||||
next.Pages[pageID] = entry
|
||||
|
||||
if entry.PID == 0 {
|
||||
if !opts.AllowCreates {
|
||||
return nil, result, next, fmt.Errorf("wiki page %q has no manifest post mapping; pass --create and --category %s=<cid> to create it", pageID, page.Namespace)
|
||||
}
|
||||
if entry.CID == 0 {
|
||||
return nil, result, next, fmt.Errorf("wiki page %q requires --category %s=<cid> for creation", pageID, page.Namespace)
|
||||
}
|
||||
result.Created++
|
||||
plans = append(plans, wikiDeployPlan{Page: page, Entry: entry, Action: "create", Content: page.Content})
|
||||
continue
|
||||
}
|
||||
|
||||
if manifest.Pages[pageID].Hash == page.Hash {
|
||||
result.Skipped++
|
||||
continue
|
||||
}
|
||||
remote, err := client.getPost(entry.PID)
|
||||
if err != nil {
|
||||
return nil, result, next, err
|
||||
}
|
||||
remoteHash := computeManagedHash(remote.Content)
|
||||
if manifest.Pages[pageID].Hash != "" && remoteHash != manifest.Pages[pageID].Hash {
|
||||
result.Drifted++
|
||||
if !opts.Force {
|
||||
continue
|
||||
}
|
||||
}
|
||||
merged := mergeManagedContent(remote.Content, page.Content)
|
||||
if merged == remote.Content {
|
||||
result.Skipped++
|
||||
continue
|
||||
}
|
||||
result.Updated++
|
||||
plans = append(plans, wikiDeployPlan{Page: page, Entry: entry, Action: "update", Content: merged, RemoteHash: remoteHash})
|
||||
}
|
||||
for pageID := range manifest.Pages {
|
||||
if _, ok := pages[pageID]; !ok {
|
||||
result.Stale++
|
||||
}
|
||||
}
|
||||
return plans, result, next, nil
|
||||
}
|
||||
|
||||
func loadDeployManifest(path string) wikiDeployManifest {
|
||||
doc := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return doc
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
|
||||
}
|
||||
if doc.Pages == nil {
|
||||
doc.Pages = map[string]wikiDeployManifestPage{}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
func saveDeployManifest(path string, manifest wikiDeployManifest) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(raw, '\n'), 0o644)
|
||||
}
|
||||
|
||||
func computeManagedHash(content string) string {
|
||||
managed, ok := extractManagedRegion(content)
|
||||
if !ok {
|
||||
managed = content
|
||||
}
|
||||
h := sha256.Sum256([]byte(strings.TrimSpace(managed)))
|
||||
return fmt.Sprintf("%x", h[:])
|
||||
}
|
||||
|
||||
func extractManagedRegion(content string) (string, bool) {
|
||||
start := strings.Index(content, managedStartMarker)
|
||||
end := strings.Index(content, managedEndMarker)
|
||||
if start < 0 || end < 0 || end < start {
|
||||
return "", false
|
||||
}
|
||||
start += len(managedStartMarker)
|
||||
return strings.TrimSpace(content[start:end]), true
|
||||
}
|
||||
|
||||
func mergeManagedContent(existing, generated string) string {
|
||||
generatedRegion, ok := extractManagedRegion(generated)
|
||||
if !ok {
|
||||
return generated
|
||||
}
|
||||
start := strings.Index(existing, managedStartMarker)
|
||||
end := strings.Index(existing, managedEndMarker)
|
||||
if start < 0 || end < 0 || end < start {
|
||||
return generated
|
||||
}
|
||||
end += len(managedEndMarker)
|
||||
return existing[:start+len(managedStartMarker)] + "\n" + generatedRegion + "\n" + existing[end:]
|
||||
}
|
||||
|
||||
func extractHTMLTitle(content, fallback string) string {
|
||||
lower := strings.ToLower(content)
|
||||
start := strings.Index(lower, "<h1>")
|
||||
end := strings.Index(lower, "</h1>")
|
||||
if start >= 0 && end > start {
|
||||
title := strings.TrimSpace(content[start+len("<h1>") : end])
|
||||
title = strings.ReplaceAll(title, "\n", " ")
|
||||
if title != "" {
|
||||
return html.UnescapeString(stripSimpleTags(title))
|
||||
}
|
||||
}
|
||||
parts := strings.Split(fallback, ":")
|
||||
return namespaceTitle(parts[len(parts)-1])
|
||||
}
|
||||
|
||||
func stripSimpleTags(text string) string {
|
||||
var out strings.Builder
|
||||
inTag := false
|
||||
for _, r := range text {
|
||||
switch r {
|
||||
case '<':
|
||||
inTag = true
|
||||
case '>':
|
||||
inTag = false
|
||||
default:
|
||||
if !inTag {
|
||||
out.WriteRune(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func sortedKeysFromMap[V any](m map[string]V) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for key := range m {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
type nodeBBClient struct {
|
||||
endpoint string
|
||||
token string
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
type nodeBBPost struct {
|
||||
PID int
|
||||
TID int
|
||||
Content string
|
||||
}
|
||||
|
||||
func newNodeBBClient(endpoint, token string) *nodeBBClient {
|
||||
return &nodeBBClient{
|
||||
endpoint: strings.TrimRight(endpoint, "/"),
|
||||
token: token,
|
||||
timeout: 30 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *nodeBBClient) getPost(pid int) (nodeBBPost, error) {
|
||||
var doc any
|
||||
if err := c.request("GET", fmt.Sprintf("/api/v3/posts/%d", pid), nil, &doc); err != nil {
|
||||
return nodeBBPost{}, err
|
||||
}
|
||||
return parseNodeBBPost(doc), nil
|
||||
}
|
||||
|
||||
func (c *nodeBBClient) createTopic(cid int, title, content, summary string) (nodeBBPost, error) {
|
||||
body := map[string]any{"cid": cid, "title": title, "content": content}
|
||||
if summary != "" {
|
||||
body["_uid_note"] = summary
|
||||
}
|
||||
var doc any
|
||||
if err := c.request("POST", "/api/v3/topics", body, &doc); err != nil {
|
||||
return nodeBBPost{}, err
|
||||
}
|
||||
post := parseNodeBBPost(doc)
|
||||
if post.TID == 0 || post.PID == 0 {
|
||||
return nodeBBPost{}, fmt.Errorf("NodeBB topic create response did not include tid and pid")
|
||||
}
|
||||
return post, nil
|
||||
}
|
||||
|
||||
func (c *nodeBBClient) updatePost(pid int, content, summary string) error {
|
||||
body := map[string]any{"content": content}
|
||||
if summary != "" {
|
||||
body["_uid_note"] = summary
|
||||
}
|
||||
return c.request("PUT", fmt.Sprintf("/api/v3/posts/%d", pid), body, nil)
|
||||
}
|
||||
|
||||
func (c *nodeBBClient) request(method, path string, payload any, out any) error {
|
||||
var body io.Reader
|
||||
if payload != nil {
|
||||
raw, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body = bytes.NewReader(raw)
|
||||
}
|
||||
req, err := http.NewRequest(method, c.endpoint+path, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if payload != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
client := http.Client{Timeout: c.timeout}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode >= 400 {
|
||||
return fmt.Errorf("NodeBB %s %s failed: HTTP %d: %s", method, path, resp.StatusCode, strings.TrimSpace(string(raw)))
|
||||
}
|
||||
if out == nil || len(bytes.TrimSpace(raw)) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(raw, out); err != nil {
|
||||
return fmt.Errorf("parse NodeBB response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseNodeBBPost(doc any) nodeBBPost {
|
||||
for {
|
||||
m, ok := doc.(map[string]any)
|
||||
if !ok {
|
||||
return nodeBBPost{}
|
||||
}
|
||||
if response, ok := m["response"]; ok {
|
||||
doc = response
|
||||
continue
|
||||
}
|
||||
if postData, ok := m["postData"]; ok {
|
||||
doc = postData
|
||||
continue
|
||||
}
|
||||
if mainPost, ok := m["mainPost"]; ok {
|
||||
post := parseNodeBBPost(mainPost)
|
||||
if post.TID == 0 {
|
||||
post.TID = intFromAny(m["tid"])
|
||||
}
|
||||
return post
|
||||
}
|
||||
return nodeBBPost{
|
||||
PID: firstInt(m, "pid", "mainPid"),
|
||||
TID: firstInt(m, "tid"),
|
||||
Content: firstString(m, "content"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func firstInt(m map[string]any, keys ...string) int {
|
||||
for _, key := range keys {
|
||||
if value := intFromAny(m[key]); value != 0 {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func intFromAny(value any) int {
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return typed
|
||||
case float64:
|
||||
return int(typed)
|
||||
case json.Number:
|
||||
i, _ := typed.Int64()
|
||||
return int(i)
|
||||
case string:
|
||||
i, _ := strconv.Atoi(typed)
|
||||
return i
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func firstString(m map[string]any, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value, ok := m[key].(string); ok {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -17,49 +17,45 @@ func TestDeployWikiDryRunDoesNotWriteRemoteOrManifest(t *testing.T) {
|
||||
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
|
||||
t.Fatalf("create source dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.txt"), []byte("Generated athletics page\n"), 0644); err != nil {
|
||||
generated := "<!-- sow-topdata-wiki:managed:start -->\n<h1>Athletics</h1>\n<p>Generated athletics page</p>\n<!-- sow-topdata-wiki:managed:end -->\n"
|
||||
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil {
|
||||
t.Fatalf("write source page: %v", err)
|
||||
}
|
||||
old := "<!-- sow-topdata-wiki:managed:start -->\n<h1>Athletics</h1>\n<p>Existing athletics page</p>\n<!-- sow-topdata-wiki:managed:end -->\n"
|
||||
oldHash := computeManagedHash(old)
|
||||
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
||||
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
|
||||
Version: "nodebb-v1",
|
||||
Pages: map[string]wikiDeployManifestPage{
|
||||
"skills:athletics": {Hash: oldHash, TID: 7, PID: 42, CID: 3},
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write deploy manifest: %v", err)
|
||||
}
|
||||
|
||||
putPageCalls := 0
|
||||
updateCalls := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Method string `json:"method"`
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer nodebb-token" {
|
||||
t.Fatalf("unexpected authorization header %q", got)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Fatalf("decode rpc request: %v", err)
|
||||
}
|
||||
|
||||
var result any
|
||||
switch req.Method {
|
||||
case "wiki.getPage":
|
||||
result = "Existing athletics page\n"
|
||||
case "dokuwiki.getPagelist":
|
||||
result = []any{}
|
||||
case "wiki.putPage":
|
||||
putPageCalls++
|
||||
result = true
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7, "content": old}})
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42":
|
||||
updateCalls++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}})
|
||||
default:
|
||||
t.Fatalf("unexpected rpc method %q", req.Method)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"result": result,
|
||||
}); err != nil {
|
||||
t.Fatalf("encode rpc response: %v", err)
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
||||
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
||||
SourceDir: sourceDir,
|
||||
Endpoint: server.URL,
|
||||
Username: "wiki-user",
|
||||
Password: "wiki-pass",
|
||||
Token: "nodebb-token",
|
||||
ManifestPath: manifestPath,
|
||||
DryRun: true,
|
||||
}, nil)
|
||||
@@ -70,10 +66,77 @@ func TestDeployWikiDryRunDoesNotWriteRemoteOrManifest(t *testing.T) {
|
||||
if result.Updated != 1 {
|
||||
t.Fatalf("expected one planned update, got %d", result.Updated)
|
||||
}
|
||||
if putPageCalls != 0 {
|
||||
t.Fatalf("expected dry run not to call wiki.putPage, got %d calls", putPageCalls)
|
||||
if updateCalls != 0 {
|
||||
t.Fatalf("expected dry run not to update NodeBB posts, got %d calls", updateCalls)
|
||||
}
|
||||
if _, err := os.Stat(manifestPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected dry run not to write manifest, got %v", err)
|
||||
after := loadDeployManifest(manifestPath)
|
||||
if after.Pages["skills:athletics"].Hash != oldHash {
|
||||
t.Fatalf("expected dry run not to rewrite manifest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeployWikiCreatesNodeBBTopicAndWritesManifest(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sourceDir := filepath.Join(root, "pages")
|
||||
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
|
||||
t.Fatalf("create source dir: %v", err)
|
||||
}
|
||||
generated := "<!-- sow-topdata-wiki:managed:start -->\n<h1>Athletics</h1>\n<p>Generated athletics page</p>\n<!-- sow-topdata-wiki:managed:end -->\n"
|
||||
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil {
|
||||
t.Fatalf("write source page: %v", err)
|
||||
}
|
||||
|
||||
createCalls := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer nodebb-token" {
|
||||
t.Fatalf("unexpected authorization header %q", got)
|
||||
}
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" {
|
||||
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
createCalls++
|
||||
var req struct {
|
||||
CID int `json:"cid"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Fatalf("decode create request: %v", err)
|
||||
}
|
||||
if req.CID != 5 || req.Title != "Athletics" || req.Content != generated {
|
||||
t.Fatalf("unexpected create request: %#v", req)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"response": map[string]any{
|
||||
"tid": 11,
|
||||
"mainPost": map[string]any{
|
||||
"pid": 42,
|
||||
"tid": 11,
|
||||
},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
||||
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
||||
SourceDir: sourceDir,
|
||||
Endpoint: server.URL,
|
||||
Token: "nodebb-token",
|
||||
ManifestPath: manifestPath,
|
||||
CategoryIDs: map[string]int{"skills": 5},
|
||||
AllowCreates: true,
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("DeployWikiWithOptions create failed: %v", err)
|
||||
}
|
||||
if result.Created != 1 || createCalls != 1 {
|
||||
t.Fatalf("expected one created topic, result=%d calls=%d", result.Created, createCalls)
|
||||
}
|
||||
manifest := loadDeployManifest(manifestPath)
|
||||
entry := manifest.Pages["skills:athletics"]
|
||||
if entry.TID != 11 || entry.PID != 42 || entry.CID != 5 || entry.Hash != computeManagedHash(generated) {
|
||||
t.Fatalf("unexpected manifest entry: %#v", entry)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -24,7 +25,7 @@ const (
|
||||
legacyWikiRootDirName = ".wiki"
|
||||
wikiPagesDirName = "pages"
|
||||
wikiStateFileName = "state.json"
|
||||
wikiGeneratorVersion = "native-v1"
|
||||
wikiGeneratorVersion = "nodebb-html-v1"
|
||||
wikiGeneratedStatus = "generated"
|
||||
wikiSkippedStatus = "skipped"
|
||||
)
|
||||
@@ -80,7 +81,7 @@ type wikiContext struct {
|
||||
implementedFeats map[string]struct{}
|
||||
}
|
||||
|
||||
func buildWiki(p *project.Project, nativeResult BuildResult, progress func(string)) (wikiResult, error) {
|
||||
func buildWiki(p *project.Project, nativeResult BuildResult, force bool, progress func(string)) (wikiResult, error) {
|
||||
if progress == nil {
|
||||
progress = func(string) {}
|
||||
}
|
||||
@@ -94,7 +95,7 @@ func buildWiki(p *project.Project, nativeResult BuildResult, progress func(strin
|
||||
}
|
||||
|
||||
state, _ := loadWikiState(statePath)
|
||||
if digest != "" && state.Digest == digest {
|
||||
if !force && digest != "" && state.Digest == digest {
|
||||
return wikiResult{
|
||||
OutputDir: outputDir,
|
||||
PageCount: state.Pages,
|
||||
@@ -125,6 +126,7 @@ func buildWiki(p *project.Project, nativeResult BuildResult, progress func(strin
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
content = renderNodeBBManagedHTML(pageID, content)
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1234,24 +1236,24 @@ func wikiStatusNoun(category string) string {
|
||||
|
||||
func generateStatusPages(outputDir string, pageStatuses map[string]string, pageTitles map[string]string) error {
|
||||
summaries := buildWikiNamespaceSummaries(pageStatuses)
|
||||
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.txt"), renderWikiStatusReportPage(summaries)); err != nil {
|
||||
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus.html"), renderWikiStatusReportPage(summaries)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, namespace := range wikiManagedNamespaces {
|
||||
summary := summaries[namespace]
|
||||
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".txt"), renderWikiNamespaceFragment(namespace, summary)); err != nil {
|
||||
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace+".html"), renderWikiNamespaceFragment(namespace, summary)); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} {
|
||||
pageIDs := filterWikiPageIDs(pageStatuses, namespace, status)
|
||||
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".txt"), renderWikiStatusListingPage(namespaceTitle(namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
|
||||
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", namespace, status+".html"), renderWikiStatusListingPage(namespaceTitle(namespace)+" "+wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, status := range []string{wikiStatusNew, wikiStatusModified, wikiStatusVanilla} {
|
||||
pageIDs := filterWikiPageIDs(pageStatuses, "", status)
|
||||
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".txt"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
|
||||
if err := writeWikiFile(filepath.Join(outputDir, "meta", "wikistatus", status+".html"), renderWikiStatusListingPage(wikiStatusLabels[status]+" Pages", pageIDs, pageTitles)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -1262,6 +1264,7 @@ func writeWikiFile(path, content string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
content = renderNodeBBManagedHTML(wikiRelPathToPageID(path), content)
|
||||
return os.WriteFile(path, []byte(ensureTrailingNewline(content)), 0o644)
|
||||
}
|
||||
|
||||
@@ -1390,7 +1393,16 @@ func wikiPageIDForKey(key string) string {
|
||||
}
|
||||
|
||||
func wikiPageIDToRelPath(pageID string) string {
|
||||
return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".txt")
|
||||
return filepath.FromSlash(strings.ReplaceAll(pageID, ":", "/") + ".html")
|
||||
}
|
||||
|
||||
func wikiRelPathToPageID(path string) string {
|
||||
path = filepath.ToSlash(path)
|
||||
path = strings.TrimSuffix(path, ".html")
|
||||
if index := strings.LastIndex(path, "/pages/"); index >= 0 {
|
||||
path = path[index+len("/pages/"):]
|
||||
}
|
||||
return strings.ReplaceAll(path, "/", ":")
|
||||
}
|
||||
|
||||
func namespaceTitle(namespace string) string {
|
||||
@@ -1567,6 +1579,184 @@ func toDokuWiki(text string) string {
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
func renderNodeBBManagedHTML(pageID, dokuText string) string {
|
||||
generated, notes := splitDokuWikiNotes(dokuText)
|
||||
body := dokuWikiToNodeBBHTML(generated)
|
||||
parts := []string{
|
||||
"<!-- sow-topdata-wiki:page=" + html.EscapeString(pageID) + " -->",
|
||||
"<!-- sow-topdata-wiki:managed:start -->",
|
||||
body,
|
||||
"<!-- sow-topdata-wiki:managed:end -->",
|
||||
}
|
||||
notesHTML := dokuWikiToNodeBBHTML("===== Notes =====\n" + notes)
|
||||
if strings.TrimSpace(notesHTML) != "" {
|
||||
parts = append(parts, notesHTML)
|
||||
}
|
||||
return ensureTrailingNewline(strings.Join(parts, "\n"))
|
||||
}
|
||||
|
||||
func splitDokuWikiNotes(text string) (string, string) {
|
||||
const delimiter = "===== Notes ====="
|
||||
before, after, ok := strings.Cut(text, delimiter)
|
||||
if !ok {
|
||||
return text, ""
|
||||
}
|
||||
return before, after
|
||||
}
|
||||
|
||||
func dokuWikiToNodeBBHTML(text string) string {
|
||||
text = strings.ReplaceAll(strings.TrimSpace(text), "\r\n", "\n")
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
var out []string
|
||||
var paragraph []string
|
||||
inList := false
|
||||
inNotice := false
|
||||
|
||||
flushParagraph := func() {
|
||||
if len(paragraph) == 0 {
|
||||
return
|
||||
}
|
||||
out = append(out, "<p>"+strings.Join(paragraph, "<br>")+"</p>")
|
||||
paragraph = nil
|
||||
}
|
||||
closeList := func() {
|
||||
if inList {
|
||||
out = append(out, "</ul>")
|
||||
inList = false
|
||||
}
|
||||
}
|
||||
closeNotice := func() {
|
||||
if inNotice {
|
||||
out = append(out, "</aside>")
|
||||
inNotice = false
|
||||
}
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" {
|
||||
flushParagraph()
|
||||
closeList()
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(trimmed, "<WRAP") {
|
||||
flushParagraph()
|
||||
closeList()
|
||||
out = append(out, `<aside class="sow-generated-note">`)
|
||||
inNotice = true
|
||||
continue
|
||||
}
|
||||
if trimmed == "</WRAP>" {
|
||||
flushParagraph()
|
||||
closeList()
|
||||
closeNotice()
|
||||
continue
|
||||
}
|
||||
if heading, level, ok := parseDokuHeading(trimmed); ok {
|
||||
flushParagraph()
|
||||
closeList()
|
||||
closeNotice()
|
||||
out = append(out, fmt.Sprintf("<h%d>%s</h%d>", level, renderInlineNodeBBHTML(heading), level))
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(trimmed, " * ") || strings.HasPrefix(trimmed, "* ") || strings.HasPrefix(trimmed, "- ") {
|
||||
flushParagraph()
|
||||
if !inList {
|
||||
out = append(out, "<ul>")
|
||||
inList = true
|
||||
}
|
||||
item := strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(trimmed, " * "), "* "), "- "))
|
||||
out = append(out, "<li>"+renderInlineNodeBBHTML(item)+"</li>")
|
||||
continue
|
||||
}
|
||||
|
||||
parts := strings.Split(trimmed, `\\`)
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
paragraph = append(paragraph, renderInlineNodeBBHTML(part))
|
||||
}
|
||||
}
|
||||
}
|
||||
flushParagraph()
|
||||
closeList()
|
||||
closeNotice()
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
func parseDokuHeading(line string) (string, int, bool) {
|
||||
if !strings.HasPrefix(line, "=") || !strings.HasSuffix(line, "=") {
|
||||
return "", 0, false
|
||||
}
|
||||
left := len(line) - len(strings.TrimLeft(line, "="))
|
||||
right := len(line) - len(strings.TrimRight(line, "="))
|
||||
if left == 0 || left != right {
|
||||
return "", 0, false
|
||||
}
|
||||
title := strings.TrimSpace(line[left : len(line)-right])
|
||||
if title == "" {
|
||||
return "", 0, false
|
||||
}
|
||||
level := 7 - left
|
||||
if level < 1 {
|
||||
level = 1
|
||||
}
|
||||
if level > 6 {
|
||||
level = 6
|
||||
}
|
||||
return title, level, true
|
||||
}
|
||||
|
||||
func renderInlineNodeBBHTML(text string) string {
|
||||
escaped := html.EscapeString(text)
|
||||
var out strings.Builder
|
||||
boldOpen := false
|
||||
for {
|
||||
index := strings.Index(escaped, "**")
|
||||
if index < 0 {
|
||||
out.WriteString(escaped)
|
||||
break
|
||||
}
|
||||
out.WriteString(escaped[:index])
|
||||
if boldOpen {
|
||||
out.WriteString("</strong>")
|
||||
} else {
|
||||
out.WriteString("<strong>")
|
||||
}
|
||||
boldOpen = !boldOpen
|
||||
escaped = escaped[index+2:]
|
||||
}
|
||||
if boldOpen {
|
||||
out.WriteString("</strong>")
|
||||
}
|
||||
return renderWikiLinks(out.String())
|
||||
}
|
||||
|
||||
func renderWikiLinks(text string) string {
|
||||
var out strings.Builder
|
||||
for {
|
||||
start := strings.Index(text, "[[")
|
||||
if start < 0 {
|
||||
out.WriteString(text)
|
||||
break
|
||||
}
|
||||
end := strings.Index(text[start+2:], "]]")
|
||||
if end < 0 {
|
||||
out.WriteString(text)
|
||||
break
|
||||
}
|
||||
end += start + 2
|
||||
out.WriteString(text[:start])
|
||||
marker := text[start : end+2]
|
||||
out.WriteString(marker)
|
||||
text = text[end+2:]
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func ensureTrailingNewline(text string) string {
|
||||
if strings.HasSuffix(text, "\n") {
|
||||
return text
|
||||
|
||||
@@ -42,24 +42,24 @@ func TestBuildNativeGeneratesAndSkipsWikiPages(t *testing.T) {
|
||||
if result.WikiPages != 1 {
|
||||
t.Fatalf("expected one generated entity page, got %d", result.WikiPages)
|
||||
}
|
||||
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.txt")
|
||||
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
|
||||
got, err := os.ReadFile(pagePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read generated page: %v", err)
|
||||
}
|
||||
text := string(got)
|
||||
if !strings.Contains(text, "====== Athletics ======") || !strings.Contains(text, "This skill is unchanged from vanilla.") {
|
||||
if !strings.Contains(text, "<h1>Athletics</h1>") || !strings.Contains(text, "This skill is unchanged from vanilla.") {
|
||||
t.Fatalf("unexpected generated page:\n%s", text)
|
||||
}
|
||||
if !strings.Contains(text, "Ability: Strength.") || !strings.Contains(text, " * Climb") {
|
||||
if !strings.Contains(text, managedStartMarker) || !strings.Contains(text, "Ability: Strength.") || !strings.Contains(text, "<li>Climb</li>") {
|
||||
t.Fatalf("expected description to be rendered into wiki page:\n%s", text)
|
||||
}
|
||||
statusPath := filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus.txt")
|
||||
statusPath := filepath.Join(root, ".cache", "wiki", "pages", "meta", "wikistatus.html")
|
||||
statusRaw, err := os.ReadFile(statusPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read status page: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(statusRaw), "**Vanilla:** 1\\\\") {
|
||||
if !strings.Contains(string(statusRaw), "<strong>Vanilla:</strong> 1") {
|
||||
t.Fatalf("expected wiki status summary to count vanilla page:\n%s", string(statusRaw))
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, ".cache", "wiki", "state.json")); err != nil {
|
||||
@@ -190,7 +190,7 @@ func TestBuildNativeRegeneratesWikiWhenTLKTextChanges(t *testing.T) {
|
||||
if _, err := BuildNative(proj, nil); err != nil {
|
||||
t.Fatalf("BuildNative after text change failed: %v", err)
|
||||
}
|
||||
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.txt")
|
||||
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
|
||||
got, err := os.ReadFile(pagePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read regenerated page: %v", err)
|
||||
@@ -232,7 +232,7 @@ func TestBuildPackageIgnoresWikiSourcesForFreshness(t *testing.T) {
|
||||
t.Fatalf("BuildNative failed: %v", err)
|
||||
}
|
||||
|
||||
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.txt")
|
||||
pagePath := filepath.Join(root, ".cache", "wiki", "pages", "skills", "athletics.html")
|
||||
newTime := time.Now().Add(2 * time.Second)
|
||||
if err := os.Chtimes(pagePath, newTime, newTime); err != nil {
|
||||
t.Fatalf("touch wiki page: %v", err)
|
||||
|
||||
Reference in New Issue
Block a user