ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Wiki deploy manifest fix

Browse Source
This commit is contained in:
archvillainette
2026-05-16 20:16:48 +02:00
parent 28bea40038
commit 8da3f42c2f
3 changed files with 328 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+5
View File
@@ -322,6 +322,11 @@ use `namespace=cid`, for example:
sow-toolkit deploy-wiki --dry-run --namespace spells --category spells=42
```
When a deploy manifest is missing but matching wiki pages already exist in
NodeBB, `deploy-wiki` checks the Westgate Wiki namespace directory API and
adopts those topic/post mappings before planning updates. `--create` is still
required for genuinely new pages that have no existing remote topic.
`build-changelog` renders a release changelog from tagged pushes. Pull-request
style commits keep their PR links and authors from the Gitea API; direct pushes
are rendered as commit links using the commit author. It defaults to
internal/topdata/wiki_deploy.go
+212
View File
@@ -9,6 +9,7 @@ import (
"html"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"slices"
@@ -324,6 +325,7 @@ func isDir(path string) bool {
func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) {
pageIDs := sortedKeysFromMap(pages)
next := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
remotePagesByCID := map[int][]nodeBBWikiPage{}
var plans []wikiDeployPlan
var result deployResult
@@ -340,6 +342,21 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
}
next.Pages[pageID] = entry
if entry.PID == 0 {
if entry.CID != 0 {
adopted, ok, err := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client)
if err != nil {
return nil, result, next, err
}
if ok {
entry.TID = adopted.TID
entry.PID = adopted.PID
entry.CID = adopted.CID
next.Pages[pageID] = entry
}
}
}
if entry.PID == 0 {
if !opts.AllowCreates {
return nil, result, next, fmt.Errorf("wiki page %q has no manifest post mapping; pass --create and --category %s=<cid> to create it", pageID, page.Namespace)
@@ -410,6 +427,120 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
return plans, result, next, nil
}
func adoptExistingNodeBBPage(page wikiDeployPage, cid int, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient) (wikiDeployManifestPage, bool, error) {
remotePages, ok := remotePagesByCID[cid]
if !ok {
var err error
remotePages, err = client.listNamespacePages(cid)
if err != nil {
return wikiDeployManifestPage{}, false, err
}
remotePagesByCID[cid] = remotePages
}
remotePage, ok := matchExistingNodeBBPage(page, remotePages)
if !ok || remotePage.TID == 0 {
return wikiDeployManifestPage{}, false, nil
}
topic, err := client.getTopic(remotePage.TID)
if err != nil {
return wikiDeployManifestPage{}, false, err
}
if topic.PID == 0 {
return wikiDeployManifestPage{}, false, fmt.Errorf("NodeBB topic %d did not include main post id for wiki page %q", remotePage.TID, page.PageID)
}
if topic.TID == 0 {
topic.TID = remotePage.TID
}
return wikiDeployManifestPage{TID: topic.TID, PID: topic.PID, CID: cid}, true, nil
}
func matchExistingNodeBBPage(page wikiDeployPage, remotePages []nodeBBWikiPage) (nodeBBWikiPage, bool) {
desiredTitles := map[string]struct{}{}
for _, title := range []string{page.Title, pageIDLeaf(page.PageID)} {
if normalized := normalizeWikiTitle(title); normalized != "" {
desiredTitles[normalized] = struct{}{}
}
}
desiredSlugs := map[string]struct{}{}
for title := range desiredTitles {
if slug := slugifyWikiTitle(title); slug != "" {
desiredSlugs[slug] = struct{}{}
}
}
for _, slug := range []string{pageIDLeaf(page.PageID), strings.ReplaceAll(pageIDRemainder(page.PageID), ":", "-")} {
if normalized := slugifyWikiTitle(slug); normalized != "" {
desiredSlugs[normalized] = struct{}{}
}
}
for _, candidate := range remotePages {
for _, title := range []string{candidate.TitleLeaf, candidate.Title} {
if _, ok := desiredTitles[normalizeWikiTitle(title)]; ok {
return candidate, true
}
}
for _, slug := range []string{nodeBBPathLeaf(candidate.Slug), nodeBBPathLeaf(candidate.WikiPath)} {
if _, ok := desiredSlugs[slugifyWikiTitle(slug)]; ok {
return candidate, true
}
}
}
return nodeBBWikiPage{}, false
}
func pageIDRemainder(pageID string) string {
_, rest, ok := strings.Cut(pageID, ":")
if !ok {
return pageID
}
return rest
}
func pageIDLeaf(pageID string) string {
pageID = strings.TrimSpace(pageID)
if pageID == "" {
return ""
}
parts := strings.FieldsFunc(pageID, func(r rune) bool {
return r == ':' || r == '/'
})
if len(parts) == 0 {
return ""
}
return parts[len(parts)-1]
}
func nodeBBPathLeaf(value string) string {
parts := strings.FieldsFunc(strings.TrimSpace(value), func(r rune) bool {
return r == '/'
})
if len(parts) == 0 {
return ""
}
return parts[len(parts)-1]
}
func normalizeWikiTitle(value string) string {
return strings.Join(strings.Fields(strings.ToLower(strings.ReplaceAll(strings.TrimSpace(value), "_", " "))), " ")
}
func slugifyWikiTitle(value string) string {
var b strings.Builder
lastDash := false
for _, r := range strings.ToLower(strings.TrimSpace(value)) {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
lastDash = false
continue
}
if !lastDash && b.Len() > 0 {
b.WriteByte('-')
lastDash = true
}
}
return strings.Trim(b.String(), "-")
}
func loadDeployManifest(path string) wikiDeployManifest {
doc := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
raw, err := os.ReadFile(path)
@@ -692,6 +823,14 @@ type nodeBBPost struct {
Content string
}
type nodeBBWikiPage struct {
TID int
Title string
TitleLeaf string
Slug string
WikiPath string
}
func newNodeBBClient(endpoint, token string) *nodeBBClient {
return &nodeBBClient{
endpoint: strings.TrimRight(endpoint, "/"),
@@ -708,6 +847,39 @@ func (c *nodeBBClient) getPost(pid int) (nodeBBPost, error) {
return parseNodeBBPost(doc), nil
}
func (c *nodeBBClient) getTopic(tid int) (nodeBBPost, error) {
var doc any
if err := c.request("GET", fmt.Sprintf("/api/v3/topics/%d", tid), nil, &doc); err != nil {
return nodeBBPost{}, err
}
post := parseNodeBBPost(doc)
if post.TID == 0 {
post.TID = tid
}
return post, nil
}
func (c *nodeBBClient) listNamespacePages(cid int) ([]nodeBBWikiPage, error) {
var out []nodeBBWikiPage
after := ""
for {
path := fmt.Sprintf("/api/v3/plugins/westgate-wiki/namespace/%d/pages?limit=80", cid)
if after != "" {
path += "&after=" + url.QueryEscape(after)
}
var doc any
if err := c.request("GET", path, nil, &doc); err != nil {
return nil, err
}
pages, next, hasMore := parseNodeBBWikiPageList(doc)
out = append(out, pages...)
if !hasMore || next == "" {
return out, nil
}
after = next
}
}
func (c *nodeBBClient) createTopic(cid int, title, content, summary string) (nodeBBPost, error) {
body := map[string]any{"cid": cid, "title": title, "content": content}
if summary != "" {
@@ -801,6 +973,46 @@ func parseNodeBBPost(doc any) nodeBBPost {
}
}
func parseNodeBBWikiPageList(doc any) ([]nodeBBWikiPage, string, bool) {
for {
m, ok := doc.(map[string]any)
if !ok {
return nil, "", false
}
if response, ok := m["response"]; ok {
doc = response
continue
}
pagesRaw, _ := m["pages"].([]any)
pages := make([]nodeBBWikiPage, 0, len(pagesRaw))
for _, raw := range pagesRaw {
row, ok := raw.(map[string]any)
if !ok {
continue
}
pages = append(pages, nodeBBWikiPage{
TID: firstInt(row, "tid"),
Title: firstString(row, "title"),
TitleLeaf: firstString(row, "titleLeaf", "title_leaf"),
Slug: firstString(row, "slug"),
WikiPath: firstString(row, "wikiPath", "wiki_path"),
})
}
return pages, firstString(m, "nextCursor", "next_cursor"), boolFromAny(m["hasMore"])
}
}
func boolFromAny(value any) bool {
switch typed := value.(type) {
case bool:
return typed
case string:
return typed == "true" || typed == "1"
default:
return false
}
}
func firstInt(m map[string]any, keys ...string) int {
for _, key := range keys {
if value := intFromAny(m[key]); value != 0 {
internal/topdata/wiki_deploy_test.go
+111
View File
@@ -92,6 +92,11 @@ func TestDeployWikiCreatesNodeBBTopicAndWritesManifest(t *testing.T) {
if got := r.Header.Get("Authorization"); got != "Bearer nodebb-token" {
t.Fatalf("unexpected authorization header %q", got)
}
if r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/5/pages" {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}})
return
}
if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
}
@@ -154,6 +159,11 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) {
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/5/pages" {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}})
return
}
if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
}
@@ -196,6 +206,107 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) {
}
}
func TestDeployWikiAdoptsExistingNodeBBPageWhenManifestIsMissingWithoutCreate(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "classes"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
generated := `<!-- sow-topdata-wiki:page=classes:acolyte -->
<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
<h1>Acolyte</h1>
<p>Generated acolyte page</p>
<!-- sow-topdata-wiki:managed:end -->
<!-- sow-topdata-wiki:manual:start id="notes" -->
<h2>Notes</h2>
<p></p>
<!-- sow-topdata-wiki:manual:end id="notes" -->
`
if err := os.WriteFile(filepath.Join(sourceDir, "classes", "acolyte.html"), []byte(generated), 0644); err != nil {
t.Fatalf("write source page: %v", err)
}
old := `<!-- sow-topdata-wiki:page=classes:acolyte -->
<!-- sow-topdata-wiki:managed:start hash="sha256:old" -->
<h1>Acolyte</h1>
<p>Old acolyte page</p>
<!-- sow-topdata-wiki:managed:end -->
<!-- sow-topdata-wiki:manual:start id="notes" -->
<h2>Notes</h2>
<p>Keep this remote note.</p>
<!-- sow-topdata-wiki:manual:end id="notes" -->
`
createCalls := 0
var updated string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/9/pages":
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"response": map[string]any{
"pages": []map[string]any{
{
"tid": 77,
"title": "Acolyte",
"titleLeaf": "Acolyte",
"slug": "77/acolyte",
"wikiPath": "/wiki/classes/acolyte",
},
},
"hasMore": false,
},
})
case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/77":
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 77, "mainPid": 88}})
case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/88":
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77, "content": old}})
case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/88":
var req struct {
Content string `json:"content"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatalf("decode update request: %v", err)
}
updated = req.Content
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77}})
case r.Method == http.MethodPost && r.URL.Path == "/api/v3/topics":
createCalls++
t.Fatalf("existing wiki page should be adopted and updated, not recreated")
default:
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
}
}))
defer server.Close()
manifestPath := filepath.Join(root, "deploy-manifest.json")
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
CategoryIDs: map[string]int{"classes": 9},
StalePolicy: "report",
NotesDelimiter: "",
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions adopt failed: %v", err)
}
if result.Created != 0 || result.Updated != 1 || createCalls != 0 {
t.Fatalf("expected one adopted update and no creates, result=%#v createCalls=%d", result, createCalls)
}
if !containsAll(updated, "<p>Generated acolyte page</p>", "<p>Keep this remote note.</p>") {
t.Fatalf("expected adopted update to merge generated and remote manual content:\n%s", updated)
}
manifest := loadDeployManifest(manifestPath)
entry := manifest.Pages["classes:acolyte"]
if entry.TID != 77 || entry.PID != 88 || entry.CID != 9 || entry.Hash != computeManagedHash(generated) {
t.Fatalf("unexpected adopted manifest entry: %#v", entry)
}
}
func TestDeployWikiMergesHTMLManagedAndManualRegions(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")