From 8da3f42c2f191b199613e2969f245552fdce4103 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Sat, 16 May 2026 20:16:48 +0200 Subject: [PATCH] Wiki deploy manifest fix --- README.md | 5 + internal/topdata/wiki_deploy.go | 212 +++++++++++++++++++++++++++ internal/topdata/wiki_deploy_test.go | 111 ++++++++++++++ 3 files changed, 328 insertions(+) diff --git a/README.md b/README.md index 6145aaf..66cd745 100644 --- a/README.md +++ b/README.md @@ -322,6 +322,11 @@ use `namespace=cid`, for example: sow-toolkit deploy-wiki --dry-run --namespace spells --category spells=42 ``` +When a deploy manifest is missing but matching wiki pages already exist in +NodeBB, `deploy-wiki` checks the Westgate Wiki namespace directory API and +adopts those topic/post mappings before planning updates. `--create` is still +required for genuinely new pages that have no existing remote topic. + `build-changelog` renders a release changelog from tagged pushes. Pull-request style commits keep their PR links and authors from the Gitea API; direct pushes are rendered as commit links using the commit author. It defaults to diff --git a/internal/topdata/wiki_deploy.go b/internal/topdata/wiki_deploy.go index d15a18b..d15c559 100644 --- a/internal/topdata/wiki_deploy.go +++ b/internal/topdata/wiki_deploy.go @@ -9,6 +9,7 @@ import ( "html" "io" "net/http" + "net/url" "os" "path/filepath" "slices" @@ -324,6 +325,7 @@ func isDir(path string) bool { func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) { pageIDs := sortedKeysFromMap(pages) next := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}} + remotePagesByCID := map[int][]nodeBBWikiPage{} var plans []wikiDeployPlan var result deployResult @@ -340,6 +342,21 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife } next.Pages[pageID] = entry + if entry.PID == 0 { + if entry.CID != 0 { + adopted, ok, err := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client) + if err != nil { + return nil, result, next, err + } + if ok { + entry.TID = adopted.TID + entry.PID = adopted.PID + entry.CID = adopted.CID + next.Pages[pageID] = entry + } + } + } + if entry.PID == 0 { if !opts.AllowCreates { return nil, result, next, fmt.Errorf("wiki page %q has no manifest post mapping; pass --create and --category %s= to create it", pageID, page.Namespace) @@ -410,6 +427,120 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife return plans, result, next, nil } +func adoptExistingNodeBBPage(page wikiDeployPage, cid int, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient) (wikiDeployManifestPage, bool, error) { + remotePages, ok := remotePagesByCID[cid] + if !ok { + var err error + remotePages, err = client.listNamespacePages(cid) + if err != nil { + return wikiDeployManifestPage{}, false, err + } + remotePagesByCID[cid] = remotePages + } + remotePage, ok := matchExistingNodeBBPage(page, remotePages) + if !ok || remotePage.TID == 0 { + return wikiDeployManifestPage{}, false, nil + } + topic, err := client.getTopic(remotePage.TID) + if err != nil { + return wikiDeployManifestPage{}, false, err + } + if topic.PID == 0 { + return wikiDeployManifestPage{}, false, fmt.Errorf("NodeBB topic %d did not include main post id for wiki page %q", remotePage.TID, page.PageID) + } + if topic.TID == 0 { + topic.TID = remotePage.TID + } + return wikiDeployManifestPage{TID: topic.TID, PID: topic.PID, CID: cid}, true, nil +} + +func matchExistingNodeBBPage(page wikiDeployPage, remotePages []nodeBBWikiPage) (nodeBBWikiPage, bool) { + desiredTitles := map[string]struct{}{} + for _, title := range []string{page.Title, pageIDLeaf(page.PageID)} { + if normalized := normalizeWikiTitle(title); normalized != "" { + desiredTitles[normalized] = struct{}{} + } + } + desiredSlugs := map[string]struct{}{} + for title := range desiredTitles { + if slug := slugifyWikiTitle(title); slug != "" { + desiredSlugs[slug] = struct{}{} + } + } + for _, slug := range []string{pageIDLeaf(page.PageID), strings.ReplaceAll(pageIDRemainder(page.PageID), ":", "-")} { + if normalized := slugifyWikiTitle(slug); normalized != "" { + desiredSlugs[normalized] = struct{}{} + } + } + + for _, candidate := range remotePages { + for _, title := range []string{candidate.TitleLeaf, candidate.Title} { + if _, ok := desiredTitles[normalizeWikiTitle(title)]; ok { + return candidate, true + } + } + for _, slug := range []string{nodeBBPathLeaf(candidate.Slug), nodeBBPathLeaf(candidate.WikiPath)} { + if _, ok := desiredSlugs[slugifyWikiTitle(slug)]; ok { + return candidate, true + } + } + } + return nodeBBWikiPage{}, false +} + +func pageIDRemainder(pageID string) string { + _, rest, ok := strings.Cut(pageID, ":") + if !ok { + return pageID + } + return rest +} + +func pageIDLeaf(pageID string) string { + pageID = strings.TrimSpace(pageID) + if pageID == "" { + return "" + } + parts := strings.FieldsFunc(pageID, func(r rune) bool { + return r == ':' || r == '/' + }) + if len(parts) == 0 { + return "" + } + return parts[len(parts)-1] +} + +func nodeBBPathLeaf(value string) string { + parts := strings.FieldsFunc(strings.TrimSpace(value), func(r rune) bool { + return r == '/' + }) + if len(parts) == 0 { + return "" + } + return parts[len(parts)-1] +} + +func normalizeWikiTitle(value string) string { + return strings.Join(strings.Fields(strings.ToLower(strings.ReplaceAll(strings.TrimSpace(value), "_", " "))), " ") +} + +func slugifyWikiTitle(value string) string { + var b strings.Builder + lastDash := false + for _, r := range strings.ToLower(strings.TrimSpace(value)) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + lastDash = false + continue + } + if !lastDash && b.Len() > 0 { + b.WriteByte('-') + lastDash = true + } + } + return strings.Trim(b.String(), "-") +} + func loadDeployManifest(path string) wikiDeployManifest { doc := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}} raw, err := os.ReadFile(path) @@ -692,6 +823,14 @@ type nodeBBPost struct { Content string } +type nodeBBWikiPage struct { + TID int + Title string + TitleLeaf string + Slug string + WikiPath string +} + func newNodeBBClient(endpoint, token string) *nodeBBClient { return &nodeBBClient{ endpoint: strings.TrimRight(endpoint, "/"), @@ -708,6 +847,39 @@ func (c *nodeBBClient) getPost(pid int) (nodeBBPost, error) { return parseNodeBBPost(doc), nil } +func (c *nodeBBClient) getTopic(tid int) (nodeBBPost, error) { + var doc any + if err := c.request("GET", fmt.Sprintf("/api/v3/topics/%d", tid), nil, &doc); err != nil { + return nodeBBPost{}, err + } + post := parseNodeBBPost(doc) + if post.TID == 0 { + post.TID = tid + } + return post, nil +} + +func (c *nodeBBClient) listNamespacePages(cid int) ([]nodeBBWikiPage, error) { + var out []nodeBBWikiPage + after := "" + for { + path := fmt.Sprintf("/api/v3/plugins/westgate-wiki/namespace/%d/pages?limit=80", cid) + if after != "" { + path += "&after=" + url.QueryEscape(after) + } + var doc any + if err := c.request("GET", path, nil, &doc); err != nil { + return nil, err + } + pages, next, hasMore := parseNodeBBWikiPageList(doc) + out = append(out, pages...) + if !hasMore || next == "" { + return out, nil + } + after = next + } +} + func (c *nodeBBClient) createTopic(cid int, title, content, summary string) (nodeBBPost, error) { body := map[string]any{"cid": cid, "title": title, "content": content} if summary != "" { @@ -801,6 +973,46 @@ func parseNodeBBPost(doc any) nodeBBPost { } } +func parseNodeBBWikiPageList(doc any) ([]nodeBBWikiPage, string, bool) { + for { + m, ok := doc.(map[string]any) + if !ok { + return nil, "", false + } + if response, ok := m["response"]; ok { + doc = response + continue + } + pagesRaw, _ := m["pages"].([]any) + pages := make([]nodeBBWikiPage, 0, len(pagesRaw)) + for _, raw := range pagesRaw { + row, ok := raw.(map[string]any) + if !ok { + continue + } + pages = append(pages, nodeBBWikiPage{ + TID: firstInt(row, "tid"), + Title: firstString(row, "title"), + TitleLeaf: firstString(row, "titleLeaf", "title_leaf"), + Slug: firstString(row, "slug"), + WikiPath: firstString(row, "wikiPath", "wiki_path"), + }) + } + return pages, firstString(m, "nextCursor", "next_cursor"), boolFromAny(m["hasMore"]) + } +} + +func boolFromAny(value any) bool { + switch typed := value.(type) { + case bool: + return typed + case string: + return typed == "true" || typed == "1" + default: + return false + } +} + func firstInt(m map[string]any, keys ...string) int { for _, key := range keys { if value := intFromAny(m[key]); value != 0 { diff --git a/internal/topdata/wiki_deploy_test.go b/internal/topdata/wiki_deploy_test.go index f1d9f1d..87ea0b4 100644 --- a/internal/topdata/wiki_deploy_test.go +++ b/internal/topdata/wiki_deploy_test.go @@ -92,6 +92,11 @@ func TestDeployWikiCreatesNodeBBTopicAndWritesManifest(t *testing.T) { if got := r.Header.Get("Authorization"); got != "Bearer nodebb-token" { t.Fatalf("unexpected authorization header %q", got) } + if r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/5/pages" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}}) + return + } if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" { t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) } @@ -154,6 +159,11 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) { } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/5/pages" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}}) + return + } if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" { t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) } @@ -196,6 +206,107 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) { } } +func TestDeployWikiAdoptsExistingNodeBBPageWhenManifestIsMissingWithoutCreate(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "classes"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Acolyte

+

Generated acolyte page

+ + +

Notes

+

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "classes", "acolyte.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + old := ` + +

Acolyte

+

Old acolyte page

+ + +

Notes

+

Keep this remote note.

+ +` + + createCalls := 0 + var updated string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/9/pages": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "pages": []map[string]any{ + { + "tid": 77, + "title": "Acolyte", + "titleLeaf": "Acolyte", + "slug": "77/acolyte", + "wikiPath": "/wiki/classes/acolyte", + }, + }, + "hasMore": false, + }, + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/77": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 77, "mainPid": 88}}) + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/88": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77, "content": old}}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/88": + var req struct { + Content string `json:"content"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode update request: %v", err) + } + updated = req.Content + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77}}) + case r.Method == http.MethodPost && r.URL.Path == "/api/v3/topics": + createCalls++ + t.Fatalf("existing wiki page should be adopted and updated, not recreated") + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + manifestPath := filepath.Join(root, "deploy-manifest.json") + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + CategoryIDs: map[string]int{"classes": 9}, + StalePolicy: "report", + NotesDelimiter: "", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions adopt failed: %v", err) + } + if result.Created != 0 || result.Updated != 1 || createCalls != 0 { + t.Fatalf("expected one adopted update and no creates, result=%#v createCalls=%d", result, createCalls) + } + if !containsAll(updated, "

Generated acolyte page

", "

Keep this remote note.

") { + t.Fatalf("expected adopted update to merge generated and remote manual content:\n%s", updated) + } + manifest := loadDeployManifest(manifestPath) + entry := manifest.Pages["classes:acolyte"] + if entry.TID != 77 || entry.PID != 88 || entry.CID != 9 || entry.Hash != computeManagedHash(generated) { + t.Fatalf("unexpected adopted manifest entry: %#v", entry) + } +} + func TestDeployWikiMergesHTMLManagedAndManualRegions(t *testing.T) { root := t.TempDir() sourceDir := filepath.Join(root, "pages")