Wiki deploy manifest fix

2026-05-16 20:16:48 +02:00
parent 28bea40038
commit 8da3f42c2f
3 changed files with 328 additions and 0 deletions
@@ -9,6 +9,7 @@ import (
 	"html"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"slices"
@@ -324,6 +325,7 @@ func isDir(path string) bool {
 func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) {
 	pageIDs := sortedKeysFromMap(pages)
 	next := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
+	remotePagesByCID := map[int][]nodeBBWikiPage{}
 	var plans []wikiDeployPlan
 	var result deployResult

@@ -340,6 +342,21 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
 		}
 		next.Pages[pageID] = entry

+		if entry.PID == 0 {
+			if entry.CID != 0 {
+				adopted, ok, err := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client)
+				if err != nil {
+					return nil, result, next, err
+				}
+				if ok {
+					entry.TID = adopted.TID
+					entry.PID = adopted.PID
+					entry.CID = adopted.CID
+					next.Pages[pageID] = entry
+				}
+			}
+		}
+
 		if entry.PID == 0 {
 			if !opts.AllowCreates {
 				return nil, result, next, fmt.Errorf("wiki page %q has no manifest post mapping; pass --create and --category %s=<cid> to create it", pageID, page.Namespace)
@@ -410,6 +427,120 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
 	return plans, result, next, nil
 }

+func adoptExistingNodeBBPage(page wikiDeployPage, cid int, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient) (wikiDeployManifestPage, bool, error) {
+	remotePages, ok := remotePagesByCID[cid]
+	if !ok {
+		var err error
+		remotePages, err = client.listNamespacePages(cid)
+		if err != nil {
+			return wikiDeployManifestPage{}, false, err
+		}
+		remotePagesByCID[cid] = remotePages
+	}
+	remotePage, ok := matchExistingNodeBBPage(page, remotePages)
+	if !ok || remotePage.TID == 0 {
+		return wikiDeployManifestPage{}, false, nil
+	}
+	topic, err := client.getTopic(remotePage.TID)
+	if err != nil {
+		return wikiDeployManifestPage{}, false, err
+	}
+	if topic.PID == 0 {
+		return wikiDeployManifestPage{}, false, fmt.Errorf("NodeBB topic %d did not include main post id for wiki page %q", remotePage.TID, page.PageID)
+	}
+	if topic.TID == 0 {
+		topic.TID = remotePage.TID
+	}
+	return wikiDeployManifestPage{TID: topic.TID, PID: topic.PID, CID: cid}, true, nil
+}
+
+func matchExistingNodeBBPage(page wikiDeployPage, remotePages []nodeBBWikiPage) (nodeBBWikiPage, bool) {
+	desiredTitles := map[string]struct{}{}
+	for _, title := range []string{page.Title, pageIDLeaf(page.PageID)} {
+		if normalized := normalizeWikiTitle(title); normalized != "" {
+			desiredTitles[normalized] = struct{}{}
+		}
+	}
+	desiredSlugs := map[string]struct{}{}
+	for title := range desiredTitles {
+		if slug := slugifyWikiTitle(title); slug != "" {
+			desiredSlugs[slug] = struct{}{}
+		}
+	}
+	for _, slug := range []string{pageIDLeaf(page.PageID), strings.ReplaceAll(pageIDRemainder(page.PageID), ":", "-")} {
+		if normalized := slugifyWikiTitle(slug); normalized != "" {
+			desiredSlugs[normalized] = struct{}{}
+		}
+	}
+
+	for _, candidate := range remotePages {
+		for _, title := range []string{candidate.TitleLeaf, candidate.Title} {
+			if _, ok := desiredTitles[normalizeWikiTitle(title)]; ok {
+				return candidate, true
+			}
+		}
+		for _, slug := range []string{nodeBBPathLeaf(candidate.Slug), nodeBBPathLeaf(candidate.WikiPath)} {
+			if _, ok := desiredSlugs[slugifyWikiTitle(slug)]; ok {
+				return candidate, true
+			}
+		}
+	}
+	return nodeBBWikiPage{}, false
+}
+
+func pageIDRemainder(pageID string) string {
+	_, rest, ok := strings.Cut(pageID, ":")
+	if !ok {
+		return pageID
+	}
+	return rest
+}
+
+func pageIDLeaf(pageID string) string {
+	pageID = strings.TrimSpace(pageID)
+	if pageID == "" {
+		return ""
+	}
+	parts := strings.FieldsFunc(pageID, func(r rune) bool {
+		return r == ':' || r == '/'
+	})
+	if len(parts) == 0 {
+		return ""
+	}
+	return parts[len(parts)-1]
+}
+
+func nodeBBPathLeaf(value string) string {
+	parts := strings.FieldsFunc(strings.TrimSpace(value), func(r rune) bool {
+		return r == '/'
+	})
+	if len(parts) == 0 {
+		return ""
+	}
+	return parts[len(parts)-1]
+}
+
+func normalizeWikiTitle(value string) string {
+	return strings.Join(strings.Fields(strings.ToLower(strings.ReplaceAll(strings.TrimSpace(value), "_", " "))), " ")
+}
+
+func slugifyWikiTitle(value string) string {
+	var b strings.Builder
+	lastDash := false
+	for _, r := range strings.ToLower(strings.TrimSpace(value)) {
+		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
+			b.WriteRune(r)
+			lastDash = false
+			continue
+		}
+		if !lastDash && b.Len() > 0 {
+			b.WriteByte('-')
+			lastDash = true
+		}
+	}
+	return strings.Trim(b.String(), "-")
+}
+
 func loadDeployManifest(path string) wikiDeployManifest {
 	doc := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
 	raw, err := os.ReadFile(path)
@@ -692,6 +823,14 @@ type nodeBBPost struct {
 	Content string
 }

+type nodeBBWikiPage struct {
+	TID       int
+	Title     string
+	TitleLeaf string
+	Slug      string
+	WikiPath  string
+}
+
 func newNodeBBClient(endpoint, token string) *nodeBBClient {
 	return &nodeBBClient{
 		endpoint: strings.TrimRight(endpoint, "/"),
@@ -708,6 +847,39 @@ func (c *nodeBBClient) getPost(pid int) (nodeBBPost, error) {
 	return parseNodeBBPost(doc), nil
 }

+func (c *nodeBBClient) getTopic(tid int) (nodeBBPost, error) {
+	var doc any
+	if err := c.request("GET", fmt.Sprintf("/api/v3/topics/%d", tid), nil, &doc); err != nil {
+		return nodeBBPost{}, err
+	}
+	post := parseNodeBBPost(doc)
+	if post.TID == 0 {
+		post.TID = tid
+	}
+	return post, nil
+}
+
+func (c *nodeBBClient) listNamespacePages(cid int) ([]nodeBBWikiPage, error) {
+	var out []nodeBBWikiPage
+	after := ""
+	for {
+		path := fmt.Sprintf("/api/v3/plugins/westgate-wiki/namespace/%d/pages?limit=80", cid)
+		if after != "" {
+			path += "&after=" + url.QueryEscape(after)
+		}
+		var doc any
+		if err := c.request("GET", path, nil, &doc); err != nil {
+			return nil, err
+		}
+		pages, next, hasMore := parseNodeBBWikiPageList(doc)
+		out = append(out, pages...)
+		if !hasMore || next == "" {
+			return out, nil
+		}
+		after = next
+	}
+}
+
 func (c *nodeBBClient) createTopic(cid int, title, content, summary string) (nodeBBPost, error) {
 	body := map[string]any{"cid": cid, "title": title, "content": content}
 	if summary != "" {
@@ -801,6 +973,46 @@ func parseNodeBBPost(doc any) nodeBBPost {
 	}
 }

+func parseNodeBBWikiPageList(doc any) ([]nodeBBWikiPage, string, bool) {
+	for {
+		m, ok := doc.(map[string]any)
+		if !ok {
+			return nil, "", false
+		}
+		if response, ok := m["response"]; ok {
+			doc = response
+			continue
+		}
+		pagesRaw, _ := m["pages"].([]any)
+		pages := make([]nodeBBWikiPage, 0, len(pagesRaw))
+		for _, raw := range pagesRaw {
+			row, ok := raw.(map[string]any)
+			if !ok {
+				continue
+			}
+			pages = append(pages, nodeBBWikiPage{
+				TID:       firstInt(row, "tid"),
+				Title:     firstString(row, "title"),
+				TitleLeaf: firstString(row, "titleLeaf", "title_leaf"),
+				Slug:      firstString(row, "slug"),
+				WikiPath:  firstString(row, "wikiPath", "wiki_path"),
+			})
+		}
+		return pages, firstString(m, "nextCursor", "next_cursor"), boolFromAny(m["hasMore"])
+	}
+}
+
+func boolFromAny(value any) bool {
+	switch typed := value.(type) {
+	case bool:
+		return typed
+	case string:
+		return typed == "true" || typed == "1"
+	default:
+		return false
+	}
+}
+
 func firstInt(m map[string]any, keys ...string) int {
 	for _, key := range keys {
 		if value := intFromAny(m[key]); value != 0 {
@@ -92,6 +92,11 @@ func TestDeployWikiCreatesNodeBBTopicAndWritesManifest(t *testing.T) {
 		if got := r.Header.Get("Authorization"); got != "Bearer nodebb-token" {
 			t.Fatalf("unexpected authorization header %q", got)
 		}
+		if r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/5/pages" {
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}})
+			return
+		}
 		if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" {
 			t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
 		}
@@ -154,6 +159,11 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) {
 	}

 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/5/pages" {
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []any{}, "hasMore": false}})
+			return
+		}
 		if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" {
 			t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
 		}
@@ -196,6 +206,107 @@ func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) {
 	}
 }

+func TestDeployWikiAdoptsExistingNodeBBPageWhenManifestIsMissingWithoutCreate(t *testing.T) {
+	root := t.TempDir()
+	sourceDir := filepath.Join(root, "pages")
+	if err := os.MkdirAll(filepath.Join(sourceDir, "classes"), 0755); err != nil {
+		t.Fatalf("create source dir: %v", err)
+	}
+	generated := `<!-- sow-topdata-wiki:page=classes:acolyte -->
+<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
+<h1>Acolyte</h1>
+<p>Generated acolyte page</p>
+<!-- sow-topdata-wiki:managed:end -->
+<!-- sow-topdata-wiki:manual:start id="notes" -->
+<h2>Notes</h2>
+<p></p>
+<!-- sow-topdata-wiki:manual:end id="notes" -->
+`
+	if err := os.WriteFile(filepath.Join(sourceDir, "classes", "acolyte.html"), []byte(generated), 0644); err != nil {
+		t.Fatalf("write source page: %v", err)
+	}
+	old := `<!-- sow-topdata-wiki:page=classes:acolyte -->
+<!-- sow-topdata-wiki:managed:start hash="sha256:old" -->
+<h1>Acolyte</h1>
+<p>Old acolyte page</p>
+<!-- sow-topdata-wiki:managed:end -->
+<!-- sow-topdata-wiki:manual:start id="notes" -->
+<h2>Notes</h2>
+<p>Keep this remote note.</p>
+<!-- sow-topdata-wiki:manual:end id="notes" -->
+`
+
+	createCalls := 0
+	var updated string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/9/pages":
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]any{
+				"response": map[string]any{
+					"pages": []map[string]any{
+						{
+							"tid":       77,
+							"title":     "Acolyte",
+							"titleLeaf": "Acolyte",
+							"slug":      "77/acolyte",
+							"wikiPath":  "/wiki/classes/acolyte",
+						},
+					},
+					"hasMore": false,
+				},
+			})
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v3/topics/77":
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"tid": 77, "mainPid": 88}})
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/88":
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77, "content": old}})
+		case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/88":
+			var req struct {
+				Content string `json:"content"`
+			}
+			if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+				t.Fatalf("decode update request: %v", err)
+			}
+			updated = req.Content
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 88, "tid": 77}})
+		case r.Method == http.MethodPost && r.URL.Path == "/api/v3/topics":
+			createCalls++
+			t.Fatalf("existing wiki page should be adopted and updated, not recreated")
+		default:
+			t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
+		}
+	}))
+	defer server.Close()
+
+	manifestPath := filepath.Join(root, "deploy-manifest.json")
+	result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
+		SourceDir:      sourceDir,
+		Endpoint:       server.URL,
+		Token:          "nodebb-token",
+		ManifestPath:   manifestPath,
+		CategoryIDs:    map[string]int{"classes": 9},
+		StalePolicy:    "report",
+		NotesDelimiter: "",
+	}, nil)
+	if err != nil {
+		t.Fatalf("DeployWikiWithOptions adopt failed: %v", err)
+	}
+	if result.Created != 0 || result.Updated != 1 || createCalls != 0 {
+		t.Fatalf("expected one adopted update and no creates, result=%#v createCalls=%d", result, createCalls)
+	}
+	if !containsAll(updated, "<p>Generated acolyte page</p>", "<p>Keep this remote note.</p>") {
+		t.Fatalf("expected adopted update to merge generated and remote manual content:\n%s", updated)
+	}
+	manifest := loadDeployManifest(manifestPath)
+	entry := manifest.Pages["classes:acolyte"]
+	if entry.TID != 77 || entry.PID != 88 || entry.CID != 9 || entry.Hash != computeManagedHash(generated) {
+		t.Fatalf("unexpected adopted manifest entry: %#v", entry)
+	}
+}
+
 func TestDeployWikiMergesHTMLManagedAndManualRegions(t *testing.T) {
 	root := t.TempDir()
 	sourceDir := filepath.Join(root, "pages")