From 58b8955e07d5a21b8fd8f0abd5eba13c079a24f4 Mon Sep 17 00:00:00 2001
From: vickydotbat <vickydotbat@tutamail.com>
Date: Sun, 24 May 2026 09:25:11 +0200
Subject: [PATCH] Pagination deploy fix

---
 internal/topdata/wiki_deploy.go      | 35 ++++++++++++++++++-
 internal/topdata/wiki_deploy_test.go | 51 ++++++++++++++++++++++------
 2 files changed, 75 insertions(+), 11 deletions(-)

diff --git a/internal/topdata/wiki_deploy.go b/internal/topdata/wiki_deploy.go
index 57dec19..57e01fa 100644
--- a/internal/topdata/wiki_deploy.go
+++ b/internal/topdata/wiki_deploy.go
@@ -1439,6 +1439,7 @@ func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nod
 	var out []nodeBBWikiPage
 	after := ""
 	seenCursors := map[string]struct{}{}
+	seenPages := map[string]struct{}{}
 	for {
 		path := fmt.Sprintf("/api/v3/plugins/westgate-wiki/namespace/%d/pages?limit=80", cid)
 		if query != "" {
@@ -1452,14 +1453,34 @@ func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nod
 			return nil, err
 		}
 		pages, next, hasMore := parseNodeBBWikiPageList(doc)
-		out = append(out, pages...)
+		newPages := 0
+		for _, page := range pages {
+			identity := nodeBBWikiPageIdentity(page)
+			if identity == "" {
+				out = append(out, page)
+				newPages++
+				continue
+			}
+			if _, ok := seenPages[identity]; ok {
+				continue
+			}
+			seenPages[identity] = struct{}{}
+			out = append(out, page)
+			newPages++
+		}
 		if !hasMore || next == "" {
 			return out, nil
 		}
 		if after != "" && next == after {
+			if newPages == 0 {
+				return out, nil
+			}
 			return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next)
 		}
 		if _, ok := seenCursors[next]; ok {
+			if newPages == 0 {
+				return out, nil
+			}
 			return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next)
 		}
 		seenCursors[next] = struct{}{}
@@ -1467,6 +1488,18 @@ func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nod
 	}
 }
 
+func nodeBBWikiPageIdentity(page nodeBBWikiPage) string {
+	if page.TID != 0 {
+		return fmt.Sprintf("tid:%d", page.TID)
+	}
+	for _, value := range []string{page.WikiPath, page.CanonicalPath, page.Slug, page.Title} {
+		if trimmed := strings.TrimSpace(value); trimmed != "" {
+			return strings.ToLower(trimmed)
+		}
+	}
+	return ""
+}
+
 func (c *nodeBBClient) createTopic(cid int, title, content, summary string) (nodeBBPost, error) {
 	body := map[string]any{"cid": cid, "title": title, "content": content, "sourceContent": content}
 	if summary != "" {
diff --git a/internal/topdata/wiki_deploy_test.go b/internal/topdata/wiki_deploy_test.go
index 1ee579b..15b784f 100644
--- a/internal/topdata/wiki_deploy_test.go
+++ b/internal/topdata/wiki_deploy_test.go
@@ -226,18 +226,52 @@ func TestDeployWikiReportsLiveExecutionProgress(t *testing.T) {
 	}
 }
 
-func TestNodeBBNamespacePaginationRejectsRepeatedCursor(t *testing.T) {
+func TestNodeBBNamespacePaginationStopsWhenRepeatedCursorReturnsNoNewPages(t *testing.T) {
 	requests := 0
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		requests++
-		if requests > 2 {
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
 		w.Header().Set("Content-Type", "application/json")
 		_ = json.NewEncoder(w).Encode(map[string]any{
 			"response": map[string]any{
-				"pages":      []map[string]any{},
+				"pages": []map[string]any{
+					{"tid": 2461, "title": "Favored Enemy: Elves", "titleLeaf": "Favored Enemy: Elves"},
+				},
+				"hasMore":    true,
+				"nextCursor": "same-cursor",
+			},
+		})
+	}))
+	defer server.Close()
+
+	client := newNodeBBClient(server.URL, "nodebb-token")
+	pages, err := client.listNamespacePages(9)
+	if err != nil {
+		t.Fatalf("expected repeated cursor with duplicate pages to stop cleanly, got %v", err)
+	}
+	if len(pages) != 1 || pages[0].TID != 2461 {
+		t.Fatalf("expected one unique page from repeated cursor response, got %#v", pages)
+	}
+	if requests != 2 {
+		t.Fatalf("expected pagination to stop after repeated cursor, got %d requests", requests)
+	}
+}
+
+func TestNodeBBNamespacePaginationRejectsRepeatedCursorWithNewPages(t *testing.T) {
+	requests := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests++
+		w.Header().Set("Content-Type", "application/json")
+		pages := []map[string]any{
+			{"tid": 1, "title": "First", "titleLeaf": "First"},
+		}
+		if requests == 2 {
+			pages = []map[string]any{
+				{"tid": 2, "title": "Second", "titleLeaf": "Second"},
+			}
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"response": map[string]any{
+				"pages":      pages,
 				"hasMore":    true,
 				"nextCursor": "same-cursor",
 			},
@@ -248,14 +282,11 @@ func TestNodeBBNamespacePaginationRejectsRepeatedCursor(t *testing.T) {
 	client := newNodeBBClient(server.URL, "nodebb-token")
 	_, err := client.listNamespacePages(9)
 	if err == nil {
-		t.Fatal("expected repeated cursor error")
+		t.Fatal("expected repeated cursor with new pages to fail")
 	}
 	if !strings.Contains(err.Error(), "repeated pagination cursor") {
 		t.Fatalf("expected repeated cursor error, got %v", err)
 	}
-	if requests != 2 {
-		t.Fatalf("expected pagination to stop after repeated cursor, got %d requests", requests)
-	}
 }
 
 func TestCollectLocalPagesReadsGeneratedPageIDFromMarkerWhenPathIsTitleBased(t *testing.T) {