Fixing release queue

internal/app/app.go

@@ -1026,6 +1026,12 @@ func (c *topdataConsole) phaseLabel(message string) string {
 		return "building wiki pages"
 	case strings.HasPrefix(message, "NodeBB wiki plan: "):
 		return "planning wiki deploy"
+	case strings.HasPrefix(message, "Planning NodeBB wiki deploy"):
+		return "planning wiki deploy"
+	case strings.HasPrefix(message, "Planning NodeBB wiki page "):
+		return "planning wiki pages"
+	case strings.HasPrefix(message, "Listing NodeBB wiki namespace category "):
+		return "listing remote wiki pages"
 	case strings.HasPrefix(message, "Collecting "):
 		return "collecting pages"
 	default:

internal/topdata/wiki_deploy.go

@@ -170,14 +170,18 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
 		manifestPath = filepath.Join(filepath.Dir(opts.SourceDir), p.EffectiveConfig().TopData.Wiki.DeployManifest)
 	}
 
+	progress(fmt.Sprintf("Collecting local wiki pages from %s", opts.SourceDir))
 	pages, err := collectLocalPages(opts.SourceDir, namespaces)
 	if err != nil {
 		return deployResult{}, err
 	}
+	progress(fmt.Sprintf("Loaded %d local wiki page(s)", len(pages)))
 	manifest := loadDeployManifest(manifestPath)
+	progress(fmt.Sprintf("Loaded wiki deploy manifest %s with %d page mapping(s)", manifestPath, len(manifest.Pages)))
 	client := newNodeBBClient(opts.Endpoint, opts.Token)
 
-	plans, result, nextManifest, err := planNodeBBDeploy(pages, manifest, opts, client)
+	progress(fmt.Sprintf("Planning NodeBB wiki deploy for %d local page(s)", len(pages)))
+	plans, result, nextManifest, err := planNodeBBDeploy(pages, manifest, opts, client, progress)
 	if err != nil {
 		return deployResult{}, err
 	}
@@ -451,7 +455,10 @@ func isDir(path string) bool {
 	return err == nil && info.IsDir()
 }
 
-func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) {
+func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient, progress func(string)) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) {
+	if progress == nil {
+		progress = func(string) {}
+	}
 	pageIDs := sortedKeysFromMap(pages)
 	next := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
 	remotePagesByCID := map[int][]nodeBBWikiPage{}
@@ -461,7 +468,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
 		if !opts.AllowCreates && len(pageIDs) > 0 {
 			return nil, result, next, errors.New("wiki managed namespace reset requires --create so current generated pages can be recreated")
 		}
-		resetPlans, purged, err := planManagedNamespaceReset(opts, client)
+		resetPlans, purged, err := planManagedNamespaceReset(opts, client, progress)
 		if err != nil {
 			return nil, result, next, err
 		}
@@ -475,7 +482,10 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
 		}
 	}
 
-	for _, pageID := range pageIDs {
+	for i, pageID := range pageIDs {
+		if i == 0 || (i+1)%100 == 0 || i+1 == len(pageIDs) {
+			progress(fmt.Sprintf("Planning NodeBB wiki page %d/%d: %s", i+1, len(pageIDs), pageID))
+		}
 		page := pages[pageID]
 		entry := manifest.Pages[pageID]
 		entry.Hash = page.Hash
@@ -491,7 +501,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
 
 		if entry.PID == 0 {
 			if entry.CID != 0 {
-				adopted, ok, err := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client)
+				adopted, ok, err := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client, progress)
 				if err != nil {
 					return nil, result, next, err
 				}
@@ -518,7 +528,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
 		}
 
 		if entry.TID != 0 && entry.CID != 0 && shouldCheckRemoteTopicTitle(manifest.Pages[pageID], page.Title, entry.TopicTitle, opts.TitlePrefixMinLength) {
-			remotePage, ok, err := findMappedRemotePage(entry, remotePagesByCID, client)
+			remotePage, ok, err := findMappedRemotePage(entry, remotePagesByCID, client, progress)
 			if err != nil {
 				return nil, result, next, err
 			}
@@ -540,7 +550,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
 			entry.TID = 0
 			entry.PID = 0
 			if entry.CID != 0 {
-				adopted, ok, adoptErr := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client)
+				adopted, ok, adoptErr := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client, progress)
 				if adoptErr != nil {
 					return nil, result, next, adoptErr
 				}
@@ -633,7 +643,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
 	return plans, result, next, nil
 }
 
-func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, int, error) {
+func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient, progress func(string)) ([]wikiDeployPlan, int, error) {
 	namespaces := slices.Clone(opts.Namespaces)
 	if len(namespaces) == 0 {
 		for namespace := range opts.CategoryIDs {
@@ -649,6 +659,7 @@ func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient) ([]
 		if cid == 0 {
 			continue
 		}
+		progress(fmt.Sprintf("Listing NodeBB wiki namespace category %d for managed reset", cid))
 		remotePages, err := client.listNamespacePages(cid)
 		if err != nil {
 			return nil, 0, err
@@ -679,10 +690,11 @@ func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient) ([]
 	return plans, len(plans), nil
 }
 
-func findMappedRemotePage(entry wikiDeployManifestPage, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient) (nodeBBWikiPage, bool, error) {
+func findMappedRemotePage(entry wikiDeployManifestPage, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient, progress func(string)) (nodeBBWikiPage, bool, error) {
 	remotePages, ok := remotePagesByCID[entry.CID]
 	if !ok {
 		var err error
+		progress(fmt.Sprintf("Listing NodeBB wiki namespace category %d", entry.CID))
 		remotePages, err = client.listNamespacePages(entry.CID)
 		if err != nil {
 			return nodeBBWikiPage{}, false, err
@@ -748,10 +760,11 @@ func recoverCreateCollision(plan wikiDeployPlan, manifest wikiDeployManifest, cl
 	return true, nil
 }
 
-func adoptExistingNodeBBPage(page wikiDeployPage, cid int, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient) (wikiDeployManifestPage, bool, error) {
+func adoptExistingNodeBBPage(page wikiDeployPage, cid int, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient, progress func(string)) (wikiDeployManifestPage, bool, error) {
 	remotePages, ok := remotePagesByCID[cid]
 	if !ok {
 		var err error
+		progress(fmt.Sprintf("Listing NodeBB wiki namespace category %d", cid))
 		remotePages, err = client.listNamespacePages(cid)
 		if err != nil {
 			return wikiDeployManifestPage{}, false, err
@@ -1406,6 +1419,7 @@ func (c *nodeBBClient) searchNamespacePages(cid int, query string) ([]nodeBBWiki
 func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nodeBBWikiPage, error) {
 	var out []nodeBBWikiPage
 	after := ""
+	seenCursors := map[string]struct{}{}
 	for {
 		path := fmt.Sprintf("/api/v3/plugins/westgate-wiki/namespace/%d/pages?limit=80", cid)
 		if query != "" {
@@ -1423,6 +1437,13 @@ func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nod
 		if !hasMore || next == "" {
 			return out, nil
 		}
+		if after != "" && next == after {
+			return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next)
+		}
+		if _, ok := seenCursors[next]; ok {
+			return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next)
+		}
+		seenCursors[next] = struct{}{}
 		after = next
 	}
 }

internal/topdata/wiki_deploy_test.go

@@ -76,6 +76,94 @@ func TestDeployWikiDryRunDoesNotWriteRemoteOrManifest(t *testing.T) {
 	}
 }
 
+func TestDeployWikiReportsPlanningProgressBeforeRemoteWork(t *testing.T) {
+	root := t.TempDir()
+	sourceDir := filepath.Join(root, "pages")
+	if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
+		t.Fatalf("create source dir: %v", err)
+	}
+	generated := `<!-- sow-topdata-wiki:page=skills:athletics -->
+<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
+<h1>Athletics</h1>
+<!-- sow-topdata-wiki:managed:end -->
+`
+	if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil {
+		t.Fatalf("write source page: %v", err)
+	}
+
+	progress := []string{}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/3/pages":
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []map[string]any{}, "hasMore": false}})
+		default:
+			t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
+		}
+	}))
+	defer server.Close()
+
+	_, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
+		SourceDir:    sourceDir,
+		Endpoint:     server.URL,
+		Token:        "nodebb-token",
+		ManifestPath: filepath.Join(root, "deploy-manifest.json"),
+		CategoryIDs:  map[string]int{"skills": 3},
+		DryRun:       true,
+		AllowCreates: true,
+	}, func(message string) {
+		progress = append(progress, message)
+	})
+	if err != nil {
+		t.Fatalf("DeployWikiWithOptions dry run failed: %v", err)
+	}
+
+	joined := strings.Join(progress, "\n")
+	for _, expected := range []string{
+		"Collecting local wiki pages from",
+		"Loaded 1 local wiki page(s)",
+		"Loaded wiki deploy manifest",
+		"Planning NodeBB wiki deploy for 1 local page(s)",
+		"Listing NodeBB wiki namespace category 3",
+	} {
+		if !strings.Contains(joined, expected) {
+			t.Fatalf("expected progress %q, got:\n%s", expected, joined)
+		}
+	}
+}
+
+func TestNodeBBNamespacePaginationRejectsRepeatedCursor(t *testing.T) {
+	requests := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests++
+		if requests > 2 {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"response": map[string]any{
+				"pages":      []map[string]any{},
+				"hasMore":    true,
+				"nextCursor": "same-cursor",
+			},
+		})
+	}))
+	defer server.Close()
+
+	client := newNodeBBClient(server.URL, "nodebb-token")
+	_, err := client.listNamespacePages(9)
+	if err == nil {
+		t.Fatal("expected repeated cursor error")
+	}
+	if !strings.Contains(err.Error(), "repeated pagination cursor") {
+		t.Fatalf("expected repeated cursor error, got %v", err)
+	}
+	if requests != 2 {
+		t.Fatalf("expected pagination to stop after repeated cursor, got %d requests", requests)
+	}
+}
+
 func TestCollectLocalPagesReadsGeneratedPageIDFromMarkerWhenPathIsTitleBased(t *testing.T) {
 	root := t.TempDir()
 	sourceDir := filepath.Join(root, "pages")