From 3d42fc70bc3f5d5f5a69171b3e3e0d09ae9456e0 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Sat, 23 May 2026 21:28:52 +0200 Subject: [PATCH] Fixing release queue --- internal/app/app.go | 6 ++ internal/topdata/wiki_deploy.go | 41 +++++++++---- internal/topdata/wiki_deploy_test.go | 88 ++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 10 deletions(-) diff --git a/internal/app/app.go b/internal/app/app.go index 7b1a07f..c525de2 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1026,6 +1026,12 @@ func (c *topdataConsole) phaseLabel(message string) string { return "building wiki pages" case strings.HasPrefix(message, "NodeBB wiki plan: "): return "planning wiki deploy" + case strings.HasPrefix(message, "Planning NodeBB wiki deploy"): + return "planning wiki deploy" + case strings.HasPrefix(message, "Planning NodeBB wiki page "): + return "planning wiki pages" + case strings.HasPrefix(message, "Listing NodeBB wiki namespace category "): + return "listing remote wiki pages" case strings.HasPrefix(message, "Collecting "): return "collecting pages" default: diff --git a/internal/topdata/wiki_deploy.go b/internal/topdata/wiki_deploy.go index d73cc0c..49c1575 100644 --- a/internal/topdata/wiki_deploy.go +++ b/internal/topdata/wiki_deploy.go @@ -170,14 +170,18 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress manifestPath = filepath.Join(filepath.Dir(opts.SourceDir), p.EffectiveConfig().TopData.Wiki.DeployManifest) } + progress(fmt.Sprintf("Collecting local wiki pages from %s", opts.SourceDir)) pages, err := collectLocalPages(opts.SourceDir, namespaces) if err != nil { return deployResult{}, err } + progress(fmt.Sprintf("Loaded %d local wiki page(s)", len(pages))) manifest := loadDeployManifest(manifestPath) + progress(fmt.Sprintf("Loaded wiki deploy manifest %s with %d page mapping(s)", manifestPath, len(manifest.Pages))) client := newNodeBBClient(opts.Endpoint, opts.Token) - plans, result, nextManifest, err := planNodeBBDeploy(pages, manifest, opts, client) + progress(fmt.Sprintf("Planning NodeBB wiki deploy for %d local page(s)", len(pages))) + plans, result, nextManifest, err := planNodeBBDeploy(pages, manifest, opts, client, progress) if err != nil { return deployResult{}, err } @@ -451,7 +455,10 @@ func isDir(path string) bool { return err == nil && info.IsDir() } -func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) { +func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient, progress func(string)) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) { + if progress == nil { + progress = func(string) {} + } pageIDs := sortedKeysFromMap(pages) next := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}} remotePagesByCID := map[int][]nodeBBWikiPage{} @@ -461,7 +468,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife if !opts.AllowCreates && len(pageIDs) > 0 { return nil, result, next, errors.New("wiki managed namespace reset requires --create so current generated pages can be recreated") } - resetPlans, purged, err := planManagedNamespaceReset(opts, client) + resetPlans, purged, err := planManagedNamespaceReset(opts, client, progress) if err != nil { return nil, result, next, err } @@ -475,7 +482,10 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife } } - for _, pageID := range pageIDs { + for i, pageID := range pageIDs { + if i == 0 || (i+1)%100 == 0 || i+1 == len(pageIDs) { + progress(fmt.Sprintf("Planning NodeBB wiki page %d/%d: %s", i+1, len(pageIDs), pageID)) + } page := pages[pageID] entry := manifest.Pages[pageID] entry.Hash = page.Hash @@ -491,7 +501,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife if entry.PID == 0 { if entry.CID != 0 { - adopted, ok, err := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client) + adopted, ok, err := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client, progress) if err != nil { return nil, result, next, err } @@ -518,7 +528,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife } if entry.TID != 0 && entry.CID != 0 && shouldCheckRemoteTopicTitle(manifest.Pages[pageID], page.Title, entry.TopicTitle, opts.TitlePrefixMinLength) { - remotePage, ok, err := findMappedRemotePage(entry, remotePagesByCID, client) + remotePage, ok, err := findMappedRemotePage(entry, remotePagesByCID, client, progress) if err != nil { return nil, result, next, err } @@ -540,7 +550,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife entry.TID = 0 entry.PID = 0 if entry.CID != 0 { - adopted, ok, adoptErr := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client) + adopted, ok, adoptErr := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client, progress) if adoptErr != nil { return nil, result, next, adoptErr } @@ -633,7 +643,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife return plans, result, next, nil } -func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, int, error) { +func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient, progress func(string)) ([]wikiDeployPlan, int, error) { namespaces := slices.Clone(opts.Namespaces) if len(namespaces) == 0 { for namespace := range opts.CategoryIDs { @@ -649,6 +659,7 @@ func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient) ([] if cid == 0 { continue } + progress(fmt.Sprintf("Listing NodeBB wiki namespace category %d for managed reset", cid)) remotePages, err := client.listNamespacePages(cid) if err != nil { return nil, 0, err @@ -679,10 +690,11 @@ func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient) ([] return plans, len(plans), nil } -func findMappedRemotePage(entry wikiDeployManifestPage, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient) (nodeBBWikiPage, bool, error) { +func findMappedRemotePage(entry wikiDeployManifestPage, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient, progress func(string)) (nodeBBWikiPage, bool, error) { remotePages, ok := remotePagesByCID[entry.CID] if !ok { var err error + progress(fmt.Sprintf("Listing NodeBB wiki namespace category %d", entry.CID)) remotePages, err = client.listNamespacePages(entry.CID) if err != nil { return nodeBBWikiPage{}, false, err @@ -748,10 +760,11 @@ func recoverCreateCollision(plan wikiDeployPlan, manifest wikiDeployManifest, cl return true, nil } -func adoptExistingNodeBBPage(page wikiDeployPage, cid int, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient) (wikiDeployManifestPage, bool, error) { +func adoptExistingNodeBBPage(page wikiDeployPage, cid int, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient, progress func(string)) (wikiDeployManifestPage, bool, error) { remotePages, ok := remotePagesByCID[cid] if !ok { var err error + progress(fmt.Sprintf("Listing NodeBB wiki namespace category %d", cid)) remotePages, err = client.listNamespacePages(cid) if err != nil { return wikiDeployManifestPage{}, false, err @@ -1406,6 +1419,7 @@ func (c *nodeBBClient) searchNamespacePages(cid int, query string) ([]nodeBBWiki func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nodeBBWikiPage, error) { var out []nodeBBWikiPage after := "" + seenCursors := map[string]struct{}{} for { path := fmt.Sprintf("/api/v3/plugins/westgate-wiki/namespace/%d/pages?limit=80", cid) if query != "" { @@ -1423,6 +1437,13 @@ func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nod if !hasMore || next == "" { return out, nil } + if after != "" && next == after { + return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next) + } + if _, ok := seenCursors[next]; ok { + return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next) + } + seenCursors[next] = struct{}{} after = next } } diff --git a/internal/topdata/wiki_deploy_test.go b/internal/topdata/wiki_deploy_test.go index 6e9ecc6..1c1aa1d 100644 --- a/internal/topdata/wiki_deploy_test.go +++ b/internal/topdata/wiki_deploy_test.go @@ -76,6 +76,94 @@ func TestDeployWikiDryRunDoesNotWriteRemoteOrManifest(t *testing.T) { } } +func TestDeployWikiReportsPlanningProgressBeforeRemoteWork(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := ` + +

Athletics

+ +` + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + + progress := []string{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/3/pages": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []map[string]any{}, "hasMore": false}}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + + _, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: filepath.Join(root, "deploy-manifest.json"), + CategoryIDs: map[string]int{"skills": 3}, + DryRun: true, + AllowCreates: true, + }, func(message string) { + progress = append(progress, message) + }) + if err != nil { + t.Fatalf("DeployWikiWithOptions dry run failed: %v", err) + } + + joined := strings.Join(progress, "\n") + for _, expected := range []string{ + "Collecting local wiki pages from", + "Loaded 1 local wiki page(s)", + "Loaded wiki deploy manifest", + "Planning NodeBB wiki deploy for 1 local page(s)", + "Listing NodeBB wiki namespace category 3", + } { + if !strings.Contains(joined, expected) { + t.Fatalf("expected progress %q, got:\n%s", expected, joined) + } + } +} + +func TestNodeBBNamespacePaginationRejectsRepeatedCursor(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if requests > 2 { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "response": map[string]any{ + "pages": []map[string]any{}, + "hasMore": true, + "nextCursor": "same-cursor", + }, + }) + })) + defer server.Close() + + client := newNodeBBClient(server.URL, "nodebb-token") + _, err := client.listNamespacePages(9) + if err == nil { + t.Fatal("expected repeated cursor error") + } + if !strings.Contains(err.Error(), "repeated pagination cursor") { + t.Fatalf("expected repeated cursor error, got %v", err) + } + if requests != 2 { + t.Fatalf("expected pagination to stop after repeated cursor, got %d requests", requests) + } +} + func TestCollectLocalPagesReadsGeneratedPageIDFromMarkerWhenPathIsTitleBased(t *testing.T) { root := t.TempDir() sourceDir := filepath.Join(root, "pages")