ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Fixing release queue

Browse Source
This commit is contained in:
archvillainette
2026-05-23 21:28:52 +02:00
parent ed2ad4f9a0
commit 3d42fc70bc
3 changed files with 125 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/topdata/wiki_deploy.go
+31 -10
View File
@@ -170,14 +170,18 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
manifestPath = filepath.Join(filepath.Dir(opts.SourceDir), p.EffectiveConfig().TopData.Wiki.DeployManifest)
}
progress(fmt.Sprintf("Collecting local wiki pages from %s", opts.SourceDir))
pages, err := collectLocalPages(opts.SourceDir, namespaces)
if err != nil {
return deployResult{}, err
}
progress(fmt.Sprintf("Loaded %d local wiki page(s)", len(pages)))
manifest := loadDeployManifest(manifestPath)
progress(fmt.Sprintf("Loaded wiki deploy manifest %s with %d page mapping(s)", manifestPath, len(manifest.Pages)))
client := newNodeBBClient(opts.Endpoint, opts.Token)
plans, result, nextManifest, err := planNodeBBDeploy(pages, manifest, opts, client)
progress(fmt.Sprintf("Planning NodeBB wiki deploy for %d local page(s)", len(pages)))
plans, result, nextManifest, err := planNodeBBDeploy(pages, manifest, opts, client, progress)
if err != nil {
return deployResult{}, err
}
@@ -451,7 +455,10 @@ func isDir(path string) bool {
return err == nil && info.IsDir()
}
func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) {
func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManifest, opts DeployWikiOptions, client *nodeBBClient, progress func(string)) ([]wikiDeployPlan, deployResult, wikiDeployManifest, error) {
if progress == nil {
progress = func(string) {}
}
pageIDs := sortedKeysFromMap(pages)
next := wikiDeployManifest{Version: "nodebb-v1", Pages: map[string]wikiDeployManifestPage{}}
remotePagesByCID := map[int][]nodeBBWikiPage{}
@@ -461,7 +468,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
if !opts.AllowCreates && len(pageIDs) > 0 {
return nil, result, next, errors.New("wiki managed namespace reset requires --create so current generated pages can be recreated")
}
resetPlans, purged, err := planManagedNamespaceReset(opts, client)
resetPlans, purged, err := planManagedNamespaceReset(opts, client, progress)
if err != nil {
return nil, result, next, err
}
@@ -475,7 +482,10 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
}
}
for _, pageID := range pageIDs {
for i, pageID := range pageIDs {
if i == 0 || (i+1)%100 == 0 || i+1 == len(pageIDs) {
progress(fmt.Sprintf("Planning NodeBB wiki page %d/%d: %s", i+1, len(pageIDs), pageID))
}
page := pages[pageID]
entry := manifest.Pages[pageID]
entry.Hash = page.Hash
@@ -491,7 +501,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
if entry.PID == 0 {
if entry.CID != 0 {
adopted, ok, err := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client)
adopted, ok, err := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client, progress)
if err != nil {
return nil, result, next, err
}
@@ -518,7 +528,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
}
if entry.TID != 0 && entry.CID != 0 && shouldCheckRemoteTopicTitle(manifest.Pages[pageID], page.Title, entry.TopicTitle, opts.TitlePrefixMinLength) {
remotePage, ok, err := findMappedRemotePage(entry, remotePagesByCID, client)
remotePage, ok, err := findMappedRemotePage(entry, remotePagesByCID, client, progress)
if err != nil {
return nil, result, next, err
}
@@ -540,7 +550,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
entry.TID = 0
entry.PID = 0
if entry.CID != 0 {
adopted, ok, adoptErr := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client)
adopted, ok, adoptErr := adoptExistingNodeBBPage(page, entry.CID, remotePagesByCID, client, progress)
if adoptErr != nil {
return nil, result, next, adoptErr
}
@@ -633,7 +643,7 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
return plans, result, next, nil
}
func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient) ([]wikiDeployPlan, int, error) {
func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient, progress func(string)) ([]wikiDeployPlan, int, error) {
namespaces := slices.Clone(opts.Namespaces)
if len(namespaces) == 0 {
for namespace := range opts.CategoryIDs {
@@ -649,6 +659,7 @@ func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient) ([]
if cid == 0 {
continue
}
progress(fmt.Sprintf("Listing NodeBB wiki namespace category %d for managed reset", cid))
remotePages, err := client.listNamespacePages(cid)
if err != nil {
return nil, 0, err
@@ -679,10 +690,11 @@ func planManagedNamespaceReset(opts DeployWikiOptions, client *nodeBBClient) ([]
return plans, len(plans), nil
}
func findMappedRemotePage(entry wikiDeployManifestPage, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient) (nodeBBWikiPage, bool, error) {
func findMappedRemotePage(entry wikiDeployManifestPage, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient, progress func(string)) (nodeBBWikiPage, bool, error) {
remotePages, ok := remotePagesByCID[entry.CID]
if !ok {
var err error
progress(fmt.Sprintf("Listing NodeBB wiki namespace category %d", entry.CID))
remotePages, err = client.listNamespacePages(entry.CID)
if err != nil {
return nodeBBWikiPage{}, false, err
@@ -748,10 +760,11 @@ func recoverCreateCollision(plan wikiDeployPlan, manifest wikiDeployManifest, cl
return true, nil
}
func adoptExistingNodeBBPage(page wikiDeployPage, cid int, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient) (wikiDeployManifestPage, bool, error) {
func adoptExistingNodeBBPage(page wikiDeployPage, cid int, remotePagesByCID map[int][]nodeBBWikiPage, client *nodeBBClient, progress func(string)) (wikiDeployManifestPage, bool, error) {
remotePages, ok := remotePagesByCID[cid]
if !ok {
var err error
progress(fmt.Sprintf("Listing NodeBB wiki namespace category %d", cid))
remotePages, err = client.listNamespacePages(cid)
if err != nil {
return wikiDeployManifestPage{}, false, err
@@ -1406,6 +1419,7 @@ func (c *nodeBBClient) searchNamespacePages(cid int, query string) ([]nodeBBWiki
func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nodeBBWikiPage, error) {
var out []nodeBBWikiPage
after := ""
seenCursors := map[string]struct{}{}
for {
path := fmt.Sprintf("/api/v3/plugins/westgate-wiki/namespace/%d/pages?limit=80", cid)
if query != "" {
@@ -1423,6 +1437,13 @@ func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nod
if !hasMore || next == "" {
return out, nil
}
if after != "" && next == after {
return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next)
}
if _, ok := seenCursors[next]; ok {
return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next)
}
seenCursors[next] = struct{}{}
after = next
}
}
internal/topdata/wiki_deploy_test.go
+88
View File
@@ -76,6 +76,94 @@ func TestDeployWikiDryRunDoesNotWriteRemoteOrManifest(t *testing.T) {
}
}
func TestDeployWikiReportsPlanningProgressBeforeRemoteWork(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
generated := `<!-- sow-topdata-wiki:page=skills:athletics -->
<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
<h1>Athletics</h1>
<!-- sow-topdata-wiki:managed:end -->
`
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil {
t.Fatalf("write source page: %v", err)
}
progress := []string{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/api/v3/plugins/westgate-wiki/namespace/3/pages":
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pages": []map[string]any{}, "hasMore": false}})
default:
t.Fatalf("unexpected request %s %s", r.Method, r.URL.String())
}
}))
defer server.Close()
_, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: filepath.Join(root, "deploy-manifest.json"),
CategoryIDs: map[string]int{"skills": 3},
DryRun: true,
AllowCreates: true,
}, func(message string) {
progress = append(progress, message)
})
if err != nil {
t.Fatalf("DeployWikiWithOptions dry run failed: %v", err)
}
joined := strings.Join(progress, "\n")
for _, expected := range []string{
"Collecting local wiki pages from",
"Loaded 1 local wiki page(s)",
"Loaded wiki deploy manifest",
"Planning NodeBB wiki deploy for 1 local page(s)",
"Listing NodeBB wiki namespace category 3",
} {
if !strings.Contains(joined, expected) {
t.Fatalf("expected progress %q, got:\n%s", expected, joined)
}
}
}
func TestNodeBBNamespacePaginationRejectsRepeatedCursor(t *testing.T) {
requests := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests++
if requests > 2 {
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"response": map[string]any{
"pages": []map[string]any{},
"hasMore": true,
"nextCursor": "same-cursor",
},
})
}))
defer server.Close()
client := newNodeBBClient(server.URL, "nodebb-token")
_, err := client.listNamespacePages(9)
if err == nil {
t.Fatal("expected repeated cursor error")
}
if !strings.Contains(err.Error(), "repeated pagination cursor") {
t.Fatalf("expected repeated cursor error, got %v", err)
}
if requests != 2 {
t.Fatalf("expected pagination to stop after repeated cursor, got %d requests", requests)
}
}
func TestCollectLocalPagesReadsGeneratedPageIDFromMarkerWhenPathIsTitleBased(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")