From f57a3c6ff26b14a9b528cfc5c4b2930fa6d738b7 Mon Sep 17 00:00:00 2001 From: vickydotbat Date: Thu, 21 May 2026 14:58:51 +0200 Subject: [PATCH] Generated Wiki Stale Purge Method (#5) Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/5 Co-authored-by: vickydotbat Co-committed-by: vickydotbat --- README.md | 8 +- internal/app/app.go | 7 +- internal/app/app_test.go | 16 +- internal/project/project.go | 2 +- internal/project/project_test.go | 59 +++++++ internal/topdata/wiki_deploy.go | 35 ++++- internal/topdata/wiki_deploy_test.go | 227 +++++++++++++++++++++++++++ 7 files changed, 342 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index f5511db..191b174 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ sow-toolkit build-top-package [--force] sow-toolkit compare-topdata sow-toolkit convert-topdata <2da-to-json|2da-to-module|json-to-2da> ... sow-toolkit build-wiki [--force] -sow-toolkit deploy-wiki [--source-dir ] [--endpoint ] [--token ] [--version ] [--namespace ] [--category ] [--manifest ] [--dry-run] [--create] [--force] +sow-toolkit deploy-wiki [--source-dir ] [--endpoint ] [--token ] [--version ] [--namespace ] [--category ] [--manifest ] [--stale-policy ] [--dry-run] [--create] [--force] sow-toolkit build-changelog [--config ] [--output ] [--current-tag ] [--previous-tag ] [--api-base-url ] [--token ] ``` @@ -413,7 +413,11 @@ rejects a create because the clean wiki URL already exists, `deploy-wiki` re-queries the namespace by page leaf, adopts the matching topic, and updates it instead. Updates and stale-page archives acquire a Westgate Wiki edit lock before saving, then send the returned `wikiEditLockToken` with the NodeBB post -edit request. +edit request. Stale generated pages are reported by default. +`--stale-policy archive` rewrites stale generated pages in place as archived +managed content. `--stale-policy purge` permanently removes only stale generated +topics already tracked in the wiki deploy manifest. Dry-run that policy before +the live run; it never discovers manual wiki pages from NodeBB for purge. For newly created NodeBB topics, `topdata.wiki.title_prefix_min_length` controls when the deployer prefixes the namespace in the topic title. The diff --git a/internal/app/app.go b/internal/app/app.go index fd7362e..c9c6418 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1089,7 +1089,7 @@ func (c *topdataConsole) emitWikiBuildResult(outputDir string, pageCount int, st fmt.Fprintf(c.stdout, "wiki status: %s\n", status) } -func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skipped, stale, archived, drifted int, manifest string) { +func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skipped, stale, archived, purged, drifted int, manifest string) { spin.linebreak() fmt.Fprintln(c.stdout, "Deploy Wiki ----------") fmt.Fprintf(c.stdout, "project: %s\n", c.projectName) @@ -1099,6 +1099,7 @@ func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skip fmt.Fprintf(c.stdout, "skipped: %d\n", skipped) fmt.Fprintf(c.stdout, "stale: %d\n", stale) fmt.Fprintf(c.stdout, "archived: %d\n", archived) + fmt.Fprintf(c.stdout, "purged: %d\n", purged) fmt.Fprintf(c.stdout, "drifted: %d\n", drifted) fmt.Fprintf(c.stdout, "manifest: %s\n", c.relPath(manifest)) } @@ -1869,7 +1870,7 @@ func runDeployWiki(ctx context) error { return err } - console.emitWikiDeployResult(result.LocalPages, result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Drifted, result.Manifest) + console.emitWikiDeployResult(result.LocalPages, result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Purged, result.Drifted, result.Manifest) return nil } @@ -1950,7 +1951,7 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO case "--create": opts.AllowCreates = true case "-h", "--help": - return opts, fmt.Errorf("usage: %s [--source-dir ] [--endpoint ] [--token ] [--version ] [--namespace ...] [--category ...] [--manifest ] [--stale-policy ] [--dry-run] [--create] [--force]", commandName) + return opts, fmt.Errorf("usage: %s [--source-dir ] [--endpoint ] [--token ] [--version ] [--namespace ...] [--category ...] [--manifest ] [--stale-policy ] [--dry-run] [--create] [--force]", commandName) default: if value, ok, err := requireInlineFlagValue(arg, "--source-dir"); ok || err != nil { if err != nil { diff --git a/internal/app/app_test.go b/internal/app/app_test.go index ba118fe..02ed9dc 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -246,21 +246,31 @@ func TestTopdataConsoleDebugProgressAndRelativePaths(t *testing.T) { level: logLevelDebug, } - console.progress("NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, drift 0") - console.emitWikiDeployResult(10, 1, 2, 3, 4, 5, 0, "/workspace/project/build/wiki/deploy-manifest.json") + console.progress("NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, purge 6, drift 0") + console.emitWikiDeployResult(10, 1, 2, 3, 4, 5, 6, 0, "/workspace/project/build/wiki/deploy-manifest.json") output := stdout.String() - if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, drift 0") { + if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, purge 6, drift 0") { t.Fatalf("expected debug progress line, got %q", output) } if !strings.Contains(output, "archived: 5") { t.Fatalf("expected archived deploy count, got %q", output) } + if !strings.Contains(output, "purged: 6") { + t.Fatalf("expected purged deploy count, got %q", output) + } if !strings.Contains(output, "manifest: build/wiki/deploy-manifest.json") { t.Fatalf("expected relative deploy manifest path, got %q", output) } } +func TestParseDeployWikiHelpListsPurgeStalePolicy(t *testing.T) { + _, err := parseDeployWikiArgs("deploy-wiki", []string{"--help"}) + if err == nil || !strings.Contains(err.Error(), "--stale-policy ") { + t.Fatalf("expected deploy-wiki help to list purge stale policy, got %v", err) + } +} + func TestProjectConsoleSuppressesBuildModuleProgressInNormalMode(t *testing.T) { var stdout bytes.Buffer console := &projectConsole{ diff --git a/internal/project/project.go b/internal/project/project.go index 71d8f8f..dcb0567 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -608,7 +608,7 @@ func (p *Project) ValidateLayout() error { "topdata.wiki.stale_pages.live_cleanup": effective.TopData.Wiki.StalePages.LiveCleanup, } { switch policy { - case "report", "archive", "unpublish": + case "report", "archive", "purge": default: failures = append(failures, fmt.Errorf("%s %q is not supported", key, policy)) } diff --git a/internal/project/project_test.go b/internal/project/project_test.go index 6a405d5..146450d 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -3,6 +3,7 @@ package project import ( "bytes" "errors" + "fmt" "os" "path/filepath" "reflect" @@ -377,6 +378,64 @@ autogen: } } +func TestProjectValidationAcceptsWikiStalePurgePolicy(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + writeProjectFile(t, filepath.Join(root, ConfigFile), ` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +topdata: + wiki: + stale_pages: + default: report + live_cleanup: purge +`) + + proj, err := Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + if err := proj.ValidateLayout(); err != nil { + t.Fatalf("validate purge stale policy: %v", err) + } +} + +func TestProjectValidationRejectsUnsupportedWikiStalePolicies(t *testing.T) { + for _, policy := range []string{"unpublish", "remove-everything"} { + t.Run(policy, func(t *testing.T) { + root := t.TempDir() + mkdirAll(t, filepath.Join(root, "src")) + mkdirAll(t, filepath.Join(root, "build")) + writeProjectFile(t, filepath.Join(root, ConfigFile), fmt.Sprintf(` +module: + name: Test Module + resref: testmod +paths: + source: src + build: build +topdata: + wiki: + stale_pages: + live_cleanup: %s +`, policy)) + + proj, err := Load(root) + if err != nil { + t.Fatalf("load project: %v", err) + } + err = proj.ValidateLayout() + if err == nil || !strings.Contains(err.Error(), policy) { + t.Fatalf("expected %q stale policy validation failure, got %v", policy, err) + } + }) + } +} + func TestValidateLayoutRejectsInvalidExtractMergeRules(t *testing.T) { tests := []struct { name string diff --git a/internal/topdata/wiki_deploy.go b/internal/topdata/wiki_deploy.go index ce591d1..ba78495 100644 --- a/internal/topdata/wiki_deploy.go +++ b/internal/topdata/wiki_deploy.go @@ -56,6 +56,7 @@ type deployResult struct { Updated int Stale int Archived int + Purged int Skipped int Drifted int Renamed int @@ -129,7 +130,7 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress if opts.Username != "" || opts.Password != "" || opts.NotesDelimiter != "" { return deployResult{}, errors.New("DokuWiki deployment options are no longer supported; use NodeBB endpoint, token, and category mappings") } - if opts.StalePolicy != "" && opts.StalePolicy != "report" && opts.StalePolicy != "archive" { + if opts.StalePolicy != "" && opts.StalePolicy != "report" && opts.StalePolicy != "archive" && opts.StalePolicy != "purge" { return deployResult{}, fmt.Errorf("wiki stale policy %q is not supported", opts.StalePolicy) } if opts.TitlePrefixMinLength <= 0 { @@ -180,7 +181,7 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress result.LocalPages = len(pages) result.Manifest = manifestPath - progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, rename %d, skip %d, stale %d, archive %d, drift %d", result.Created, result.Updated, result.Renamed, result.Skipped, result.Stale, result.Archived, result.Drifted)) + progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, rename %d, skip %d, stale %d, archive %d, purge %d, drift %d", result.Created, result.Updated, result.Renamed, result.Skipped, result.Stale, result.Archived, result.Purged, result.Drifted)) if opts.DryRun { return result, nil } @@ -222,6 +223,10 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress if err := client.updatePost(plan.Entry.TID, plan.Entry.PID, plan.Content, summary); err != nil { return result, fmt.Errorf("deploy wiki page %q: archive NodeBB post %d: %w", plan.Page.PageID, plan.Entry.PID, err) } + case "purge": + if err := client.purgeTopic(plan.Entry.TID); err != nil { + return result, fmt.Errorf("deploy wiki page %q: purge NodeBB topic %d: %w", plan.Page.PageID, plan.Entry.TID, err) + } case "rename": if err := client.renameWikiPage(plan.Entry.TID, plan.Entry.CID, plan.Title); err != nil { return result, fmt.Errorf("deploy wiki page %q: rename NodeBB topic %d to %q: %w", plan.Page.PageID, plan.Entry.TID, plan.Title, err) @@ -447,7 +452,8 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife if policy == "" { policy = "report" } - if policy == "archive" { + switch policy { + case "archive": body := renderArchivedWikiPage(pageID, entry.Title) entry.Hash = computeManagedHash(body) entry.ArchivedHash = entry.Hash @@ -459,6 +465,17 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife Action: "archive", Content: body, }) + case "purge": + if entry.TID == 0 { + return nil, result, next, fmt.Errorf("stale generated wiki page %q cannot be purged without tracked NodeBB topic id", pageID) + } + result.Purged++ + delete(next.Pages, pageID) + plans = append(plans, wikiDeployPlan{ + Page: wikiDeployPage{PageID: pageID, Title: entry.Title, Namespace: entry.Namespace}, + Entry: entry, + Action: "purge", + }) } } } @@ -1086,6 +1103,18 @@ func (c *nodeBBClient) renameWikiPage(tid, cid int, title string) error { return c.request("PUT", "/api/v3/plugins/westgate-wiki/page/move", body, nil) } +func (c *nodeBBClient) purgeTopic(tid int) error { + if tid == 0 { + return fmt.Errorf("NodeBB topic purge requires topic id") + } + err := c.request(http.MethodDelete, fmt.Sprintf("/api/v3/topics/%d", tid), nil, nil) + var httpErr nodeBBHTTPError + if errors.As(err, &httpErr) && (httpErr.Status == http.StatusNotFound || httpErr.Status == http.StatusGone) { + return nil + } + return err +} + func (c *nodeBBClient) acquireEditLock(tid int) (nodeBBEditLock, error) { if tid == 0 { return nodeBBEditLock{}, fmt.Errorf("NodeBB wiki post update requires a topic id for edit locking") diff --git a/internal/topdata/wiki_deploy_test.go b/internal/topdata/wiki_deploy_test.go index dd6ba88..0e39045 100644 --- a/internal/topdata/wiki_deploy_test.go +++ b/internal/topdata/wiki_deploy_test.go @@ -842,6 +842,233 @@ func TestDeployWikiReportsAndArchivesStalePages(t *testing.T) { } } +func TestDeployWikiDryRunPlansTrackedStalePagePurge(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("dry-run purge must not call NodeBB, got %s %s", r.Method, r.URL.Path) + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + DryRun: true, + StalePolicy: "purge", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions stale purge dry run failed: %v", err) + } + if result.Stale != 1 || result.Purged != 1 { + t.Fatalf("expected one planned stale purge, got %#v", result) + } + if _, ok := loadDeployManifest(manifestPath).Pages["skills:retired"]; !ok { + t.Fatalf("dry-run purge must not change the deploy manifest") + } +} + +func TestDeployWikiPurgeRefusesStaleManifestEntryWithoutTopicID(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("missing-topic-id purge must not call NodeBB, got %s %s", r.Method, r.URL.Path) + })) + defer server.Close() + + _, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + DryRun: true, + StalePolicy: "purge", + }, nil) + if err == nil || !strings.Contains(err.Error(), "topic id") { + t.Fatalf("expected stale purge to require tracked topic id, got %v", err) + } +} + +func TestDeployWikiPurgesTrackedStaleGeneratedTopic(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + purgeCalls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete || r.URL.Path != "/api/v3/topics/7" { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + purgeCalls++ + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]any{"code": "ok"}}) + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + StalePolicy: "purge", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions stale purge failed: %v", err) + } + if result.Stale != 1 || result.Purged != 1 { + t.Fatalf("expected one stale purge, got %#v", result) + } + if purgeCalls != 1 { + t.Fatalf("expected one NodeBB topic purge call, got %d", purgeCalls) + } + if _, ok := loadDeployManifest(manifestPath).Pages["skills:retired"]; ok { + t.Fatalf("expected purged stale manifest entry to be removed") + } +} + +func TestDeployWikiPurgeTreatsAlreadyMissingTrackedTopicAsSuccess(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3}, + "skills:gone": {Hash: "old-hash", Title: "Gone Skill", Namespace: "skills", TID: 8, PID: 43, CID: 3}, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + seen := map[string]int{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + seen[r.URL.Path]++ + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v3/topics/7": + http.Error(w, `{"status":{"message":"topic not found"}}`, http.StatusNotFound) + case "/api/v3/topics/8": + _ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]any{"code": "ok"}}) + default: + t.Fatalf("unexpected purge path %s", r.URL.Path) + } + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + StalePolicy: "purge", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions stale purge with missing topic failed: %v", err) + } + if result.Stale != 2 || result.Purged != 2 { + t.Fatalf("expected two stale purges, got %#v", result) + } + if seen["/api/v3/topics/7"] != 1 || seen["/api/v3/topics/8"] != 1 { + t.Fatalf("expected one purge call for each tracked topic, got %#v", seen) + } + manifest := loadDeployManifest(manifestPath) + if _, ok := manifest.Pages["skills:retired"]; ok { + t.Fatalf("expected already-missing stale manifest entry to be removed") + } + if _, ok := manifest.Pages["skills:gone"]; ok { + t.Fatalf("expected purged stale manifest entry to be removed") + } +} + +func TestDeployWikiPurgeDoesNotTargetCurrentGeneratedPages(t *testing.T) { + root := t.TempDir() + sourceDir := filepath.Join(root, "pages") + if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil { + t.Fatalf("create source dir: %v", err) + } + generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Athletics\n\nGenerated athletics page\n[//]: # (sow-topdata-wiki:managed:end)\n" + if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.md"), []byte(generated), 0644); err != nil { + t.Fatalf("write source page: %v", err) + } + manifestPath := filepath.Join(root, "deploy-manifest.json") + if err := saveDeployManifest(manifestPath, wikiDeployManifest{ + Version: "nodebb-v1", + Pages: map[string]wikiDeployManifestPage{ + "skills:athletics": { + Hash: computeManagedHash(generated), + Title: "Athletics", + Namespace: "skills", + TID: 7, + PID: 42, + CID: 3, + }, + }, + }); err != nil { + t.Fatalf("write deploy manifest: %v", err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("current generated page must not call NodeBB during matching-hash deploy, got %s %s", r.Method, r.URL.Path) + })) + defer server.Close() + + result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{ + SourceDir: sourceDir, + Endpoint: server.URL, + Token: "nodebb-token", + ManifestPath: manifestPath, + StalePolicy: "purge", + }, nil) + if err != nil { + t.Fatalf("DeployWikiWithOptions current page purge deploy failed: %v", err) + } + if result.Stale != 0 || result.Purged != 0 || result.Skipped != 1 { + t.Fatalf("expected current generated page to skip purge, got %#v", result) + } + if _, ok := loadDeployManifest(manifestPath).Pages["skills:athletics"]; !ok { + t.Fatalf("expected current generated page to remain in deploy manifest") + } +} + func containsAll(haystack string, needles ...string) bool { for _, needle := range needles { if !strings.Contains(haystack, needle) {