ShadowsOverWestgate/sow-tools
7
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 38 Activity

Generated Wiki Stale Purge Method (#5)

Browse Source 
Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/5
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
This commit is contained in:
archvillainette
2026-05-21 14:58:51 +02:00
committed by archvillainette
parent a64dbfb085
commit f57a3c6ff2
7 changed files with 342 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6 -2
View File
@@ -71,7 +71,7 @@ sow-toolkit build-top-package [--force]
sow-toolkit compare-topdata
sow-toolkit convert-topdata <2da-to-json|2da-to-module|json-to-2da> ...
sow-toolkit build-wiki [--force]
sow-toolkit deploy-wiki [--source-dir <path>] [--endpoint <url>] [--token <token>] [--version <ver>] [--namespace <ns>] [--category <namespace=cid>] [--manifest <path>] [--dry-run] [--create] [--force]
sow-toolkit deploy-wiki [--source-dir <path>] [--endpoint <url>] [--token <token>] [--version <ver>] [--namespace <ns>] [--category <namespace=cid>] [--manifest <path>] [--stale-policy <report|archive|purge>] [--dry-run] [--create] [--force]
sow-toolkit build-changelog [--config <path>] [--output <path>] [--current-tag <tag>] [--previous-tag <tag>] [--api-base-url <url>] [--token <token>]
```
@@ -413,7 +413,11 @@ rejects a create because the clean wiki URL already exists, `deploy-wiki`
re-queries the namespace by page leaf, adopts the matching topic, and updates it
instead. Updates and stale-page archives acquire a Westgate Wiki edit lock
before saving, then send the returned `wikiEditLockToken` with the NodeBB post
edit request.
edit request. Stale generated pages are reported by default.
`--stale-policy archive` rewrites stale generated pages in place as archived
managed content. `--stale-policy purge` permanently removes only stale generated
topics already tracked in the wiki deploy manifest. Dry-run that policy before
the live run; it never discovers manual wiki pages from NodeBB for purge.
For newly created NodeBB topics, `topdata.wiki.title_prefix_min_length`
controls when the deployer prefixes the namespace in the topic title. The
internal/app/app.go
+4 -3
View File
@@ -1089,7 +1089,7 @@ func (c *topdataConsole) emitWikiBuildResult(outputDir string, pageCount int, st
fmt.Fprintf(c.stdout, "wiki status: %s\n", status)
}
func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skipped, stale, archived, drifted int, manifest string) {
func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skipped, stale, archived, purged, drifted int, manifest string) {
spin.linebreak()
fmt.Fprintln(c.stdout, "Deploy Wiki ----------")
fmt.Fprintf(c.stdout, "project: %s\n", c.projectName)
@@ -1099,6 +1099,7 @@ func (c *topdataConsole) emitWikiDeployResult(localPages, created, updated, skip
fmt.Fprintf(c.stdout, "skipped: %d\n", skipped)
fmt.Fprintf(c.stdout, "stale: %d\n", stale)
fmt.Fprintf(c.stdout, "archived: %d\n", archived)
fmt.Fprintf(c.stdout, "purged: %d\n", purged)
fmt.Fprintf(c.stdout, "drifted: %d\n", drifted)
fmt.Fprintf(c.stdout, "manifest: %s\n", c.relPath(manifest))
}
@@ -1869,7 +1870,7 @@ func runDeployWiki(ctx context) error {
return err
}
console.emitWikiDeployResult(result.LocalPages, result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Drifted, result.Manifest)
console.emitWikiDeployResult(result.LocalPages, result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Purged, result.Drifted, result.Manifest)
return nil
}
@@ -1950,7 +1951,7 @@ func parseDeployWikiArgs(commandName string, args []string) (topdata.DeployWikiO
case "--create":
opts.AllowCreates = true
case "-h", "--help":
return opts, fmt.Errorf("usage: %s [--source-dir <path>] [--endpoint <url>] [--token <token>] [--version <ver>] [--namespace <ns> ...] [--category <namespace=cid> ...] [--manifest <path>] [--stale-policy <report|archive>] [--dry-run] [--create] [--force]", commandName)
return opts, fmt.Errorf("usage: %s [--source-dir <path>] [--endpoint <url>] [--token <token>] [--version <ver>] [--namespace <ns> ...] [--category <namespace=cid> ...] [--manifest <path>] [--stale-policy <report|archive|purge>] [--dry-run] [--create] [--force]", commandName)
default:
if value, ok, err := requireInlineFlagValue(arg, "--source-dir"); ok || err != nil {
if err != nil {
internal/app/app_test.go
+13 -3
View File
@@ -246,21 +246,31 @@ func TestTopdataConsoleDebugProgressAndRelativePaths(t *testing.T) {
level: logLevelDebug,
}
console.progress("NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, drift 0")
console.emitWikiDeployResult(10, 1, 2, 3, 4, 5, 0, "/workspace/project/build/wiki/deploy-manifest.json")
console.progress("NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, purge 6, drift 0")
console.emitWikiDeployResult(10, 1, 2, 3, 4, 5, 6, 0, "/workspace/project/build/wiki/deploy-manifest.json")
output := stdout.String()
if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, drift 0") {
if !strings.Contains(output, "[debug] NodeBB wiki plan: create 1, update 2, skip 3, stale 4, archive 5, purge 6, drift 0") {
t.Fatalf("expected debug progress line, got %q", output)
}
if !strings.Contains(output, "archived: 5") {
t.Fatalf("expected archived deploy count, got %q", output)
}
if !strings.Contains(output, "purged: 6") {
t.Fatalf("expected purged deploy count, got %q", output)
}
if !strings.Contains(output, "manifest: build/wiki/deploy-manifest.json") {
t.Fatalf("expected relative deploy manifest path, got %q", output)
}
}
func TestParseDeployWikiHelpListsPurgeStalePolicy(t *testing.T) {
_, err := parseDeployWikiArgs("deploy-wiki", []string{"--help"})
if err == nil || !strings.Contains(err.Error(), "--stale-policy <report|archive|purge>") {
t.Fatalf("expected deploy-wiki help to list purge stale policy, got %v", err)
}
}
func TestProjectConsoleSuppressesBuildModuleProgressInNormalMode(t *testing.T) {
var stdout bytes.Buffer
console := &projectConsole{
internal/project/project.go
+1 -1
View File
@@ -608,7 +608,7 @@ func (p *Project) ValidateLayout() error {
"topdata.wiki.stale_pages.live_cleanup": effective.TopData.Wiki.StalePages.LiveCleanup,
} {
switch policy {
case "report", "archive", "unpublish":
case "report", "archive", "purge":
default:
failures = append(failures, fmt.Errorf("%s %q is not supported", key, policy))
}
internal/project/project_test.go
+59
View File
@@ -3,6 +3,7 @@ package project
import (
"bytes"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
@@ -377,6 +378,64 @@ autogen:
}
}
func TestProjectValidationAcceptsWikiStalePurgePolicy(t *testing.T) {
root := t.TempDir()
mkdirAll(t, filepath.Join(root, "src"))
mkdirAll(t, filepath.Join(root, "build"))
writeProjectFile(t, filepath.Join(root, ConfigFile), `
module:
name: Test Module
resref: testmod
paths:
source: src
build: build
topdata:
wiki:
stale_pages:
default: report
live_cleanup: purge
`)
proj, err := Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
if err := proj.ValidateLayout(); err != nil {
t.Fatalf("validate purge stale policy: %v", err)
}
}
func TestProjectValidationRejectsUnsupportedWikiStalePolicies(t *testing.T) {
for _, policy := range []string{"unpublish", "remove-everything"} {
t.Run(policy, func(t *testing.T) {
root := t.TempDir()
mkdirAll(t, filepath.Join(root, "src"))
mkdirAll(t, filepath.Join(root, "build"))
writeProjectFile(t, filepath.Join(root, ConfigFile), fmt.Sprintf(`
module:
name: Test Module
resref: testmod
paths:
source: src
build: build
topdata:
wiki:
stale_pages:
live_cleanup: %s
`, policy))
proj, err := Load(root)
if err != nil {
t.Fatalf("load project: %v", err)
}
err = proj.ValidateLayout()
if err == nil || !strings.Contains(err.Error(), policy) {
t.Fatalf("expected %q stale policy validation failure, got %v", policy, err)
}
})
}
}
func TestValidateLayoutRejectsInvalidExtractMergeRules(t *testing.T) {
tests := []struct {
name string
internal/topdata/wiki_deploy.go
+32 -3
View File
@@ -56,6 +56,7 @@ type deployResult struct {
Updated int
Stale int
Archived int
Purged int
Skipped int
Drifted int
Renamed int
@@ -129,7 +130,7 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
if opts.Username != "" || opts.Password != "" || opts.NotesDelimiter != "" {
return deployResult{}, errors.New("DokuWiki deployment options are no longer supported; use NodeBB endpoint, token, and category mappings")
}
if opts.StalePolicy != "" && opts.StalePolicy != "report" && opts.StalePolicy != "archive" {
if opts.StalePolicy != "" && opts.StalePolicy != "report" && opts.StalePolicy != "archive" && opts.StalePolicy != "purge" {
return deployResult{}, fmt.Errorf("wiki stale policy %q is not supported", opts.StalePolicy)
}
if opts.TitlePrefixMinLength <= 0 {
@@ -180,7 +181,7 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
result.LocalPages = len(pages)
result.Manifest = manifestPath
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, rename %d, skip %d, stale %d, archive %d, drift %d", result.Created, result.Updated, result.Renamed, result.Skipped, result.Stale, result.Archived, result.Drifted))
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, rename %d, skip %d, stale %d, archive %d, purge %d, drift %d", result.Created, result.Updated, result.Renamed, result.Skipped, result.Stale, result.Archived, result.Purged, result.Drifted))
if opts.DryRun {
return result, nil
}
@@ -222,6 +223,10 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
if err := client.updatePost(plan.Entry.TID, plan.Entry.PID, plan.Content, summary); err != nil {
return result, fmt.Errorf("deploy wiki page %q: archive NodeBB post %d: %w", plan.Page.PageID, plan.Entry.PID, err)
}
case "purge":
if err := client.purgeTopic(plan.Entry.TID); err != nil {
return result, fmt.Errorf("deploy wiki page %q: purge NodeBB topic %d: %w", plan.Page.PageID, plan.Entry.TID, err)
}
case "rename":
if err := client.renameWikiPage(plan.Entry.TID, plan.Entry.CID, plan.Title); err != nil {
return result, fmt.Errorf("deploy wiki page %q: rename NodeBB topic %d to %q: %w", plan.Page.PageID, plan.Entry.TID, plan.Title, err)
@@ -447,7 +452,8 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
if policy == "" {
policy = "report"
}
if policy == "archive" {
switch policy {
case "archive":
body := renderArchivedWikiPage(pageID, entry.Title)
entry.Hash = computeManagedHash(body)
entry.ArchivedHash = entry.Hash
@@ -459,6 +465,17 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
Action: "archive",
Content: body,
})
case "purge":
if entry.TID == 0 {
return nil, result, next, fmt.Errorf("stale generated wiki page %q cannot be purged without tracked NodeBB topic id", pageID)
}
result.Purged++
delete(next.Pages, pageID)
plans = append(plans, wikiDeployPlan{
Page: wikiDeployPage{PageID: pageID, Title: entry.Title, Namespace: entry.Namespace},
Entry: entry,
Action: "purge",
})
}
}
}
@@ -1086,6 +1103,18 @@ func (c *nodeBBClient) renameWikiPage(tid, cid int, title string) error {
return c.request("PUT", "/api/v3/plugins/westgate-wiki/page/move", body, nil)
}
func (c *nodeBBClient) purgeTopic(tid int) error {
if tid == 0 {
return fmt.Errorf("NodeBB topic purge requires topic id")
}
err := c.request(http.MethodDelete, fmt.Sprintf("/api/v3/topics/%d", tid), nil, nil)
var httpErr nodeBBHTTPError
if errors.As(err, &httpErr) && (httpErr.Status == http.StatusNotFound || httpErr.Status == http.StatusGone) {
return nil
}
return err
}
func (c *nodeBBClient) acquireEditLock(tid int) (nodeBBEditLock, error) {
if tid == 0 {
return nodeBBEditLock{}, fmt.Errorf("NodeBB wiki post update requires a topic id for edit locking")
internal/topdata/wiki_deploy_test.go
+227
View File
@@ -842,6 +842,233 @@ func TestDeployWikiReportsAndArchivesStalePages(t *testing.T) {
}
}
func TestDeployWikiDryRunPlansTrackedStalePagePurge(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
manifestPath := filepath.Join(root, "deploy-manifest.json")
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
Version: "nodebb-v1",
Pages: map[string]wikiDeployManifestPage{
"skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3},
},
}); err != nil {
t.Fatalf("write deploy manifest: %v", err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Fatalf("dry-run purge must not call NodeBB, got %s %s", r.Method, r.URL.Path)
}))
defer server.Close()
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
DryRun: true,
StalePolicy: "purge",
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions stale purge dry run failed: %v", err)
}
if result.Stale != 1 || result.Purged != 1 {
t.Fatalf("expected one planned stale purge, got %#v", result)
}
if _, ok := loadDeployManifest(manifestPath).Pages["skills:retired"]; !ok {
t.Fatalf("dry-run purge must not change the deploy manifest")
}
}
func TestDeployWikiPurgeRefusesStaleManifestEntryWithoutTopicID(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
manifestPath := filepath.Join(root, "deploy-manifest.json")
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
Version: "nodebb-v1",
Pages: map[string]wikiDeployManifestPage{
"skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", PID: 42, CID: 3},
},
}); err != nil {
t.Fatalf("write deploy manifest: %v", err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Fatalf("missing-topic-id purge must not call NodeBB, got %s %s", r.Method, r.URL.Path)
}))
defer server.Close()
_, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
DryRun: true,
StalePolicy: "purge",
}, nil)
if err == nil || !strings.Contains(err.Error(), "topic id") {
t.Fatalf("expected stale purge to require tracked topic id, got %v", err)
}
}
func TestDeployWikiPurgesTrackedStaleGeneratedTopic(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
manifestPath := filepath.Join(root, "deploy-manifest.json")
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
Version: "nodebb-v1",
Pages: map[string]wikiDeployManifestPage{
"skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3},
},
}); err != nil {
t.Fatalf("write deploy manifest: %v", err)
}
purgeCalls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete || r.URL.Path != "/api/v3/topics/7" {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
}
purgeCalls++
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]any{"code": "ok"}})
}))
defer server.Close()
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
StalePolicy: "purge",
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions stale purge failed: %v", err)
}
if result.Stale != 1 || result.Purged != 1 {
t.Fatalf("expected one stale purge, got %#v", result)
}
if purgeCalls != 1 {
t.Fatalf("expected one NodeBB topic purge call, got %d", purgeCalls)
}
if _, ok := loadDeployManifest(manifestPath).Pages["skills:retired"]; ok {
t.Fatalf("expected purged stale manifest entry to be removed")
}
}
func TestDeployWikiPurgeTreatsAlreadyMissingTrackedTopicAsSuccess(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
manifestPath := filepath.Join(root, "deploy-manifest.json")
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
Version: "nodebb-v1",
Pages: map[string]wikiDeployManifestPage{
"skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3},
"skills:gone": {Hash: "old-hash", Title: "Gone Skill", Namespace: "skills", TID: 8, PID: 43, CID: 3},
},
}); err != nil {
t.Fatalf("write deploy manifest: %v", err)
}
seen := map[string]int{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
}
seen[r.URL.Path]++
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/v3/topics/7":
http.Error(w, `{"status":{"message":"topic not found"}}`, http.StatusNotFound)
case "/api/v3/topics/8":
_ = json.NewEncoder(w).Encode(map[string]any{"status": map[string]any{"code": "ok"}})
default:
t.Fatalf("unexpected purge path %s", r.URL.Path)
}
}))
defer server.Close()
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
StalePolicy: "purge",
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions stale purge with missing topic failed: %v", err)
}
if result.Stale != 2 || result.Purged != 2 {
t.Fatalf("expected two stale purges, got %#v", result)
}
if seen["/api/v3/topics/7"] != 1 || seen["/api/v3/topics/8"] != 1 {
t.Fatalf("expected one purge call for each tracked topic, got %#v", seen)
}
manifest := loadDeployManifest(manifestPath)
if _, ok := manifest.Pages["skills:retired"]; ok {
t.Fatalf("expected already-missing stale manifest entry to be removed")
}
if _, ok := manifest.Pages["skills:gone"]; ok {
t.Fatalf("expected purged stale manifest entry to be removed")
}
}
func TestDeployWikiPurgeDoesNotTargetCurrentGeneratedPages(t *testing.T) {
root := t.TempDir()
sourceDir := filepath.Join(root, "pages")
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
t.Fatalf("create source dir: %v", err)
}
generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Athletics\n\nGenerated athletics page\n[//]: # (sow-topdata-wiki:managed:end)\n"
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.md"), []byte(generated), 0644); err != nil {
t.Fatalf("write source page: %v", err)
}
manifestPath := filepath.Join(root, "deploy-manifest.json")
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
Version: "nodebb-v1",
Pages: map[string]wikiDeployManifestPage{
"skills:athletics": {
Hash: computeManagedHash(generated),
Title: "Athletics",
Namespace: "skills",
TID: 7,
PID: 42,
CID: 3,
},
},
}); err != nil {
t.Fatalf("write deploy manifest: %v", err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Fatalf("current generated page must not call NodeBB during matching-hash deploy, got %s %s", r.Method, r.URL.Path)
}))
defer server.Close()
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
SourceDir: sourceDir,
Endpoint: server.URL,
Token: "nodebb-token",
ManifestPath: manifestPath,
StalePolicy: "purge",
}, nil)
if err != nil {
t.Fatalf("DeployWikiWithOptions current page purge deploy failed: %v", err)
}
if result.Stale != 0 || result.Purged != 0 || result.Skipped != 1 {
t.Fatalf("expected current generated page to skip purge, got %#v", result)
}
if _, ok := loadDeployManifest(manifestPath).Pages["skills:athletics"]; !ok {
t.Fatalf("expected current generated page to remain in deploy manifest")
}
}
func containsAll(haystack string, needles ...string) bool {
for _, needle := range needles {
if !strings.Contains(haystack, needle) {