Implemented the wiki pipeline to a shippable local state across `toolkit/`, `module/`, and the NodeBB wiki plugin.
**Scope Audited**
- Toolkit config/project loading, wiki renderer, deployer, app command wiring.
- Module `nwn-tool.yaml`, wiki source/docs, release script, topdata docs.
- Plugin sanitizer/API storage contract docs and sanitizer tests.
**Changes Made**
- Added topdata-owned wiki declarations under `module/topdata/wiki/`.
- Added `topdata.wiki.*` config fields, validation, effective config output.
- Switched generated wiki pages to `.html` with HTML comment managed/manual markers.
- Added `page-index.json` generation and deterministic metadata.
- Added manual-section merge preservation and stale page `report`/`archive` handling.
- Added namespace `category_env` loading from `topdata/wiki/namespaces.yaml`, while keeping `--category`/`NODEBB_WIKI_CATEGORIES` overrides.
- Updated release deployment to keep dry-run first and make stale cleanup opt-in via `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY`.
- Updated plugin sanitizer to preserve only `sow-topdata-wiki` comments.
**Configuration/Schema Impact**
- `module/nwn-tool.yaml` now declares wiki source, renderer, link strategy, templates, manual sections, managed marker policy, and stale policy.
- Deploy manifest entries now support stale/archive metadata: `stale`, `last_seen_hash`, `archived_hash`, `title`, `namespace`, `tid`, `pid`, `cid`.
**Compatibility Impact**
- Legacy Markdown managed markers are still readable for migration.
- Generated output is now HTML, not Markdown.
- Existing mapped pages update by `pid`; creates still require explicit `--create`.
**Tests Added/Updated**
- Go tests for wiki config validation, HTML rendering, page index, manual merge, stale report/archive, app summary output.
- Plugin sanitizer test for generated topdata bot HTML markers/subset.
**Validation Performed**
- `go test ./internal/project ./internal/topdata ./internal/app` passed.
- `./build-tool.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./validate-topdata.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force` passed, generating 1345 entity pages.
- `deploy-wiki --dry-run --create` with dummy endpoint/token and namespace CID env vars passed locally: 1377 planned creates, 0 stale/drift.
- `node tests/wiki-html-sanitizer.test.js` passed.
**Remaining Risks**
- Full plugin `npm test` is blocked by an existing unrelated drawer CSS contract failure in `tests/wiki-article-drawers.test.js`.
- Controlled live migration was not performed because it requires a non-production NodeBB namespace and credentials.
- Direct `PUT /api/v3/posts/{pid}` save-filter behavior is documented as needing live confirmation against the deployed NodeBB/plugin stack.
Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/3
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
360 lines
12 KiB
Go
360 lines
12 KiB
Go
package topdata
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
|
)
|
|
|
|
func TestDeployWikiDryRunDoesNotWriteRemoteOrManifest(t *testing.T) {
|
|
root := t.TempDir()
|
|
sourceDir := filepath.Join(root, "pages")
|
|
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
|
|
t.Fatalf("create source dir: %v", err)
|
|
}
|
|
generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Athletics\n\nGenerated athletics page\n[//]: # (sow-topdata-wiki:managed:end)\n"
|
|
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.md"), []byte(generated), 0644); err != nil {
|
|
t.Fatalf("write source page: %v", err)
|
|
}
|
|
old := "<!-- sow-topdata-wiki:managed:start -->\n<h1>Athletics</h1>\n<p>Existing athletics page</p>\n<!-- sow-topdata-wiki:managed:end -->\n"
|
|
oldHash := computeManagedHash(old)
|
|
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
|
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
|
|
Version: "nodebb-v1",
|
|
Pages: map[string]wikiDeployManifestPage{
|
|
"skills:athletics": {Hash: oldHash, TID: 7, PID: 42, CID: 3},
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("write deploy manifest: %v", err)
|
|
}
|
|
|
|
updateCalls := 0
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if got := r.Header.Get("Authorization"); got != "Bearer nodebb-token" {
|
|
t.Fatalf("unexpected authorization header %q", got)
|
|
}
|
|
switch {
|
|
case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42":
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7, "content": old}})
|
|
case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42":
|
|
updateCalls++
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}})
|
|
default:
|
|
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
|
SourceDir: sourceDir,
|
|
Endpoint: server.URL,
|
|
Token: "nodebb-token",
|
|
ManifestPath: manifestPath,
|
|
DryRun: true,
|
|
}, nil)
|
|
if err != nil {
|
|
t.Fatalf("DeployWikiWithOptions dry run failed: %v", err)
|
|
}
|
|
|
|
if result.Updated != 1 {
|
|
t.Fatalf("expected one planned update, got %d", result.Updated)
|
|
}
|
|
if updateCalls != 0 {
|
|
t.Fatalf("expected dry run not to update NodeBB posts, got %d calls", updateCalls)
|
|
}
|
|
after := loadDeployManifest(manifestPath)
|
|
if after.Pages["skills:athletics"].Hash != oldHash {
|
|
t.Fatalf("expected dry run not to rewrite manifest")
|
|
}
|
|
}
|
|
|
|
func TestDeployWikiCreatesNodeBBTopicAndWritesManifest(t *testing.T) {
|
|
root := t.TempDir()
|
|
sourceDir := filepath.Join(root, "pages")
|
|
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
|
|
t.Fatalf("create source dir: %v", err)
|
|
}
|
|
generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Athletics\n\nGenerated athletics page\n[//]: # (sow-topdata-wiki:managed:end)\n"
|
|
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.md"), []byte(generated), 0644); err != nil {
|
|
t.Fatalf("write source page: %v", err)
|
|
}
|
|
|
|
createCalls := 0
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if got := r.Header.Get("Authorization"); got != "Bearer nodebb-token" {
|
|
t.Fatalf("unexpected authorization header %q", got)
|
|
}
|
|
if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" {
|
|
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
|
}
|
|
createCalls++
|
|
var req struct {
|
|
CID int `json:"cid"`
|
|
Title string `json:"title"`
|
|
Content string `json:"content"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
t.Fatalf("decode create request: %v", err)
|
|
}
|
|
if req.CID != 5 || req.Title != "Athletics" || req.Content != generated {
|
|
t.Fatalf("unexpected create request: %#v", req)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"response": map[string]any{
|
|
"tid": 11,
|
|
"mainPost": map[string]any{
|
|
"pid": 42,
|
|
"tid": 11,
|
|
},
|
|
},
|
|
})
|
|
}))
|
|
defer server.Close()
|
|
|
|
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
|
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
|
SourceDir: sourceDir,
|
|
Endpoint: server.URL,
|
|
Token: "nodebb-token",
|
|
ManifestPath: manifestPath,
|
|
CategoryIDs: map[string]int{"skills": 5},
|
|
AllowCreates: true,
|
|
}, nil)
|
|
if err != nil {
|
|
t.Fatalf("DeployWikiWithOptions create failed: %v", err)
|
|
}
|
|
if result.Created != 1 || createCalls != 1 {
|
|
t.Fatalf("expected one created topic, result=%d calls=%d", result.Created, createCalls)
|
|
}
|
|
manifest := loadDeployManifest(manifestPath)
|
|
entry := manifest.Pages["skills:athletics"]
|
|
if entry.TID != 11 || entry.PID != 42 || entry.CID != 5 || entry.Hash != computeManagedHash(generated) {
|
|
t.Fatalf("unexpected manifest entry: %#v", entry)
|
|
}
|
|
}
|
|
|
|
func TestDeployWikiCreatesNodeBBTopicWithFallbackForShortTitle(t *testing.T) {
|
|
root := t.TempDir()
|
|
sourceDir := filepath.Join(root, "pages")
|
|
if err := os.MkdirAll(filepath.Join(sourceDir, "spells"), 0755); err != nil {
|
|
t.Fatalf("create source dir: %v", err)
|
|
}
|
|
generated := "[//]: # (sow-topdata-wiki:managed:start)\n# Aid\n\nGenerated spell page\n[//]: # (sow-topdata-wiki:managed:end)\n"
|
|
if err := os.WriteFile(filepath.Join(sourceDir, "spells", "aid.md"), []byte(generated), 0644); err != nil {
|
|
t.Fatalf("write source page: %v", err)
|
|
}
|
|
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost || r.URL.Path != "/api/v3/topics" {
|
|
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
|
}
|
|
var req struct {
|
|
Title string `json:"title"`
|
|
Content string `json:"content"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
t.Fatalf("decode create request: %v", err)
|
|
}
|
|
if req.Title != "Spells: Aid" {
|
|
t.Fatalf("expected fallback title, got %q", req.Title)
|
|
}
|
|
if req.Content != generated {
|
|
t.Fatalf("expected page body to keep short h1 unchanged:\n%s", req.Content)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
|
"response": map[string]any{
|
|
"tid": 11,
|
|
"mainPost": map[string]any{
|
|
"pid": 42,
|
|
"tid": 11,
|
|
},
|
|
},
|
|
})
|
|
}))
|
|
defer server.Close()
|
|
|
|
_, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
|
SourceDir: sourceDir,
|
|
Endpoint: server.URL,
|
|
Token: "nodebb-token",
|
|
ManifestPath: filepath.Join(root, "deploy-manifest.json"),
|
|
CategoryIDs: map[string]int{"spells": 5},
|
|
AllowCreates: true,
|
|
}, nil)
|
|
if err != nil {
|
|
t.Fatalf("DeployWikiWithOptions create failed: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDeployWikiMergesHTMLManagedAndManualRegions(t *testing.T) {
|
|
root := t.TempDir()
|
|
sourceDir := filepath.Join(root, "pages")
|
|
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
|
|
t.Fatalf("create source dir: %v", err)
|
|
}
|
|
generated := `<!-- sow-topdata-wiki:page=skills:athletics -->
|
|
<!-- sow-topdata-wiki:managed:start hash="sha256:local" -->
|
|
<h1>Athletics</h1>
|
|
<p>Generated athletics page</p>
|
|
<!-- sow-topdata-wiki:managed:end -->
|
|
<!-- sow-topdata-wiki:manual:start id="notes" -->
|
|
<h2>Notes</h2>
|
|
<p></p>
|
|
<!-- sow-topdata-wiki:manual:end id="notes" -->
|
|
<!-- sow-topdata-wiki:manual:start id="see_also" -->
|
|
<h2>See Also</h2>
|
|
<p></p>
|
|
<!-- sow-topdata-wiki:manual:end id="see_also" -->
|
|
`
|
|
if err := os.WriteFile(filepath.Join(sourceDir, "skills", "athletics.html"), []byte(generated), 0644); err != nil {
|
|
t.Fatalf("write source page: %v", err)
|
|
}
|
|
old := `<!-- sow-topdata-wiki:page=skills:athletics -->
|
|
<!-- sow-topdata-wiki:managed:start hash="sha256:old" -->
|
|
<h1>Athletics</h1>
|
|
<p>Old generated text</p>
|
|
<!-- sow-topdata-wiki:managed:end -->
|
|
<!-- sow-topdata-wiki:manual:start id="notes" -->
|
|
<h2>Notes</h2>
|
|
<p>Keep this human note.</p>
|
|
<!-- sow-topdata-wiki:manual:end id="notes" -->
|
|
`
|
|
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
|
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
|
|
Version: "nodebb-v1",
|
|
Pages: map[string]wikiDeployManifestPage{
|
|
"skills:athletics": {Hash: computeManagedHash(old), TID: 7, PID: 42, CID: 3},
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("write deploy manifest: %v", err)
|
|
}
|
|
|
|
var updated string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch {
|
|
case r.Method == http.MethodGet && r.URL.Path == "/api/v3/posts/42":
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7, "content": old}})
|
|
case r.Method == http.MethodPut && r.URL.Path == "/api/v3/posts/42":
|
|
var req struct {
|
|
Content string `json:"content"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
t.Fatalf("decode update request: %v", err)
|
|
}
|
|
updated = req.Content
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}})
|
|
default:
|
|
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
|
|
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
|
SourceDir: sourceDir,
|
|
Endpoint: server.URL,
|
|
Token: "nodebb-token",
|
|
ManifestPath: manifestPath,
|
|
}, nil)
|
|
if err != nil {
|
|
t.Fatalf("DeployWikiWithOptions update failed: %v", err)
|
|
}
|
|
if result.Updated != 1 {
|
|
t.Fatalf("expected one update, got %#v", result)
|
|
}
|
|
if !containsAll(updated, "<p>Generated athletics page</p>", "<p>Keep this human note.</p>", `manual:start id="see_also"`) {
|
|
t.Fatalf("expected update to replace managed HTML and preserve/bootstrap manual regions:\n%s", updated)
|
|
}
|
|
}
|
|
|
|
func TestDeployWikiReportsAndArchivesStalePages(t *testing.T) {
|
|
root := t.TempDir()
|
|
sourceDir := filepath.Join(root, "pages")
|
|
if err := os.MkdirAll(filepath.Join(sourceDir, "skills"), 0755); err != nil {
|
|
t.Fatalf("create source dir: %v", err)
|
|
}
|
|
manifestPath := filepath.Join(root, "deploy-manifest.json")
|
|
if err := saveDeployManifest(manifestPath, wikiDeployManifest{
|
|
Version: "nodebb-v1",
|
|
Pages: map[string]wikiDeployManifestPage{
|
|
"skills:retired": {Hash: "old-hash", Title: "Retired Skill", Namespace: "skills", TID: 7, PID: 42, CID: 3},
|
|
},
|
|
}); err != nil {
|
|
t.Fatalf("write deploy manifest: %v", err)
|
|
}
|
|
|
|
var archived string
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPut || r.URL.Path != "/api/v3/posts/42" {
|
|
t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path)
|
|
}
|
|
var req struct {
|
|
Content string `json:"content"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
t.Fatalf("decode archive request: %v", err)
|
|
}
|
|
archived = req.Content
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"response": map[string]any{"pid": 42, "tid": 7}})
|
|
}))
|
|
defer server.Close()
|
|
|
|
report, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
|
SourceDir: sourceDir,
|
|
Endpoint: server.URL,
|
|
Token: "nodebb-token",
|
|
ManifestPath: manifestPath,
|
|
DryRun: true,
|
|
StalePolicy: "report",
|
|
}, nil)
|
|
if err != nil {
|
|
t.Fatalf("DeployWikiWithOptions stale report failed: %v", err)
|
|
}
|
|
if report.Stale != 1 {
|
|
t.Fatalf("expected one stale report, got %#v", report)
|
|
}
|
|
|
|
result, err := DeployWikiWithOptions(&project.Project{}, DeployWikiOptions{
|
|
SourceDir: sourceDir,
|
|
Endpoint: server.URL,
|
|
Token: "nodebb-token",
|
|
ManifestPath: manifestPath,
|
|
StalePolicy: "archive",
|
|
}, nil)
|
|
if err != nil {
|
|
t.Fatalf("DeployWikiWithOptions stale archive failed: %v", err)
|
|
}
|
|
if result.Stale != 1 || result.Archived != 1 {
|
|
t.Fatalf("expected one stale archive, got %#v", result)
|
|
}
|
|
if !containsAll(archived, "<h1>Retired Skill</h1>", "no longer present", `<!-- sow-topdata-wiki:managed:start hash="sha256:`) {
|
|
t.Fatalf("unexpected archived body:\n%s", archived)
|
|
}
|
|
manifest := loadDeployManifest(manifestPath)
|
|
entry := manifest.Pages["skills:retired"]
|
|
if !entry.Stale || entry.ArchivedHash == "" || entry.PID != 42 {
|
|
t.Fatalf("expected archived stale manifest entry, got %#v", entry)
|
|
}
|
|
}
|
|
|
|
func containsAll(haystack string, needles ...string) bool {
|
|
for _, needle := range needles {
|
|
if !strings.Contains(haystack, needle) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|