Wiki pipeline (#3)
Implemented the wiki pipeline to a shippable local state across `toolkit/`, `module/`, and the NodeBB wiki plugin.
**Scope Audited**
- Toolkit config/project loading, wiki renderer, deployer, app command wiring.
- Module `nwn-tool.yaml`, wiki source/docs, release script, topdata docs.
- Plugin sanitizer/API storage contract docs and sanitizer tests.
**Changes Made**
- Added topdata-owned wiki declarations under `module/topdata/wiki/`.
- Added `topdata.wiki.*` config fields, validation, effective config output.
- Switched generated wiki pages to `.html` with HTML comment managed/manual markers.
- Added `page-index.json` generation and deterministic metadata.
- Added manual-section merge preservation and stale page `report`/`archive` handling.
- Added namespace `category_env` loading from `topdata/wiki/namespaces.yaml`, while keeping `--category`/`NODEBB_WIKI_CATEGORIES` overrides.
- Updated release deployment to keep dry-run first and make stale cleanup opt-in via `SOW_MODULE_WIKI_DEPLOY_STALE_POLICY`.
- Updated plugin sanitizer to preserve only `sow-topdata-wiki` comments.
**Configuration/Schema Impact**
- `module/nwn-tool.yaml` now declares wiki source, renderer, link strategy, templates, manual sections, managed marker policy, and stale policy.
- Deploy manifest entries now support stale/archive metadata: `stale`, `last_seen_hash`, `archived_hash`, `title`, `namespace`, `tid`, `pid`, `cid`.
**Compatibility Impact**
- Legacy Markdown managed markers are still readable for migration.
- Generated output is now HTML, not Markdown.
- Existing mapped pages update by `pid`; creates still require explicit `--create`.
**Tests Added/Updated**
- Go tests for wiki config validation, HTML rendering, page index, manual merge, stale report/archive, app summary output.
- Plugin sanitizer test for generated topdata bot HTML markers/subset.
**Validation Performed**
- `go test ./internal/project ./internal/topdata ./internal/app` passed.
- `./build-tool.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./validate-topdata.sh` passed.
- `SOW_TOOLS_DEV_BINARY=../toolkit/tools/sow-toolkit ./build-wiki.sh --force` passed, generating 1345 entity pages.
- `deploy-wiki --dry-run --create` with dummy endpoint/token and namespace CID env vars passed locally: 1377 planned creates, 0 stale/drift.
- `node tests/wiki-html-sanitizer.test.js` passed.
**Remaining Risks**
- Full plugin `npm test` is blocked by an existing unrelated drawer CSS contract failure in `tests/wiki-article-drawers.test.js`.
- Controlled live migration was not performed because it requires a non-production NodeBB namespace and credentials.
- Direct `PUT /api/v3/posts/{pid}` save-filter behavior is documented as needing live confirmation against the deployed NodeBB/plugin stack.
Reviewed-on: https://gitea.westgate.pw/ShadowsOverWestgate/sow-tools/pulls/3
Co-authored-by: vickydotbat <vickydotbat@tutamail.com>
Co-committed-by: vickydotbat <vickydotbat@tutamail.com>
This commit is contained in:
+270
-13
@@ -17,13 +17,18 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
managedStartMarker = "[//]: # (sow-topdata-wiki:managed:start)"
|
||||
managedEndMarker = "[//]: # (sow-topdata-wiki:managed:end)"
|
||||
legacyManagedStartMarker = "<!-- sow-topdata-wiki:managed:start -->"
|
||||
legacyManagedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
|
||||
managedStartMarker = "<!-- sow-topdata-wiki:managed:start"
|
||||
managedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
|
||||
legacyMarkdownStartMarker = "[//]: # (sow-topdata-wiki:managed:start)"
|
||||
legacyMarkdownEndMarker = "[//]: # (sow-topdata-wiki:managed:end)"
|
||||
legacyManagedStartMarker = "<!-- sow-topdata-wiki:managed:start -->"
|
||||
legacyManagedEndMarker = "<!-- sow-topdata-wiki:managed:end -->"
|
||||
manualStartMarkerPrefix = "<!-- sow-topdata-wiki:manual:start"
|
||||
manualEndMarkerPrefix = "<!-- sow-topdata-wiki:manual:end"
|
||||
)
|
||||
|
||||
type DeployWikiOptions struct {
|
||||
@@ -48,6 +53,7 @@ type deployResult struct {
|
||||
Created int
|
||||
Updated int
|
||||
Stale int
|
||||
Archived int
|
||||
Skipped int
|
||||
Drifted int
|
||||
Manifest string
|
||||
@@ -67,10 +73,15 @@ type wikiDeployManifest struct {
|
||||
}
|
||||
|
||||
type wikiDeployManifestPage struct {
|
||||
Hash string `json:"hash"`
|
||||
TID int `json:"tid,omitempty"`
|
||||
PID int `json:"pid,omitempty"`
|
||||
CID int `json:"cid,omitempty"`
|
||||
Hash string `json:"hash"`
|
||||
LastSeenHash string `json:"last_seen_hash,omitempty"`
|
||||
ArchivedHash string `json:"archived_hash,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
Stale bool `json:"stale,omitempty"`
|
||||
TID int `json:"tid,omitempty"`
|
||||
PID int `json:"pid,omitempty"`
|
||||
CID int `json:"cid,omitempty"`
|
||||
}
|
||||
|
||||
type wikiDeployPlan struct {
|
||||
@@ -81,6 +92,19 @@ type wikiDeployPlan struct {
|
||||
RemoteHash string
|
||||
}
|
||||
|
||||
type wikiNamespacesDocument struct {
|
||||
Namespaces []wikiNamespaceDeclaration `json:"namespaces" yaml:"namespaces"`
|
||||
}
|
||||
|
||||
type wikiNamespaceDeclaration struct {
|
||||
ID string `json:"id" yaml:"id"`
|
||||
Title string `json:"title" yaml:"title"`
|
||||
CategoryEnv string `json:"category_env" yaml:"category_env"`
|
||||
PageTitle string `json:"page_title" yaml:"page_title"`
|
||||
Tags []string `json:"tags" yaml:"tags"`
|
||||
EditPolicy string `json:"edit_policy" yaml:"edit_policy"`
|
||||
}
|
||||
|
||||
func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress func(string)) (deployResult, error) {
|
||||
if progress == nil {
|
||||
progress = func(string) {}
|
||||
@@ -100,10 +124,33 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
|
||||
if opts.Username != "" || opts.Password != "" || opts.NotesDelimiter != "" {
|
||||
return deployResult{}, errors.New("DokuWiki deployment options are no longer supported; use NodeBB endpoint, token, and category mappings")
|
||||
}
|
||||
if opts.StalePolicy != "" && opts.StalePolicy != "report" && opts.StalePolicy != "archive" {
|
||||
return deployResult{}, fmt.Errorf("wiki stale policy %q is not supported", opts.StalePolicy)
|
||||
}
|
||||
|
||||
namespaces := opts.Namespaces
|
||||
if len(namespaces) == 0 {
|
||||
namespaces = p.EffectiveConfig().TopData.Wiki.ManagedNamespaces
|
||||
declarations, err := loadWikiNamespaceDeclarations(p)
|
||||
if err != nil {
|
||||
return deployResult{}, err
|
||||
}
|
||||
for _, declaration := range declarations {
|
||||
namespaces = append(namespaces, declaration.ID)
|
||||
}
|
||||
if len(namespaces) == 0 {
|
||||
namespaces = p.EffectiveConfig().TopData.Wiki.ManagedNamespaces
|
||||
}
|
||||
envCategories, err := categoryIDsFromNamespaceEnv(declarations)
|
||||
if err != nil {
|
||||
return deployResult{}, err
|
||||
}
|
||||
if len(envCategories) > 0 {
|
||||
merged := envCategories
|
||||
for namespace, cid := range opts.CategoryIDs {
|
||||
merged[namespace] = cid
|
||||
}
|
||||
opts.CategoryIDs = merged
|
||||
}
|
||||
}
|
||||
|
||||
manifestPath := opts.ManifestPath
|
||||
@@ -125,7 +172,7 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
|
||||
result.LocalPages = len(pages)
|
||||
result.Manifest = manifestPath
|
||||
|
||||
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, skip %d, drift %d", result.Created, result.Updated, result.Skipped, result.Drifted))
|
||||
progress(fmt.Sprintf("NodeBB wiki plan: create %d, update %d, skip %d, stale %d, archive %d, drift %d", result.Created, result.Updated, result.Skipped, result.Stale, result.Archived, result.Drifted))
|
||||
if opts.DryRun {
|
||||
return result, nil
|
||||
}
|
||||
@@ -152,6 +199,10 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
|
||||
if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil {
|
||||
return result, err
|
||||
}
|
||||
case "archive":
|
||||
if err := client.updatePost(plan.Entry.PID, plan.Content, summary); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := saveDeployManifest(manifestPath, nextManifest); err != nil {
|
||||
@@ -160,6 +211,62 @@ func DeployWikiWithOptions(p *project.Project, opts DeployWikiOptions, progress
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func loadWikiNamespaceDeclarations(p *project.Project) ([]wikiNamespaceDeclaration, error) {
|
||||
cfg := p.EffectiveConfig().TopData.Wiki
|
||||
path := filepath.Join(p.TopDataWikiSourceDir(), filepath.FromSlash(cfg.NamespacesFile))
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("read wiki namespaces: %w", err)
|
||||
}
|
||||
var doc wikiNamespacesDocument
|
||||
if err := yaml.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, fmt.Errorf("parse wiki namespaces: %w", err)
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
for _, declaration := range doc.Namespaces {
|
||||
id := strings.TrimSpace(declaration.ID)
|
||||
if id == "" {
|
||||
return nil, errors.New("topdata wiki namespace id is required")
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
return nil, fmt.Errorf("duplicate topdata wiki namespace id %q", id)
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
if strings.TrimSpace(declaration.CategoryEnv) == "" {
|
||||
return nil, fmt.Errorf("topdata wiki namespace %q category_env is required", id)
|
||||
}
|
||||
switch strings.TrimSpace(declaration.EditPolicy) {
|
||||
case "", "preserve_manual_sections", "generated_only":
|
||||
default:
|
||||
return nil, fmt.Errorf("topdata wiki namespace %q edit_policy %q is not supported", id, declaration.EditPolicy)
|
||||
}
|
||||
}
|
||||
return doc.Namespaces, nil
|
||||
}
|
||||
|
||||
func categoryIDsFromNamespaceEnv(declarations []wikiNamespaceDeclaration) (map[string]int, error) {
|
||||
categories := map[string]int{}
|
||||
for _, declaration := range declarations {
|
||||
envName := strings.TrimSpace(declaration.CategoryEnv)
|
||||
if envName == "" {
|
||||
continue
|
||||
}
|
||||
raw := strings.TrimSpace(os.Getenv(envName))
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
cid, err := strconv.Atoi(raw)
|
||||
if err != nil || cid <= 0 {
|
||||
return nil, fmt.Errorf("topdata wiki namespace %q env %s has invalid cid %q", declaration.ID, envName, raw)
|
||||
}
|
||||
categories[declaration.ID] = cid
|
||||
}
|
||||
return categories, nil
|
||||
}
|
||||
|
||||
func collectLocalPages(sourceDir string, namespaces []string) (map[string]wikiDeployPage, error) {
|
||||
namespaceSet := map[string]struct{}{}
|
||||
for _, namespace := range namespaces {
|
||||
@@ -224,6 +331,10 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
|
||||
page := pages[pageID]
|
||||
entry := manifest.Pages[pageID]
|
||||
entry.Hash = page.Hash
|
||||
entry.LastSeenHash = page.Hash
|
||||
entry.Title = page.Title
|
||||
entry.Namespace = page.Namespace
|
||||
entry.Stale = false
|
||||
if entry.CID == 0 {
|
||||
entry.CID = opts.CategoryIDs[page.Namespace]
|
||||
}
|
||||
@@ -267,7 +378,33 @@ func planNodeBBDeploy(pages map[string]wikiDeployPage, manifest wikiDeployManife
|
||||
}
|
||||
for pageID := range manifest.Pages {
|
||||
if _, ok := pages[pageID]; !ok {
|
||||
entry := manifest.Pages[pageID]
|
||||
entry.Stale = true
|
||||
if entry.Namespace == "" {
|
||||
entry.Namespace = strings.SplitN(pageID, ":", 2)[0]
|
||||
}
|
||||
if entry.Title == "" {
|
||||
entry.Title = extractHTMLTitle("", pageID)
|
||||
}
|
||||
next.Pages[pageID] = entry
|
||||
result.Stale++
|
||||
policy := strings.TrimSpace(opts.StalePolicy)
|
||||
if policy == "" {
|
||||
policy = "report"
|
||||
}
|
||||
if policy == "archive" {
|
||||
body := renderArchivedWikiPage(pageID, entry.Title)
|
||||
entry.Hash = computeManagedHash(body)
|
||||
entry.ArchivedHash = entry.Hash
|
||||
next.Pages[pageID] = entry
|
||||
result.Archived++
|
||||
plans = append(plans, wikiDeployPlan{
|
||||
Page: wikiDeployPage{PageID: pageID, Title: entry.Title, Namespace: entry.Namespace, Content: body, Hash: entry.Hash},
|
||||
Entry: entry,
|
||||
Action: "archive",
|
||||
Content: body,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return plans, result, next, nil
|
||||
@@ -322,17 +459,34 @@ func mergeManagedContent(existing, generated string) string {
|
||||
if !ok {
|
||||
return generated
|
||||
}
|
||||
_, endMarker, start, end := managedRegionBounds(existing)
|
||||
startMarker, endMarker, start, end := managedRegionBounds(existing)
|
||||
if start < 0 || end < 0 || end < start {
|
||||
return generated
|
||||
}
|
||||
if strings.Contains(generated, manualStartMarkerPrefix) {
|
||||
return mergeManualRegions(generated, existing)
|
||||
}
|
||||
end += len(endMarker)
|
||||
return existing[:start] + managedStartMarker + "\n" + generatedRegion + "\n" + managedEndMarker + existing[end:]
|
||||
generatedStart, generatedEnd, ok := extractManagedMarkers(generated)
|
||||
if !ok {
|
||||
generatedStart = startMarker
|
||||
generatedEnd = endMarker
|
||||
}
|
||||
merged := existing[:start] + generatedStart + "\n" + generatedRegion + "\n" + generatedEnd + existing[end:]
|
||||
return mergeManualRegions(merged, existing)
|
||||
}
|
||||
|
||||
func managedRegionBounds(content string) (string, string, int, int) {
|
||||
if start := strings.Index(content, managedStartMarker); start >= 0 {
|
||||
startEnd := strings.Index(content[start:], "-->")
|
||||
end := strings.Index(content, managedEndMarker)
|
||||
if startEnd >= 0 && end >= 0 {
|
||||
startMarker := content[start : start+startEnd+len("-->")]
|
||||
return startMarker, managedEndMarker, start, end
|
||||
}
|
||||
}
|
||||
for _, markers := range [][2]string{
|
||||
{managedStartMarker, managedEndMarker},
|
||||
{legacyMarkdownStartMarker, legacyMarkdownEndMarker},
|
||||
{legacyManagedStartMarker, legacyManagedEndMarker},
|
||||
} {
|
||||
start := strings.Index(content, markers[0])
|
||||
@@ -344,6 +498,109 @@ func managedRegionBounds(content string) (string, string, int, int) {
|
||||
return "", "", -1, -1
|
||||
}
|
||||
|
||||
func extractManagedMarkers(content string) (string, string, bool) {
|
||||
startMarker, endMarker, start, end := managedRegionBounds(content)
|
||||
return startMarker, endMarker, start >= 0 && end >= 0
|
||||
}
|
||||
|
||||
func mergeManualRegions(merged, existing string) string {
|
||||
existingManual := extractManualRegions(existing)
|
||||
if len(existingManual) == 0 {
|
||||
return merged
|
||||
}
|
||||
return replaceManualRegions(merged, existingManual)
|
||||
}
|
||||
|
||||
func extractManualRegions(content string) map[string]string {
|
||||
regions := map[string]string{}
|
||||
offset := 0
|
||||
for {
|
||||
startRel := strings.Index(content[offset:], manualStartMarkerPrefix)
|
||||
if startRel < 0 {
|
||||
break
|
||||
}
|
||||
start := offset + startRel
|
||||
startEndRel := strings.Index(content[start:], "-->")
|
||||
if startEndRel < 0 {
|
||||
break
|
||||
}
|
||||
startMarker := content[start : start+startEndRel+len("-->")]
|
||||
id := markerID(startMarker)
|
||||
if id == "" {
|
||||
offset = start + len(startMarker)
|
||||
continue
|
||||
}
|
||||
endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\""
|
||||
end := strings.Index(content[start+len(startMarker):], endPrefix)
|
||||
if end < 0 {
|
||||
offset = start + len(startMarker)
|
||||
continue
|
||||
}
|
||||
end += start + len(startMarker)
|
||||
endEndRel := strings.Index(content[end:], "-->")
|
||||
if endEndRel < 0 {
|
||||
break
|
||||
}
|
||||
endMarker := content[end : end+endEndRel+len("-->")]
|
||||
regions[id] = startMarker + strings.TrimRight(content[start+len(startMarker):end], "\n") + "\n" + endMarker
|
||||
offset = end + len(endMarker)
|
||||
}
|
||||
return regions
|
||||
}
|
||||
|
||||
func replaceManualRegions(content string, replacements map[string]string) string {
|
||||
for id, replacement := range replacements {
|
||||
start := strings.Index(content, manualStartMarkerPrefix+" id=\""+id+"\"")
|
||||
if start < 0 {
|
||||
continue
|
||||
}
|
||||
endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\""
|
||||
end := strings.Index(content[start:], endPrefix)
|
||||
if end < 0 {
|
||||
continue
|
||||
}
|
||||
end += start
|
||||
endEndRel := strings.Index(content[end:], "-->")
|
||||
if endEndRel < 0 {
|
||||
continue
|
||||
}
|
||||
endEnd := end + endEndRel + len("-->")
|
||||
content = content[:start] + replacement + content[endEnd:]
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
func markerID(marker string) string {
|
||||
const prefix = `id="`
|
||||
start := strings.Index(marker, prefix)
|
||||
if start < 0 {
|
||||
return ""
|
||||
}
|
||||
start += len(prefix)
|
||||
end := strings.Index(marker[start:], `"`)
|
||||
if end < 0 {
|
||||
return ""
|
||||
}
|
||||
return marker[start : start+end]
|
||||
}
|
||||
|
||||
func renderArchivedWikiPage(pageID, title string) string {
|
||||
if title == "" {
|
||||
title = extractHTMLTitle("", pageID)
|
||||
}
|
||||
body := strings.Join([]string{
|
||||
"<h1>" + html.EscapeString(title) + "</h1>",
|
||||
"<p>This generated page is no longer present in the current Shadows Over Westgate topdata wiki source.</p>",
|
||||
}, "\n")
|
||||
hash := computeManagedHash(body)
|
||||
return ensureTrailingNewline(strings.Join([]string{
|
||||
"<!-- sow-topdata-wiki:page=" + pageID + " -->",
|
||||
"<!-- sow-topdata-wiki:managed:start hash=\"sha256:" + hash + "\" -->",
|
||||
body,
|
||||
"<!-- sow-topdata-wiki:managed:end -->",
|
||||
}, "\n"))
|
||||
}
|
||||
|
||||
func extractPageTitle(content, fallback string) string {
|
||||
if title := extractMarkdownTitle(content); title != "" {
|
||||
return title
|
||||
|
||||
Reference in New Issue
Block a user