package topdata import ( "bytes" "crypto/sha256" "encoding/json" "errors" "fmt" "html" "io" "net/http" "net/url" "os" "path/filepath" "slices" "strconv" "strings" "time" "gitea.westgate.pw/ShadowsOverWestgate/sow-tools/internal/project" "gopkg.in/yaml.v3" ) const ( managedStartMarker = "" legacyMarkdownStartMarker = "[//]: # (sow-topdata-wiki:managed:start)" legacyMarkdownEndMarker = "[//]: # (sow-topdata-wiki:managed:end)" legacyManagedStartMarker = "" legacyManagedEndMarker = "" manualStartMarkerPrefix = "") end := strings.Index(content, managedEndMarker) if startEnd >= 0 && end >= 0 { startMarker := content[start : start+startEnd+len("-->")] return startMarker, managedEndMarker, start, end } } for _, markers := range [][2]string{ {legacyMarkdownStartMarker, legacyMarkdownEndMarker}, {legacyManagedStartMarker, legacyManagedEndMarker}, } { start := strings.Index(content, markers[0]) end := strings.Index(content, markers[1]) if start >= 0 && end >= 0 { return markers[0], markers[1], start, end } } return "", "", -1, -1 } func extractManagedMarkers(content string) (string, string, bool) { startMarker, endMarker, start, end := managedRegionBounds(content) return startMarker, endMarker, start >= 0 && end >= 0 } func mergeManualRegions(merged, existing string) string { existingManual := extractManualRegions(existing) if len(existingManual) == 0 { return merged } return replaceManualRegions(merged, existingManual) } func extractManualRegions(content string) map[string]string { regions := map[string]string{} offset := 0 for { startRel := strings.Index(content[offset:], manualStartMarkerPrefix) if startRel < 0 { break } start := offset + startRel startEndRel := strings.Index(content[start:], "-->") if startEndRel < 0 { break } startMarker := content[start : start+startEndRel+len("-->")] id := markerID(startMarker) if id == "" { offset = start + len(startMarker) continue } endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\"" end := strings.Index(content[start+len(startMarker):], endPrefix) if end < 0 { offset = start + len(startMarker) continue } end += start + len(startMarker) endEndRel := strings.Index(content[end:], "-->") if endEndRel < 0 { break } endMarker := content[end : end+endEndRel+len("-->")] regions[id] = startMarker + strings.TrimRight(content[start+len(startMarker):end], "\n") + "\n" + endMarker offset = end + len(endMarker) } return regions } func replaceManualRegions(content string, replacements map[string]string) string { for id, replacement := range replacements { start := strings.Index(content, manualStartMarkerPrefix+" id=\""+id+"\"") if start < 0 { continue } endPrefix := manualEndMarkerPrefix + " id=\"" + id + "\"" end := strings.Index(content[start:], endPrefix) if end < 0 { continue } end += start endEndRel := strings.Index(content[end:], "-->") if endEndRel < 0 { continue } endEnd := end + endEndRel + len("-->") content = content[:start] + replacement + content[endEnd:] } return content } func markerID(marker string) string { const prefix = `id="` start := strings.Index(marker, prefix) if start < 0 { return "" } start += len(prefix) end := strings.Index(marker[start:], `"`) if end < 0 { return "" } return marker[start : start+end] } func renderArchivedWikiPage(pageID, title string) string { if title == "" { title = extractHTMLTitle("", pageID) } body := strings.Join([]string{ "
This generated page is no longer present in the current Shadows Over Westgate topdata wiki source.
", }, "\n") hash := computeManagedHash(body) return ensureTrailingNewline(strings.Join([]string{ "", "", body, "", }, "\n")) } func extractWikiPageID(content string) string { const pageMarker = "sow-topdata-wiki:page=" start := strings.Index(content, pageMarker) if start < 0 { return "" } value := content[start+len(pageMarker):] if end := strings.Index(value, "-->"); end >= 0 { value = value[:end] } if fields := strings.Fields(value); len(fields) > 0 { return strings.Trim(strings.TrimSpace(fields[0]), `"`) } return "" } func extractWikiPagePublicPath(content string) string { const pageMarker = "sow-topdata-wiki:page=" const pathField = "public_path=" start := strings.Index(content, pageMarker) if start < 0 { return "" } end := strings.Index(content[start:], "-->") if end < 0 { return "" } marker := content[start : start+end] fieldStart := strings.Index(marker, pathField) if fieldStart < 0 { return "" } value := marker[fieldStart+len(pathField):] if fields := strings.Fields(value); len(fields) > 0 { path := strings.Trim(strings.TrimSpace(fields[0]), `"`) if path != "" && canonicalWikiSlashPath(path) == path { return path } } return "" } func extractPageTitle(content, fallback string) string { if title := extractMarkdownTitle(content); title != "" { return title } return extractHTMLTitle(content, fallback) } func extractMarkdownTitle(content string) string { for _, line := range strings.Split(content, "\n") { trimmed := strings.TrimSpace(line) if !strings.HasPrefix(trimmed, "# ") { continue } title := strings.TrimSpace(strings.TrimPrefix(trimmed, "# ")) if title != "" { return title } } return "" } func extractHTMLTitle(content, fallback string) string { lower := strings.ToLower(content) for _, tag := range []string{"h1", "h2"} { open := "<" + tag + ">" close := "" + tag + ">" start := strings.Index(lower, open) end := strings.Index(lower, close) if start >= 0 && end > start { title := strings.TrimSpace(content[start+len(open) : end]) title = strings.ReplaceAll(title, "\n", " ") if title != "" { return html.UnescapeString(stripSimpleTags(title)) } } } parts := strings.Split(fallback, ":") return namespaceTitle(parts[len(parts)-1]) } func nodeBBTopicTitle(page wikiDeployPage, minLength int) string { if minLength <= 0 { minLength = project.DefaultTopDataWikiTitlePrefixMinLength } title := strings.TrimSpace(page.Title) if len([]rune(title)) >= minLength { return title } namespace := strings.TrimSpace(namespaceTitle(page.Namespace)) if namespace == "" { namespace = "Wiki" } if title == "" { parts := strings.Split(page.PageID, ":") title = namespaceTitle(parts[len(parts)-1]) } return namespace + ": " + title } func stripSimpleTags(text string) string { var out strings.Builder inTag := false for _, r := range text { switch r { case '<': inTag = true case '>': inTag = false default: if !inTag { out.WriteRune(r) } } } return out.String() } func sortedKeysFromMap[V any](m map[string]V) []string { keys := make([]string, 0, len(m)) for key := range m { keys = append(keys, key) } slices.Sort(keys) return keys } type nodeBBClient struct { endpoint string token string timeout time.Duration } type nodeBBHTTPError struct { Method string Path string Status int Body string } func (e nodeBBHTTPError) Error() string { return fmt.Sprintf("NodeBB %s %s failed: HTTP %d: %s", e.Method, e.Path, e.Status, e.Body) } func isNodeBBMissingResource(err error) bool { var httpErr nodeBBHTTPError return errors.As(err, &httpErr) && (httpErr.Status == http.StatusNotFound || httpErr.Status == http.StatusGone) } type nodeBBPost struct { PID int TID int Title string Content string SourceContent string } type nodeBBEditLock struct { Token string } type nodeBBWikiPage struct { TID int Title string TitleLeaf string Slug string WikiPath string CanonicalPath string } func newNodeBBClient(endpoint, token string) *nodeBBClient { return &nodeBBClient{ endpoint: strings.TrimRight(endpoint, "/"), token: token, timeout: 30 * time.Second, } } func (c *nodeBBClient) getPost(pid int) (nodeBBPost, error) { var doc any if err := c.request("GET", fmt.Sprintf("/api/v3/posts/%d", pid), nil, &doc); err != nil { return nodeBBPost{}, err } return parseNodeBBPost(doc), nil } func (c *nodeBBClient) getTopic(tid int) (nodeBBPost, error) { var doc any if err := c.request("GET", fmt.Sprintf("/api/v3/topics/%d", tid), nil, &doc); err != nil { return nodeBBPost{}, err } post := parseNodeBBPost(doc) if post.TID == 0 { post.TID = tid } return post, nil } func (c *nodeBBClient) listNamespacePages(cid int) ([]nodeBBWikiPage, error) { return c.listNamespacePagesWithQuery(cid, "") } func (c *nodeBBClient) searchNamespacePages(cid int, query string) ([]nodeBBWikiPage, error) { return c.listNamespacePagesWithQuery(cid, query) } func (c *nodeBBClient) listNamespacePagesWithQuery(cid int, query string) ([]nodeBBWikiPage, error) { var out []nodeBBWikiPage after := "" seenCursors := map[string]struct{}{} seenPages := map[string]struct{}{} for { path := fmt.Sprintf("/api/v3/plugins/westgate-wiki/namespace/%d/pages?limit=80", cid) if query != "" { path += "&q=" + url.QueryEscape(query) } if after != "" { path += "&after=" + url.QueryEscape(after) } var doc any if err := c.request("GET", path, nil, &doc); err != nil { return nil, err } pages, next, hasMore := parseNodeBBWikiPageList(doc) newPages := 0 for _, page := range pages { identity := nodeBBWikiPageIdentity(page) if identity == "" { out = append(out, page) newPages++ continue } if _, ok := seenPages[identity]; ok { continue } seenPages[identity] = struct{}{} out = append(out, page) newPages++ } if !hasMore || next == "" { return out, nil } if after != "" && next == after { if newPages == 0 { return out, nil } return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next) } if _, ok := seenCursors[next]; ok { if newPages == 0 { return out, nil } return nil, fmt.Errorf("NodeBB wiki namespace %d returned repeated pagination cursor %q", cid, next) } seenCursors[next] = struct{}{} after = next } } func nodeBBWikiPageIdentity(page nodeBBWikiPage) string { if page.TID != 0 { return fmt.Sprintf("tid:%d", page.TID) } for _, value := range []string{page.WikiPath, page.CanonicalPath, page.Slug, page.Title} { if trimmed := strings.TrimSpace(value); trimmed != "" { return strings.ToLower(trimmed) } } return "" } func (c *nodeBBClient) createTopic(cid int, title, content, summary string) (nodeBBPost, error) { body := map[string]any{"cid": cid, "title": title, "content": content, "sourceContent": content} if summary != "" { body["_uid_note"] = summary } var doc any if err := c.request("POST", "/api/v3/topics", body, &doc); err != nil { return nodeBBPost{}, err } post := parseNodeBBPost(doc) if post.TID == 0 || post.PID == 0 { return nodeBBPost{}, fmt.Errorf("NodeBB topic create response did not include tid and pid") } return post, nil } func (c *nodeBBClient) updatePost(tid, pid int, content, summary string) error { if tid == 0 { post, err := c.getPost(pid) if err != nil { return err } tid = post.TID } lock, err := c.acquireEditLock(tid) if err != nil { return err } body := map[string]any{"content": content, "sourceContent": content} if lock.Token != "" { body["wikiEditLockToken"] = lock.Token } if summary != "" { body["_uid_note"] = summary } return c.request("PUT", fmt.Sprintf("/api/v3/posts/%d", pid), body, nil) } func (c *nodeBBClient) renameWikiPage(tid, cid int, title string) error { if tid == 0 || cid == 0 { return fmt.Errorf("NodeBB wiki page rename requires topic id and category id") } body := map[string]any{"tid": tid, "cid": cid, "title": title} return c.request("PUT", "/api/v3/plugins/westgate-wiki/page/move", body, nil) } func (c *nodeBBClient) purgeTopic(tid int) error { if tid == 0 { return fmt.Errorf("NodeBB topic purge requires topic id") } err := c.request(http.MethodDelete, fmt.Sprintf("/api/v3/topics/%d", tid), nil, nil) var httpErr nodeBBHTTPError if errors.As(err, &httpErr) && (httpErr.Status == http.StatusNotFound || httpErr.Status == http.StatusGone) { return nil } return err } func (c *nodeBBClient) acquireEditLock(tid int) (nodeBBEditLock, error) { if tid == 0 { return nodeBBEditLock{}, fmt.Errorf("NodeBB wiki post update requires a topic id for edit locking") } body := map[string]any{"tid": tid} var doc any if err := c.request("PUT", "/api/v3/plugins/westgate-wiki/edit-lock", body, &doc); err != nil { return nodeBBEditLock{}, err } lock := parseNodeBBEditLock(doc) if lock.Token == "" { return nodeBBEditLock{}, fmt.Errorf("NodeBB wiki edit lock response for topic %d did not include token", tid) } return lock, nil } func (c *nodeBBClient) request(method, path string, payload any, out any) error { var body io.Reader if payload != nil { raw, err := json.Marshal(payload) if err != nil { return err } body = bytes.NewReader(raw) } req, err := http.NewRequest(method, c.endpoint+path, body) if err != nil { return err } req.Header.Set("Authorization", "Bearer "+c.token) req.Header.Set("Accept", "application/json") if payload != nil { req.Header.Set("Content-Type", "application/json") } client := http.Client{Timeout: c.timeout} resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() raw, err := io.ReadAll(resp.Body) if err != nil { return err } if resp.StatusCode >= 400 { return nodeBBHTTPError{Method: method, Path: path, Status: resp.StatusCode, Body: strings.TrimSpace(string(raw))} } if out == nil || len(bytes.TrimSpace(raw)) == 0 { return nil } if err := json.Unmarshal(raw, out); err != nil { return fmt.Errorf("parse NodeBB response: %w", err) } return nil } func parseNodeBBPost(doc any) nodeBBPost { for { m, ok := doc.(map[string]any) if !ok { return nodeBBPost{} } if response, ok := m["response"]; ok { doc = response continue } if postData, ok := m["postData"]; ok { doc = postData continue } if mainPost, ok := m["mainPost"]; ok { post := parseNodeBBPost(mainPost) if post.TID == 0 { post.TID = intFromAny(m["tid"]) } if post.PID == 0 { post.PID = firstInt(m, "mainPid") } if post.Title == "" { post.Title = firstString(m, "title", "titleRaw", "title_raw") } return post } sourceContent := firstString(m, "sourceContent", "source_content") content := firstString(m, "content") if sourceContent != "" { content = sourceContent } return nodeBBPost{ PID: firstInt(m, "pid", "mainPid"), TID: firstInt(m, "tid"), Title: firstString(m, "title", "titleRaw", "title_raw"), Content: content, SourceContent: sourceContent, } } } func parseNodeBBEditLock(doc any) nodeBBEditLock { for { m, ok := doc.(map[string]any) if !ok { return nodeBBEditLock{} } if response, ok := m["response"]; ok { doc = response continue } return nodeBBEditLock{Token: firstString(m, "token")} } } func parseNodeBBWikiPageList(doc any) ([]nodeBBWikiPage, string, bool) { for { m, ok := doc.(map[string]any) if !ok { return nil, "", false } if response, ok := m["response"]; ok { doc = response continue } pagesRaw, _ := m["pages"].([]any) pages := make([]nodeBBWikiPage, 0, len(pagesRaw)) for _, raw := range pagesRaw { row, ok := raw.(map[string]any) if !ok { continue } pages = append(pages, nodeBBWikiPage{ TID: firstInt(row, "tid"), Title: firstString(row, "title"), TitleLeaf: firstString(row, "titleLeaf", "title_leaf"), Slug: firstString(row, "slug"), WikiPath: firstString(row, "wikiPath", "wiki_path"), CanonicalPath: firstString(row, "canonicalPath", "canonical_path"), }) } return pages, firstString(m, "nextCursor", "next_cursor"), boolFromAny(m["hasMore"]) } } func isNodeBBWikiPageCollision(err error) bool { var httpErr nodeBBHTTPError if !errors.As(err, &httpErr) { return false } if httpErr.Status != http.StatusBadRequest { return false } body := strings.ToLower(httpErr.Body) return strings.Contains(body, "wiki page with this url already exists") || strings.Contains(body, "page-collision") || strings.Contains(body, "namespace-page-collision") } func boolFromAny(value any) bool { switch typed := value.(type) { case bool: return typed case string: return typed == "true" || typed == "1" default: return false } } func firstInt(m map[string]any, keys ...string) int { for _, key := range keys { if value := intFromAny(m[key]); value != 0 { return value } } return 0 } func intFromAny(value any) int { switch typed := value.(type) { case int: return typed case float64: return int(typed) case json.Number: i, _ := typed.Int64() return int(i) case string: i, _ := strconv.Atoi(typed) return i default: return 0 } } func firstString(m map[string]any, keys ...string) string { for _, key := range keys { if value, ok := m[key].(string); ok { return value } } return "" }