Slug joiner normalization

internal/topdata/wiki_slug.go

@@ -0,0 +1,48 @@
+package topdata
+
+import (
+	"html"
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/unicode/norm"
+)
+
+func slugifyWikiText(value string, fallback string) string {
+	value = html.UnescapeString(strings.TrimSpace(value))
+	var b strings.Builder
+	lastDash := false
+	for _, r := range norm.NFKD.String(strings.ToLower(value)) {
+		switch {
+		case r >= 'a' && r <= 'z':
+			b.WriteRune(r)
+			lastDash = false
+		case r >= '0' && r <= '9':
+			b.WriteRune(r)
+			lastDash = false
+		case unicode.Is(unicode.Mn, r):
+			continue
+		case isWikiSlugJoinerPunctuation(r):
+			continue
+		default:
+			if b.Len() > 0 && !lastDash {
+				b.WriteByte('-')
+				lastDash = true
+			}
+		}
+	}
+	slug := strings.Trim(b.String(), "-")
+	if slug == "" {
+		return fallback
+	}
+	return slug
+}
+
+func isWikiSlugJoinerPunctuation(r rune) bool {
+	switch r {
+	case '\'', '"', '`', '´', '‘', '’', '‚', '‛', '“', '”', '„', '‟', '‹', '›', '«', '»', '′', '″', '＇', '＂':
+		return true
+	default:
+		return false
+	}
+}