package topdata import ( "html" "strings" "unicode" "golang.org/x/text/unicode/norm" ) func canonicalWikiSegment(value string) string { value = html.UnescapeString(strings.TrimSpace(value)) var b strings.Builder lastSeparator := false for _, r := range norm.NFKD.String(value) { transliterated := transliterateWikiSlugRune(unicode.ToLower(r)) switch { case (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9'): b.WriteRune(r) lastSeparator = false case r == '_': if b.Len() > 0 && !lastSeparator { b.WriteRune(r) lastSeparator = true } case unicode.Is(unicode.Mn, r): continue case isWikiSlugJoinerPunctuation(r): continue case transliterated != "": b.WriteString(matchWikiTransliterationCase(r, transliterated)) lastSeparator = false default: if b.Len() > 0 && !lastSeparator { b.WriteByte('_') lastSeparator = true } } } return strings.Trim(b.String(), "_") } func matchWikiTransliterationCase(r rune, value string) string { if unicode.IsUpper(r) { if len(value) == 1 { return strings.ToUpper(value) } return strings.ToUpper(value[:1]) + value[1:] } return value } func canonicalWikiPath(segments ...string) string { out := []string{} for _, segment := range segments { canonical := canonicalWikiSegment(segment) if canonical != "" { out = append(out, canonical) } } return strings.Join(out, "/") } func slugifyWikiText(value string, fallback string) string { value = html.UnescapeString(strings.TrimSpace(value)) var b strings.Builder lastDash := false for _, r := range norm.NFKD.String(strings.ToLower(value)) { transliterated := transliterateWikiSlugRune(r) switch { case r >= 'a' && r <= 'z': b.WriteRune(r) lastDash = false case r >= '0' && r <= '9': b.WriteRune(r) lastDash = false case unicode.Is(unicode.Mn, r): continue case isWikiSlugJoinerPunctuation(r): continue case transliterated != "": b.WriteString(transliterated) lastDash = false default: if b.Len() > 0 && !lastDash { b.WriteByte('-') lastDash = true } } } slug := strings.Trim(b.String(), "-") if slug == "" { return fallback } return slug } func isWikiSlugJoinerPunctuation(r rune) bool { switch r { case '\'', '"', '`', '´', '‘', '’', '‚', '‛', '“', '”', '„', '‟', '‹', '›', '«', '»', '′', '″', ''', '"': return true default: return false } } func transliterateWikiSlugRune(r rune) string { switch r { case 'æ': return "ae" case 'œ': return "oe" case 'ø': return "o" case 'ß': return "ss" case 'þ': return "th" case 'ð', 'đ': return "d" case 'ł': return "l" case 'ħ': return "h" case 'ı': return "i" case 'ŋ': return "n" case 'ŧ': return "t" default: return "" } }