package topdata import ( "html" "strings" "unicode" "golang.org/x/text/unicode/norm" ) var reservedCanonicalWikiFirstSegments = map[string]struct{}{ "admin": {}, "api": {}, "category": {}, "compose": {}, "edit": {}, "namespace": {}, "search": {}, } func canonicalWikiSegment(value string) string { value = html.UnescapeString(strings.TrimSpace(value)) var b strings.Builder lastSeparator := false for _, r := range norm.NFKD.String(value) { transliterated := transliterateWikiSlugRune(unicode.ToLower(r)) switch { case (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9'): b.WriteRune(r) lastSeparator = false case r == '_': if b.Len() > 0 && !lastSeparator { b.WriteRune(r) lastSeparator = true } case unicode.Is(unicode.Mn, r): continue case isWikiSlugJoinerPunctuation(r): continue case transliterated != "": b.WriteString(matchWikiTransliterationCase(r, transliterated)) lastSeparator = false default: if b.Len() > 0 && !lastSeparator { b.WriteByte('_') lastSeparator = true } } } return strings.Trim(b.String(), "_") } func canonicalWikiSegmentFoldedKey(value string) string { return strings.Join(strings.Fields(strings.ToLower(strings.ReplaceAll(canonicalWikiSegment(value), "_", " "))), " ") } func canonicalWikiSlashPath(value string) string { segments := []string{} for _, part := range strings.Split(strings.Trim(strings.TrimSpace(value), "/"), "/") { canonical := canonicalWikiSegment(part) if canonical != "" { segments = append(segments, canonical) } } return strings.Join(segments, "/") } func canonicalWikiSlashPathFoldedKey(value string) string { segments := []string{} for _, part := range strings.Split(strings.Trim(strings.TrimSpace(value), "/"), "/") { folded := canonicalWikiSegmentFoldedKey(part) if folded != "" { segments = append(segments, folded) } } return strings.Join(segments, "/") } func canonicalWikiSlashPathSegments(value string) []string { segments := []string{} for _, part := range strings.Split(strings.Trim(strings.TrimSpace(value), "/"), "/") { canonical := canonicalWikiSegment(part) if canonical != "" { segments = append(segments, canonical) } } return segments } func isReservedCanonicalWikiFirstSegment(path string) bool { segments := canonicalWikiSlashPathSegments(path) if len(segments) == 0 { return false } first := strings.ToLower(segments[0]) if isCanonicalWikiNumericSegment(first) { return true } _, reserved := reservedCanonicalWikiFirstSegments[first] return reserved } func isCanonicalWikiNumericSegment(segment string) bool { if segment == "" { return false } for _, r := range segment { if r < '0' || r > '9' { return false } } return true } func isNestedCanonicalWikiPath(path string) bool { return len(canonicalWikiSlashPathSegments(path)) > 2 } func matchWikiTransliterationCase(r rune, value string) string { if unicode.IsUpper(r) { if len(value) == 1 { return strings.ToUpper(value) } return strings.ToUpper(value[:1]) + value[1:] } return value } func canonicalWikiPath(segments ...string) string { out := []string{} for _, segment := range segments { canonical := canonicalWikiSegment(segment) if canonical != "" { out = append(out, canonical) } } return strings.Join(out, "/") } func slugifyWikiText(value string, fallback string) string { value = html.UnescapeString(strings.TrimSpace(value)) var b strings.Builder lastDash := false for _, r := range norm.NFKD.String(strings.ToLower(value)) { transliterated := transliterateWikiSlugRune(r) switch { case r >= 'a' && r <= 'z': b.WriteRune(r) lastDash = false case r >= '0' && r <= '9': b.WriteRune(r) lastDash = false case unicode.Is(unicode.Mn, r): continue case isWikiSlugJoinerPunctuation(r): continue case transliterated != "": b.WriteString(transliterated) lastDash = false default: if b.Len() > 0 && !lastDash { b.WriteByte('-') lastDash = true } } } slug := strings.Trim(b.String(), "-") if slug == "" { return fallback } return slug } func isWikiSlugJoinerPunctuation(r rune) bool { switch r { case '\'', '"', '`', '´', '‘', '’', '‚', '‛', '“', '”', '„', '‟', '‹', '›', '«', '»', '′', '″', ''', '"': return true default: return false } } func transliterateWikiSlugRune(r rune) string { switch r { case 'æ': return "ae" case 'œ': return "oe" case 'ø': return "o" case 'ß': return "ss" case 'þ': return "th" case 'ð', 'đ': return "d" case 'ł': return "l" case 'ħ': return "h" case 'ı': return "i" case 'ŋ': return "n" case 'ŧ': return "t" default: return "" } }