1import re
2
3# emoji regex from https://stackoverflow.com/questions/26568722/remove-unicode-emoji-using-re-in-python
4re_emoji = re.compile('[' # lgtm[py/overly-large-range]
5 '\U0001F300-\U0001F64F'
6 '\U0001F680-\U0001F6FF'
7 '\u2600-\u26FF\u2700-\u27BF]+')
8
9# Invisible bidirectional formatting characters: ALM, LRM, RLM, the
10# embedding/override marks (LRE/RLE/PDF/LRO/RLO) and the isolates
11# (LRI/RLI/FSI/PDI). Copy-pasted RTL names often carry these; they render
12# as nothing but stick to name parts and break equality/lookups.
13re_bidi = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+')
14
15EMPTY_REGEX = re.compile('')
16
17REGEXES = {
18 "spaces": re.compile(r"\s+"),
19 "word": re.compile(r"(\w|\.)+"),
20 "mac": re.compile(r'^(ma?c)(\w{2,})', re.I),
21 "initial": re.compile(r'^(\w\.|[A-Z])?$'),
22 "quoted_word": re.compile(r'(?<!\w)\'([^\s]*?)\'(?!\w)'),
23 "double_quotes": re.compile(r'\"(.*?)\"'),
24 "parenthesis": re.compile(r'\((.*?)\)'),
25 "roman_numeral": re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I),
26 "no_vowels": re.compile(r'^[^aeyiuo]+$', re.I),
27 "period_not_at_end": re.compile(r'.*\..+$', re.I),
28 "emoji": re_emoji,
29 "bidi": re_bidi,
30 "phd": re.compile(r'\s(ph\.?\s+d\.?)', re.I),
31 "space_before_comma": re.compile(r'\s+,'),
32 # ASCII comma plus its Arabic (U+060C) and fullwidth CJK (U+FF0C)
33 # counterparts, used to split "Last, First" format and to strip a
34 # trailing comma before parsing (#265).
35 "commas": re.compile(r'[,،,]'),
36 "east_slavic_patronymic": re.compile(
37 r'(ovich|ovna|evich|evna|ichna|ilyich|kuzmich|lukich|fomich|fokich)$',
38 re.I,
39 ),
40 "east_slavic_patronymic_cyrillic": re.compile(
41 r'(ович|овна|евич|евна|ична|ильич|кузьмич|лукич|фомич|фокич)$',
42 re.I,
43 ),
44 "turkic_patronymic_marker": re.compile(
45 r"^(oglu|oğlu|ogly|ogli|o['’ʻ]g['’ʻ]li"
46 r"|qizi|qızı|kizi|kyzy|gyzy|uly|uulu)$",
47 re.I,
48 ),
49 "turkic_patronymic_marker_cyrillic": re.compile(
50 r'^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$',
51 re.I,
52 ),
53 "period_abbreviation": re.compile(r'^[^\W\d_]{2,}\.$'),
54}
55"""
56All regular expressions used by the parser are precompiled and stored in the config.
57"""