Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage: tokenize.
3Consumes: original, masked (regions to skip), extracted (regions that
4tokenize with a pre-set role).
5Produces: tokens (span-sorted WorkTokens; text always == original
6slice), comma_offsets (segmentation points of the main stream; a comma
7there is never a token, one inside an extracted region is word text),
8interpunct_offsets (间隔号 and fullwidth-stop transcription markers, #298,
9#1312; never tokens).
10Reads: Policy.strip_emoji, Policy.strip_bidi.
12Implements rules T1, T2, T3 and T6 of docs/design/rules/, cited at
13their code below. There is NO text-rewriting normalize stage: all
14four are character-classification rules, so spans always index the
15original exactly as given (v1 contrast: decisions/T1).
17Beside the stage function, `_ignorable` is shared: extract asks it
18where a listed particle's word ends (rules/N2, #1241), so the two
19stages cannot disagree about which characters separate a word.
20"""
21from __future__ import annotations
23import bisect
24import re
26from nameparser._pipeline._state import (
27 COMMA_CHARS, ParseState, WorkToken, copy_with,
28)
29from nameparser._policy import Policy, _classified_char
30from nameparser._types import Role, Span
32# Ported from v1's "emoji" and "bidi" patterns (the 1.x and 2.x
33# nameparser.config.regexes, removed in 3.0, #1008); this is now the
34# only copy. Integer ranges, not a regex character class:
35# the per-char test needs no regex, and CodeQL's py/overly-large-range
36# false-positives on literal astral ranges (surrogate decomposition).
37_EMOJI_RANGES = ((0x1F300, 0x1F64F), (0x1F680, 0x1F6FF),
38 (0x2600, 0x26FF), (0x2700, 0x27BF))
39_BIDI = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+')
41# rules/T2: "the katakana middle dot, its halfwidth twin and the
42# Ethiopic wordspace divide a name like whitespace, always." They
43# record no offset: the nakaguro also divides kanji roster pairs
44# (高橋・一郎, read family-first by the script license, #272), so it is
45# not a transcription marker; only the Chinese 间隔号 is (#298). The
46# Ethiopic wordspace U+1361 joined them (decisions/T2, #1202).
47_NAME_DOT_SEPARATORS = frozenset({"\u30FB", "\uFF65", "\u1361"})
49# rules/T6: "a narrow no-break space between two Mongolian letters
50# joins them into one word" -- the whole rule, T1's Except: anywhere
51# else U+202F is whitespace and divides (decisions/T6, #1236).
52_NARROW_NBSP = "\u202F"
53_MONGOLIAN_BLOCK = (0x1800, 0x18AF)
55# rules/T1: "a zero-width space, an invisible separator and a
56# byte-order mark have no such opt-out" -- the Except clause: they
57# separate under every policy, carrying nothing a caller would keep.
58# Not here: U+200C and U+200D, which decide how the letters around
59# them join, the word joiner U+2060, which means no break, the soft
60# hyphen U+00AD, which marks a break inside one word, the invisible
61# math operators U+2061, U+2062 and U+2064, and the Mongolian vowel
62# separator U+180E, which shapes the letters beside it
63# (decisions/T1, #851).
64_INVISIBLE_SEPARATORS = frozenset({"\u200B", "\u2063", "\uFEFF"})
66_INTERPUNCT = "\u00B7"
67# The fullwidth full stop is the same transcription dot where Taiwan's
68# writers type it between two Han names (`台邦．撒沙勒`, #1312), and the
69# same plain full stop everywhere else, so it takes the interpunct's
70# flank guard whole: it divides, and records, only between two characters
71# of a classified script. The ideographic full stop U+3002 is not one:
72# it is the sentence stop (decisions/T3, #1312).
73_TRANSCRIPTION_DOTS = frozenset({_INTERPUNCT, "\uFF0E"})
74# rules/T3: "the interpunct divides a name only between two
75# characters of a classified East Asian script; anywhere else it is
76# part of the word" (history: decisions/T3). The flank guard asks
77# _policy._classified_char per CHAR: a one-char string is
78# wholly-classified iff the character is. Context-sensitivity is why the
79# guard lives in _tokenize_region (where the index exists) and not in
80# _ignorable.
83def _mongolian_letter(ch: str) -> bool:
84 """A character of the Mongolian block that is not its punctuation
85 (U+1800-U+180A) or a digit (U+1810-U+1819): its letters, free
86 variation selectors and vowel separator, each of which stands
87 inside a written word, and the block's unassigned codepoints."""
88 cp = ord(ch)
89 return (_MONGOLIAN_BLOCK[0] <= cp <= _MONGOLIAN_BLOCK[1]
90 and not 0x1800 <= cp <= 0x180A and not 0x1810 <= cp <= 0x1819)
93def _is_emoji(ch: str) -> bool:
94 cp = ord(ch)
95 return any(lo <= cp <= hi for lo, hi in _EMOJI_RANGES)
98# rules/T1: "an ignorable character separates its neighbors and
99# never joins them" (v1 contrast: decisions/T1)
100def _stripped(ch: str, policy: Policy) -> bool:
101 """True when the strip policy removes `ch` from the token stream.
102 The ONE definition of that set, shared by _ignorable and _flank: a
103 character the strip policy removes must not occupy a flank
104 position either, so a new strip class added here stays transparent
105 to the interpunct guard by construction. The separators (whitespace,
106 the name dots and _INVISIBLE_SEPARATORS) are not in this set and do
107 occupy one."""
108 if policy.strip_bidi and _BIDI.match(ch):
109 return True
110 return bool(policy.strip_emoji and _is_emoji(ch))
113def _flank(text: str, indices: range, policy: Policy) -> str | None:
114 """The nearest flank character the strip policy would keep, or
115 None if the range exhausts. Stripped invisibles are TRANSPARENT
116 here: an RTL document quoting a transcription puts U+200F beside
117 the dot in visually identical text. Whitespace and the other
118 separators stay guard-defeating: a dot beside a space is not
119 between characters."""
120 for i in indices:
121 ch = text[i]
122 if not _stripped(ch, policy):
123 return ch
124 return None
127def _ignorable(ch: str, state: ParseState) -> bool:
128 if ch.isspace():
129 return True
130 if ch.isascii():
131 # both strip classes and the unconditional separators are
132 # entirely non-ASCII (bidi >= U+061C, emoji >= U+2600, the
133 # invisibles >= U+200B, dividers >= U+1361): skip the checks below
134 # for every ASCII letter
135 return False
136 # unconditional, like whitespace -- not policy-gated, so this sits
137 # ahead of the policy check rather than in _tokenize_region beside
138 # COMMA_CHARS (commas RECORD an offset; these must not -- only the
139 # 间隔号 marks a transcription and records, #298, see
140 # _NAME_DOT_SEPARATORS above). The only load-bearing constraint is
141 # "after the isascii fast path" (every member of both sets is
142 # non-ASCII).
143 if ch in _NAME_DOT_SEPARATORS or ch in _INVISIBLE_SEPARATORS:
144 return True
145 return _stripped(ch, state.policy)
148def _tokenize_region(state: ParseState, start: int, end: int,
149 role: Role | None, record_offsets: bool,
150 tokens: list[WorkToken], commas: list[int],
151 interpuncts: list[int]) -> None:
152 text = state.original
153 tok_start: int | None = None
154 for i in range(start, end):
155 ch = text[i]
156 # rules/N1: "a comma inside the clause is kept as written: the
157 # comma divides nothing there" -- the Then bullet on a comma: a
158 # comma separates only where it records a segmentation offset,
159 # the main stream; inside an extracted region it is the clause's
160 # text, like ';' (decisions/N1, #653)
161 is_separator = ((record_offsets and ch in COMMA_CHARS)
162 or _ignorable(ch, state))
163 if (ch == _NARROW_NBSP and start < i < end - 1
164 and _mongolian_letter(text[i - 1])
165 and _mongolian_letter(text[i + 1])):
166 is_separator = False
167 if ch in _TRANSCRIPTION_DOTS:
168 # Region-local flanks: a dot at a region edge stays token
169 # text, and the bound also stops a custom delimiter's
170 # classified edge character from acting as a flank across
171 # a mask seam. The scan reads raw characters, not the NFC
172 # copy classification reads; the script table holds the
173 # conjoining jamo a decomposed syllable is written in, so
174 # decomposed hangul flanks as composed hangul does
175 # (decisions/nfd-shapes, #1087).
176 left = _flank(text, range(i - 1, start - 1, -1),
177 state.policy)
178 if left is not None and _classified_char(left):
179 right = _flank(text, range(i + 1, end), state.policy)
180 is_separator = (right is not None
181 and _classified_char(right))
182 if is_separator:
183 if tok_start is not None:
184 tokens.append(WorkToken(text[tok_start:i],
185 Span(tok_start, i), role=role))
186 tok_start = None
187 if record_offsets:
188 if ch in COMMA_CHARS:
189 commas.append(i)
190 elif ch in _TRANSCRIPTION_DOTS:
191 interpuncts.append(i)
192 continue
193 if tok_start is None:
194 tok_start = i
195 if tok_start is not None:
196 tokens.append(WorkToken(text[tok_start:end],
197 Span(tok_start, end), role=role))
200def tokenize(state: ParseState) -> ParseState:
201 tokens: list[WorkToken] = []
202 commas: list[int] = []
203 interpuncts: list[int] = []
204 # main stream: everything outside masked regions
205 boundaries = [0]
206 for m in state.masked:
207 boundaries.extend((m.start, m.end))
208 boundaries.append(len(state.original))
209 for start, end in zip(boundaries[::2], boundaries[1::2]):
210 _tokenize_region(state, start, end, None, True, tokens, commas,
211 interpuncts)
212 # extracted regions: pre-set role; a comma is word text there and an
213 # interpunct a mere separator, neither recording an offset
214 for role, inner in state.extracted:
215 _tokenize_region(state, inner.start, inner.end, role, False,
216 tokens, commas, interpuncts)
217 tokens.sort(key=lambda t: t.span)
218 # extract_delimited runs before tokens exist, so its ambiguities
219 # carry a character offset instead of an index. Resolve them now
220 # that the stray character has landed in a token ('"Nick', 'Smith)')
221 # -- without this the ambiguity is locatable only by parsing the
222 # offset back out of its detail string. An offset inside a masked
223 # region belongs to no token; those keep an empty tuple, which the
224 # kind's contract already allows.
225 # Bisect rather than rescan: the closer sweep can emit one
226 # ambiguity per delimiter character, so a linear scan per ambiguity
227 # is quadratic on pathological input (') ' * 1600 spent 180ms here,
228 # against 9ms before the sweep existed). Spans are non-overlapping
229 # and sorted just above, so the candidate is the last token whose
230 # start is <= the offset.
231 # Nothing to resolve on the overwhelmingly common no-ambiguity
232 # parse, and building starts + the closure is not free.
233 ambiguities = state.ambiguities
234 if any(a.origin is not None for a in ambiguities):
235 starts = [t.span.start for t in tokens]
237 def _containing(offset: int) -> tuple[int, ...]:
238 i = bisect.bisect_right(starts, offset) - 1
239 if i >= 0 and offset < tokens[i].span.end:
240 return (i,)
241 return ()
243 ambiguities = tuple(
244 a if a.origin is None
245 else copy_with(a, indices=_containing(a.origin))
246 for a in ambiguities)
247 return copy_with(state, tokens=tuple(tokens),
248 comma_offsets=tuple(sorted(commas)),
249 interpunct_offsets=tuple(sorted(interpuncts)),
250 ambiguities=ambiguities)