Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage: tokenize.
3Consumes: original, masked (regions to skip), extracted (regions that
4tokenize with a pre-set role).
5Produces: tokens (span-sorted WorkTokens; text always == original
6slice), comma_offsets (segmentation points; never tokens),
7interpunct_offsets (间隔号 transcription markers, #298; never tokens).
8Reads: Policy.strip_emoji, Policy.strip_bidi.
10Implements rules T1, T2 and T3 of docs/design/rules.md, cited at
11their code below. There is NO text-rewriting normalize stage: all
12three are character-classification rules, so spans always index the
13original exactly as given (v1 contrast: decisions.md#T1).
14"""
15from __future__ import annotations
17import bisect
18import dataclasses
19import re
21from nameparser._pipeline._state import (
22 COMMA_CHARS, ParseState, WorkToken,
23)
24from nameparser._policy import Policy, _SCRIPT_RANGES, _script_matcher
25from nameparser._types import Role, Span
27# Ported from v1 (nameparser/config/regexes.py, "emoji" and "bidi") --
28# layering forbids importing the config package here, so the tables are
29# duplicated by design with this provenance note. When editing, keep
30# both copies in sync (regexes.py builds its public re_emoji from the
31# SAME codepoint pairs). Integer ranges, not a regex character class:
32# the per-char test needs no regex, and CodeQL's py/overly-large-range
33# false-positives on literal astral ranges (surrogate decomposition).
34_EMOJI_RANGES = ((0x1F300, 0x1F64F), (0x1F680, 0x1F6FF),
35 (0x2600, 0x26FF), (0x2700, 0x27BF))
36_BIDI = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+')
38# rules.md#T2: "the katakana middle dot and its halfwidth twin divide
39# a name like whitespace, always." They record no offset: the
40# nakaguro also divides kanji roster pairs (高橋・一郎, read
41# family-first by the script license, #272), so it is not a
42# transcription marker; only the Chinese 间隔号 is (#298).
43_NAME_DOT_SEPARATORS = frozenset({"\u30FB", "\uFF65"})
45_INTERPUNCT = "\u00B7"
46# rules.md#T3: "the interpunct divides a name only between two
47# characters of a classified East Asian script; anywhere else it is
48# part of the word" (history: decisions.md#T3). Per-CHAR classifier
49# for the flank guard: a one-char string is wholly-classified iff the
50# character is. Context-sensitivity is why this lives in
51# _tokenize_region (where the index exists) and not in _ignorable.
52_classified_char = _script_matcher(*_SCRIPT_RANGES, whole=True)
55def _is_emoji(ch: str) -> bool:
56 cp = ord(ch)
57 return any(lo <= cp <= hi for lo, hi in _EMOJI_RANGES)
60# rules.md#T1: "an ignorable character separates its neighbors and
61# never joins them" (v1 contrast: decisions.md#T1)
62def _stripped(ch: str, policy: Policy) -> bool:
63 """True when the strip policy removes `ch` from the token stream.
64 The ONE definition of that set, shared by _ignorable and _flank: a
65 character that vanishes from tokens must not occupy a flank
66 position either, so a new strip class added here stays transparent
67 to the interpunct guard by construction."""
68 if policy.strip_bidi and _BIDI.match(ch):
69 return True
70 return bool(policy.strip_emoji and _is_emoji(ch))
73def _flank(text: str, indices: range, policy: Policy) -> str | None:
74 """The nearest flank character the strip policy would keep, or
75 None if the range exhausts. Stripped invisibles are TRANSPARENT
76 here: an RTL document quoting a transcription puts U+200F beside
77 the dot in visually identical text. Whitespace and the other
78 separators stay guard-defeating: a dot beside a space is not
79 between characters."""
80 for i in indices:
81 ch = text[i]
82 if not _stripped(ch, policy):
83 return ch
84 return None
87def _ignorable(ch: str, state: ParseState) -> bool:
88 if ch.isspace():
89 return True
90 if ch.isascii():
91 # both strip classes and the name-dot separators are entirely
92 # non-ASCII (bidi >= U+061C, emoji >= U+2600, dots >= U+30FB):
93 # skip the checks below for every ASCII letter
94 return False
95 # unconditional, like whitespace -- not policy-gated, so this sits
96 # ahead of the policy check rather than in _tokenize_region beside
97 # COMMA_CHARS (commas RECORD an offset; these dots must not --
98 # only the 间隔号 marks a transcription and records, #298, see
99 # _NAME_DOT_SEPARATORS above). The only load-bearing constraint is
100 # "after the isascii fast path" (both dots are non-ASCII).
101 if ch in _NAME_DOT_SEPARATORS:
102 return True
103 return _stripped(ch, state.policy)
106def _tokenize_region(state: ParseState, start: int, end: int,
107 role: Role | None, record_offsets: bool,
108 tokens: list[WorkToken], commas: list[int],
109 interpuncts: list[int]) -> None:
110 text = state.original
111 tok_start: int | None = None
112 for i in range(start, end):
113 ch = text[i]
114 is_separator = ch in COMMA_CHARS or _ignorable(ch, state)
115 if ch == _INTERPUNCT:
116 # Region-local flanks: a B7 at a region edge stays token
117 # text, and the bound also stops a custom delimiter's
118 # classified edge character from acting as a flank across
119 # a mask seam. The scan reads raw text like segmentation
120 # matching does (#272's stance): NFD hangul degrades to
121 # no-split, never wrong-split -- classification
122 # NFC-normalizes but the guard does not.
123 left = _flank(text, range(i - 1, start - 1, -1),
124 state.policy)
125 if left is not None and _classified_char(left):
126 right = _flank(text, range(i + 1, end), state.policy)
127 is_separator = (right is not None
128 and _classified_char(right))
129 if is_separator:
130 if tok_start is not None:
131 tokens.append(WorkToken(text[tok_start:i],
132 Span(tok_start, i), role=role))
133 tok_start = None
134 if record_offsets:
135 if ch in COMMA_CHARS:
136 commas.append(i)
137 elif ch == _INTERPUNCT:
138 interpuncts.append(i)
139 continue
140 if tok_start is None:
141 tok_start = i
142 if tok_start is not None:
143 tokens.append(WorkToken(text[tok_start:end],
144 Span(tok_start, end), role=role))
147def tokenize(state: ParseState) -> ParseState:
148 tokens: list[WorkToken] = []
149 commas: list[int] = []
150 interpuncts: list[int] = []
151 # main stream: everything outside masked regions
152 boundaries = [0]
153 for m in state.masked:
154 boundaries.extend((m.start, m.end))
155 boundaries.append(len(state.original))
156 for start, end in zip(boundaries[::2], boundaries[1::2]):
157 _tokenize_region(state, start, end, None, True, tokens, commas,
158 interpuncts)
159 # extracted regions: pre-set role, commas and interpuncts are mere
160 # separators
161 for role, inner in state.extracted:
162 _tokenize_region(state, inner.start, inner.end, role, False,
163 tokens, commas, interpuncts)
164 tokens.sort(key=lambda t: t.span)
165 # extract_delimited runs before tokens exist, so its ambiguities
166 # carry a character offset instead of an index. Resolve them now
167 # that the stray character has landed in a token ('"Nick', 'Smith)')
168 # -- without this the ambiguity is locatable only by parsing the
169 # offset back out of its detail string. An offset inside a masked
170 # region belongs to no token; those keep an empty tuple, which the
171 # kind's contract already allows.
172 # Bisect rather than rescan: the closer sweep can emit one
173 # ambiguity per delimiter character, so a linear scan per ambiguity
174 # is quadratic on pathological input (') ' * 1600 spent 180ms here,
175 # against 9ms before the sweep existed). Spans are non-overlapping
176 # and sorted just above, so the candidate is the last token whose
177 # start is <= the offset.
178 # Nothing to resolve on the overwhelmingly common no-ambiguity
179 # parse, and building starts + the closure is not free.
180 ambiguities = state.ambiguities
181 if any(a.origin is not None for a in ambiguities):
182 starts = [t.span.start for t in tokens]
184 def _containing(offset: int) -> tuple[int, ...]:
185 i = bisect.bisect_right(starts, offset) - 1
186 if i >= 0 and offset < tokens[i].span.end:
187 return (i,)
188 return ()
190 ambiguities = tuple(
191 a if a.origin is None
192 else dataclasses.replace(a, indices=_containing(a.origin))
193 for a in ambiguities)
194 return dataclasses.replace(state, tokens=tuple(tokens),
195 comma_offsets=tuple(sorted(commas)),
196 interpunct_offsets=tuple(sorted(interpuncts)),
197 ambiguities=ambiguities)