Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage: extract_delimited.
3Consumes: ParseState.original.
4Produces: extracted (role + inner span per delimited region), masked
5(full regions incl. delimiter chars, skipped by tokenize),
6UNBALANCED_DELIMITER ambiguities for opens with no close.
7A Role.MAIDEN region is the WHOLE inner span, marker included (all of
8it, a phrase marker being several words) -- nothing here strips one.
9classify tags a marker inside it like any other token, and group drops
10it from a multi-token clause (#329).
11A region reaches that role two ways: the pair that matched sits in
12Policy.maiden_delimiters (M1), or the content itself opens with a
13marker word (M3), which reassigns the role after the match and so is
14the one thing here that a bucket alone does not decide.
15Reads: Policy.nickname_delimiters, Policy.maiden_delimiters,
16Lexicon.maiden_markers, and
17Lexicon.suffix_words / suffix_acronyms / suffix_acronyms_ambiguous
18through _suffix_shaped.
20Implements rules N1, N2, S1, M1 and M3 of docs/design/rules.md (the #273
21matching mechanism); each is cited at its code below. One scan
22mechanic worth stating up front: matching is one left-to-right pass,
23no nesting, and delimiter characters inside a matched region are
24literal content for every other pair.
26Bucket precedence is NOT decided here: Policy canonicalizes overlap
27away before parsing, so the two buckets are always disjoint by the
28time this stage runs. The nickname-before-maiden candidate order
29below is only a same-position tie-break for exotic configs where two
30pairs share an OPEN character.
31"""
32from __future__ import annotations
34import bisect
35import dataclasses
36import functools
38from nameparser._lexicon import Lexicon, _normalize
39from nameparser._pipeline._state import (
40 COMMA_CHARS, ParseState, PendingAmbiguity,
41)
42from nameparser._pipeline._vocab import maiden_marker_run
43from nameparser._types import AmbiguityKind, Role, Span
46# rules.md#S1: "a bracketed clause whose content is suffix-shaped is
47# not a nickname: the brackets are dropped and the content reads
48# exactly as if written bare"
49def _suffix_shaped(content: str, lexicon: Lexicon) -> bool:
50 """v1 parse_nicknames' escape (parser.py:1125-1141): an unambiguous
51 suffix_words member (edge-normalized), an unambiguous acronym
52 (period-free form), or anything ending in a period. No initial
53 veto -- v1 deliberately skipped it here."""
54 stripped = _normalize(content)
55 acronym = stripped.replace(".", "")
56 return (stripped in lexicon.suffix_words
57 or (acronym in lexicon.suffix_acronyms
58 and acronym not in lexicon.suffix_acronyms_ambiguous)
59 or content.endswith("."))
62# rules.md#M3: "a bracketed clause whose content opens with a
63# recognized marker and carries a word after it reads as the
64# maiden name, whichever bucket the enclosing pair sits in"
65def _maiden_marked(content: str, lexicon: Lexicon) -> bool:
66 """The clause says 'maiden' out loud, so the caller does not have to
67 say it in Policy. Requires a word AFTER the marker: a lone marker in
68 brackets is a word in brackets, and M1 deliberately keeps a one-word
69 clause's word (it may be the surname Nee). A PHRASE marker is one
70 marker, so the word after is the word after the whole run --
71 '(z domu Nowak)' has one, '(z domu)' has none. The word after is
72 not tested for anything -- M3's Accepted line, and the reason a
73 bracketed '(née V)' reads maiden 'V' where the bare 'née V' gives
74 M2 a suffix.
76 This stage runs before tokenize, so it calls maiden_marker_run
77 itself rather than reading the tags classify will set. The two
78 questions are deliberately not the same one: this splits on
79 WHITESPACE, so a marker the writer glued to punctuation is not one
80 here ('née,'); the tokenizer splits that comma off and classify
81 still tags the token, which is what keeps _group's Role.MAIDEN
82 filter reachable. Sharing the PREDICATE does not merge the two
83 questions -- each hands it a different sequence of words."""
84 words = content.split()
85 # A one-word clause cannot satisfy the condition whatever the
86 # vocabulary says -- `run > 0 and 1 > run` is unsatisfiable -- so
87 # refuse it before any fold or lookup, as the pre-#434 words[0]
88 # test did for free.
89 if len(words) < 2:
90 return False
91 run = maiden_marker_run(words, lexicon.maiden_markers)
92 return run > 0 and len(words) > run
95# rules.md#N2: "a quote whose open and close are the same character
96# opens only at a word start and closes only at a word end, so an
97# apostrophe inside or at the end of a word is literal"
98def _open_ok(text: str, i: int) -> bool:
99 return i == 0 or text[i - 1].isspace()
102def _close_ok(text: str, j: int, width: int) -> bool:
103 k = j + width
104 return k >= len(text) or text[k].isspace() or text[k] in COMMA_CHARS
107# Delimiters that also occur INSIDE and at the end of real name parts,
108# so a dangling one is literal rather than unbalanced. Only the straight
109# apostrophe qualifies: quotes do not appear inside names, so the same
110# position with a quote genuinely is ambiguous. #273 dropped the curly
111# apostrophe from the defaults outright for this reason; the straight
112# one has to stay a delimiter (v1's quoted_word), so it is carved out
113# here instead. Deliberately not widened to a configured delimiter set.
114WORD_INTERNAL_DELIMITERS = frozenset({"'"})
117def _word_internal(text: str, j: int, close: str) -> bool:
118 """A word-internal delimiter directly after a word character is part
119 of the word, not a dangling close -- "Mari' Aube'", "Ali Baba'"."""
120 return (close in WORD_INTERNAL_DELIMITERS and j > 0
121 and (text[j - 1].isalnum() or text[j - 1] == "."))
124def _overlaps(span: Span, taken: list[Span], starts: list[int]) -> bool:
125 """`taken` sorted and non-overlapping (both hold by construction),
126 `starts` its start offsets. Bisect rather than scan: the closer
127 sweep tests one span per delimiter character found, so a linear
128 probe is quadratic in the number of matched pairs (400 pairs spent
129 5.4ms here, against 2.5ms before the sweep existed). Same idiom, and
130 the same reason, as the origin resolution in _tokenize."""
131 i = bisect.bisect_right(starts, span.start) - 1
132 # the only candidates are the last span starting at or before us and
133 # its successor -- anything earlier ends before it, anything later
134 # starts after us
135 for k in (i, i + 1):
136 if 0 <= k < len(taken) and span.start < taken[k].end and (
137 taken[k].start < span.end):
138 return True
139 return False
142@functools.lru_cache(maxsize=128)
143def _delimiter_chars(
144 nickname_pairs: frozenset[tuple[str, str]],
145 maiden_pairs: frozenset[tuple[str, str]],
146) -> frozenset[str]:
147 """Every character appearing in any configured delimiter, cached on
148 the (hashable) policy frozensets: the common no-delimiter name pays
149 one isdisjoint() instead of a per-pair scan."""
150 return frozenset(
151 ch
152 for pairs in (nickname_pairs, maiden_pairs)
153 for pair in pairs
154 for part in pair
155 for ch in part
156 )
159def _unmatched(open_: str, offset: int) -> tuple[int, PendingAmbiguity]:
160 return (offset, PendingAmbiguity(
161 AmbiguityKind.UNBALANCED_DELIMITER,
162 f"unmatched {open_!r} at offset {offset}; treated as literal text",
163 origin=offset,
164 ))
167# rules.md#N1: "a clause enclosed by a configured nickname delimiter
168# pair reads as the nickname and is lifted out of the name; an empty
169# enclosure is simply dropped"
170# rules.md#M1: "with a delimiter pair configured for maiden names, its
171# enclosed clause reads as the maiden name" (history: decisions.md#M1)
172def extract_delimited(state: ParseState) -> ParseState:
173 text = state.original
174 policy = state.policy
175 if _delimiter_chars(policy.nickname_delimiters,
176 policy.maiden_delimiters).isdisjoint(text):
177 return state
178 # Candidate order matters only as a same-position tie-break (see
179 # module docstring); the scan itself is position-driven.
180 order = tuple(
181 (role, open_, close)
182 for role, pairs in ((Role.NICKNAME, policy.nickname_delimiters),
183 (Role.MAIDEN, policy.maiden_delimiters))
184 for open_, close in sorted(pairs)
185 )
186 extracted: list[tuple[Role, Span]] = []
187 masked: list[Span] = []
188 # candidates, not final: each carries the offset of the unmatched
189 # open so ones consumed by a later match can be filtered at the end
190 unbalanced: list[tuple[int, PendingAmbiguity]] = []
191 # per-candidate cursor cache: next boundary-valid open at or after
192 # the position it was computed for. find() calls only ever move
193 # forward, keeping the whole scan linear in len(text) per pair.
194 cursors: dict[tuple[Role, str, str], int] = {}
195 exhausted: set[tuple[Role, str, str]] = set()
196 pos = 0
197 while pos < len(text):
198 best: tuple[int, Role, str, str] | None = None
199 for key in order:
200 if key in exhausted:
201 continue
202 _, open_, close = key
203 i = cursors.get(key, -2)
204 if i != -1 and i < pos:
205 i = text.find(open_, pos)
206 while (i != -1 and open_ == close
207 and not _open_ok(text, i)):
208 i = text.find(open_, i + 1)
209 cursors[key] = i
210 if i != -1 and (best is None or i < best[0]):
211 best = (i, *key)
212 if best is None:
213 break
214 i, role, open_, close = best
215 j = text.find(close, i + len(open_))
216 while (open_ == close and j != -1
217 and not _close_ok(text, j, len(close))):
218 j = text.find(close, j + 1)
219 if j == -1:
220 # No (boundary-valid) close exists anywhere to the right --
221 # the walk above ran to end of text -- so every remaining
222 # open of this pair is unmatched too. Record them all in
223 # one forward pass and retire the pair; other pairs keep
224 # scanning from the same position.
225 unbalanced.append(_unmatched(open_, i))
226 scan = i + len(open_)
227 while (k := text.find(open_, scan)) != -1:
228 if open_ != close or _open_ok(text, k):
229 unbalanced.append(_unmatched(open_, k))
230 scan = k + 1
231 exhausted.add((role, open_, close))
232 continue
233 inner = Span(i + len(open_), j)
234 if inner.start < inner.end and _suffix_shaped(
235 text[inner.start:inner.end], state.lexicon):
236 # v1 parse_nicknames: suffix-shaped delimited content is
237 # left IN PLACE (undelimited) for normal downstream parsing
238 # -- 'Andrew Perkins (MBA)' keeps MBA a suffix, not a
239 # nickname. Spans index the original (anti-#100), so the v2
240 # spelling masks only the two delimiter spans and lets the
241 # inner content join the main token stream.
242 masked.append(Span(i, i + len(open_)))
243 masked.append(Span(j, j + len(close)))
244 else:
245 if inner.start < inner.end:
246 # M3 upgrades a nickname clause; a configured maiden
247 # pair is M1's and is left alone. The role test is
248 # False whenever a maiden pair matched, but it cannot
249 # change the OUTCOME, and no test can catch its
250 # removal: `order` above holds exactly two roles, so a
251 # role that is not NICKNAME is already MAIDEN and the
252 # assignment would be a no-op either way. It is kept
253 # for the day `order` gains a third bucket, when it
254 # becomes the difference between M3 claiming that
255 # bucket's clauses and leaving them. Measured
256 # 2026-08-26: dropping it leaves the suite and all
257 # three gates green.
258 if (role is Role.NICKNAME and _maiden_marked(
259 text[inner.start:inner.end], state.lexicon)):
260 role = Role.MAIDEN
261 extracted.append((role, inner))
262 masked.append(Span(i, j + len(close)))
263 # position-driven scanning makes overlapping matches
264 # impossible by construction: every later open is found at or
265 # after this match's end
266 pos = j + len(close)
267 extracted.sort(key=lambda pair: pair[1])
268 masked.sort()
269 # An unmatched-open candidate whose character was consumed by a
270 # later successful match (the bulk pass above runs ahead of the
271 # main scan) is literal content there, not a dangling delimiter.
272 # offsets already claimed as unbalanced, by an open above or by a
273 # close in the sweep below -- either way, do not report one twice
274 reported = {offset for offset, _ in unbalanced}
275 mask_starts = [s.start for s in masked]
276 ambiguities = [
277 a for offset, a in unbalanced
278 if not _overlaps(Span(offset, offset + 1), masked, mask_starts)]
279 # The scan above is opener-driven: it searches for an open and then
280 # looks rightward for its close, so a close with no open to its
281 # LEFT is never in its search space. Sweep for those separately --
282 # they signal the same malformed input, and the kind's contract has
283 # always covered them ("opened without closing, or closed without
284 # opening"). Same boundary test as the matched path, which is what
285 # keeps the apostrophe in "O'connor" out of it.
286 # Distinct closes only: the defaults list '”' twice (from both
287 # ('“','”') and ('”','”')), and a repeat can only rediscover
288 # offsets the first pass already handled.
289 for close in sorted({c for _, _, c in order}):
290 start = 0
291 while (j := text.find(close, start)) != -1:
292 start = j + 1
293 if (j in reported
294 or not _close_ok(text, j, len(close))
295 or _word_internal(text, j, close)
296 or _overlaps(Span(j, j + len(close)), masked,
297 mask_starts)):
298 continue
299 reported.add(j)
300 ambiguities.append(_unmatched(close, j)[1])
301 return dataclasses.replace(
302 state, extracted=tuple(extracted), masked=tuple(masked),
303 ambiguities=state.ambiguities + tuple(ambiguities))