Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage: extract_delimited.
3Consumes: ParseState.original.
4Produces: extracted (role + inner span per delimited region), masked
5(full regions incl. delimiter chars, skipped by tokenize),
6UNBALANCED_DELIMITER ambiguities for opens with no close.
7A Role.MAIDEN region is the WHOLE inner span, marker included (all of
8it, a phrase marker being several words) -- nothing here strips one.
9classify tags a marker inside it like any other token, and group drops
10it from a multi-token clause (#329).
11A region reaches that role two ways: the pair that matched sits in
12Policy.maiden_delimiters (M1), or the content itself opens with a
13marker word (M3), which reassigns the role after the match and so is
14the one thing here that a bucket alone does not decide.
15Reads: Policy.nickname_delimiters, Policy.maiden_delimiters,
16Lexicon.maiden_markers, Lexicon.particles (an apostrophe opening a
17listed particle, rules/N2), Policy.strip_emoji and Policy.strip_bidi
18(through `_tokenize._ignorable`, where that particle's word ends), and
19through `_vocab.set_apart_kind` and
20`set_apart` (rules/S1) Lexicon.suffix_words / suffix_words_ambiguous /
21suffix_acronyms / suffix_acronyms_ambiguous / titles /
22title_modifiers / title_styles / given_name_titles /
23after_given_titles.
25Implements rules N1, N2, S1, M1 and M3 of docs/design/rules/ (the #273
26matching mechanism); each is cited at its code below. One scan
27mechanic worth stating up front: matching is one left-to-right pass,
28no nesting, and delimiter characters inside a matched region are
29literal content for every other pair.
31Bucket precedence is NOT decided here: Policy canonicalizes overlap
32away before parsing, so the two buckets are always disjoint by the
33time this stage runs. The nickname-before-maiden candidate order
34below is only a same-position tie-break for exotic configs where two
35pairs share an OPEN character.
36"""
37from __future__ import annotations
39import bisect
40import functools
42from nameparser._lexicon import Lexicon, _normalize
43from nameparser._pipeline._state import (
44 COMMA_CHARS, ParseState, PendingAmbiguity, copy_with,
45)
46from nameparser._pipeline._tokenize import _ignorable
47from nameparser._pipeline._vocab import (
48 comma_follows_marker, maiden_marker_run, set_apart, set_apart_kind,
49)
50from nameparser._types import AmbiguityKind, Role, Span
53# rules/S1: "loses its brackets and reads exactly as if written bare,
54# the escape v1 made for any period-final content" -- asked only of a
55# clause `_set_apart` declined (decisions/S1, #792)
56def _period_escape(content: str) -> bool:
57 """v1 parse_nicknames' escape (parser.py:1125-1141) for content
58 ending in a period. v1's other two arms, an unambiguous suffix word
59 or acronym standing alone, are `_set_apart`'s since #792, which
60 takes every such clause first."""
61 return content.endswith(".")
64def _set_apart(text: str, inner: Span,
65 lexicon: Lexicon) -> bool:
66 """Whether a delimited clause is wholly of words the vocabulary
67 reads as no name, by `_vocab.set_apart_kind` and `set_apart`: the
68 unambiguous half of S1's set-apart reading, which needs no role. A
69 member of the ambiguous class settled by its writing (S13) is
70 weighed in post_rules, once it is known where the name ends, and
71 post_rules may still decline a clause this sets apart (titles
72 alone behind a credential, initials before a name word), which then
73 keeps its pair's reading and not the period escape."""
74 words = text[inner.start:inner.end].split()
75 kinds = []
76 for word in words:
77 kind = set_apart_kind(word, lexicon)
78 if kind is None or kind == "member":
79 return False
80 kinds.append(kind)
81 return bool(words) and set_apart(words, kinds, False, lexicon)
84# rules/M3: "a bracketed clause whose content opens with a
85# recognized marker and carries a word after it reads as the
86# maiden name, whichever bucket the enclosing pair sits in"
87def _maiden_marked(content: str, lexicon: Lexicon) -> bool:
88 """The clause says 'maiden' out loud, so the caller does not have to
89 say it in Policy. Requires a word AFTER the marker: a lone marker in
90 brackets is a word in brackets, and M1 deliberately keeps a one-word
91 clause's word (it may be the surname Nee). A PHRASE marker is one
92 marker, so the word after is the word after the whole run --
93 '(z domu Nowak)' has one, '(z domu)' has none. The word after is
94 not tested for anything, which is
95 rules/M3's Accepted bullet "The word taken after the marker is not tested"
96 (decisions/M3) and the reason a bracketed '(née PhD)' reads maiden
97 'PhD' where the bare 'née PhD' gives M2 a suffix.
99 This stage runs before tokenize, so it calls maiden_marker_run
100 itself rather than reading the tags classify will set. The two
101 questions are deliberately not the same one: this splits on
102 WHITESPACE, so a marker the writer glued to punctuation is not one
103 here ('née,') -- nor to classify, since #653 keeps a clause's comma
104 on its word. A nakaguro still divides the two: the tokenizer splits
105 '旧姓・田中' at it (rules/T2) and classify tags '旧姓', which is
106 what keeps _group's Role.MAIDEN filter reachable (cases.py's
107 marker_divided_by_a_nakaguro_keeps_the_clause_a_nickname). Sharing
108 the PREDICATE does not merge the two
109 questions -- each hands it a different sequence of words."""
110 words = content.split()
111 # A one-word clause cannot satisfy the condition whatever the
112 # vocabulary says -- `run > 0 and 1 > run` is unsatisfiable -- so
113 # refuse it before any fold or lookup, as the pre-#434 words[0]
114 # test did for free.
115 if len(words) < 2:
116 return False
117 run = maiden_marker_run(words, lexicon._sets.maiden_markers)
118 return (run > 0 and len(words) > run
119 and not comma_follows_marker(words[run]))
122# rules/N2: "a quote whose open and close are the same character
123# opens only at a word start and closes only at a word end, so an
124# apostrophe inside or at the end of a word is literal"
125def _open_ok(text: str, i: int, state: ParseState) -> bool:
126 return ((i == 0 or text[i - 1].isspace())
127 and not _opens_a_listed_particle(text, i, state))
130# rules/N2: "An apostrophe opening a word the vocabulary lists as a
131# particle" -- the Except clause: it opens no quote, so Dutch 't
132# reports nothing (decisions/N2, #1241). Asked of any same-character
133# open, the curly ’t being listed too. The word ends at a comma or a
134# character tokenize's _ignorable separates at. A delimited region's
135# edge is not asked, so 'Pieter \'t(Bob) Hart' still reports, a shape
136# no measured name writes; nor is T6's join, no particle holding U+202F.
137def _opens_a_listed_particle(text: str, i: int, state: ParseState) -> bool:
138 # Only where the NEXT same character cannot close the quote, being
139 # absent or after whitespace: 'T Bone' and 'T-Bone Jones' keep their
140 # nicknames though 't is listed (decisions/N2). Asked first, a C
141 # call, so a closed quoted nickname pays neither the word scan nor
142 # a _normalize frame.
143 j = text.find(text[i], i + 1)
144 if j >= 0 and not text[j - 1].isspace():
145 return False
146 k = i + 1
147 while k < len(text):
148 ch = text[k]
149 # the narrower question inline: a letter, or an ASCII character
150 # other than whitespace and the comma, never ends the word (no
151 # letter is _ignorable or a comma), so a quoted nickname pays a
152 # frame only per mark, digit or symbol beyond ASCII
153 if ch.isalpha() or (ch.isascii() and not ch.isspace()
154 and ch != ","):
155 k += 1
156 continue
157 if ch in COMMA_CHARS or _ignorable(ch, state):
158 break
159 k += 1
160 word = text[i:k]
161 particles = state.lexicon._sets.particles
162 if _normalize(word) in particles:
163 return True
164 # a particle hyphened to its surname ('s-Gravesande), its head the
165 # word before the FIRST hyphen; a word without one pays no second
166 # _normalize frame
167 head, hyphen, _ = word.partition("-")
168 return bool(hyphen) and _normalize(head) in particles
171def _close_ok(text: str, j: int, width: int) -> bool:
172 k = j + width
173 return k >= len(text) or text[k].isspace() or text[k] in COMMA_CHARS
176# Delimiters that also occur INSIDE and at the end of real name parts,
177# so a dangling one is literal rather than unbalanced. Only the straight
178# apostrophe qualifies: quotes do not appear inside names, so the same
179# position with a quote genuinely is ambiguous. #273 dropped the curly
180# apostrophe from the defaults outright for this reason; the straight
181# one has to stay a delimiter (v1's quoted_word), so it is carved out
182# here instead. Deliberately not widened to a configured delimiter set.
183WORD_INTERNAL_DELIMITERS = frozenset({"'"})
186def _word_internal(text: str, j: int, close: str) -> bool:
187 """A word-internal delimiter directly after a word character is part
188 of the word, not a dangling close -- "Mari' Aube'", "Ali Baba'"."""
189 return (close in WORD_INTERNAL_DELIMITERS and j > 0
190 and (text[j - 1].isalnum() or text[j - 1] == "."))
193def _overlaps(span: Span, taken: list[Span], starts: list[int]) -> bool:
194 """`taken` sorted and non-overlapping (both hold by construction),
195 `starts` its start offsets. Bisect rather than scan: the closer
196 sweep tests one span per delimiter character found, so a linear
197 probe is quadratic in the number of matched pairs (400 pairs spent
198 5.4ms here, against 2.5ms before the sweep existed). Same idiom, and
199 the same reason, as the origin resolution in _tokenize."""
200 i = bisect.bisect_right(starts, span.start) - 1
201 # the only candidates are the last span starting at or before us and
202 # its successor -- anything earlier ends before it, anything later
203 # starts after us
204 for k in (i, i + 1):
205 if 0 <= k < len(taken) and span.start < taken[k].end and (
206 taken[k].start < span.end):
207 return True
208 return False
211@functools.lru_cache(maxsize=128)
212def _delimiter_chars(
213 nickname_pairs: frozenset[tuple[str, str]],
214 maiden_pairs: frozenset[tuple[str, str]],
215) -> frozenset[str]:
216 """Every character appearing in any configured delimiter, cached on
217 the (hashable) policy frozensets: the common no-delimiter name pays
218 one isdisjoint() instead of a per-pair scan."""
219 return frozenset(
220 ch
221 for pairs in (nickname_pairs, maiden_pairs)
222 for pair in pairs
223 for part in pair
224 for ch in part
225 )
228def _unmatched(open_: str, offset: int) -> tuple[int, PendingAmbiguity]:
229 return (offset, PendingAmbiguity(
230 AmbiguityKind.UNBALANCED_DELIMITER,
231 f"unmatched {open_!r} at offset {offset}; treated as literal text",
232 origin=offset,
233 ))
236# rules/N1: "a clause enclosed by a configured nickname delimiter
237# pair reads as the nickname and is lifted out of the name" -- Then,
238# and the empty enclosure it drops; its punctuation is tokenize's half
239# of the rule (decisions/N1)
240# rules/M1: "its enclosed clause reads as the maiden name" -- Then,
241# the pair being one the caller configured for maiden names; the
242# marker drop is group's half (decisions/M1)
243def extract_delimited(state: ParseState) -> ParseState:
244 text = state.original
245 policy = state.policy
246 if _delimiter_chars(policy.nickname_delimiters,
247 policy.maiden_delimiters).isdisjoint(text):
248 return state
249 # Candidate order matters only as a same-position tie-break (see
250 # module docstring); the scan itself is position-driven.
251 order = tuple(
252 (role, open_, close)
253 for role, pairs in ((Role.NICKNAME, policy.nickname_delimiters),
254 (Role.MAIDEN, policy.maiden_delimiters))
255 for open_, close in sorted(pairs)
256 )
257 extracted: list[tuple[Role, Span]] = []
258 masked: list[Span] = []
259 # candidates, not final: each carries the offset of the unmatched
260 # open so ones consumed by a later match can be filtered at the end
261 unbalanced: list[tuple[int, PendingAmbiguity]] = []
262 # per-candidate cursor cache: next boundary-valid open at or after
263 # the position it was computed for. find() calls only ever move
264 # forward, keeping the whole scan linear in len(text) per pair.
265 cursors: dict[tuple[Role, str, str], int] = {}
266 exhausted: set[tuple[Role, str, str]] = set()
267 pos = 0
268 while pos < len(text):
269 best: tuple[int, Role, str, str] | None = None
270 for key in order:
271 if key in exhausted:
272 continue
273 _, open_, close = key
274 i = cursors.get(key, -2)
275 if i != -1 and i < pos:
276 i = text.find(open_, pos)
277 while (i != -1 and open_ == close
278 and not _open_ok(text, i, state)):
279 i = text.find(open_, i + 1)
280 cursors[key] = i
281 if i != -1 and (best is None or i < best[0]):
282 best = (i, *key)
283 if best is None:
284 break
285 i, role, open_, close = best
286 j = text.find(close, i + len(open_))
287 while (open_ == close and j != -1
288 and not _close_ok(text, j, len(close))):
289 j = text.find(close, j + 1)
290 if j == -1:
291 # No (boundary-valid) close exists anywhere to the right --
292 # the walk above ran to end of text -- so every remaining
293 # open of this pair is unmatched too. Record them all in
294 # one forward pass and retire the pair; other pairs keep
295 # scanning from the same position.
296 unbalanced.append(_unmatched(open_, i))
297 scan = i + len(open_)
298 while (k := text.find(open_, scan)) != -1:
299 if open_ != close or _open_ok(text, k, state):
300 unbalanced.append(_unmatched(open_, k))
301 scan = k + 1
302 exhausted.add((role, open_, close))
303 continue
304 inner = Span(i + len(open_), j)
305 if inner.start < inner.end and _set_apart(text, inner, state.lexicon):
306 # rules/S1: "it is no nickname and no maiden name, and the
307 # words around it read as they would beside a nickname"
308 # -- extracted as its pair reads it, post_rules giving its
309 # words their roles once the name's words are placed
310 # (decisions/S1, #792)
311 extracted.append((role, inner))
312 masked.append(Span(i, j + len(close)))
313 elif inner.start < inner.end and _period_escape(
314 text[inner.start:inner.end]):
315 # v1 parse_nicknames: period-final content S1 does not set
316 # apart is left IN PLACE (undelimited) for normal downstream
317 # parsing -- 'John Smith (M.A.)' keeps M.A. a suffix, not a
318 # nickname. Spans index the original (anti-#100), so the v2
319 # spelling masks only the two delimiter spans and lets the
320 # inner content join the main token stream.
321 masked.append(Span(i, i + len(open_)))
322 masked.append(Span(j, j + len(close)))
323 else:
324 if inner.start < inner.end:
325 # M3 upgrades a nickname clause; a configured maiden
326 # pair is M1's and is left alone. The role test is
327 # False whenever a maiden pair matched, but it cannot
328 # change the OUTCOME, and no test can catch its
329 # removal: `order` above holds exactly two roles, so a
330 # role that is not NICKNAME is already MAIDEN and the
331 # assignment would be a no-op either way. It is kept
332 # for the day `order` gains a third bucket, when it
333 # becomes the difference between M3 claiming that
334 # bucket's clauses and leaving them. Measured
335 # 2026-08-26: dropping it leaves the suite and all
336 # three gates green.
337 if (role is Role.NICKNAME and _maiden_marked(
338 text[inner.start:inner.end], state.lexicon)):
339 role = Role.MAIDEN
340 extracted.append((role, inner))
341 masked.append(Span(i, j + len(close)))
342 # position-driven scanning makes overlapping matches
343 # impossible by construction: every later open is found at or
344 # after this match's end
345 pos = j + len(close)
346 extracted.sort(key=lambda pair: pair[1])
347 masked.sort()
348 # An unmatched-open candidate whose character was consumed by a
349 # later successful match (the bulk pass above runs ahead of the
350 # main scan) is literal content there, not a dangling delimiter.
351 # offsets already claimed as unbalanced, by an open above or by a
352 # close in the sweep below -- either way, do not report one twice
353 reported = {offset for offset, _ in unbalanced}
354 mask_starts = [s.start for s in masked]
355 ambiguities = [
356 a for offset, a in unbalanced
357 if not _overlaps(Span(offset, offset + 1), masked, mask_starts)]
358 # The scan above is opener-driven: it searches for an open and then
359 # looks rightward for its close, so a close with no open to its
360 # LEFT is never in its search space. Sweep for those separately --
361 # they signal the same malformed input, and the kind's contract has
362 # always covered them ("opened without closing, or closed without
363 # opening"). Same boundary test as the matched path, which is what
364 # keeps the apostrophe in "O'connor" out of it.
365 # Distinct closes only: the defaults list '”' twice (from both
366 # ('“','”') and ('”','”')), and a repeat can only rediscover
367 # offsets the first pass already handled.
368 for close in sorted({c for _, _, c in order}):
369 start = 0
370 while (j := text.find(close, start)) != -1:
371 start = j + 1
372 if (j in reported
373 or not _close_ok(text, j, len(close))
374 or _word_internal(text, j, close)
375 or _overlaps(Span(j, j + len(close)), masked,
376 mask_starts)):
377 continue
378 reported.add(j)
379 ambiguities.append(_unmatched(close, j)[1])
380 return copy_with(
381 state, extracted=tuple(extracted), masked=tuple(masked),
382 ambiguities=state.ambiguities + tuple(ambiguities))