Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage: post_rules.
3Consumes: tokens (roles assigned), plus pieces and structure -- the
4particle fold reads the opening piece of segment 0, or of segment 1
5under a family comma (#359). structure was always read here, for the
6rotation gate. Also comma_offsets and dropped, which R1's entry pass
7below reads to find the separators the writer typed (#436/#437).
8Produces: tokens with roles adjusted by the post rules, the stable
9"joined" tag on a post-nominal continuing the entry before it, and the
10ambiguity P6's attachment reports for the fork it decides (#405).
11Reads: Policy.patronymic_rules, Policy.middle_as_family,
12Policy.extra_suffix_delimiters (R1's entry pass, for the delimiter
13cores group drops); Lexicon.given_name_titles.
15Implements rules H1, M4, P1, O1, O2, O3 and R1 of docs/design/rules.md;
16each is cited at its code below, and H1/P1/O1/O2's history lives in
17docs/design/decisions.md. `suffix_entries` is the R1 entry pass as a
18state-in/state-out function, for Parser.revise to run over a
19forced-role sub-parse (#511); `post_rules` runs the same worker last.
20"""
21from __future__ import annotations
23import dataclasses
24import re
26from nameparser._lexicon import _run_addresses_by_given
27from nameparser._pipeline._assign import _name_positions
28from nameparser._pipeline._state import (
29 ParseState, PendingAmbiguity, Structure, WorkToken, _NEVER_FLIPPED,
30 comma_bucket,
31)
32from nameparser._pipeline._vocab import delimiter_cores
33from nameparser._policy import PatronymicRule
34from nameparser._types import (
35 FOLDED_TAG, UNJOINED_TAG, AmbiguityKind, Role,
36)
38# Ported verbatim from v1 (nameparser/config/regexes.py) -- layering
39# forbids the config import; keep in sync by hand.
40_EAST_SLAVIC = re.compile(
41 r"(ovich|ovna|evich|evna|ichna|ilyich|kuzmich|lukich|fomich|fokich)$",
42 re.I)
43_EAST_SLAVIC_CYR = re.compile(
44 r"(ович|овна|евич|евна|ична|ильич|кузьмич|лукич|фомич|фокич)$",
45 re.I)
46_TURKIC = re.compile(
47 r"^(oglu|oğlu|ogly|ogli|o['’ʻ]g['’ʻ]li"
48 r"|qizi|qızı|kizi|kyzy|gyzy|uly|uulu)$", re.I)
49_TURKIC_CYR = re.compile(
50 r"^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$", re.I)
53_NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY)
55#: The roles that are transparent to a run of post-nominals (R1's
56#: entry pass below). These three roles render into fields other than
57#: the name and the suffix, so a run of
58#: post-nominals is not parted by them -- 'Smith, MD Dr. PhD',
59#: 'Smith, MD "Doc" PhD' and 'Smith, MD nee Jones PhD' are each one
60#: entry. GIVEN/MIDDLE/FAMILY are name words and part it.
61_RENDERS_ELSEWHERE = frozenset({Role.TITLE, Role.NICKNAME, Role.MAIDEN})
64def _rotations_apply(state: ParseState) -> bool:
65 # Both patronymic rotations RESTORE the given-first reading a
66 # family-first listing hides, so rules.md#O1's scope clause holds
67 # them to the default order: a caller who declared family-first has
68 # already said what the rotation would infer, and position decides
69 # (decisions.md#O1, the 2026-09-07 entry on #384). `state.order`,
70 # not policy.name_order, for the reason the P1 fold gives -- a
71 # script_orders entry can override the policy, and the roles the
72 # rotations read are the ones assign actually made. None means
73 # assign positioned nothing: a family comma (which the NO_COMMA
74 # test already excludes) or an early return with no name piece to
75 # position, so there is no declaration to defer to and the
76 # rotation's own shape test decides.
77 return state.structure is Structure.NO_COMMA and (
78 state.order is None or state.order[0] is Role.GIVEN)
81def _mark_suffix_entries(tokens: list[WorkToken], state: ParseState) -> None:
82 # In place over the caller's token list, the way every other rule
83 # in post_rules writes: a state-in/state-out spelling here cost
84 # three calls per parse for the second ParseState build, against
85 # the call-count band tests/v2/test_benchmark.py holds (measured
86 # 2026-09-06 with tools/perf/call_count.py, py3.11: 450 calls/name
87 # before the move, 451 with this worker, 454 with the state-wrapper
88 # draft; the facade band tops at 455.9 and parse, the tighter row,
89 # sits at 414 in a 402-418 band). suffix_entries() below is the
90 # state wrapper; its docstring says for whom.
91 # Reads dropped, comma_offsets and the policy off `state`, which
92 # post_rules does not change, so the un-rebuilt state is current.
93 #
94 # rules.md#R1: "a run of post-nominals written with spaces renders
95 # with spaces, and one written with commas keeps them"
96 #
97 # The entry boundary, read off the text the writer typed rather
98 # than off the shape of the segments (#436/#437). Two consecutive
99 # SUFFIX tokens are one entry iff they sit in the same comma
100 # bucket AND nothing between them parts the run -- what parts it
101 # and what does not is spelled out below. The
102 # comma is the separator the rule names -- comma_bucket is the
103 # function segment BUILDS segments with and classify asks about
104 # boundaries, so "same part" here is an identity with segment's
105 # answer rather than a resemblance to it
106 # (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
107 #
108 # What parts a run: a name word between the two post-nominals
109 # (GIVEN/MIDDLE/FAMILY), or a dropped delimiter core (#206). What
110 # does NOT part it: a token whose role is in _RENDERS_ELSEWHERE,
111 # because it renders into another field entirely and so is not
112 # standing in the run at all -- 'Smith, MD Dr. PhD',
113 # 'Smith, MD "Doc" PhD' and 'Smith, MD (nee Jones) PhD' are each
114 # one entry; and a dropped token that is not a core, which is the
115 # maiden MARKER of 'Smith, MD nee Jones PhD' (the marker is
116 # dropped with no role at all, so the role test cannot see it).
117 # That is why the dropped arm reads the core set instead of
118 # treating every dropped index as a boundary: a core is the one
119 # dropped token the writer typed AS a separator, and the set is
120 # `delimiter_cores` -- group's own derivation off
121 # Policy.extra_suffix_delimiters, imported rather than repeated
122 # (mechanisms.md#ONE-PREDICATE-PER-QUESTION). The two sites read
123 # ONE derivation and differ only in a gate: group drops a core
124 # only on a `tail` segment, through `seg_cores`, while this arm
125 # reads `delimiter_cores` whole and asks by TEXT alone. So a
126 # dropped token whose text the policy names as a delimiter parts
127 # the run whatever dropped it. The gate is not needed here: a core
128 # dropped BY GROUP was on a tail segment by construction, and the
129 # only case the ungated read adds is a maiden marker the policy
130 # ALSO lists -- under
131 # `Policy(extra_suffix_delimiters=frozenset({" nee "}))`,
132 # 'John Doe, MD nee Jones PhD' renders 'MD, PhD' where the default
133 # policy renders 'MD PhD' (measured 2026-09-06). That is the
134 # policy's own declaration deciding it: the writer's configuration
135 # named that text a separator, so the run parts there.
136 #
137 # Four shapes were declined, recorded in decisions.md#C1 by the
138 # bundle that landed this pass: marking the boundary at the
139 # core-drop site (`dropped` already holds the fact with its
140 # span, so a second recording of it is the duplication
141 # MARK-DONT-STRIP exists to prevent); making the "joined" tag
142 # role-aware (within a piece it is role-blind and right for every
143 # role -- 'Smith, Ph. D. Smith' gives first_list ['Ph. D.']);
144 # scanning spans at render time instead of reading the tag; and
145 # adding a third shape-derived branch inside group's block.
146 #
147 # AFTER assign, and THAT is the load-bearing constraint: this keys
148 # on Role.SUFFIX, and the same span rule run role-blind would join
149 # the A and B of 'John A B Smith' into one middle_list element
150 # (test_the_pass_runs_after_roles_are_settled pins it).
151 #
152 # Within post_rules the pass runs last by convention, not by
153 # necessity. It reads SUFFIX and _RENDERS_ELSEWHERE, and no rule
154 # in post_rules writes either: every retag in post_rules targets a
155 # NAME role and nothing else -- `_retag` is called with
156 # Role.FAMILY, Role.GIVEN, Role.MIDDLE, and with
157 # `_name_positions`' return, which is those same three; the three
158 # `role=Role.FAMILY` replaces (P6's attachment on both arms, O3's
159 # fold) are FAMILY as well. So no rule here
160 # moves a token into or out of SUFFIX, or into or out of
161 # {TITLE, NICKNAME, MAIDEN}, and this predicate reads the same
162 # answer wherever in the stage it stands (measured 2026-09-06 by
163 # reading the stage's retag targets).
164 #
165 # RECORDED as a tag rather than recomputed by the render, because
166 # the render cannot see a span: _facade.__setstate__ and
167 # ParsedName.replace() build span-less tokens AFTER the pipeline,
168 # so an unpickled name has nothing to scan and the tag IS the
169 # entry structure the pickle carries
170 # (mechanisms.md#MARK-DONT-STRIP). Every token here has a span --
171 # tokenize is the sole producer of a WorkToken, and WorkToken.span
172 # is not Optional -- so `span.start` is read unguarded.
173 #
174 # The `i not in dropped` filter is belt-and-braces: a dropped token
175 # never carries a SUFFIX role. Cores leave `pieces` before assign
176 # runs, so assign gives them no role at all; the one class of
177 # dropped token that arrives already roled is the MAIDEN one that
178 # tokenize roles from an extracted clause ('Smith, MD (nee Jones)
179 # PhD' drops index 2, the marker, and it is Role.MAIDEN). The
180 # filter is here so that `suffixes` and the `parted` scan below
181 # cannot disagree about what a dropped index is.
182 dropped = set(state.dropped)
183 cores = delimiter_cores(state.policy.extra_suffix_delimiters)
184 suffixes = [i for i, tok in enumerate(tokens)
185 if tok.role is Role.SUFFIX and i not in dropped]
186 for previous, current in zip(suffixes, suffixes[1:]):
187 same_part = (comma_bucket(tokens[previous].span.start,
188 state.comma_offsets)
189 == comma_bucket(tokens[current].span.start,
190 state.comma_offsets))
191 parted = any(
192 tokens[between].text in cores if between in dropped
193 else tokens[between].role not in _RENDERS_ELSEWHERE
194 for between in range(previous + 1, current))
195 if same_part and not parted:
196 tokens[current] = dataclasses.replace(
197 tokens[current], tags=tokens[current].tags | {"joined"})
200def suffix_entries(state: ParseState) -> ParseState:
201 """rules.md#R1's entry pass over a whole state, wrapping
202 _mark_suffix_entries: tag a SUFFIX token "joined" when it
203 continues the entry of the SUFFIX token before it. post_rules runs
204 the worker last, over the roles assign settled; this wrapper exists
205 for Parser.revise (#511), which runs it over a SUB-PARSE of a field
206 value whose every non-dropped token it has forced to the named
207 role, so a suffix value derives its entries from its own commas by
208 the rule a whole name uses. The pass ADDS the tag and never removes
209 one; a caller forcing roles keeps what the sub-parse marked, a
210 within-piece mark being role-blind (decisions.md#C1, the #436
211 DECLINED list) and every between-piece mark on a suffix value
212 being one this pass sets again. Reads comma_offsets, dropped and
213 Policy.extra_suffix_delimiters; writes the "joined" tag and
214 nothing else."""
215 tokens = list(state.tokens)
216 _mark_suffix_entries(tokens, state)
217 return dataclasses.replace(state, tokens=tuple(tokens))
220def _idx(tokens: list[WorkToken], role: Role) -> list[int]:
221 return [i for i, t in enumerate(tokens) if t.role is role]
224def _leading_name_piece(state: ParseState,
225 tokens: list[WorkToken]) -> tuple[int, ...]:
226 """The piece that OPENS the name, whatever role name_order gave it:
227 the first piece holding a GIVEN, MIDDLE or FAMILY token, in the
228 segment the positional read governs. Every piece holding none of
229 those is walked past -- title and suffix pieces, but NICKNAME and
230 MAIDEN as well, and anything assign left unroled -- and any number
231 of them, not only a single leading title. The segment is 0, except
232 under a family comma that fixed the surname, where the name
233 continues in segment 1 -- assign records no order there. A family
234 comma followed by no name word fixed nothing, and assign reads
235 segment 0 positionally and records the order (#296's bundle), so
236 the name is segment 0 again. Empty on either of two exits: that
237 segment does not exist, or none of its pieces holds a name
238 token."""
239 seg = (1 if state.structure is Structure.FAMILY_COMMA
240 and state.order is None else 0)
241 if seg >= len(state.pieces):
242 return ()
243 for piece in state.pieces[seg]:
244 if any(tokens[i].role in _NAME_ROLES for i in piece):
245 return piece
246 return ()
249def _retag(tokens: list[WorkToken], i: int, role: Role) -> None:
250 tokens[i] = dataclasses.replace(tokens[i], role=role)
253# rules.md#P2: "a particle joins the words after it into one name
254# part, the join running until the next particle starts a group of
255# its own, a trailing suffix begins"
256# rules.md#P3: "the joined part is ONE name word wherever another
257# rule counts them"
258# rules.md#P5: "a recognized bound given-name word joins the word
259# after it into one given name"
260def _unit_end(tokens: list[WorkToken], idx: list[int], i: int) -> int:
261 """One past the end of the unit starting at `idx[i]`.
263 RECURSIVE, and that is the whole point: what a conjunction or a
264 bound given-name word joins is the next UNIT, not the next word.
265 Absorbing a single index instead strands a particle at the end of
266 the unit, severed from the words it chains -- "de la Vega y la
267 Vega" cut between `la` and `Vega`, reporting family
268 "de la Vega y la", which is the same defect as a bare particle
269 opening the given name, mirrored."""
270 if "particle" in tokens[idx[i]].tags:
271 j = i
272 while j + 1 < len(idx) and "particle" in tokens[idx[j + 1]].tags:
273 j += 1
274 # ... then the words it joins, stopping where the next
275 # particle starts a group of its own, at a suffix word (the
276 # stop _group's chain uses), or at a conjunction, which the
277 # shared loop below joins to the whole unit after it rather
278 # than to the one word after it.
279 while (j + 1 < len(idx)
280 and "particle" not in tokens[idx[j + 1]].tags
281 and "conjunction" not in tokens[idx[j + 1]].tags
282 and "vocab:suffix" not in tokens[idx[j + 1]].tags):
283 j += 1
284 end = j + 1
285 else:
286 end = i + 1
287 if "vocab:bound-given" in tokens[idx[i]].tags and end < len(idx):
288 end = _unit_end(tokens, idx, end)
289 while end + 1 < len(idx) and "conjunction" in tokens[idx[end]].tags:
290 end = _unit_end(tokens, idx, end + 1)
291 return end
294def _units(tokens: list[WorkToken], idx: list[int]) -> list[list[int]]:
295 """`idx` split into the units other rules COUNT: one name word
296 each, except where another rule has already made several words one
297 name. Three do -- a particle and the words it chains, a
298 conjunction-joined run, and a bound given-name word with the word
299 it completes -- so `van der Berg` and `abdul Rahman` are each one
300 unit and the fold cannot leave half of either behind.
302 Read off the TAGS rather than the pieces, for two different
303 reasons. A conjunction join grouping DID build and the prefix
304 chain then swallowed: "de la Vega y Santos Juan" reaches this as
305 [de][la Vega y Santos Juan], the ambiguous particle having chained
306 forward over the join, so the join's own boundary is gone. (The
307 leading particle keeps its piece -- it has to, or the fold's
308 lone-piece site test would not fire at all.) The bound-given join grouping
309 never built at all: P5 joins only where the bound word is the
310 first non-title piece, and at a fold site the first piece is the
311 particle -- "ibn Awf abdul Rahman" reaches here as four separate
312 pieces. The tag is the only witness in both cases.
314 The particle chain is what keeps this partition agreeing with the
315 one `assign` reads off pieces: strip the folded run and `assign`
316 gives the same tail one role, so the fold must not hand its words
317 out separately."""
318 units: list[list[int]] = []
319 i = 0
320 while i < len(idx):
321 end = _unit_end(tokens, idx, i)
322 units.append(list(idx[i:end]))
323 i = end
324 return units
327def _fold_reach(tokens: list[WorkToken], name_idx: list[int]) -> int:
328 """How many of `name_idx` the fold takes: the particle run, plus
329 the one name word it attaches to -- one UNIT, so a conjunction
330 join goes whole ("de la Vega y Santos Juan" keeps Vega y Santos
331 together). All of them when the name is nothing but particles."""
332 i = 0
333 while i < len(name_idx) and "particle" in tokens[name_idx[i]].tags:
334 i += 1
335 if i == len(name_idx):
336 return i
337 return i + len(_units(tokens, name_idx[i:])[0])
340def _is_lone_never_given_particle(site: tuple[int, ...],
341 tokens: list[WorkToken]) -> bool:
342 return (len(site) == 1
343 and "particle" in tokens[site[0]].tags
344 and "vocab:particle-ambiguous" not in tokens[site[0]].tags)
347def _addressing_run(titles: list[int], name_word: int) -> list[int]:
348 """The title run H1 asks about: the LEADING one where one stands,
349 else the whole (trailing) run -- rules.md#H1 -- the run standing
350 BEFORE the one name word being the run that addresses, and a run
351 standing behind it deciding that word's field only when none
352 stands before.
354 Every title token is in the TITLE role by the time this runs, both
355 ends of `Sir John Prof.` among them, so `titles` is not a run --
356 keeping the two ends apart is what makes a trailing title
357 TRANSPARENT (rules.md#H5): `Sir John Prof.` is `Sir John` plus a
358 title, and reading both ends as one run keyed 'sir prof' made
359 adding the title flip the name word's field (#489, #316).
361 The split is at the NAME WORD, not at the first token of another
362 role, and the difference is a nickname or a maiden name written
363 among the titles. H1's own rationale says what stands beside the
364 name word "does not make the name any longer, so it does not
365 decide this reading", and that has to hold for WHICH run
366 addresses as well as for how many words the name has: `Dr.
367 'Smitty' Sir John` is one run written around a nickname and reads
368 given 'John' as `Dr. Sir John` does, while `'Smitty' Dr. Jones
369 Sir.` keeps family 'Jones' as `Dr. Jones Sir.` does. Splitting on
370 the first non-title token got both wrong (measured 2026-09-09).
372 Called only from inside H1's guard, after the role counts have
373 short-circuited, so a name with a family never builds this list;
374 `name_word` is the first GIVEN, which that guard has already
375 proved is the only name word there is."""
376 return [i for i in titles if i < name_word] or titles
379def post_rules(state: ParseState) -> ParseState:
380 tokens = list(state.tokens)
381 ambiguities = list(state.ambiguities)
382 titles = _idx(tokens, Role.TITLE)
383 givens = _idx(tokens, Role.GIVEN)
384 middles = _idx(tokens, Role.MIDDLE)
385 families = _idx(tokens, Role.FAMILY)
387 # rules.md#H1: "a title followed by exactly one name word makes
388 # that word the family name, whatever suffix, nickname or maiden
389 # name stands beside it, unless the title is a given-name title,
390 # which keeps it the given name; a run of several titles addresses
391 # as its last title does" -- counting suffix, nickname and maiden
392 # as further name words is what emptied the family (#410)
393 # (known gap: the guard tests which roles are unoccupied, it does
394 # not count units -- decisions.md#H1) (v1 handle_firstnames)
395 #
396 # rules.md#H1: "a run of several titles addresses as its last
397 # title does, and where a run stands BEFORE the one name word it
398 # is the run that addresses, a run standing behind it deciding
399 # that word's field only when none stands before" -- #489. WHICH
400 # run that is, and why the two ends of a name are not one, are
401 # _addressing_run's; its docstring carries the history.
402 if (titles and givens and not middles and not families
403 and not _run_addresses_by_given(
404 (tokens[i].text
405 for i in _addressing_run(titles, givens[0])),
406 state.lexicon.given_name_titles)):
407 for i in givens:
408 _retag(tokens, i, Role.FAMILY)
409 # every rule below reads these lists; recompute after any
410 # retag so no guard can inspect a name that has already
411 # moved -- a stale index list is the bug shape #359 fixed
412 givens = _idx(tokens, Role.GIVEN)
413 middles = _idx(tokens, Role.MIDDLE)
414 families = _idx(tokens, Role.FAMILY)
416 # rules.md#M4: "a maiden name standing beside exactly one name
417 # word makes that word the family name, whatever suffix or
418 # nickname stands beside it" (#445) -- the MAIDEN role, not a
419 # marker: M1's configured pair produces one with no marker
420 # anywhere ("Smith (Jones)" under maiden_delimiters), and the
421 # rationale carries over unchanged, so the rule is keyed on the
422 # maiden name as the statement is.
423 #
424 # H1's sibling, and placed under it so the interaction the rule
425 # states is decidable by reading: where H1 fired, `givens` is
426 # empty here and this cannot fire; where H1 declined because the
427 # title addresses by given name, `titles` is what keeps this rule
428 # off the same word. A titled name is H1's at both outcomes.
429 #
430 # Sibling EXCEPT in what it counts, and that is a known gap
431 # rather than a boundary: this guard counts GIVEN tokens where
432 # H1 counts nothing (it tests which roles are unoccupied), so a
433 # name word another rule joined counts as several here and the
434 # rule declines -- `Dr. Dean of Chemistry` reads family, while
435 # `Dean of Chemistry née Jones` keeps given 'Dean of Chemistry'.
436 # rules.md#P3 says a joined part is one name word wherever
437 # another rule counts them, so the two disagree; widening moves
438 # no corpus name and is not this change's to make
439 # (decisions.md#M4).
440 # rules.md#M4: "a word the vocabulary has claimed as a given name
441 # keeps that reading, and so does a word read as an initial"
442 # -- read off the tags classify already recorded rather than a
443 # predicate of this rule's own, because this rule changes what
444 # POSITION decided and must not reach what a word IS
445 # (mechanisms.md#TWO-LAYER-ASSIGN).
446 if (not titles and len(givens) == 1 and not middles and not families
447 and any(t.role is Role.MAIDEN for t in tokens)
448 and not (_NEVER_FLIPPED & tokens[givens[0]].tags)):
449 _retag(tokens, givens[0], Role.FAMILY)
450 # recomputed for H1's reason, stated at H1: a stale index list
451 # is the bug shape #359 fixed
452 givens = _idx(tokens, Role.GIVEN)
453 middles = _idx(tokens, Role.MIDDLE)
454 families = _idx(tokens, Role.FAMILY)
456 # rules.md#P1: "A never-given particle opening the name marks the
457 # name as surname-only: the particle run and the name words it
458 # attaches to are the family." (v1 handle_non_first_name_prefix;
459 # history: decisions.md#P1)
460 # How far the fold reaches depends on the order the name was READ
461 # under (#395; decisions.md#P1, 2026-08-17): declaring a
462 # family-first order asserts that what follows the family is not
463 # more surname, which is the very question of where the run stops.
464 # Under that declaration the run takes its own particles and ONE
465 # name word; under the default order it keeps taking the rest of
466 # the name, nothing having marked where the surname ends. Note what
467 # actually holds the family-comma shape back, since it is NOT
468 # that test -- "Smith, de Mesnil Jean" DOES fire the leading site,
469 # segment 1 opening with the particle. It keeps the old reach
470 # because assign records no order after a family comma, the comma
471 # having already fixed the surname, so `order is None` here.
472 # Anything that later gives that path an order turns the
473 # narrowing on for it.
474 # ONE site, the opening piece. A particle standing in the GIVEN
475 # position was a second site until #467, and it was the parser
476 # overriding a declaration rather than reading one: under a
477 # family-first order that slot holds what the caller SAID is the
478 # given name, and the never-given vocabulary supplies a default
479 # where position leaves the question open, not a veto over an
480 # answer position already gave. So "Menil de" under either
481 # family-first order reports given "de" -- P6 takes the trailing
482 # particle only where it landed in a MIDDLE, which means nothing.
483 # Code-local: a lone PIECE is the test, so a particle group
484 # already chained forward is not a lone particle, and rule H1
485 # above cannot be what produces the fold's family reading -- H1 is
486 # gated on `not families`.
487 lead = _leading_name_piece(state, tokens)
488 lead_fires = _is_lone_never_given_particle(lead, tokens)
489 if len(givens) + len(middles) + len(families) > 1 and lead_fires:
490 order = state.order
491 if lead_fires and order is not None and order[0] is Role.FAMILY:
492 # `state.order`, not policy.name_order: a script_orders
493 # entry can put the family first under a given-first
494 # policy, and the roles below have to match the read
495 # assign actually made.
496 name_idx = sorted(givens + middles + families)
497 cut = _fold_reach(tokens, name_idx)
498 for i in name_idx[:cut]:
499 _retag(tokens, i, Role.FAMILY)
500 # What is left is a shorter name of the same order: one
501 # family already placed, so drop that slot and lay the
502 # rest out as _name_positions would for n + 1 pieces.
503 rest = _units(tokens, name_idx[cut:])
504 for unit, role in zip(rest, _name_positions(
505 order, len(rest) + 1)[1:]):
506 for i in unit:
507 _retag(tokens, i, role)
508 else:
509 for i in givens + middles:
510 _retag(tokens, i, Role.FAMILY)
511 # downstream rules key on the role counts: recompute
512 givens = _idx(tokens, Role.GIVEN)
513 middles = _idx(tokens, Role.MIDDLE)
514 families = _idx(tokens, Role.FAMILY)
516 # v1 gates both rotations on `not self._had_comma`; the
517 # middle_as_family fold below runs comma or not (v1 order:
518 # patronymics first, then handle_middle_name_as_last)
519 rules = state.policy.patronymic_rules
520 rotations_apply = _rotations_apply(state)
521 # rules.md#O1: "a name of exactly three name words — titles,
522 # suffixes and nicknames aside — whose last name word carries a
523 # patronymic ending and whose middle name word does not reads as
524 # family-first" (history: decisions.md#O1)
525 if rotations_apply and PatronymicRule.EAST_SLAVIC in rules and \
526 len(givens) == 1 and len(middles) == 1 and len(families) == 1:
527 tail = tokens[families[0]].text
528 mid = tokens[middles[0]].text
529 if (_EAST_SLAVIC.search(tail) or _EAST_SLAVIC_CYR.search(tail)) \
530 and not (_EAST_SLAVIC.search(mid)
531 or _EAST_SLAVIC_CYR.search(mid)):
532 g, m, f = givens[0], middles[0], families[0]
533 _retag(tokens, m, Role.GIVEN)
534 _retag(tokens, f, Role.MIDDLE)
535 _retag(tokens, g, Role.FAMILY)
536 # rules.md#O2: "a name of exactly four name words — titles,
537 # suffixes and nicknames aside — ending in a standalone
538 # patronymic marker reads family-first: the first name word is
539 # the family name" (history: decisions.md#O2)
540 if rotations_apply and PatronymicRule.TURKIC in rules and \
541 len(givens) == 1 and len(middles) == 2 and len(families) == 1:
542 tail = tokens[families[0]].text
543 if _TURKIC.match(tail) or _TURKIC_CYR.match(tail):
544 g, m1, m2, f = givens[0], middles[0], middles[1], families[0]
545 _retag(tokens, m1, Role.GIVEN)
546 _retag(tokens, m2, Role.MIDDLE)
547 _retag(tokens, f, Role.MIDDLE)
548 _retag(tokens, g, Role.FAMILY)
549 # rules.md#P6: "a particle ending the name attaches to that family
550 # name and is written before it" -- the no-comma site (#467).
551 # A comma is not the only thing that names the family; a declared
552 # family-first order does too, and the same listing is written
553 # both ways ("Jong, Anke de" and "Jong Anke de" under
554 # FAMILY_FIRST both give family "de Jong").
555 #
556 # Keyed on the SLOT the run landed in, not on its vocabulary.
557 # MIDDLE is the one position that means nothing here: middles are
558 # further given names, and a particle is not one. Only
559 # FAMILY_FIRST puts a trailing piece there --
560 # FAMILY_FIRST_GIVEN_LAST puts it in GIVEN, where the caller's own
561 # declaration says it IS the given name, and P1's site above no
562 # longer overrides that either.
563 #
564 # Which is why no vocabulary test appears here and none is wanted.
565 # The comma path needs one because a comma cannot separate the
566 # Dutch reading from the Vietnamese; the declared ORDER can, when
567 # the caller declares the right one for the name: "Beethoven
568 # Ludwig van" under FAMILY_FIRST gives family "van Beethoven"
569 # though `van` is ambiguous vocabulary, and "Nguyen Thi Van" under
570 # FAMILY_FIRST_GIVEN_LAST keeps given "Van" though the same word
571 # is in the same set. (Under the WRONG order each loses: the
572 # Vietnamese name read as FAMILY_FIRST gives family "Van Nguyen".
573 # rules.md#P6 records that as accepted -- it is one order, not
574 # both.) A never-given test here would have excluded every
575 # ambiguous particle -- von, di, da, del, le and `van` itself,
576 # the flagship word of the listing this rule is named for.
577 #
578 # NO RE-LAYOUT, and that is a property of ONE order rather than a
579 # simplification: under FAMILY_FIRST the roles run family, given,
580 # middle, middle..., so dropping a trailing MIDDLE leaves every
581 # other piece's role untouched. It is not true of
582 # FAMILY_FIRST_GIVEN_LAST (family, middle..., given) and not true
583 # of the default order, which is why this site tests the order
584 # rather than trusting the slot to imply it -- review found the
585 # first draft firing on 52 default-order names, where a
586 # conjunction stops a particle's forward chain and leaves it
587 # standing in a middle ("Maria Luisa y de la Cruz"). An earlier
588 # draft still (#466) removed a piece and re-laid the leftover out,
589 # which lost a given name outright on "van Berg Jan de" and
590 # promoted a post-nominal into the given slot on
591 # "Berg Jan Jr. de"; neither is reachable without a re-layout.
592 #
593 # The family must still hold a base of its own -- R2's invariant,
594 # that a non-empty family has a non-empty base, not a ban on
595 # particle-before-particle. Without it "van Berg Jan de" reports
596 # family 'de van', whose words R2 reads as ordinary name words,
597 # and no rule reorders those. It does NOT stop a second particle
598 # joining a family that merely OPENS with one: "de Mesnil Jan de"
599 # gives 'de de Mesnil', which keeps its base and is accepted.
600 mids = _idx(tokens, Role.MIDDLE)
601 # O1 and O2 above retag between roles WITHOUT recomputing, so this
602 # is the first consumer of a stale list -- the bug shape #359 fixed,
603 # and every other retagging block in this stage recomputes for it.
604 families = _idx(tokens, Role.FAMILY)
605 names = _idx(tokens, Role.GIVEN) + mids + families
606 if mids and families and state.order is not None:
607 run: list[int] = []
608 i = len(mids)
609 while i and "particle" in tokens[mids[i - 1]].tags:
610 i -= 1
611 run.append(mids[i])
612 # ENDING the name is the rule's own word, and it is the whole
613 # test: the trailing MIDDLE is not always the trailing NAME
614 # word. Under FAMILY_FIRST_GIVEN_LAST the given name stands
615 # behind the middles, and under the default order the family
616 # does, so in both a middle that ends the name is impossible
617 # and this declines without asking which order was declared.
618 # Only FAMILY_FIRST can put a name's last word in a middle.
619 #
620 # A draft tested the SLOT alone and fired on 52 default-order
621 # names, where a conjunction stops a particle's forward chain
622 # and leaves it standing in a middle ("Maria Luisa y de la
623 # Cruz"), and on 366 FAMILY_FIRST_GIVEN_LAST middles with the
624 # given name still behind them. A later draft tested the order
625 # as well; measured over 542,592 generated parses, that second
626 # test never decides anything this one has not already decided,
627 # so it is not here.
628 trailing = bool(run) and max(run) == max(names)
629 if trailing and any("particle" not in tokens[j].tags
630 for j in families):
631 # mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE: "Emit at
632 # the site that takes the branch, not where an ambiguous
633 # tag sits". decisions.md#P6 (#405) settled that this
634 # attachment reports the fork it decides, and the state it
635 # fixed is the one this site would otherwise recreate --
636 # `Beethoven, Ludwig van` reporting while `Beethoven Ludwig
637 # van` under FAMILY_FIRST decides the identical fork in
638 # silence. ONE arm here where the comma path has two: the
639 # suffix reading stands without a comma, so no post-nominal
640 # reading is ever overridden for this site to report.
641 ambiguous = [j for j in run
642 if "vocab:particle-ambiguous" in tokens[j].tags]
643 if ambiguous:
644 word = tokens[ambiguous[0]].text
645 text = " ".join(tokens[j].text for j in sorted(run))
646 ambiguities.append(PendingAmbiguity(
647 AmbiguityKind.PARTICLE_OR_GIVEN,
648 f"{word!r} is both a family-name particle and an "
649 f"ordinary given name; ending a name read "
650 f"family-first, {text!r} joins the family that "
651 f"order named rather than standing as a name word "
652 f"of its own",
653 tuple(sorted(run))))
654 for j in run:
655 tokens[j] = dataclasses.replace(
656 tokens[j], role=Role.FAMILY,
657 tags=tokens[j].tags | {FOLDED_TAG})
658 # recomputed for H1's reason, stated at H1: a stale index
659 # list is the bug shape #359 fixed
660 givens = _idx(tokens, Role.GIVEN)
661 middles = _idx(tokens, Role.MIDDLE)
662 families = _idx(tokens, Role.FAMILY)
664 # rules.md#P6: "a particle ending the name attaches to that family
665 # name and is written before it" -- where a family comma has
666 # already named the family, and provided at least one given word
667 # remains (history: decisions.md#P6). The Dutch alphabetized
668 # listing:
669 # "Beethoven, Ludwig van" is how "Ludwig van Beethoven" is filed.
670 #
671 # Keyed on the token's VOCABULARY, not its assigned role, which is
672 # what gives the attachment its stated precedence over S2. `vd`,
673 # `mc` and `do` are the three words in both vocabularies; assign
674 # reads a trailing `vd` or `mc` as a post-nominal, so those two
675 # need the override. `do` is in the AMBIGUOUS acronym half, which
676 # already leaves it a name word, so it attaches by the plain rule.
677 # After a family comma the tussenvoegsel is the commoner reading.
678 #
679 # The words-to-spare guard is a piece test, not a count: every
680 # trailing piece that is wholly particles attaches, and the run
681 # must leave a GIVEN word ahead of it, so "Nguyen, Van" keeps its
682 # only given word rather than being left with none. Only the
683 # DEGENERATE Vietnamese listing is protected by that -- "Nguyen,
684 # Thi Van" has a given word to spare, so `Van` attaches and the
685 # given name is lost. rules.md#P6 records why that is accepted.
686 #
687 # mechanisms.md#FOLDED_TAG does the rest: tokens never move, so
688 # the family view reads the tag and renders these before the base.
689 if state.structure is Structure.FAMILY_COMMA and len(state.pieces) > 1:
690 seg = state.pieces[1]
691 # A post-nominal sits BEHIND the tussenvoegsel in this listing
692 # ("Berg, Jan van Jr."), so the run is found by walking past a
693 # trailing piece that holds no name -- but only one that is not
694 # itself particle vocabulary, since `vd` arrives suffix-roled
695 # and IS the run. Without this the same name parsed two ways on
696 # whether a comma preceded the credential.
697 end = len(seg)
698 while (end
699 and not any(tokens[i].role in _NAME_ROLES
700 for i in seg[end - 1])
701 and not all("particle" in tokens[i].tags
702 for i in seg[end - 1])):
703 end -= 1
704 k = end
705 while k and all("particle" in tokens[i].tags for i in seg[k - 1]):
706 k -= 1
707 # GIVEN alone, which is what P6 says ("provided at least one
708 # given word remains"). Not `_NAME_ROLES`: P1's fold runs
709 # earlier in this function and retags all of segment 1 to
710 # FAMILY, so a test for "some name word remains" passes on
711 # family text P1 just produced, and the rule then hoists the
712 # particle in front of a base it never preceded ("Smith, de
713 # Mesnil van" -> 'van Smith de Mesnil'). MIDDLE was in this
714 # test until review found no input where it decides anything
715 # -- 0 hits over 740,552 instrumented guard sites. The reason
716 # is structural: the only rule that can leave a MIDDLE with no
717 # GIVEN ahead of it in segment 1 is P1's family-first
718 # redistribution, which is gated on `state.order`. On the
719 # FAMILY_COMMA path assign records an order only where
720 # segment 1 holds no name word (#296's positional read), and
721 # a no-name segment holds no MIDDLE for the fold to leave
722 # either. P6 runs only on that path, so the branch cannot be
723 # reached from here.
724 if k and any(tokens[i].role is Role.GIVEN
725 for piece in seg[:k] for i in piece):
726 # A range, though only ever one piece today: grouping's
727 # prefix chain makes a non-leading particle absorb what
728 # follows, so a trailing run splits into several pieces
729 # only where nothing ahead of it holds a given role --
730 # the run opening the segment ("Berg, de van"), or only
731 # titles ahead of it ("Berg, Sir de la", 8% of them). The
732 # guard then declines either way. Measured over 95,180
733 # generated multi-piece runs: `end - k` is never above 1
734 # where the guard passes. Written as a range because the
735 # guard, not this loop, is what bounds it.
736 run = [i for piece in seg[k:end] for i in piece]
737 # mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE: "Emit at
738 # the site that takes the branch, not where an ambiguous
739 # tag sits" -- this attachment is where the fork is
740 # decided, so the report is raised here rather than in
741 # assign, whose own emitter is scoped to the no-comma
742 # shapes (#405).
743 #
744 # Two arms, keyed on what the attachment OVERRODE rather
745 # than on what the words are, so each names a branch the
746 # parse actually weighed. `declined_suffix` reads the role
747 # assign gave the run, which is why both lists are built
748 # BEFORE the re-roling loop below overwrites it.
749 #
750 # ORDERED, not asserted disjoint. The two cannot both hold
751 # under the shipped vocabulary -- `vd` and `mc` are the
752 # only words in both the particle and the unambiguous
753 # suffix vocabularies, and neither is ambiguous particle
754 # vocabulary -- but a caller's Lexicon may put one word in
755 # both, and rules.md#A1 says parsing never fails on any
756 # input, so this decides instead of raising. (Measured: an
757 # assert here DID fire on `Berg, Jan zz` under a Lexicon
758 # adding `zz` to particles_ambiguous and suffix_acronyms.)
759 # The suffix arm wins because it names the reading the
760 # parse actually took: assign had read the word as a
761 # post-nominal, so the name-word reading was never on the
762 # table for the attachment to decline.
763 #
764 # Both details name the ATTACHMENT and stop there. What
765 # P6 decides is that the run joins the family the comma
766 # named instead of standing on its own; how the joined
767 # words then READ is R2's call, taken by the UNJOINED_TAG
768 # loop at the end of this stage. The two can disagree:
769 # `de la, Jan van` attaches `van` and leaves an
770 # all-particle family, which R2 marks, so every other view
771 # -- `family_base`, the initials, case repair -- reads
772 # those words as ordinary name words ('Jan Van De La'
773 # forced). A detail promising "read as the family's
774 # particle" would contradict all three.
775 ambiguous = [i for i in run
776 if "vocab:particle-ambiguous" in tokens[i].tags]
777 declined_suffix = [i for i in run
778 if tokens[i].role is Role.SUFFIX]
779 text = " ".join(tokens[i].text for i in run)
780 if declined_suffix:
781 ambiguities.append(PendingAmbiguity(
782 AmbiguityKind.SUFFIX_OR_NAME,
783 f"{text!r} written without periods is both a "
784 f"post-nominal and a family-name particle; after a "
785 f"family comma it joins the family the comma named "
786 f"rather than standing as a post-nominal",
787 tuple(run)))
788 elif ambiguous:
789 word = tokens[ambiguous[0]].text
790 ambiguities.append(PendingAmbiguity(
791 AmbiguityKind.PARTICLE_OR_GIVEN,
792 f"{word!r} is both a family-name particle and an "
793 f"ordinary given name; after a family comma "
794 f"{text!r} joins the family the comma named "
795 f"rather than standing as a name word of its own",
796 tuple(run)))
797 for i in run:
798 tokens[i] = dataclasses.replace(
799 tokens[i], role=Role.FAMILY,
800 tags=tokens[i].tags | {FOLDED_TAG})
802 # rules.md#O3: "every middle word joins the family name and is
803 # rendered before it" (v1 handle_middle_name_as_last). v1
804 # PREPENDED middle_list to last_list; mechanisms.md#FOLDED_TAG:
805 # "tokens never move: a rule that needs different rendering order
806 # tags the token, and the rendering views consult the tag"
807 if state.policy.middle_as_family:
808 for i in _idx(tokens, Role.MIDDLE):
809 tokens[i] = dataclasses.replace(
810 tokens[i], role=Role.FAMILY,
811 tags=tokens[i].tags | {FOLDED_TAG})
812 # rules.md#R2: "a name part whose every word is particle
813 # vocabulary is a part where none of them is doing a particle's
814 # work — nothing joins them to a name — so they read as ordinary
815 # name words"
816 #
817 # Last of the rules that MOVE a token, because every rule above
818 # can still move one between parts:
819 # P1's fold, P6's attachment and O3's fold all rewrite roles, and
820 # this reads the roles they settle on.
821 #
822 # Marked, not untagged: `particle` is stable API and says the word
823 # IS particle vocabulary wherever it lands, which stays true.
824 #
825 # All three roles for uniformity with the rule, not because all
826 # three are observable: no view filters tags on GIVEN (initials
827 # exempt that role outright), so restricting this loop to MIDDLE
828 # and FAMILY moves 0 of 4,506 parses. The GIVEN arm is marked so a
829 # future view reading the mark gets a consistent answer.
830 for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY):
831 part = _idx(tokens, role)
832 if part and all("particle" in tokens[i].tags for i in part):
833 for i in part:
834 tokens[i] = dataclasses.replace(
835 tokens[i], tags=tokens[i].tags | {UNJOINED_TAG})
836 _mark_suffix_entries(tokens, state)
837 return dataclasses.replace(state, tokens=tuple(tokens),
838 ambiguities=tuple(ambiguities))