Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage: segment.
3Consumes: tokens (role-None main stream), comma_offsets, one_case
4(where an earlier stage recorded it).
5Produces: segments (runs of main-token indices; interior segments may
6be EMPTY -- doubled commas keep their structural position), structure
7-- NO_COMMA, or FAMILY_COMMA for every comma form: whether the part
8after the first comma is the postnominal part is decided at the head
9of group once classify has tagged the words (#613, `_comma.decide`),
10which may set SUFFIX_COMMA -- one_case where C2's flag asked for it,
11and COMMA_STRUCTURE ambiguities for unrecognized extra segments (a
12part of title and suffix words is recognized, #603).
13Reads: for C2's flag on a part past the second comma only: Lexicon
14suffix vocabulary and Policy through _vocab.is_wholly_suffix
15(Policy.lenient_comma_suffixes picks the lenient or strict token test;
16Policy.extra_suffix_delimiters makes a delimiter-core token
17transparent, as the built-in cores are), with the credential lean the lazy case gate supplies
18(Lexicon.maiden_markers DIRECTLY, for the own-words span
19_pieces.own_words takes); Policy.unlisted_dotted_suffixes through
20_vocab.ambiguous_class_candidate and _vocab.ambiguous_class_member,
21for a part of by-shape credentials; and Lexicon.titles DIRECTLY, with
22_vocab.period_joined_vocab for a period-joined title, and
23Policy.extra_suffix_delimiters through _vocab.glued_cores for a
24title with a delimiter glued to it (rules/C10), for a part of titles
25and suffixes (#603).
27Implements rule C2 of docs/design/rules/, and C1's segmentation;
28C1's decision is `_comma.decide`'s. History in decisions/C1.
29"""
30from __future__ import annotations
32from nameparser._lexicon import _normalize
33from nameparser._pipeline._pieces import own_words
34from nameparser._pipeline._state import (
35 ParseState, PendingAmbiguity, Structure, comma_bucket, copy_with,
36)
37from nameparser._pipeline._vocab import (
38 ambiguous_class_candidate, ambiguous_class_member, glued_cores,
39 glued_stem, period_joined_vocab, is_one_case, is_wholly_suffix,
40)
41from nameparser._types import AmbiguityKind
47def segment(state: ParseState) -> ParseState:
48 main = [i for i, t in enumerate(state.tokens) if t.role is None]
49 if not main:
50 return copy_with(state, segments=(),
51 structure=Structure.NO_COMMA)
52 if not state.comma_offsets:
53 return copy_with(state, segments=(tuple(main),),
54 structure=Structure.NO_COMMA)
55 buckets: list[list[int]] = [[] for _ in range(len(state.comma_offsets) + 1)]
56 for i in main:
57 # _state.comma_bucket, not a local bisect: classify asks the
58 # same question of the same offsets to keep a marker run inside
59 # one segment, and the two must be one expression rather than
60 # two that agree
61 buckets[comma_bucket(state.tokens[i].span.start,
62 state.comma_offsets)].append(i)
63 groups = [tuple(b) for b in buckets]
64 # v1 strips exactly ONE trailing comma as cosmetic (parser.py's
65 # collapse_whitespace); every other empty bucket is STRUCTURAL and
66 # keeps its position -- in 'Doe,, Jr.' the given segment is empty,
67 # so 'Jr.' stays a tail suffix instead of masquerading as a lone
68 # post-comma title (v1 parity, pinned live 2026-07-16)
69 if len(groups) > 1 and not groups[-1]:
70 groups.pop()
71 if len(groups) <= 1:
72 segs = tuple(groups) if groups and groups[0] else (tuple(main),)
73 return copy_with(state, segments=segs,
74 structure=Structure.NO_COMMA)
76 # The case fact, asked LAZILY: here only C2's flag on a part past
77 # the second comma turns on it. A name that never asks pays nothing;
78 # classify asks for itself later where this did not
79 # (decisions/S6, and #429's precedent for paying a predicate
80 # twice rather than plumbing a field two sites would not otherwise
81 # share).
82 one_case = state.one_case
84 def case_class() -> bool:
85 nonlocal one_case
86 if one_case is None:
87 one_case = is_one_case(own_words(
88 state.tokens, state.comma_offsets,
89 state.lexicon._sets.maiden_markers)[0])
90 return one_case
92 # The texts are built inline rather than through a helper
93 # (measured, #289/#516's eager-gate fix round): on 3.11 a list
94 # comprehension IS a frame
95 # (PEP 709 inlines it only from 3.12 on; see
96 # tools/perf/call_count.py's docstring), so wrapping it in another
97 # call doubles the cost of every tail segment asked.
98 #
99 # `case` is ParseState.one_case: None is the case-FREE reading,
100 # which is what every caller on this path wants, and the tail
101 # walk below passes `case_class()` for the leaning one. One
102 # closure with a default rather than two whose bodies differ only
103 # in that argument.
104 # The strict test also reads the case fact, for its initial veto
105 # (rules/T4, #727): asked only under that opt-in policy, so the
106 # default pays nothing for it.
107 def suffixy(seg: tuple[int, ...], case: bool | None = None) -> bool:
108 return is_wholly_suffix(
109 [state.tokens[i].text for i in seg], state.lexicon,
110 state.policy, one_case=case,
111 letters_one_case=(None if state.policy.lenient_comma_suffixes
112 else case_class()))
114 def class_run(seg: tuple[int, ...]) -> bool:
115 # Every token of the run joins the ambiguous credential class
116 # BY SHAPE -- a candidate that is not a LISTED member, which is
117 # the one half `is_wholly_suffix` cannot see (its own docstring
118 # says so, and the blindness is what keeps a by-shape token out
119 # of C1's whole-name suffix fallback in `_comma.decide`). A tail
120 # segment is consumed as suffix either way, so what this decides is only
121 # whether the parse says it did not RECOGNIZE the segment -- and
122 # a run the parse itself reads as a credential run by shape is
123 # recognized. Without it, the narrow roman retirement
124 # (rules/S3) moved 'R.A.I.', 'X.Y.I.' and 'J.u.n.i.o.r.' out
125 # of the vocabulary verdict and into the shape class, and every
126 # one of them gained a COMMA_STRUCTURE flag in a third segment
127 # that 2.3 did not raise -- a report about the parser's own new
128 # reading rather than about the name (review round, #289/#516).
129 #
130 # The LISTED half is excluded rather than folded in: a listed
131 # member reaches this reading through the LEAN (the leaning
132 # `suffixy` call below), and that is the whole of what quiets
133 # it -- 'STEVEN HARDMAN, MD, DO, DDS' is written in one case,
134 # leans nothing, and keeps its flag, the recorded negative
135 # control for the lean's own effect here. Admitting membership
136 # alone would silence it and leave the control measuring
137 # nothing.
138 #
139 # Policy-sensitive by construction: with
140 # `unlisted_dotted_suffixes` off the token is name material, is
141 # no candidate, and the flag stands.
142 return all(ambiguous_class_candidate(state.tokens[i].text,
143 state.lexicon, state.policy)
144 and not ambiguous_class_member(state.tokens[i].text,
145 state.lexicon)
146 for i in seg)
148 def titles_and_suffixes(seg: tuple[int, ...]) -> bool:
149 # rules/C2: "a word of the title vocabulary there that is not
150 # also suffix vocabulary is a title" -- at least one title
151 # word, and every other word a suffix word
152 # period_joined_vocab for 'Lt.Gov.', one token that classify
153 # (not yet run) tags a title by this same call, and that assign
154 # then reads as one; a word without a period cannot be one
155 # rules/C10: "the word keeps the delimiter in its text and reads
156 # as it would without it" -- a title with one glued ('Dr;')
157 # reads as the title, as classify tags it (decisions/C10)
158 titles = state.lexicon._sets.titles
159 cores = glued_cores(state.policy.extra_suffix_delimiters)
160 rest = tuple([i for i in seg
161 if _normalize(t := state.tokens[i].text) not in titles
162 and ("." not in t
163 or period_joined_vocab(t, state.lexicon)
164 != "title")
165 # the pre-check keeps glued_stem's frame off a
166 # word that holds no core (its docstring's terms)
167 and not ((";" in t or len(cores) > 1)
168 and (g := glued_stem(t, cores))
169 and _normalize(g) in titles)])
170 return len(rest) < len(seg) and (not rest or suffixy(rest))
172 # rules/C1's decision is not this stage's since #613: every comma
173 # form leaves here as the FAMILY comma -- the listing form, what a
174 # comma means until the part after it has been read -- and group
175 # decides at its head, once classify has tagged every word, whether
176 # that part is the postnominal part (`_comma.decide`). Deciding here
177 # meant predicting the later stages: group's particle chain (#562's
178 # pair) and assign's reading of a run the capitals settle, with a
179 # second count in assign besides. What stays here is the
180 # segmentation and C2's flag on a part past the second comma.
181 ambiguities = list(state.ambiguities)
182 # rules/C2: "a non-empty extra part that is not entirely suffix
183 # words is flagged as a structural ambiguity rather than rejected"
184 # -- parts[2:] are consumed as suffixes either way, their title
185 # words as titles (#603), so a tail segment that is neither all
186 # suffix words nor title words beside suffix words gets the
187 # COMMA_STRUCTURE flag, not a structure veto. The lean reaches this reading too:
188 # a tail of leaning credentials is a credential run, which is the
189 # one place this design quiets a report rather than adding one.
190 for seg in groups[2:]:
191 # empty segments are consumed silently (v1 skips them without
192 # comment); only non-empty non-suffix tails get flagged.
193 # The case-FREE `suffixy(seg)` first: by construction the
194 # leaning call can only ADD a disjunct to it
195 # (`is_wholly_suffix`'s credential-lean branch), never remove
196 # one -- so a seg that already reads wholly suffix case-free
197 # reads so case-aware too, and the case fact is worth forcing
198 # only when the case-free answer was False (measured
199 # regression: the leaning call forced the fact for every tail
200 # segment, suffix or not).
201 # `class_run` last, for the same lazy reason and one step
202 # further out: it is the only one of the three that walks the
203 # by-shape class, and it is asked only of a run BOTH suffix
204 # readings have already declined.
205 # #603: a part assign reads as titles and suffixes is
206 # recognized too -- word by word, off the title vocabulary
207 # (classify has not run yet), so a part holding a connective ('Secretary of State') keeps the
208 # flag: knowing that group's title chain takes it would be a
209 # model of group, the shape mechanisms.md's
210 # READ-WITHOUT-THEN-BIND exists to remove. Last, as the
211 # rarest reading.
212 if (seg and not suffixy(seg) and not suffixy(seg, case_class())
213 and not class_run(seg) and not titles_and_suffixes(seg)):
214 texts_joined = " ".join([state.tokens[i].text for i in seg])
215 # Names no field: since #603 a title word in the part reads
216 # as a title, so 'consumed as suffix' had told
217 # 'John Smith, Jr., Dr. Bart' its 'Dr.' was a suffix, and
218 # the fields themselves say where each word went (#629).
219 ambiguities.append(PendingAmbiguity(
220 AmbiguityKind.COMMA_STRUCTURE,
221 f"segment {texts_joined!r} beyond the recognized comma "
222 f"structures; read best-effort",
223 tuple(seg)))
224 return copy_with(state, segments=tuple(groups),
225 structure=Structure.FAMILY_COMMA,
226 ambiguities=tuple(ambiguities),
227 one_case=one_case)