Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage: classify.
3Consumes: tokens, comma_offsets (with token roles, the two halves of
4the structural-boundary test the marker pass applies -- see
5_tag_marker_runs).
6Produces: tokens with vocabulary tags added (text/span/role unchanged),
7plus ambiguities (SUFFIX_OR_NICKNAME, CONJUNCTION_OR_INITIAL).
8Reads: every Lexicon vocabulary field except surnames and
9honorific_tails, which script_segment consumes upstream; no Policy
10FIELD is consulted (is_initial does consult the _policy module's
11_NO_INITIALS constant, which is not configuration -- nothing here
12varies by Policy value).
14Tags emitted -- stable (API): "particle", "conjunction", "initial";
15namespaced (unstable): "vocab:title", "vocab:given-title",
16"vocab:suffix", "vocab:suffix-word", "vocab:suffix-ambiguous",
17"vocab:particle-ambiguous", "vocab:bound-given", "vocab:maiden-marker",
18"vocab:maiden-marker-cont".
19"vocab:maiden-marker" tags the HEAD of a maiden marker, which is a
20whole marker whenever the marker is one word; the continuation tag
21carries the rest of a PHRASE marker ("z domu"), so a site asking
22"does a marker start here" reads the same tag it always did and a site
23asking "where does it end" walks the continuations.
24"vocab:suffix" means "counts as a suffix as written": unambiguous
25suffix vocabulary, or an ambiguous acronym written with periods --
26at the TAG level 'M.A.' gets "vocab:suffix" while 'Ma' gets only
27"vocab:suffix-ambiguous"; what assign then does with a trailing
28ambiguous tag is the rest of rule S2's statement (the
29words-to-spare guard) and its Accepted consequences.
30The initial veto is assign's job, not classify's: 'V' carries both
31"vocab:suffix" and "initial".
32"""
33from __future__ import annotations
35import dataclasses
36from collections.abc import Sequence
38from nameparser._lexicon import _normalize
39from nameparser._pipeline._state import (
40 ParseState, PendingAmbiguity, WorkToken, comma_bucket,
41)
42from nameparser._types import AmbiguityKind, Role
43from nameparser._pipeline._vocab import (
44 _longest_marker, is_initial, is_one_case, maiden_marker_head,
45 maiden_marker_run, period_joined_vocab, suffix_as_written,
46)
51# rules.md#S2: "a trailing word of the suffix vocabulary reads as a
52# suffix — generational forms and credential acronyms alike, and an
53# ambiguous acronym written with its periods, one after each
54# letter, counts unambiguously; a single trailing period is the
55# abbreviation shape any word can wear and does not. A
56# bare ambiguous acronym is consumed only when the name has words to
57# spare"
58def _tags_for(token: WorkToken, n: str, state: ParseState,
59 marker_tag: str | None, one_case_own: bool) -> frozenset[str]:
60 """`n` is _normalize(token.text), folded once by the caller and
61 shared with the marker pass; `marker_tag` is what that pass decided
62 for this token, or None. The marker DECISION is entirely
63 _tag_marker_runs'; only the writing happens here, so the two tokens
64 of a phrase are built once rather than replaced twice.
66 `one_case_own` is true when the name's OWN words are written in one
67 case AND this token is one of the name's own words -- a maiden clause
68 and any delimited (nickname) content are not, so the fork never
69 reads them either (rules.md#P3): a clause's words are not the
70 name's own words, and appending one must not change how THIS token
71 reads."""
72 lex = state.lexicon
73 tags = set(token.tags)
74 if marker_tag is not None:
75 tags.add(marker_tag)
76 if n in lex.titles:
77 tags.add("vocab:title")
78 if n in lex.given_name_titles:
79 tags.add("vocab:given-title")
80 if suffix_as_written(n, token.text, lex):
81 tags.add("vocab:suffix")
82 if n in lex.suffix_words:
83 tags.add("vocab:suffix-word")
84 if n in lex.suffix_acronyms_ambiguous:
85 tags.add("vocab:suffix-ambiguous")
86 if n in lex.particles:
87 tags.add("particle")
88 if n in lex.particles_ambiguous:
89 tags.add("vocab:particle-ambiguous")
90 # rules.md#P3: "a single-letter connective reads as an initial
91 # where the writing says so: written as a bare Latin capital in a
92 # name that is not written wholly in one case, or — in a name
93 # written wholly in one case, where nothing says so — where the
94 # letter is one the vocabulary marks as reading both ways"
95 # (#383/#479; history: decisions.md#P3)
96 single_letter_connective = (len(token.text) == 1
97 and token.text.upper() != token.text.lower()
98 and n in lex.conjunctions)
99 if single_letter_connective and one_case_own:
100 # No case evidence, so the vocabulary decides. No namespaced
101 # tag beside it: the emitted ambiguity IS the record of the
102 # decision (mechanisms.md#MARK-DONT-STRIP is satisfied by the
103 # report) -- mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE says
104 # emit where the branch is taken, not where an ambiguous tag
105 # sits, and a vocab: tag records MEMBERSHIP, not the branch
106 # taken, so it is the wrong shape of record here.
107 if n in lex.conjunctions_ambiguous:
108 tags.add("initial")
109 else:
110 # a bare capital y joins here, where mixed case vetoes it
111 tags.add("conjunction")
112 else:
113 # the mixed-case rule, unchanged. v1's is_conjunction excludes
114 # initials: 'e.' in 'john e. smith' is a middle initial, not
115 # the Spanish conjunction 'e'
116 initial = is_initial(token.text)
117 if n in lex.conjunctions and not initial:
118 tags.add("conjunction")
119 if initial:
120 tags.add("initial")
121 if n in lex.bound_given_names:
122 tags.add("vocab:bound-given")
123 # maiden markers are NOT tagged here: an entry may be a phrase whose
124 # words are not markers on their own, and this function sees one
125 # token with no neighbours. _tag_marker_runs below does the whole
126 # field, single words included, so there is one place that decides
127 # it (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
128 # v1's period-joined derivation (parse_pieces): a token with a
129 # period not at the end, ANY of whose period chunks is a title, is
130 # a title as a whole ('Lt.Gov.', and by the ANY rule 'Mr.Smith');
131 # else ANY suffix chunk makes it a suffix ('JD.CPA'). Title wins
132 # (v1's continue). Skipped when the whole token already matched.
133 if "vocab:title" not in tags and "vocab:suffix" not in tags:
134 derived = period_joined_vocab(token.text, lex)
135 if derived == "title":
136 tags.add("vocab:title")
137 elif derived == "suffix":
138 tags.add("vocab:suffix")
139 return frozenset(tags)
142def _tag_marker_runs(state: ParseState,
143 folded: Sequence[str]) -> dict[int, str]:
144 """Which tokens are maiden marker runs: index -> "vocab:maiden-marker"
145 for a run's head, "vocab:maiden-marker-cont" for the rest.
147 Returns the decision rather than rewriting the tokens; classify
148 writes it into the one pass that builds them, so a marker token is
149 not replaced twice. `folded` is _normalize per token, computed once
150 for this pass and the vocabulary tags alike.
152 The one sequence pass in this stage, and it has to be one: a marker
153 entry may be a PHRASE whose words are not markers individually
154 ('z', 'domu'), so no per-token membership test can find it.
155 Left to right, longest first at each position, then skip past what
156 the run claimed -- a second marker cannot start inside the first.
158 This is where the tag is DECIDED for the two stages that read it
159 afterwards. group runs later and asks its questions of these tags
160 rather than re-deriving the run (the recorded-answer half of
161 mechanisms.md#ONE-PREDICATE-PER-QUESTION); extract runs EARLIER,
162 before tokens exist, so it calls the predicate itself over the
163 clause's whitespace words.
165 A tagged run is structurally contiguous, and the test is
166 one-directional: a role change IS a clause edge, so no run spans
167 one, but not every clause edge is a role change -- two ADJACENT
168 clauses of the same role are indistinguishable here, and
169 'Jane (z) (domu) Jones' does tag a run across them. Both consumers
170 refuse that run for reasons of their own (the piece walk never sees
171 role-bearing tokens at all; the clause drop is scoped to one
172 clause's span), so no reading depends on it today, and the claim
173 this pass can honestly make is the weaker one. What it does
174 guarantee is what _group._marker_run_pieces needs: a run inside the
175 MAIN stream stays inside one segment. Without it this pass walked
176 the whole span-sorted stream while group walked one segment --
177 _segment keeps only role-less tokens and buckets them by the commas
178 before them -- so a run half inside a bracketed clause was tagged
179 whole and consumed as a proper PREFIX of itself, and
180 'Anna z (domu) Nowak' read family 'Anna', maiden 'Nowak': the bare
181 preposition eating the name, which is the exact damage the phrase
182 entry exists to prevent. Refusing to tag such a run is the fix;
183 truncating it instead would hand M2 the same wrong prefix one word
184 shorter.
185 """
186 markers = state.lexicon.maiden_markers
187 # the lookahead the vocabulary actually needs; 0 for an empty set,
188 # which skips the pass entirely
189 cap = _longest_marker(markers)
190 if not cap:
191 return {}
192 tokens = state.tokens
193 n_tokens = len(tokens)
194 # Deferred, not computed up front: only the contiguity walk reads
195 # it, only a phrase vocabulary runs that walk, and only at a token
196 # that opens an entry -- so a single-word vocabulary, and a
197 # phrase vocabulary over a name holding no marker, never pay the
198 # sweep at all.
199 buckets: list[int] | None = None
200 tags: dict[int, str] = {}
201 i = 0
202 while i < n_tokens:
203 # The predicate's own head test first, over the fold the caller
204 # already has: almost no token opens any entry, and for those
205 # there is nothing to assemble. Same function maiden_marker_run
206 # consults, so a token skipped here is one it would refuse.
207 if not maiden_marker_head(folded[i], markers):
208 i += 1
209 continue
210 # Bound the lookahead at the first structural boundary, so the
211 # predicate is asked over the words that could form one run and
212 # answers longest-first WITHIN them -- a two-word entry refused
213 # at a clause edge still leaves a one-word entry starting there
214 # free to match.
215 limit = 1
216 if cap > 1:
217 if buckets is None:
218 buckets = [comma_bucket(t.span.start, state.comma_offsets)
219 for t in tokens]
220 role, bucket = tokens[i].role, buckets[i]
221 while (limit < cap and i + limit < n_tokens
222 and tokens[i + limit].role is role
223 and buckets[i + limit] == bucket):
224 limit += 1
225 run = maiden_marker_run(
226 [tokens[k].text for k in range(i, i + limit)], markers)
227 if not run:
228 i += 1
229 continue
230 tags[i] = "vocab:maiden-marker"
231 for k in range(i + 1, i + run):
232 tags[k] = "vocab:maiden-marker-cont"
233 i += run
234 return tags
237def classify(state: ParseState) -> ParseState:
238 # One fold per token, shared by the marker pass and the vocabulary
239 # tags -- the shape suffix_as_written already asks for ("n is
240 # _normalize(text), passed in so callers normalize once").
241 folded = [_normalize(t.text) for t in state.tokens]
242 marker_tags = _tag_marker_runs(state, folded)
243 # rules.md#P3 says a maiden marker, taken as one, and the words it
244 # takes, are not among the name's own words -- so clause_at is the
245 # smallest index tagged as a marker HEAD, and everything from there
246 # on is the clause. A plain loop, not a generator handed to min():
247 # marker_tags is almost always empty, and its keys arrive in index
248 # order (_tag_marker_runs walks left to right), so the first head a
249 # forward walk finds is already the smallest. Filtering on the HEAD
250 # tag specifically (never "-cont") is what makes that answer right
251 # independent of _tag_marker_runs's insertion order too: every
252 # matching entry is a clause start, so a min() over them in any
253 # order would agree with this walk -- the walk just takes the
254 # cheaper path given the order this dict happens to arrive in.
255 clause_at = len(state.tokens)
256 for i, tag in marker_tags.items():
257 # A marker word already carrying a role arrived pre-set by
258 # extract (WorkToken.role's docstring) -- it is the CLAUSE's
259 # word, not a bare one opening a new clause, so it must not
260 # move clause_at: a delimited/maiden clause's own marker
261 # content is excluded from "own" by its role already, and
262 # letting it also set clause_at truncates the OWN words that
263 # follow the clause ("JUAN (NEE JONES) GARCIA Y LOPEZ"'s
264 # trailing "GARCIA Y LOPEZ" is such own text).
265 if tag == "vocab:maiden-marker" and state.tokens[i].role is None:
266 clause_at = i
267 break
268 # ONE fact per parse, taken over the name's OWN words (rules.md#P3):
269 # not a delimited clause's tokens, which arrive with `role` already
270 # set by extract (WorkToken.role's docstring), and not the maiden
271 # clause itself, which starts at clause_at. Appending a clause must
272 # not flip the reading of words that did not change. Not stored on
273 # ParseState: nothing downstream reads it today, and #289/#516 can
274 # promote it the way `order` was recorded rather than recomputed.
275 # is_one_case's own `Sequence` parameter is where the frame-cost
276 # argument for handing it a built list lives (_vocab.py, #475).
277 own = [t.text for t in state.tokens[:clause_at] if t.role is None]
278 one_case = is_one_case(own)
279 # The fork itself must not read a clause's words either, so the
280 # `one_case and ...` argument below repeats `own`'s membership test
281 # per token, and the fork and its emitter then agree with the case
282 # class they consult. No extra frame -- it is one more boolean in a
283 # comprehension that already walks every token.
284 tokens = tuple(
285 dataclasses.replace(
286 t, tags=_tags_for(t, folded[i], state, marker_tags.get(i),
287 one_case and i < clause_at
288 and t.role is None))
289 for i, t in enumerate(state.tokens))
290 # Delimited content whose vocabulary cannot settle it: extract's
291 # escape sends an UNambiguous suffix straight through ("(MBA)" ->
292 # suffix) and keeps everything else as a nickname, so an AMBIGUOUS
293 # acronym in there was a coin the parser had to call. Reported here
294 # rather than at the escape itself, which runs before tokenize and
295 # so has no token index to point at.
296 ambiguities = list(state.ambiguities)
297 for i, token in enumerate(tokens):
298 if (token.role is Role.NICKNAME
299 and "vocab:suffix-ambiguous" in token.tags):
300 ambiguities.append(PendingAmbiguity(
301 AmbiguityKind.SUFFIX_OR_NICKNAME,
302 f"delimited {token.text!r} is also a post-nominal; read "
303 f"as a nickname rather than a suffix",
304 (i,)))
305 # #383/#479: narrows the fork's own "initial" tag with the same
306 # inputs the fork used, rather than re-deciding from scratch.
307 # Only the casedness test is inherited from the tag --
308 # `is_initial(token.text)` also tags a bare capital "initial"
309 # in the fork's else branch, so the tag alone does not tell
310 # this apart from that.
311 #
312 # Of the two clauses beside it, one is load-bearing and one is
313 # not. `conjunctions_ambiguous` is IMPLIED by the others for a
314 # fresh parse -- the contrapositive of what it looks like:
315 # `is_initial` matches an ASCII capital only, so ONLY the
316 # fork's branch can tag a bare LOWERCASE letter "initial"
317 # ("jose e maria santos" tags a lowercase e), and it does so
318 # only for a conjunctions_ambiguous member. So for a letter of
319 # EITHER case, "initial" plus len 1 implies membership. It is
320 # kept only because `_tags_for` starts from `set(token.tags)`, so
321 # this clause is what keeps the emitter honest if a runner ever
322 # hands classify tokens it did not build; no such path exists
323 # today. `conjunctions` is the clause doing real work: it keeps
324 # an orphan marker (a conjunctions_ambiguous entry no longer in
325 # conjunctions) inert rather than reported (decisions.md#P3).
326 # `i < clause_at and token.role is None` mirrors the fork's own
327 # "own words" test above, for the same reason (rules.md#P3): a
328 # clause's words were never eligible for the fork, so they must
329 # never be eligible to report either. Emitted at the decision
330 # site (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE), per
331 # token -- 'e and e' reports twice.
332 if (one_case and "initial" in token.tags
333 and len(token.text) == 1
334 and folded[i] in state.lexicon.conjunctions_ambiguous
335 and folded[i] in state.lexicon.conjunctions
336 and i < clause_at and token.role is None):
337 ambiguities.append(PendingAmbiguity(
338 AmbiguityKind.CONJUNCTION_OR_INITIAL,
339 f"{token.text!r} is both a connective and an initial; "
340 f"the name is written in one case, so nothing marks "
341 f"which, and it is read as an initial",
342 (i,)))
343 return dataclasses.replace(state, tokens=tokens,
344 ambiguities=tuple(ambiguities))