Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Shared vocabulary predicates for pipeline stages.
3Text-level tests used by more than one stage; piece-level ones live
4in _pieces, the sibling layer over tokens-plus-tags. All take
5normalized-or-raw text explicitly and no state, with the departures
6named below.
8is_wholly_suffix departs from that shape twice, deliberately. It is
9RUN-level rather than text-level, because the question it answers is
10genuinely about a run: the Ph./D. merge spans two tokens, so no
11per-token predicate composed with all() can express it. And it takes
12the Policy OBJECT, where delimiter_cores takes a pre-extracted
13frozenset so its caller hands in one field rather than the config --
14is_wholly_suffix needs TWO policy fields (lenient_comma_suffixes and
15extra_suffix_delimiters), and threading both past every caller costs
16more than the config parameter saves. Still no state: Policy is frozen
17config, not pipeline state.
19maiden_marker_run is run-level for the first of those reasons and not
20the second: a maiden marker may be a PHRASE ('z domu'), so how far one
21reaches is a question about a run of words that no per-word membership
22test can answer, and the vocabulary reaches it as a plain frozenset
23field like every other predicate here.
25tag_marker_runs is the third departure, and the reason this module also
26imports _state (WorkToken, comma_bucket): deciding which tokens open a
27maiden-marker RUN needs each token's role and span, not its text alone,
28because a run must not cross a role change or a comma. It moved here
29from classify (#289/#516) so _pieces.own_words -- which may import
30only _state and _vocab (tests/v2/test_layering.py) -- can call the
31SAME function classify does rather than approximating it with a
32text-only walk. Two callers building a map from the same tokens with
33the same function cannot disagree, which a from-scratch approximation
34could (measured: 'ANNA z Nowak, MD' flips the one-case verdict when
35the approximation and the real run-completion test disagree on
36whether 'z' opens a run that completes).
38suffix_reading, name_reading and the two reading constants are the
39fourth: they answer no question about a word but BUILD the reading a
40report did not take (#752 phase 2m), from a token's text and tags,
41which is why they import _state's PendingReading and the public enums
42it re-exports. They sit here because every stage that reports a fork
43builds one, and the tags they read are this module's vocabulary.
45Layering: imports _lexicon, _policy, and _pipeline._state (WorkToken,
46comma_bucket, PendingReading -- read-only, never a whole ParseState).
47"""
48from __future__ import annotations
50import functools
51import re
52import string
53import unicodedata
54from collections.abc import Callable, Iterable, Sequence, Set
55from typing import Literal
57from nameparser._lexicon import (
58 FULL_STOPS, Lexicon, _VOCAB_FIELDS, _normalize, _spells_an_initial,
59)
60from nameparser._policy import (CapsSuffixes, Policy, Script, _JA_SCRIPTS, _NO_INITIALS,
61 _SCRIPT_RANGES, _script_matcher)
62from nameparser._pipeline._state import (
63 AGNOMEN_TAG, AMBIGUOUS_ACRONYM_TAG, AMBIGUOUS_WORD_TAG, COMMA_CHARS, COMMA_STRIP,
64 CREDENTIAL_WORD_TAG, Deferred, GENERATION_WORD_TAG,
65 GIVEN_ABBREVIATION_TAG, HONORIFIC_WORD_TAG, PATRONYMIC_TAG, SHAPE_ACRONYM_TAG,
66 TITLE_AMBIGUOUS_TAG, PendingReading, ReadingKind, ReadingSource, Role, WorkToken,
67 comma_bucket,
68)
70# Ported verbatim from v1's "initial" pattern (the 1.x and 2.x
71# nameparser.config.regexes, removed in 3.0, #1008) minus its
72# empty-string alternative -- WorkToken text is never empty. _render
73# holds a hand copy, pinned to this one by tests/v2/test_regex_sync.py.
74# "Verbatim" is a promise about the PATTERN, not about the predicate:
75# since #320 is_initial is this SHAPE test ANDed with a repertoire test
76# (in_initialless_script, below), so _INITIAL.fullmatch(text) and
77# is_initial(text) are no longer the same question -- '씨.' answers yes
78# to the first and no to the second. Call is_initial; the bare pattern
79# is not the thing to ask. The narrowing lives in the predicate
80# precisely so this copy can stay exactly as verbatim as it ever was,
81# the pattern _render's copy must equal.
82_INITIAL = re.compile(r"^(\w\.|[A-Z])$")
84# Ported verbatim from v1's "period_not_at_end" pattern (the 1.x and
85# 2.x nameparser.config.regexes, removed in 3.0, #1008).
86_PERIOD_NOT_AT_END = re.compile(r".*\..+$", re.I)
88# The fix_phd credential pair ('Ph.' + 'D.' as adjacent tokens),
89# `split_pair`'s first answer, below.
90_PH = re.compile(r"^ph\.?$", re.IGNORECASE)
91_D = re.compile(r"^d\.?$", re.IGNORECASE)
92#: `split_pair`'s necessary half on its FIRST word for the Ph./D.
93#: answer alone, asked C-level by `is_wholly_suffix` ahead of the
94#: call, the one caller that asks only that answer and has no tags to
95#: read. The same compiled object the predicate asks, so the two
96#: cannot drift.
97SPLIT_PAIR_OPENS = _PH
100# The codepoint table lives in _policy beside Script -- one copy
101# importable from the pipeline and the locale packs alike; everything
102# here DERIVES from it. (single_script's sweep was first written
103# per-char on the _EMOJI_RANGES precedent in _tokenize.py, on the
104# theory that a range test needs no regex; measured at token scale the
105# compiled regex wins by 3-9x, and by roughly 70x on long tokens.)
106# Derived, never hand-written -- even the class construction goes
107# through the shared factory: one wholly-of predicate per script, in
108# the table's key order -- the FIRST-covering-entry rule _classify
109# documents.
110_SCRIPT_MATCHERS: dict[Script, Callable[[str], bool]] = {
111 script: _script_matcher(script, whole=True)
112 for script in _SCRIPT_RANGES
113}
115# The whole-token matcher over _policy's _JA_SCRIPTS union, backing
116# effective_script's kana license.
117_wholly_ja = _script_matcher(*_JA_SCRIPTS, whole=True)
119# The four kana voicing marks U+3099-U+309C (#596). None belongs to
120# either syllabary -- each voices the kana it follows -- but the script
121# table classifies by block and all four sit in the HIRAGANA one, so a
122# mark NFC cannot compose (ア + U+3099) made a katakana token read as
123# katakana plus hiragana and took the kana license.
124# _normalized_for_script drops every mark standing after a KANA, so the
125# mark takes its base's script. After anything else -- hangul, Han,
126# Latin, or nothing at all -- the mark is not voicing a kana and keeps
127# the table's answer: deleting it there would hand 김゙민준 a script the
128# surname site then divides on the RAW text, cutting the mark off its
129# base and leaving it to open the given name.
130_VOICING_MARKS = frozenset("\u3099\u309a\u309b\u309c")
132# Whole-text kana test for one character: the base a voicing mark may
133# be dropped after.
134_is_kana = _script_matcher(Script.HIRAGANA, Script.KATAKANA, whole=True)
137def _drop_kana_voicing_marks(text: str) -> str:
138 """`text` without the voicing marks that follow a kana (a run of
139 them included); every other mark kept. Called only on text holding
140 a mark, so names without one never pay for the walk."""
141 kept: list[str] = []
142 after_kana = False
143 for ch in text:
144 if ch in _VOICING_MARKS:
145 if after_kana:
146 continue
147 else:
148 after_kana = _is_kana(ch)
149 kept.append(ch)
150 return "".join(kept)
152# The repertoire half of is_initial (_policy._NO_INITIALS), kept apart
153# from _INITIAL's SHAPE half so the pattern itself stays v1-verbatim
154# and _render's copy stays pinned to it by tests/v2/test_regex_sync.py.
155# contains-any, not whole=True: the shape half has already admitted the
156# trailing period, so the text reaching here is '씨.' rather than '씨'
157# and a wholly-of match would be False for every case this exists for.
158# The second caller, is_title_shaped below, admits two or more
159# characters, so contains-any there means one CJK character anywhere
160# vetoes the whole word -- 'Kim김.' is refused as a title along with
161# '田中.' -- and that is deliberate: a word carrying a script with no
162# abbreviations is not wearing an abbreviation's period.
163_has_initialless_char = _script_matcher(*_NO_INITIALS, whole=False)
166def in_initialless_script(text: str) -> bool:
167 """Whether TEXT holds a character of a script that writes no
168 initials (#320), asked of the composed spelling, the one every
169 shape test beside it reads, so the veto and the shape read one
170 spelling (decisions/nfd-shapes, #665, #1087). ASCII answers without the
171 matcher's frame, as the callers' own `isascii()` gates do."""
172 return (not text.isascii()
173 and _has_initialless_char(unicodedata.normalize("NFC", text)))
176# H2's own shape, text-level: an unlisted period-marked abbreviation,
177# two or more letters, one trailing period. Moved here from _pieces
178# (#289/#516, quality-review finding) so _vocab.name_word_count can
179# ask the SAME question _pieces.is_leading_title asks -- measured
180# divergence before the move: 'Dr. Smith, Ed' (LISTED title 'Dr.') and
181# 'Xyz. Smith, Ed' (UNLISTED, H2-shaped) both read family 'Dr.
182# Smith'/'Xyz. Smith', given 'Ed' at the peel (since #701 the listed
183# 'Dr.' then peels into the title, rules/H6), because is_leading_title
184# reads H2's shape and a bare `folded in lexicon._sets.titles` lookup does
185# not -- but name_word_count's OWN count disagreed: 'Xyz.' counted as
186# a name word where 'Dr.' did not, so 'Xyz. Smith, Ed' alone flipped
187# the comma structure to a credential run
188# (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
189#
190# v1's "period_abbreviation" shape, composed from `_LETTER` below, so
191# the text is v1's plus the marks. This is the only definition:
192# `_pieces._PERIOD_ABBREV` is an import of it, which
193# tests/v2/test_regex_sync.py declares as such.
195# `_LETTER` is the letter every shape regex in this module spells
196# (`_PERIOD_ABBREV`, `_DOTTED`, `_CHUNKED`, `_PAIRED_INITIALS`): one
197# letter (a word character that is not a digit or underscore) plus the
198# combining marks written after it (#652). `inž.` typed as i, n, z,
199# U+030C, '.' is the same word as the composed one, and `\w` alone ends
200# it at the z. `_MARKS` is the general combining blocks -- what NFD of
201# a Latin, Greek or Cyrillic letter produces -- and not the marks that
202# only some scripts write (Indic vowel signs, Arabic and Hebrew
203# points); rules/H2's Accepted bullet "The shape is only recognizable
204# as an unbroken run of letters" owns that limit (decisions/H2).
205_MARKS = '[\u0300-\u036f\u1ab0-\u1aff\u1dc0-\u1dff\u20d0-\u20ff\ufe20-\ufe2f]'
206_LETTER = r'[^\W\d_]' + _MARKS + '*'
207_PERIOD_ABBREV = re.compile(r'^(?:' + _LETTER + r'){2,}\.$')
209# rules/H2: "A word in one of the ten Brahmic scripts from Devanagari
210# to Sinhala" "never reads as a title by shape, even where it is a run
211# of letters" -- their Unicode blocks, U+0900 to U+0DFF. A vowel sign
212# already breaks `_LETTER`'s run, but a spelled initial written with
213# none (एम. for M., এম., ਐਮ.) is a run of two letters, and the shape
214# read it as a title; these scripts write initials, so the veto is the
215# title shape's alone and not `is_initial`'s (decisions/H2, #1191).
216# Searched on the raw text: a nukta letter decomposes inside the range.
217_INDIC_BRAHMIC = re.compile('[\u0900-\u0dff]')
219# rules/H2: "A transliteration digraph" "never reads as a title by
220# shape" -- the digraphs that Except names, lowercase and periodless,
221# so the test is a C-level membership of the word less its period;
222# the Except itself names the digraphs it leaves out (Ts, Ch, Sh) --
223# decisions/H2 (#1136).
224_DIGRAPH_INITIALS = frozenset({"yu", "ya", "ye", "yo", "zh", "kh",
225 "shch", "dzh"})
227#: The hyphens and the dashes that join two family names in one written
228#: word (rules/O6), and the parts of an abbreviation written hyphenated
229#: (rules/H5, 'Dipl.‐Ing.' as a word processor writes it): hyphen-minus,
230#: hyphen, non-breaking hyphen, figure dash, en dash, minus sign (a word
231#: processor's substitute), and the small and fullwidth hyphen-minus.
232#: The em dashes are punctuation between words, not joiners, and are
233#: left out. Hyphen-minus first, so it is literal inside a class.
234_HYPHEN_CHARS = "-\u2010\u2011\u2012\u2013\u2212\ufe63\uff0d"
235_HYPHENS = re.compile("[" + _HYPHEN_CHARS + "]")
237# rules/H5: "A word that ends in a period and is written with a
238# period or a hyphen inside it wears the abbreviation shape too" --
239# the trailing chain's shape, _PERIOD_ABBREV widened to 'Priv.-Doz.',
240# 'Dipl.Ing.' and 'Lt.Col.'. It opens and closes on a letter and ends
241# in one period, so every _PERIOD_ABBREV match is one of its matches;
242# the front slot's H2 shape stays _PERIOD_ABBREV (decisions/H5, #1116).
243_TRAILING_TITLE_SHAPE = re.compile(
244 r'^' + _LETTER + r'(?:' + _LETTER + r'|[' + _HYPHEN_CHARS + r'.])*'
245 + _LETTER + r'\.$')
247# rules/H5: "a title the vocabulary lists whole with a period inside
248# it chains without a final one" -- the letters of such a listing
249# joined by periods and hyphens ('dott.ssa', 'priv.-doz'), read by
250# classify on the normalized word, which has no edge periods, ANDed
251# there with a period test, since the pattern alone admits a word a
252# hyphen alone joins ('co-founder'); a digit keeps a rank written with
253# a period (a caller's 'cwo.3') out (decisions/H5, #1104).
254_JOINED_WORD = re.compile(
255 r'^(?:' + _LETTER + r')+(?:[' + _HYPHEN_CHARS + r'.]+(?:'
256 + _LETTER + r')+)+$')
259# rules/H2: "an abbreviation opening the part of the name that
260# carries the given name — the whole name, or the part after a
261# family comma — reads as a title even when unlisted"
262# (history: decisions/H2)
263def is_title_shaped(text: str, lexicon: Lexicon) -> bool:
264 """Whether TEXT reads as a title by H2's shape -- the LISTED half
265 being a separate lookup at each caller's own site
266 (is_title_piece/lexicon._sets.titles), and the one vocabulary read here
267 the given name written short, which outranks the shape (rules/H2,
268 #789: 'Chas.' is no title). `name_word_count`, `title_readings`
269 (classify's rules/H9 phrase test) and `_assemble`'s source of a
270 title no vocabulary tags are its callers; `_pieces.is_leading_title`
271 asks the same question but keeps this body inline, its path being
272 hot (every leading piece of every parse) where these run only on a
273 comma name, a name holding a contested title or modifier, or an
274 untagged title token
275 -- the measured figure lives at that inline copy and is not
276 restated here, two homes for one measurement being how they come
277 to disagree. Touch one, touch both:
278 `test_pieces.test_is_title_shaped_and_is_leading_title_agree`
279 runs both over the union of the two example tables, so the
280 agreement is checked rather than asserted in prose
281 (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
283 The shape reads a Latin convention: a period marks an
284 abbreviation. Scripts with no initials have no period
285 abbreviations either (_policy._NO_INITIALS, the #320 veto
286 is_initial carries), so a CJK word wearing a period is a name
287 word, not a title -- a lone '田中.' is the family name (#323).
288 `_PERIOD_ABBREV` stays ASCII-period only: a word wearing '。' never
289 matches it, and the veto is what makes the ASCII spelling agree.
290 ASCII text can carry no _NO_INITIALS character (every range sits
291 above U+3000), so the C-level test declines before the regex
292 search runs -- four frames per unlisted-abbreviation opener per
293 parse, this running four times per piece.
295 A word holding a letter of the ten Brahmic scripts from Devanagari
296 to Sinhala is refused too (`_INDIC_BRAHMIC`, rules/H2's Except,
297 #1191): those scripts write initials, and spell a Latin one out by
298 its sound with no vowel sign (एम. for M.), a run of letters the
299 shape would misread. Its search is C-level, so it costs no frame.
301 A transliteration digraph (`Yu.`, `Zh.`, `Kh.`, `_DIGRAPH_INITIALS`)
302 declines the shape too, being the initial of one Cyrillic letter
303 (rules/H2's transliteration-digraph Except, #1136).
304 """
305 return (bool(_PERIOD_ABBREV.match(text))
306 and (text.isascii() or not (_INDIC_BRAHMIC.search(text)
307 or in_initialless_script(text)))
308 and text[:-1].lower() not in _DIGRAPH_INITIALS
309 and _normalize(text) not in lexicon._sets.given_name_abbreviations)
312def is_initial_shaped(text: str) -> bool:
313 """v1's is_an_initial verbatim: the SHAPE half alone -- one word
314 character plus a period, or a bare ASCII capital.
316 Callers asking whether a token is STRUCTURALLY part of an initial
317 run want this; callers asking whether it can really stand in for a
318 name want is_initial (#320). The two answers differ only inside
319 _NO_INITIALS scripts, where '씨.' is initial-SHAPED but is not an
320 initial -- see assign's roman-numeral fork, the shape caller, for
321 what picking the wrong one costs.
323 Asked of the composed spelling, as `_render` asks it (#542): `\\w`
324 does not match a combining mark, so a decomposed 'É.' is three
325 characters to the pattern and no initial (#665). Composing rather
326 than widening keeps the pattern v1-verbatim, and both calls are
327 C-level, so the predicate costs no frame it did not before."""
328 if not text.isascii():
329 text = unicodedata.normalize("NFC", text)
330 return bool(_INITIAL.fullmatch(text))
333# v1's "roman_numeral" pattern; _render's hand copy is pinned to this
334# one by tests/v2/test_regex_sync.py.
335_ROMAN = re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I)
336#: The same pattern, shared rather than copied: `_comma` filters on
337#: it before asking `reads_as_generation` (rules/C12).
338ROMAN = _ROMAN
341# Ported verbatim from v1's patterns (the 1.x and 2.x
342# nameparser.config.regexes, removed in 3.0, #1008), the RU and TR_AZ
343# packs' copies pinned to these by tests/v2/test_locales.py: the East
344# Slavic patronymic endings and the standalone Turkic patronymic
345# markers, which rules/O1's and O2's rotations and rules/R9's reading
346# ask through the two predicates below.
347_EAST_SLAVIC = re.compile(
348 r"(ovich|ovna|evich|evna|ichna|ilyich|kuzmich|lukich|fomich|fokich|nikitich|savvich)$",
349 re.I)
350_EAST_SLAVIC_CYR = re.compile(
351 r"(ович|овна|евич|евна|ична|ильич|кузьмич|лукич|фомич|фокич|никитич|саввич)$",
352 re.I)
353_TURKIC = re.compile(
354 r"^(oglu|oğlu|ogly|ogli|o['’ʻ‘ʼ`]g['’ʻ‘ʼ`]li"
355 r"|qizi|qızı|kizi|kyzy|gyzy|uly|uulu)$", re.I)
356_TURKIC_CYR = re.compile(
357 r"^(оглу|оглы|оғлу|ўғли|угли|кызы|гызы|қызы|қизи|улы|ұлы|уулу)$", re.I)
360def has_patronymic_ending(text: str) -> bool:
361 """Whether `text` ends in an East Slavic patronymic ending, Latin or
362 Cyrillic ("Ivanovich", "Петрович"): rules/O1's test and, under
363 `PatronymicRule.EAST_SLAVIC`, rules/R9's."""
364 return bool(_EAST_SLAVIC.search(text) or _EAST_SLAVIC_CYR.search(text))
367def is_patronymic_marker(text: str) -> bool:
368 """Whether `text` is a standalone Turkic patronymic marker ("oglu",
369 "кызы"): rules/O2's test and, under `PatronymicRule.TURKIC`,
370 rules/R9's. Asked of the word NFC-composed (#1316): `oğlu` written
371 with a combining breve is the same marker, the vocabulary path's fold
372 applied to a shape test; the fold is inline behind `isascii()` so a
373 plain word costs no frame."""
374 if not text.isascii():
375 text = unicodedata.normalize("NFC", text)
376 return bool(_TURKIC.match(text) or _TURKIC_CYR.match(text))
379# rules/O1: "whose last name word carries a patronymic ending and whose
380# middle name word does not reads as family-first" -- the word test of
381# the rotation, shared with the group suggester (nameparser/_suggest.py),
382# which counts a patronymic vote where this rule can read one
383# (decisions/group-suggestion)
384def east_slavic_rotates(middle: str, last: str) -> bool:
385 """Whether the middle and last of three name words have rules/O1's
386 family-first shape: the ending on the last word and not on the
387 middle one. The word count and the initials beside it are the
388 caller's."""
389 return has_patronymic_ending(last) and not has_patronymic_ending(middle)
392# rules/S10: "the numeral reads as the generation: the numerals the
393# vocabulary lists, and those up to X it does not, by their shape
394# alone" -- its Then clause, the `_ROMAN` test; and rules/S10's Except
395# bullet "An initial in front makes it a name word too", the two
396# is_initial_shaped tests on `preceding`. The words-to-spare count,
397# rules/S10's Except bullet "A numeral of one letter needs words to
398# spare", is the callers'. decisions/S10 (#718, #721)
399# mechanisms.md#ONE-PREDICATE-PER-QUESTION ("exactly one predicate answers it and every other site calls that one")
400# -- the numeral's predicate, which every numeral rule asks
401# (decisions/vocabulary-model, #752; #784, #774, #819)
402def reads_as_generation(text: str, tags: Set[str], preceding: str | None,
403 one_case: bool | None) -> bool:
404 """Whether a word reads as the generation: S10's roman-numeral
405 fork, asked by every rule that weighs a numeral (#752). For a
406 one-letter numeral a necessary condition only: the words-to-spare
407 count is the caller's, which alone knows where the name's words
408 are (rules/S10, #718).
410 A numeral is I to X by shape, written with its periods or without
411 ('VI.', the German ordinal), save that a single letter written
412 with its period is an initial (rules/T5). The vocabulary's own
413 numerals of two or more letters ('ii' to 'iv', `tags` carrying
414 `vocab:suffix`) are the generation whatever stands around them,
415 the vocabulary settling them; the rest ('vi' to 'x', and the
416 one-letter 'i' and 'v') are weighed:
418 - an initial in front makes the numeral a name word: `preceding`
419 is the first token of the piece in front, read for layout, and
420 None where the slot never asks (the given part after a family
421 comma, where 'Chang, Andy C I' reads suffix 'I', v1's reading);
422 - a Title-case numeral in a name of mixed case leans to the name,
423 as a Title-case member of the ambiguous class does
424 (`numeral_leans_to_the_name`).
426 In the given part after a family comma (`preceding` None) a
427 one-letter numeral is the generation only where the vocabulary
428 lists it, rules/C9's "one-character suffix word": no count is
429 asked there (S5), and an unlisted letter is the middle initial
430 ('SMITH, JOHN X').
432 is_initial_shaped, not is_initial: this asks whether the preceding
433 piece looks like part of an initial run, which is a question
434 about layout, and #320 narrowed the tag to initials that can
435 really stand in for a name. Reading the tag here made '씨.' stop
436 suppressing the fork and cost 'John 씨. V' its family name.
438 `one_case` is ParseState.one_case: in a name written in one case
439 the word in front is read as its capital would be, so a bare 'j'
440 is initial-shaped as 'J' is (rules/S10, #721's review: 'john j v'
441 reads as 'JOHN J V'), while 'é', 'ж' or an uncased word, whose
442 capitals are not, stay as they were. None (the maiden take's
443 superset view passes it) asks neither case question, which admits
444 more and so stays a superset."""
445 stem = text.rstrip(".")
446 if not stem or _ROMAN.match(stem) is None:
447 return False
448 if len(stem) == 1:
449 # rules/T5: "A one-letter word written with its period is an
450 # initial"
451 if len(text) > 1:
452 return False
453 if preceding is None:
454 return "vocab:suffix" in tags
455 elif "vocab:suffix" in tags:
456 return True
457 elif one_case is False and stem.istitle():
458 return False
459 return (preceding is None
460 or not (is_initial_shaped(preceding)
461 or one_case and is_initial_shaped(preceding.upper())))
464# rules/C3: "a numeral ending the part, or standing before the listed
465# suffix words it ends with, that the trailing fork reads as the generation is
466# no name word" -- asked of the shape-only numerals,
467# the listed ones being suffix words to every count that asks it already;
468# decisions/C3 (#893)
469def ends_in_a_generation(texts: Sequence[str], one_case: bool | None,
470 ) -> bool:
471 """Whether the second of `texts`, two words of a name, is a
472 numeral of two or more letters that the trailing fork reads as the
473 generation behind the first. It reads no tags, so a listed numeral
474 ('III') answers True too: each caller passes only a word it has not
475 already read as a suffix, which leaves the unlisted VI to IX, and
476 finds the two words itself, the numeral ending the name words past
477 any suffix words behind it."""
478 last = texts[-1]
479 return (len(last.rstrip(".")) >= 2
480 and reads_as_generation(last, frozenset(), texts[-2], one_case))
483#: What a word of a clause S1 sets apart is: a title, a suffix, or a
484#: word that is both, its place in the name choosing (rules/S1).
485SetApartKind = Literal["title", "suffix", "dual"]
488# rules/S13: "Written with a period inside it, whatever case the name is
489# in" -- the writing that settles a member, asked of its text since a
490# clause word carries no shape tag (decisions/S13, #810, #1088)
491def dotted_member(word: str, lexicon: Lexicon) -> bool:
492 """Whether a word of a delimited clause is a member of the
493 ambiguous class (S4) written with a period inside it ('M.A',
494 'M.A.'), a comma the clause keeps on it left off (rules/N1). A
495 period ending the clause is extract's escape, not this."""
496 bare = word.strip(COMMA_STRIP)
497 return ("." in bare.strip(".")
498 and _normalize(bare).replace(".", "")
499 in lexicon._sets.suffix_acronyms_ambiguous)
502# rules/S1: "a clause wholly of words the vocabulary reads as no name
503# is set apart from the name" -- the word's half; `set_apart` is the
504# clause's. Its suffix half is `suffix_as_written`'s, so a credential
505# joined by a slash or a hyphen is one here as it is bare; a word its
506# periods join is asked first, then a dotted member, and a joined word
507# holding a member after the suffix test, each member left to S13's
508# position test (decisions/S1, #792, #1088; decisions/S3, #1131, #1138)
509def set_apart_kind(word: str, lexicon: Lexicon,
510 ) -> SetApartKind | Literal["member"] | None:
511 """What a word in a delimited clause is to S1's set-apart reading:
512 a title, a suffix (a word `suffix_as_written` reads, joined
513 credentials included, or one its periods join,
514 `period_joined_credential`), both, a member of the ambiguous class
515 written with a period inside it (`dotted_member`), a joined word
516 holding one (`joined_member`, 'MA/PhD') or a word its periods join
517 holding one ('Msc.Ed'), which S13 settles after the name only, or
518 None where the vocabulary reads it as a name or not at
519 all. A comma the clause keeps on its word is no part of it
520 (rules/N1)."""
521 bare = word.strip(COMMA_STRIP)
522 folded = _normalize(bare)
523 acronym = folded.replace(".", "")
524 sets = lexicon._sets
525 # rules/S1: "one joined by a slash, a hyphen or a period (S3) too,
526 # every chunk counting" -- a word with interior periods read here as
527 # at the end of a name ('(Msc.Ed)', '(Lt.Gov)'; not '(Bob.PhD)'),
528 # the period test asked at C level first (decisions/S3, #1131)
529 joined = ("." in folded and folded not in sets.titles
530 and acronym not in sets.suffix_acronyms)
531 title = folded in sets.titles or joined and period_joined_title(
532 bare, lexicon)
533 if joined and not title and period_joined_credential(bare, lexicon):
534 # rules/S3: "a joined word holding a member is a member itself"
535 # -- 'Msc.Ed' as 'MA/PhD', settled by S13's position (#1138)
536 return "member" if _holds_a_member(bare, lexicon) else "suffix"
537 # a member of the ambiguous class, the set asked inline as the
538 # necessary half of `dotted_member`, so no other word pays its
539 # frame: `suffix_as_written` would read a dotted member as a suffix
540 # wherever it stands, and reads no other member as one
541 if acronym in sets.suffix_acronyms_ambiguous:
542 if title:
543 return "title"
544 return "member" if "." in bare and dotted_member(
545 bare, lexicon) else None
546 # the sets and the joiners asked first, at C level, the necessary
547 # half of `suffix_as_written`, so a word it could never read (a
548 # nickname) pays no frame
549 suffix = ((acronym in sets.suffix_acronyms or folded in sets.suffix_words
550 or "-" in folded or "/" in folded)
551 and suffix_as_written(folded, bare, lexicon))
552 # rules/S3: "a joined word holding a member is a member itself" --
553 # settled after the name only, as S13 settles one written alone
554 # (decisions/S3, #1138)
555 if suffix and not title and joined_member(bare, lexicon):
556 return "member"
557 if title:
558 return "dual" if suffix else "title"
559 return "suffix" if suffix else None
562#: Single letters each followed by a period ('R.A.'), the writing that
563#: makes a word that is a title too a set of initials (rules/S1).
564_DOTTED_INITIALS = re.compile(r"(?:[^\W\d_]\.)+")
566#: A word written as initials: one or two letters, with or without a
567#: period, or single letters each followed by a period ('JP', 'Md',
568#: 'MD.', 'A.M.', 'R.E.S.').
569_INITIALS_SHAPE = re.compile(r"[^\W\d_]{1,2}\.?|" + _DOTTED_INITIALS.pattern)
572# rules/S1: "A credential written as initials before a name word stays
573# the nickname, and so does any word of its clause" -- asked of each
574# chunk of a credential joined by a slash or a hyphen, so it reads
575# before a name word as its split spelling does (decisions/S1, #792,
576# #1088)
577def reads_as_initials(word: str, kind: SetApartKind,
578 one_case: bool | None, lexicon: Lexicon) -> bool:
579 """Whether a word of a clause S1 would set apart counts, before a
580 name word of its comma part, as the initials the person goes by: a
581 credential written in their shape that is no generation word, and a
582 word that is a title too only written with a period after each
583 letter ('R.A.') or in capitals in a name of mixed case ('RA'),
584 Title case reading as the title it also is and capitals in a name
585 written in one case saying nothing (S6); a title alone never. A
586 member of the ambiguous class counts as the credential it is
587 settled as. `one_case` is ParseState.one_case. A joined credential
588 counts where any of its chunks would ('JP/MD', 'MD/PhD'), as its
589 words written apart do, and one the vocabulary lists whole never
590 ('NP-C'), a run of its chunks listed whole being one chunk
591 ('PhD/NP-C' is 'PhD', 'NP-C', as `_joined_chunks` cuts it). A
592 chunk is asked as the word it is, with no further cut by this
593 function, so its loop is one level deep."""
594 bare = word.strip(COMMA_STRIP)
595 # a credential the vocabulary lists whole ('NP-C', 'SHRM-CP') is one
596 # word, not two S3 joins, and has no other spelling written apart
597 if kind == "suffix" and ("/" in bare or "-" in bare) and not (
598 _listed_whole(bare, lexicon)):
599 # a word with no cut S3 admits has no chunk to ask; unreached by
600 # a parse, where a joined word not listed whole is a suffix only
601 # through a cut
602 for chunk in _joined_chunks(bare, lexicon) or ():
603 if _shaped_as_initials(chunk, set_apart_kind(chunk, lexicon),
604 one_case, lexicon):
605 return True
606 return False
607 # rules/S1: "a credential S3 reads through chunks joined by a slash,
608 # a hyphen or a period counts where any of its chunks would" -- a
609 # period-joined word's chunks asked as a slash's are ('JP.MD' as
610 # 'JP/MD'), one listed whole ('M.D.', 'MD.PhD') never (#1131)
611 if (kind == "suffix" and "." in bare.strip(".")
612 and _normalize(bare).replace(".", "")
613 not in lexicon._sets.suffix_acronyms
614 and period_joined_credential(bare, lexicon)):
615 # a single letter before a glued honorific is the initial it is
616 # written as ('J.씨', the one chunk the every-chunk test lets
617 # count as nothing)
618 return any(len(chunk) == 1 and chunk.isalpha()
619 or _shaped_as_initials(chunk, set_apart_kind(chunk, lexicon),
620 one_case, lexicon)
621 for chunk in _period_chunks(bare) or ())
622 return _shaped_as_initials(bare, kind, one_case, lexicon)
625def _shaped_as_initials(bare: str,
626 kind: SetApartKind | Literal["member"] | None,
627 one_case: bool | None, lexicon: Lexicon) -> bool:
628 """`reads_as_initials` for one word, or one listed run of a joined
629 word, its kind as `set_apart_kind` gives it; a run holding a joiner
630 never matches the initials shape."""
631 return ((kind == "suffix" or kind == "member" or kind == "dual" and (
632 _DOTTED_INITIALS.fullmatch(bare) is not None
633 or one_case is False and bare.isupper()))
634 and _INITIALS_SHAPE.fullmatch(bare) is not None
635 and _normalize(bare) not in lexicon._sets.suffix_words)
638def set_apart(words: Sequence[str],
639 kinds: Sequence[SetApartKind],
640 behind_a_credential: bool, lexicon: Lexicon) -> bool:
641 """rules/S1's clause tests over the words of a delimited clause,
642 each already found to be no name by `set_apart_kind` (or, after
643 the name, a member its writing settles, rules/S13): a clause of
644 nothing but title modifiers and styles is no title ('(UK)',
645 '(Hon)'), a lone title of address may be the name the person goes
646 by ('(Sir)', '(Mr)': a title addressing the given name, or a
647 salutation that may end the given part, rules/H8), and a clause
648 of titles alone straight behind a credential qualifies that
649 credential ('FRCS (SN)'). `behind_a_credential` is whether the
650 word in front of the clause was read as a suffix, which extract,
651 running before any word is read, passes as False."""
652 sets = lexicon._sets
653 folded = [_normalize(w.strip(COMMA_STRIP)) for w in words]
654 if all(f in sets.title_modifiers or f in sets.title_styles
655 for f in folded):
656 return False
657 if len(folded) == 1 and (folded[0] in sets.given_name_titles
658 or folded[0] in sets.after_given_titles):
659 return False
660 return not (behind_a_credential and "suffix" not in kinds
661 and "dual" not in kinds)
664# rules/S10: "a Title-case numeral in a name of mixed case leans to the
665# name" -- the fork weighed it and the writing declined it, so it is
666# reported as a declined member of the ambiguous class is (#752)
667def numeral_leans_to_the_name(text: str, tags: Set[str],
668 preceding: str | None,
669 one_case: bool | None) -> bool:
670 """Whether `reads_as_generation` declined a numeral for its Title
671 case alone: two or more letters, unlisted, in a name of mixed case,
672 and no initial in front (which makes it a name word unweighed)."""
673 return (one_case is False and text.rstrip(".").istitle()
674 and len(text.rstrip(".")) >= 2 and "vocab:suffix" not in tags
675 and reads_as_generation(text, tags, preceding, None))
678def is_listed_numeral(text: str, tags: Set[str]) -> bool:
679 """A numeral the vocabulary lists in two or more letters ('III'):
680 not in question, so its reading is never reported (rules/S10)."""
681 stem = text.rstrip(".")
682 return (len(stem) >= 2 and "vocab:suffix" in tags
683 and _ROMAN.match(stem) is not None)
686# rules/S10: "a numeral written in lower case in a name of mixed case is
687# the generation and reports nothing" -- the one writing that settles
688# the fork, since nobody writes an initial in lower case beside
689# capitals. Asked by the walk, which takes such a numeral as any suffix
690# word and records no call, and by the report on a numeral before a
691# credential (`reads_as_a_reported_numeral`).
692def numeral_settled_by_case(text: str, one_case: bool | None) -> bool:
693 """Whether a numeral the trailing fork reads is lower case in a
694 name written in more than one case. `one_case` is
695 ParseState.one_case; None (nobody asked) settles nothing."""
696 return one_case is False and text.islower()
699# rules/S10: "either reading is reported" -- the one question the two
700# reports on a numeral a post-nominal follows ask (assign's, and the
701# maiden take's for a numeral its clause keeps): the fork would read
702# this numeral, and its writing does not settle it (#721)
703def reads_as_a_reported_numeral(text: str, tags: Set[str],
704 preceding: str,
705 one_case: bool | None) -> bool:
706 """Whether a numeral the walk left a name word, with `preceding`
707 in front of it, is one the fork would weigh and so reports: one it
708 reads as the generation, or declines for its Title case."""
709 return ((reads_as_generation(text, tags, preceding, one_case)
710 or numeral_leans_to_the_name(text, tags, preceding, one_case))
711 and not numeral_settled_by_case(text, one_case))
714# #544: a single-letter roman numeral ('V', 'v', 'I.'), in any case,
715# is excluded for its SHAPE -- one letter is how a middle initial is
716# written, and rules/S3 retires single-character matches for the
717# same reason -- not for being a generation: a multi-letter suffix
718# word ('III', 'Jr') anchors and runs like any other. So it neither
719# anchors an ambiguous member behind it nor counts toward C4's
720# multi-word credential run. It still peels exactly as before; this
721# answers only those two questions.
722def is_single_letter_numeral(text: str) -> bool:
723 """One letter, optionally followed by periods, that is a roman
724 numeral ('V', 'v', 'I.', 'X')."""
725 letters = text.rstrip(".")
726 return len(letters) == 1 and _ROMAN.match(letters) is not None
729# rules/T5: "A one-letter word written with its period is an initial,
730# in a script that writes initials, and so is a bare unaccented Latin
731# capital, A to Z"
732def is_initial(text: str) -> bool:
733 """'A.' / 'j.' / bare capital -- v1's is_an_initial, narrowed to
734 scripts that HAVE initials (#320). v1's \\w is Unicode-aware and
735 matched CJK too, which made period-written CJK honorifics ('씨.')
736 fail is_suffix_strict -- the veto in _is_suffix_strict_n, NOT the
737 vocabulary: suffix_as_written has no veto, so classify tagged '씨.'
738 'vocab:suffix' either way, and is_suffix_lenient took it either way
739 too. Downstream of that one strict-test No, the glued honorific in
740 a name carrying such a token went unpeeled ('田中さん 様.')."""
741 return is_initial_shaped(text) and not in_initialless_script(text)
744def is_one_case(texts: Sequence[str]) -> bool:
745 """Whether a name is written wholly in ONE case -- all upper or all
746 lower alike -- and so carries no case EVIDENCE about any letter in
747 it (rules/P8, #383/#479). The caller passes the name's OWN
748 words: a maiden marker's clause and any delimited (nickname)
749 content are not among them, and appending one must not flip the
750 reading of words that did not change.
752 Mirrors the SHAPE of the comparison the R5 gate in
753 `_render.capitalized` makes, not its SPAN: R5 joins every token but
754 the suffixes, nickname and maiden content included, while
755 classify's caller hands in only the name's own words (rules/P3's
756 own-words doctrine, see above) -- so a clause-bearing name can be
757 one-case to this function and mixed to R5 (measured:
758 `'JUAN GARCIA Y LOPEZ née Jones'` is one case here, mixed there),
759 and a name with a cased suffix mixed here and one-case there
760 (`'john e jones III'`). Not shared by import today -- render is a
761 layer this module does not reach into -- and #492 left the two
762 spans apart by decision (decisions/R5, "THE R5 / PARSER SPLIT").
764 `Sequence`, not `Iterable`: the caller passes a list it already
765 built rather than a fresh generator, so `is_one_case` costs one
766 profiler frame per parse rather than one per token (#475).
768 A CASELESS script answers True, harmlessly: `'محمد و علي'.upper()`
769 is the string itself, so the comparison holds. Its callers are
770 classify, segment (only where C2's flag on a tail part asks) and
771 script_segment's `_one_case`; the first two record the answer as
772 `ParseState.one_case`, and script_segment does not; the decisions
773 that read it each also require the token they judge to be CASED:
774 classify's connective-or-initial fork asks for a token whose own
775 `upper()` and `lower()` differ, `ambiguous_lean` -- which the
776 suffix readers in `_pieces`, `_group` and `_assign` reach through
777 `listed_lean` -- returns no lean for a caseless token, and
778 `caps_shape_candidate` requires
779 `isupper()`. A caseless letter is none of those, so a caseless
780 name never reaches a decision this gates, and "one case" is the
781 honest verdict for text that has only one.
782 """
783 joined = " ".join(texts)
784 return joined in (joined.upper(), joined.lower())
787#: Which way the WRITING leans for a member of the ambiguous
788#: credential class; None is no lean at all. Named so the two
789#: predicates that answer it -- `ambiguous_lean` here and
790#: `_pieces.listed_lean`, which wraps it -- carry the same three-value
791#: type rather than a bare `str`. Under mypy's `strict_equality` that
792#: makes a misspelled comparison (`== "credentail"`) an error at the
793#: two call sites that branch on the answer, where a `str` return
794#: leaves it silently False forever -- which is what the value is FOR:
795#: `_pieces.peel_trailing` peels or declines on it.
796#: `period_joined_vocab`'s own verdict is spelled inline for the same
797#: reason; it has one caller shape and no wrapper to keep in step.
798Lean = Literal["credential", "name"]
800#: How a member's writing leans, by what the member is (rules/S11):
801#: an acronym's capitals are the credential, the lower case of a suffix
802#: word written `title_case_is_name` is the suffix, and any other suffix
803#: word's writing (an agnomen's, `Junior`'s) says nothing (#1039).
804Cue = Literal["acronym", "word", "none"]
807# #289's credential lean, rules/S6: "Written case is evidence beside
808# the count (S4), and it speaks only in a name written in more than
809# one case" -- the WRITING evidence that decides ahead of the
810# words-to-spare count (history: decisions/S6).
811def ambiguous_lean(text: str, one_case: bool,
812 cue: Cue = "acronym") -> Lean | None:
813 """Which way the WRITING leans for a member of the ambiguous
814 credential set: "credential", "name", or None for no lean at all.
816 None is the fall-through to rules/S4's count, and it is the
817 answer for three inputs. A name written wholly in one case says
818 nothing about any word in it. A member written wholly in lower
819 case says nothing either -- lower is how most of a mixed-case
820 name is written, so it is not a contrast. And a CASELESS token
821 ('씨', '毛') can be written against nothing, so neither test
822 fires and the count decides, which is what keeps this rule out of
823 a caseless script by construction.
825 `one_case` is the recorded fact (ParseState.one_case), taken over
826 the name's OWN words -- handed in rather than recomputed, because
827 three sites read this and a fact two of them derived apart is the
828 bug the field exists to prevent
829 (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
831 The caller decides MEMBERSHIP: this answers only about the
832 writing. Periods do not disturb either test, '.' having no case,
833 so 'MA.' leans as 'MA' does -- the lean reads CASE, not periods,
834 and the dotted spelling is the vocabulary's own question
835 (suffix_as_written).
837 `cue` is what the member is (rules/S11). A suffix WORD that is
838 also a name and whose Title case is its writing as a name ("word",
839 'fils', 'kc', written `title_case_is_name`) leans the other way at
840 one end: its lower case in a mixed-case name is the suffix
841 ("credential", the suffix reading), its capitals say nothing (a
842 surname is often written so), and Title case leans to the name as
843 an acronym's does. Any other ("none": an agnomen, 'Neto', written
844 in the case of the name around it; 'Junior', whose Title case is
845 the generation's writing too) never leans (#1039).
846 """
847 if one_case or cue == "none":
848 return None
849 if text.upper() == text.lower(): # caseless: no contrast to read
850 return None
851 if text.isupper():
852 return "credential" if cue == "acronym" else None
853 if text.islower():
854 return "credential" if cue == "word" else None
855 return "name"
858def lexicon_cue(n: str, lexicon: Lexicon) -> Cue:
859 """The `ambiguous_lean` cue of the folded member `n`, read off the
860 lexicon by a caller that has no tags to read (`is_wholly_suffix`);
861 `_pieces.listed_lean` reads the same answer off classify's tags."""
862 if n in lexicon._sets.suffix_words_ambiguous:
863 return ("word" if n in lexicon._sets.title_case_name_words
864 else "none")
865 return "acronym"
868_DOTTED = re.compile(r"(?:" + _LETTER + r"\.)+")
869# rules/T7: "a run of single letters joined by full stops" -- two or
870# more letters, each closed by a stop but the last, whose stop is
871# optional ('E.T.', 'J.R.R', #1296), and the same run rules/H10 reads as
872# a title ('U.S.', 'U.S', the closing stop optional as
873# `period_joined_vocab`'s shape verdict lets it be); _RUN_LETTER is one letter of it
874# with its stop. _render holds a copy of each (tests/v2/test_regex_sync.py)
875_LETTER_RUN = re.compile(r"(?:" + _LETTER + r"\.)+" + _LETTER + r"\.?")
878_RUN_LETTER = re.compile(_LETTER + r"\.?")
881def is_letter_run(text: str) -> bool:
882 """Two or more single letters joined by full stops ('J.P.', 'J.R.R'),
883 the way a person's initials are written run together: rules/T7's
884 run, and the shape rules/C16 reads as a name after the comma behind
885 one name word whatever it spells (#1238)."""
886 return _LETTER_RUN.fullmatch(text) is not None
889# #544: the CHUNKED spelling -- two or more runs of letters, each
890# closed by a period ('M.Eng.', 'L.Ac.'). Both callers compare the
891# period-free letters against the listed member, so the chunks must
892# spell it; one chunk ('Ma.', 'Ed.') is the single trailing period and
893# stays outside the gate.
894_CHUNKED = re.compile(r"(?:(?:" + _LETTER + r")+\.){2,}")
897def _dotted(text: str) -> bool:
898 """Written with its periods: one after each letter ('M.A.',
899 'J.D.'), the acronym's own spelling, or -- since #544 -- two or
900 more letter chunks each closed by a period ('M.Eng.', 'L.Ac.'), the
901 spelling a member with a lower-case tail is written in. A single
902 trailing period ('Ma.', 'Ed.', 'Ms.') is the abbreviation shape any
903 word can wear -- the honorific's, a name's -- and is not the gate's
904 "written with periods" (rules/S3). Until #296's review the gate
905 was "any period", and 'Smith, Ms.' passed it as the degree."""
906 return (_DOTTED.fullmatch(text) is not None
907 or _CHUNKED.fullmatch(text) is not None)
910# rules/T8: "glued to the next at its full stop" -- the shapes
911# `glued_cuts` reads (decisions/T8 (#1230, #1235)): one chunk of letters
912# closed by a stop with something after it
913_GLUED_CHUNK = re.compile(r"((?:" + _LETTER + r")+)\.(?=\S)")
914# a run of two or more single letters closed by stops, a letter after
915# it (the name is `glued_cuts`' to find)
916_GLUED_RUN = re.compile(r"(?:" + _LETTER + r"\.){2,}(?=[^\W\d_])")
917#: a letter (its marks with it, so NFD text passes), a full stop and a
918#: letter: the C-level test `_script_segment` asks before `glued_cuts`
919GLUED_SHAPE = re.compile(_LETTER + r"\.[^\W\d_]")
920# the Thai vowel and tone marks (Mn), which `_LETTER` does not count:
921# nearly every Thai name writes one ('สุดา', 'ใจดี'), so without them the
922# name after a Thai title's stop is never found (#1166)
923_THAI_MARKS = "[\u0e31\u0e34-\u0e3a\u0e47-\u0e4e]"
924# one word of letters, apostrophes and hyphens: what can follow a chunk
925# and be a name
926_GLUED_WORD = re.compile(
927 r"(?:" + _LETTER + r")(?:" + _LETTER + "|" + _THAI_MARKS + r"|['’\-])*")
930def _is_cased_letter(chunk: str) -> bool:
931 """One letter in a script with case: the initial T8 reads alone
932 (rules/T8); a single letter in a script without case is read alone
933 only where it is listed, as a title or a given name written short."""
934 return len(chunk) == 1 and chunk.upper() != chunk.lower()
937def _reads_alone(chunk: str, lexicon: Lexicon) -> bool:
938 """A chunk the parser reads on its own before a full stop: one
939 letter in a script with case, a transliteration digraph rules/H2
940 names, a listed title that is no saint particle, or a listed given
941 name written short (decisions/period-joined-words, #1255). A single
942 letter in a script without case is read alone only where it is
943 listed (Thai 'ศ.', #1166); a run of them is T7's run in any script."""
944 chunk = unicodedata.normalize("NFC", chunk)
945 if _is_cased_letter(chunk):
946 return True
947 n = _normalize(chunk)
948 sets = lexicon._sets
949 if n in _DIGRAPH_INITIALS or n in sets.given_name_abbreviations:
950 return True
951 return is_title_chunk(chunk, lexicon)
954def glued_cuts(text: str, lexicon: Lexicon) -> tuple[int, ...]:
955 """Where a word glued to the next at a full stop divides (T8,
956 #1230, #1235): the offset after each stop at which a chunk the
957 parser reads alone (`_reads_alone`) meets a name, so the word reads
958 as it does with a space there ('Dr.John' as 'Dr. John',
959 'Ц.Сандаг-Очир' as 'Ц. Сандаг-Очир', 'Yu.A.Chibrikova' as 'Yu. A.
960 Chibrikova'). A run of single letters stays one chunk before the name
961 ('J.R.Smith' as 'J.R. Smith', rules/T7), while titles glued to each
962 other divide at each stop, as they read written apart ('Lt.Gov.John'
963 as 'Lt. Gov. John'). Empty where the vocabulary reads the word whole
964 (a suffix spelled without its stops, 'M.Sc', 'M.Sc.'; a title listed
965 whole), and wherever the
966 word does not divide through to its end: at a word in lower case
967 ('Dr.med', 'G.ppe'), at one closed by its own stop or followed by
968 another chunk ('Dr.Med.', 'J.Smith.', 'Dr.John.Smith'), and at an
969 unambiguous title or credential after the stop ('Dr.MD')."""
970 sets = lexicon._sets
971 n = _normalize(text)
972 bare = n.replace(".", "")
973 # a title listed whole is read whole, as a suffix spelled without its
974 # stops is: with its stops, or without them where no initial ends it
975 # ('Dr.Ma' where a caller lists 'drma'; 'Dr.A.' divides, 'dra' being
976 # the title and the initial T8 reads behind one)
977 if (bare in sets.suffix_acronyms or n in sets.titles
978 or bare in sets.titles and len(unicodedata.normalize(
979 "NFC", n.rsplit(".", 1)[-1])) > 1):
980 return ()
981 low = text.lower()
982 dotted = _dotted_titles(sets.titles)
983 if len(low) != len(text):
984 # offsets in `low` would not be the text's
985 dotted = ()
986 cuts: list[int] = []
987 at = 0
988 titled = False
989 # the cuts stand only where the word divides through to its end: a
990 # name closing it ('Yu.A.Chibrikova'), or initials closing one a
991 # title opens ('Dr.J.', 'Dr.J.R.'); never initials alone ('Kim
992 # Yu.A.', the surname-first initials #1254 asks about), an initial
993 # opening a chunk that stays whole ('Q.R.ST.'), nor a word holding a
994 # delimiter or a digit, which no chunk or name matches ('M.D./Ph.D.')
995 while True:
996 if _LETTER_RUN.fullmatch(text, at):
997 return tuple(cuts) if titled else ()
998 d = (dotted_title_end(low, at, dotted)
999 if dotted and low.startswith(dotted, at) else None)
1000 if d is not None and (
1001 (run := _GLUED_RUN.match(text, at)) is not None
1002 and run.end() > d):
1003 # a run of single letters running past the title is T7's
1004 # run of initials ('พ.ต.ส.สมชาย')
1005 d = None
1006 if d is not None:
1007 # a title listed with its stops is one chunk, read alone
1008 # (#1319: 'พล.ต.ต.สุรศักดิ์', 'พ.อ.หญิงวาสนา')
1009 titled = titled or not cuts
1010 if d == len(text):
1011 return tuple(cuts)
1012 nxt = _GLUED_CHUNK.match(text, d)
1013 if (low.startswith(dotted, d)
1014 and dotted_title_end(low, d, dotted) is not None) or (
1015 nxt is not None and is_title_chunk(nxt.group(1), lexicon)):
1016 cuts.append(d)
1017 at = d
1018 continue
1019 end, letter = d, False
1020 elif (run := _GLUED_RUN.match(text, at)) is not None:
1021 # a run of initials before the name stays one word (T7)
1022 end, letter = run.end(), True
1023 else:
1024 m = _GLUED_CHUNK.match(text, at)
1025 if m is None or not _reads_alone(m.group(1), lexicon):
1026 return ()
1027 end = m.end()
1028 letter = _is_cased_letter(
1029 unicodedata.normalize("NFC", m.group(1)))
1030 if not letter and is_title_chunk(m.group(1), lexicon):
1031 titled = titled or not cuts
1032 nxt = _GLUED_CHUNK.match(text, end)
1033 if nxt is not None and is_title_chunk(nxt.group(1), lexicon):
1034 # titles glued to each other divide at each stop, as
1035 # the titles written apart read ('Lt.Gov.John',
1036 # 'Maj.Gen.John', a phrase H9 reads)
1037 cuts.append(end)
1038 at = end
1039 continue
1040 word = _GLUED_WORD.match(text, end)
1041 # its first letter with case, so a modifier letter opening it
1042 # ('m.ʿali') does not hide a word in lower case; a script with
1043 # no case gives none, and divides
1044 if word is None or next(
1045 (c.islower() for c in word.group() if c.upper() != c.lower()),
1046 False):
1047 return ()
1048 initial = len(unicodedata.normalize("NFC", word.group())) == 1
1049 tail = text[word.end():]
1050 # a word closed by its own stop is a field or a name closed by
1051 # one ('Dr.Med.', 'J.Smith.'), save an initial behind a title
1052 # ('Dr.J.'); one followed by another chunk stays whole
1053 # ('Dr.John.Smith'), save an initial opening the next division
1054 # ('Yu.A.Chibrikova')
1055 if tail == "." and not (initial and not letter):
1056 return ()
1057 if len(tail) > 1 and not initial:
1058 return ()
1059 w = _normalize(word.group())
1060 if w in sets.titles or (
1061 w in sets.suffix_acronyms
1062 and w not in sets.suffix_acronyms_ambiguous) or (
1063 (w in sets.suffix_words or w in sets.credential_words)
1064 and w not in sets.suffix_words_ambiguous):
1065 return ()
1066 cuts.append(end)
1067 at = end
1068 if word.end() == len(text) or tail == ".":
1069 return tuple(cuts) if titled or not initial else ()
1072@functools.lru_cache(maxsize=16)
1073def _dotted_titles(titles: frozenset[str]) -> tuple[str, ...]:
1074 """The titles of a script without case listed with a full stop
1075 inside ('พ.ต.อ', 'พ.อ.หญิง'), longest first: one tuple per
1076 vocabulary, so a word opening with none of them is told by a C-level
1077 `startswith` alone. A cased title written with its stops ('dr.c',
1078 'mgr.a') is left out: glued, it keeps the reading decisions/T8's
1079 #1305 entry decided, a title and the initial behind it
1080 ('Dr.C.Meera'), where a caseless script writes no initials."""
1081 return tuple(sorted((t for t in titles if "." in t and t == t.upper()),
1082 key=len, reverse=True))
1085@functools.lru_cache(maxsize=16)
1086def title_prefixes(titles: frozenset[str]) -> tuple[str, ...]:
1087 """Every listed title, a tuple for a C-level `startswith`: what may
1088 stand behind a title only in a phrase glued in front (rules/W6)."""
1089 return tuple(titles)
1092def dotted_title_end(low: str, at: int,
1093 dotted: tuple[str, ...]) -> int | None:
1094 """Where a title listed with its stops, opening `low` at `at`, ends
1095 as a word of its own: past its own full stop, or, where its last
1096 chunk is more than one letter ('พ.อ.หญิง'), right before a letter
1097 of a script without case. A last chunk of one letter needs its
1098 stop, being the initial it spells otherwise ('น.สมชาย' is no
1099 'น.ส' + 'มชาย', 'พ.ต.อนุชา' no 'พ.ต.อ' + 'นุชา'), and a letter with
1100 case after a longer chunk goes on the word ('Lt.Colin', not 'Lt.Col'
1101 + 'in'); None where none does (#1319)."""
1102 for t in dotted:
1103 if not low.startswith(t, at):
1104 continue
1105 end = at + len(t)
1106 if end == len(low):
1107 return end
1108 if low[end] == ".":
1109 return end + 1
1110 c = low[end]
1111 last = t.rsplit(".", 1)[-1]
1112 if len(unicodedata.normalize("NFC", last)) > 1 and c.isalpha() and (
1113 c.upper() == c.lower()):
1114 return end
1115 return None
1118def glued_saint_cut(text: str, lexicon: Lexicon) -> int | None:
1119 """The offset after the stop where a listed saint particle is glued
1120 to the name it opens ('Sta.Ana', 'St.John', 'STE.MARIE'): the
1121 particle, one full stop and one name word whose first letter with
1122 case is a capital, nothing after it; None for any other word
1123 (rules/T8, #1071). The word is one name word wherever it stands,
1124 and divides here only as the last word of a part (`_script_segment`
1125 asks that)."""
1126 m = _GLUED_CHUNK.match(text)
1127 if m is None or _normalize(m.group(1)) not in lexicon._sets.saint_particles:
1128 return None
1129 word = _GLUED_WORD.fullmatch(text, m.end())
1130 if word is None or next(
1131 (c.islower() for c in word.group() if c.upper() != c.lower()),
1132 True):
1133 return None
1134 # an unambiguous title or suffix word after the stop is no name
1135 # ('St.MD', 'St.Jr')
1136 sets = lexicon._sets
1137 w = _normalize(word.group())
1138 if w in sets.titles or (
1139 w in sets.suffix_acronyms
1140 and w not in sets.suffix_acronyms_ambiguous) or (
1141 (w in sets.suffix_words or w in sets.credential_words)
1142 and w not in sets.suffix_words_ambiguous):
1143 return None
1144 return m.end()
1147def is_initial_chunk(chunk: str) -> bool:
1148 """Initials before a full stop, as T8's report asks of the chunks it
1149 cut (rules/T8): one letter in a script with case, a run of them
1150 ('J.R'), or a transliteration digraph rules/H2 names ('Yu'), and no
1151 title or given name written short."""
1152 parts = chunk.split(".")
1153 if len(parts) == 1 and _normalize(chunk) in _DIGRAPH_INITIALS:
1154 return True
1155 for part in parts:
1156 part = unicodedata.normalize("NFC", part)
1157 if len(part) != 1 or part.upper() == part.lower():
1158 return False
1159 return True
1162def is_title_chunk(chunk: str, lexicon: Lexicon) -> bool:
1163 """A listed title that is no saint particle, read as a chunk before
1164 a full stop (rules/T8; a saint particle glued to a name is one name
1165 word, `glued_saint_cut`)."""
1166 sets = lexicon._sets
1167 n = _normalize(chunk)
1168 return n not in sets.saint_particles and (
1169 n in sets.titles or n in sets.titles_ambiguous
1170 or n in sets.title_modifiers or n in sets.title_styles)
1173def is_dotted_title(text: str, n: str, lexicon: Lexicon) -> bool:
1174 """A run of single letters joined by full stops ('U.S.', 'P.D.')
1175 whose spelling without them is a title written `dotted`
1176 (`Lexicon.dotted_titles`, rules/H10, #490); `n` is the folded
1177 text, which both callers already have."""
1178 return (n.replace(".", "") in lexicon._sets.dotted_titles
1179 and _LETTER_RUN.fullmatch(text) is not None)
1182# #563: exactly two single letters, each closed by a period -- the way
1183# a given and a middle initial are written run together ('M.J.',
1184# 'G.J.'). The trailing period is optional, as `period_joined_vocab`'s
1185# shape verdict lets it be.
1186_PAIRED_INITIALS = re.compile(_LETTER + r"\." + _LETTER + r"\.?")
1189def is_paired_initials(text: str) -> bool:
1190 """Two single letters run together with periods ('M.J.'): the one
1191 unlisted dotted shape a person's own initials take, so after a
1192 comma it is a name until something in front of it, or another
1193 dotted word beside it, says it is a credential (rules/C5).
1194 Three or more letters ('X.Y.Z.') and any longer chunk ('B.Tech.')
1195 are outside it."""
1196 return _PAIRED_INITIALS.fullmatch(text) is not None
1199# rules/S3: "Paired initials are the exception, as C5 defines them, in a
1200# script that writes capitals" -- the script half of that clause, which
1201# classify asked inline until #752 (decisions/S3 (#864),
1202# decisions/vocabulary-model (#752)). C5's comma reading asks the bare
1203# `is_paired_initials`: after a comma a caseless pair reads as a name
1204# too ('John Smith, พ.บ.' given 'พ.บ.'), and moving it here would change
1205# that reading.
1206def is_cased_paired_initials(text: str) -> bool:
1207 """Paired initials (`is_paired_initials`) in a script that writes
1208 capitals: a text with a case mapping, so 'M.J.' and 'm.j.' but not
1209 Thai 'พ.บ.', which is how that script writes an abbreviation.
1210 The same compiled pattern as `is_paired_initials`, asked here
1211 rather than through it so a dotted token pays one frame, not
1212 two."""
1213 return (_PAIRED_INITIALS.fullmatch(text) is not None
1214 and text.upper() != text.lower())
1217# rules/S12: "a credential written split across a space reads as one
1218# word" -- one chunk of up to five letters, with or without its period
1219# ('Pharm.', 'D', 'Sc.'), the shape both words of a split credential
1220# take. The C-level half classify asks of every adjacent pair before
1221# the fold is looked up.
1222SPLIT_CHUNK = re.compile(r"(?:" + _LETTER + r"){1,5}\.?")
1225# mechanisms.md#ONE-PREDICATE-PER-QUESTION ("exactly one predicate answers it and every other site calls that one")
1226# -- the split credential's one test, asked by classify, which tags
1227# the pair for group's merge and `_comma._pieces`, and by
1228# `is_wholly_suffix` for the Ph./D. answer. decisions/phd-merge,
1229# decisions/vocabulary-model (#752)
1230def split_pair(first: str, second: str, lexicon: Lexicon) -> str | None:
1231 """Whether two adjacent words are one credential written split
1232 across a space, and so read as one unit -- answered with the PIECE
1233 TAG the merged pair carries, or None.
1235 "suffix": 'Ph.' then 'D.', with or without their periods, in any
1236 case, where the lexicon lists `phd` as unambiguous suffix
1237 vocabulary (a credential, or a suffix word) -- v1's fix_phd,
1238 merged as an unambiguous suffix wherever it stands but the head
1239 of the string (decisions/phd-merge); under a lexicon that does
1240 not, None, the pair being the two words it is written as.
1242 "split": any other pair of short chunks (`SPLIT_CHUNK`) whose
1243 period-free concatenation is unambiguous suffix vocabulary while
1244 the two are not both suffix vocabulary on their own ('Pharm. D.',
1245 'Pharm D', 'MB ChB' where 'chb' is unlisted) and neither is a
1246 particle or a connective, which join name words. The merged piece is
1247 a member of the ambiguous class, which the count decides (rules/S4,
1248 rules/C4). Its first chunk has two or more letters: a single
1249 letter opening the pair is a middle initial far more often (FEC
1250 writes 'SMITH, JOHN G. MR.', and 'G. MR.' folds to the listed
1251 'gmr'), so 'M. Sc.' and 'M. D.' stay two words. Two dotted single
1252 letters are the one opening letter asked further, and asked of
1253 #872's predicate rather than a copy: in a script with capitals
1254 they are paired initials (`is_cased_paired_initials`), the reading
1255 rules/S3 gives them joined, so only a caseless script's pair can
1256 merge, and not one in a script that writes no abbreviations (Han,
1257 hangul, kana: '博. 士.' is no dotted '博士'), which leaves no
1258 default entry such a fold.
1260 Where the merge may happen (never at the head of the string,
1261 rules/S2; a member pair only in a main segment and in the part
1262 after a comma) is the caller's question, not this one's."""
1263 if _PH.fullmatch(first) is not None and _D.fullmatch(second) is not None:
1264 # rules/S12: "is a credential only where the lexicon lists `phd`"
1265 # -- as unambiguous suffix vocabulary of either kind, as the
1266 # joined spelling reads it; elsewhere the two words, the fold
1267 # below being `phd` itself, which an ambiguous suffix word would
1268 # admit as a member. decisions/phd-merge (#1000)
1269 sets = lexicon._sets
1270 return ("suffix" if "phd" in sets.suffix_acronyms
1271 and "phd" not in sets.suffix_acronyms_ambiguous
1272 or "phd" in sets.suffix_words
1273 and "phd" not in sets.suffix_words_ambiguous else None)
1274 if (SPLIT_CHUNK.fullmatch(first) is None
1275 or SPLIT_CHUNK.fullmatch(second) is None):
1276 return None
1277 a, b = first.rstrip("."), second.rstrip(".")
1278 if composed_len(a) < 2 and (
1279 composed_len(b) != 1 or not first.endswith(".")
1280 or not second.endswith(".")
1281 or is_cased_paired_initials(first + second)
1282 # a script that writes no abbreviations is not wearing an
1283 # abbreviation's periods, as rules/S3 reads it joined
1284 or in_initialless_script(first + second)):
1285 return None
1286 fold = _normalize(a + b)
1287 if (fold in lexicon._sets.suffix_acronyms_ambiguous
1288 or not (fold in lexicon._sets.suffix_acronyms
1289 or fold in lexicon._sets.suffix_words)):
1290 return None
1291 na, nb = _normalize(a), _normalize(b)
1292 if ((na in lexicon._sets.suffix_acronyms or na in lexicon._sets.suffix_words)
1293 and (nb in lexicon._sets.suffix_acronyms
1294 or nb in lexicon._sets.suffix_words)):
1295 return None
1296 # a particle or a connective joins name words, which is what it is
1297 # doing beside one ('DE A JUNIOR', the A a surname written short);
1298 # written with its period a letter is an initial, not the connective
1299 # it spells ('SAM E.', rules/T5)
1300 for written, n in ((first, na), (second, nb)):
1301 if not written.endswith(".") and (n in lexicon._sets.particles
1302 or n in lexicon._sets.conjunctions):
1303 return None
1304 return "split"
1307def suffix_as_written(n: str, text: str, lexicon: Lexicon) -> bool:
1308 """Counts as a suffix as written, with NO initial veto (the veto
1309 differs by caller): unambiguous suffix vocabulary, an ambiguous
1310 acronym written with periods ('M.A.' yes, 'Ma' no), or a credential
1311 joined by '-' or '/' (`_joined_credential`, 'MD/PhD'). `n` is
1312 _normalize(text), passed in so callers normalize once.
1314 Single source for classify's "vocab:suffix" tag and the segment/
1315 assign predicates. The ambiguous subset is EXCLUDED from the plain
1316 membership test: in the real data suffix_acronyms_ambiguous is a
1317 subset of suffix_acronyms, and without the exclusion the period
1318 gate is dead code (bare 'Ed'/'Jd' would silently become suffixes).
1319 """
1320 # acronyms may be written with periods ('M.B.A.'): the ACRONYM
1321 # membership alone uses the period-free form (v1's is_suffix
1322 # removed periods only for the suffix_acronyms test); suffix WORDS
1323 # match on the plain normalized form
1324 a = n.replace(".", "")
1325 if a in lexicon._sets.suffix_acronyms_ambiguous and _dotted(text):
1326 return True
1327 if ((a in lexicon._sets.suffix_acronyms
1328 and a not in lexicon._sets.suffix_acronyms_ambiguous)
1329 or (n in lexicon._sets.suffix_words
1330 and n not in lexicon._sets.suffix_words_ambiguous)):
1331 return True
1332 # the joiners asked in C first, so a word holding neither pays no
1333 # frame
1334 return ("-" in n or "/" in n) and _joined_credential(text, lexicon)
1337#: What joins the letter chunks of the caps shape's joined spelling
1338#: ('PU-PH', `_joined_capitals`); a joined credential is cut by
1339#: `_JOINER_SPLIT`, which keeps the joiners.
1340_JOINERS = re.compile(r"[-/]")
1343def _member(chunk: str, lexicon: Lexicon) -> bool:
1344 """Whether CHUNK is a member of the ambiguous class (S4), however
1345 written ('MA', 'Ma', 'M.A', 'M.A.'): an acronym once its periods are
1346 removed, or a suffix word, that is a name too."""
1347 folded = _normalize(chunk)
1348 return (folded.replace(".", "") in lexicon._sets.suffix_acronyms_ambiguous
1349 or folded in lexicon._sets.suffix_words_ambiguous)
1352def _settles(chunk: str, lexicon: Lexicon) -> bool:
1353 """Whether CHUNK, one word or one listed run of a joined word's cut,
1354 settles a member beside it: a credential that is no name
1355 (`settles_a_member`), or a member written with a period inside it
1356 ('M.A')."""
1357 return (settles_a_member(chunk, lexicon)
1358 or "." in chunk and dotted_member(chunk, lexicon))
1361# rules/S1: "a credential that is no name, or a member its periods
1362# settle, says which" -- the credential half, asked of a clause word
1363# and of a joined word's chunk alike: a generation, a numeral, an
1364# honorific or a word that is a title too says nothing of the kind,
1365# 'Ed Jr' being a name a person goes by (decisions/S1, #1138)
1366def settles_a_member(word: str, lexicon: Lexicon) -> bool:
1367 """Whether WORD, a comma it is written against left off (rules/N1),
1368 is a credential that is no name: it counts as a suffix as written
1369 (a member its periods settle, 'M.A.', included) and is no
1370 generation, no honorific and no title. A joined word not listed
1371 whole is one where a chunk of its cut is ('PhD/Jr', not 'Jr/Sr')."""
1372 bare = word.strip(COMMA_STRIP)
1373 if ("/" in bare or "-" in bare) and not _listed_whole(bare, lexicon):
1374 return any(_settles(c, lexicon)
1375 for c in _joined_chunks(bare, lexicon) or ())
1376 folded = _normalize(bare)
1377 sets = lexicon._sets
1378 return (suffix_as_written(folded, bare, lexicon)
1379 and folded not in sets.generation_words
1380 and folded not in sets.honorific_words
1381 and folded not in sets.titles)
1384def joined_member(word: str, lexicon: Lexicon) -> bool:
1385 """Whether WORD, joined by '/' or '-' and not listed whole ('NP-C'),
1386 cuts (`_joined_chunks`) into pieces one of which is a member of the
1387 ambiguous class that does not count as a suffix as written
1388 ('MA/PhD', 'M.A/PhD', 'MA/MBA'; not 'M.A./PhD'). `set_apart_kind`
1389 asks it of a word `suffix_as_written` reads, making that word a
1390 member S13 settles by its position; `_report_delimited_members`
1391 asks it of any joined nickname, so a joined word of members alone
1392 reports as its words written apart do."""
1393 if not ("/" in word or "-" in word) or _listed_whole(word, lexicon):
1394 return False
1395 return any(_member(c, lexicon)
1396 and not suffix_as_written(_normalize(c), c, lexicon)
1397 for c in _joined_chunks(word, lexicon) or ())
1400def member_word(word: str, lexicon: Lexicon) -> bool:
1401 """Whether a word of a delimited clause, a comma the clause keeps
1402 on it left off (rules/N1), is a member of the ambiguous class (S4),
1403 however written."""
1404 return _member(word.strip(COMMA_STRIP), lexicon)
1407def _listed_whole(word: str, lexicon: Lexicon) -> bool:
1408 """Whether the suffix vocabulary lists WORD whole, its joiners
1409 included ('NP-C', 'OTR/L'): as a suffix word, or as an acronym once
1410 its periods are removed. A member of an ambiguous class counts as
1411 listed."""
1412 folded = _normalize(word)
1413 return (folded in lexicon._sets.suffix_words
1414 or folded.replace(".", "") in lexicon._sets.suffix_acronyms)
1417#: The joiners of a joined credential, kept: 'MS/OTR/L' splits to
1418#: 'MS', '/', 'OTR', '/', 'L', the chunks at the even places.
1419_JOINER_SPLIT = re.compile(r"([-/])")
1422# bounded as `_longest_marker` is, for the reason given there: a set
1423# per lexicon, so 16 bounds many-lexicon churn without evicting in
1424# normal use
1425@functools.lru_cache(maxsize=16)
1426def _joined_span(words: frozenset[str]) -> int:
1427 """How many chunks the most-joined of WORDS holds: the longest run a
1428 listed word can be. Cached per set, which hashes once."""
1429 return max((w.count("/") + w.count("-") for w in words), default=0) + 1
1432# rules/S3: "a run of chunks the vocabulary lists whole, its slash or
1433# hyphen included, is one credential" -- the joiner inside 'NP-C' or
1434# 'OTR/L' is how that credential is spelled, not a joiner between two
1435# (decisions/S3, #1103)
1436def _joined_chunks(text: str, lexicon: Lexicon) -> list[str] | None:
1437 """A cut of TEXT at some of its joiners into pieces that each count
1438 as a suffix as written or are a member of the ambiguous class
1439 however written ('MA', 'M.A'), a run of chunks the vocabulary lists
1440 whole among them, with the fewest pieces ('MS/OTR/L' is 'MS', 'OTR/L';
1441 'CCC/SLP/L' is 'CCC', 'SLP/L'; 'APRN-FNP-C' is 'APRN', 'FNP-C';
1442 'MD-PhD' is 'MD', 'PhD'), or None where no cut gives such pieces
1443 ('MD-Jones', 'MD/'). No piece is the whole of TEXT: each caller has
1444 asked TEXT whole already, and a word listed whole that it refused
1445 (one that is a name too) would otherwise be asked again for ever.
1446 A piece is no longer than the most-joined listed word, so a long
1447 chain of joiners costs linear time."""
1448 parts = _JOINER_SPLIT.split(text)
1449 n = (len(parts) + 1) // 2 # the chunks
1450 span = max(_joined_span(lexicon._sets.suffix_words),
1451 _joined_span(lexicon._sets.suffix_acronyms))
1452 # pieces[k]: the fewest pieces cutting the first k chunks (0 where
1453 # none does, past the start); start[k]: where the last of them starts
1454 pieces = [0] * (n + 1)
1455 start = [0] * (n + 1)
1456 for i in range(n):
1457 if i and not pieces[i]:
1458 continue
1459 for j in range(i + 1, min(n - (i == 0), i + span) + 1):
1460 piece = "".join(parts[2 * i:2 * j - 1])
1461 # a run must be listed whole, which a chunk alone need not be
1462 if not piece or j > i + 1 and not _listed_whole(piece, lexicon):
1463 continue
1464 # a member is a chunk however written; `_joined_credential`
1465 # asks that a chunk beside it settles it (#1138)
1466 if ((not pieces[j] or pieces[i] + 1 < pieces[j])
1467 and (suffix_as_written(_normalize(piece), piece, lexicon)
1468 or _member(piece, lexicon))):
1469 pieces[j] = pieces[i] + 1
1470 start[j] = i
1471 if not pieces[n]:
1472 return None
1473 cut: list[str] = []
1474 j = n
1475 while j:
1476 i = start[j]
1477 cut.append("".join(parts[2 * i:2 * j - 1]))
1478 j = i
1479 cut.reverse()
1480 return cut
1483# rules/S3: "a word whose chunks are joined by a hyphen or a slash
1484# reads as a suffix when every chunk counts as a suffix as written" --
1485# the stricter test, every chunk rather than any, because a hyphen also
1486# joins names (decisions/S3 (#752)); a chunk is a run the vocabulary
1487# lists whole where one is (#1103)
1488def _joined_credential(text: str, lexicon: Lexicon) -> bool:
1489 """Whether TEXT is two or more chunks joined by '-' or '/', cut so
1490 that each is non-empty and counts as a suffix as written (listed,
1491 and not a bare member of the ambiguous class, a run listed whole
1492 counting as one chunk) or is a member of that class however
1493 written, where one chunk at least settles every such member
1494 (`_settles`), and not every chunk a single ASCII character, and --
1495 joined by a hyphen -- not written as a hyphenated name is ('MD/PhD', 'M.A.-Ph.D.', 'MA/PhD',
1496 'M.A/PhD', 'PhD-Md', 'md-phd', 'PhD/NP-C', but not 'MA/MBA',
1497 'MA/Jones', 'Smith-Jones', 'I-V', 'Ae-Ra', 'Ae-ra' or 'MD/'). The
1498 writing is read first, so a hyphenated name ('Smith-Jones') is
1499 never cut."""
1500 # a slash joins no name, so a word holding one takes no cue
1501 if "/" not in text:
1502 # rules/S3: "a word joined by a hyphen and written the way a
1503 # hyphenated name is, its first chunk in Title case and every
1504 # other in Title case or lower case, is a name word" -- where a
1505 # credential's chunks are not (decisions/S3 (#752)); its periods
1506 # are not the chunk's ('I.' is 'i'), as in a word split at its
1507 # periods
1508 named = True
1509 for k, c in enumerate(text.split("-")):
1510 bare = c.replace(".", "")
1511 named = (len(bare) > 1 and bare[0].isupper() and bare[1:].islower()
1512 or k > 0 and bare.islower())
1513 if not named:
1514 break
1515 if named:
1516 return False
1517 chunks = _joined_chunks(text, lexicon)
1518 if chunks is None:
1519 return False
1520 # rules/S3: "a member of the ambiguous class (S4) is a chunk too,
1521 # however it is written, where the word holds a chunk that settles
1522 # it" -- a credential that is no name, or a member its periods
1523 # settle; a word of bare members alone has nothing to settle it
1524 # (decisions/S3, #1138)
1525 unsettled = any(_member(c, lexicon)
1526 and not suffix_as_written(_normalize(c), c, lexicon)
1527 for c in chunks)
1528 if unsettled and not any(_settles(c, lexicon) for c in chunks):
1529 return False
1530 # rules/S3: "Every chunk the vocabulary matches is a single ASCII
1531 # character" -- the numerals and the digit, which are about
1532 # generations, make no joined word a suffix either ('Jack I-V')
1533 for c in chunks:
1534 bare = c.replace(".", "")
1535 if len(bare) > 1 or not bare.isascii():
1536 return True
1537 return False
1540# rules/S3: "a word with interior periods reads as a suffix when
1541# every one of its period-separated chunks counts as a suffix as
1542# written" -- the joined words' every chunk, the periods being the one
1543# sign the word is joined at all (decisions/S3, #1131)
1544def period_joined_credential(text: str, lexicon: Lexicon) -> bool:
1545 """Whether TEXT, cut at its periods (`_period_chunks`), is chunks
1546 that each count as a suffix as written or are a member of the ambiguous
1547 class, one chunk at least settling every such member (`_settles`),
1548 and not every chunk the vocabulary matches a single ASCII character
1549 ('Msc.Ed', 'JD.CPA'; not 'Bob.PhD', 'Ma.Ed' or 'X.Y.I.'). A word
1550 whose last chunk is a glued honorific ('J.씨') reads as one too:
1551 the honorific is the suffix there and the chunk before it the
1552 initial it is written as (rules/W2). The chunks are two or more,
1553 save one where the word's periods all stand at its edge beside an
1554 ampersand or a semicolon ('MD.&', as 'MD.& PhD' writes it)."""
1555 chunks = _period_chunks(text)
1556 if chunks is None:
1557 return False
1558 sets = lexicon._sets
1559 if _normalize(chunks[-1]) in sets.honorific_tails:
1560 return True
1561 unsettled = False
1562 for c in chunks:
1563 folded = _normalize(c)
1564 if suffix_as_written(folded, c, lexicon):
1565 continue
1566 # rules/S11: "the word is part of that one word and is weighed
1567 # only by the shape the joined word wears" -- no chunk claim
1568 # for a suffix word that is also a name ('MD.Filho', whose Filho
1569 # MD would otherwise settle as a member)
1570 if folded in sets.suffix_words_ambiguous or not _member(c, lexicon):
1571 return False
1572 unsettled = True
1573 if unsettled and not any(_settles(c, lexicon) for c in chunks):
1574 return False
1575 # rules/S3: "Every chunk the vocabulary matches is a single ASCII
1576 # character" -- a numeral or the digit says nothing of a credential
1577 return not all(len(c) == 1 and c.isascii() for c in chunks)
1580def _period_chunks(text: str) -> list[str] | None:
1581 """TEXT cut at its periods, or None where it holds none inside it
1582 (one chunk where every period stands at the word's edge and an '&'
1583 or ';' there keeps one off the outer end: 'MD.&', ';.MD').
1584 ASCII punctuation beside an interior period is no part of the chunk
1585 ('MD.&PhD'); at the word's outer edges only its periods, a glued
1586 semicolon (rules/C10, 'Jr.;') and an ampersand joining the next
1587 credential ('MD.& PhD') are, so a word wearing any other there
1588 is no period-joined word at all, as a single word or a slash word
1589 wearing it is a name word ('Msc.Ed!', 'Msc.Ed.]', '[Msc.Ed]' as
1590 'MD!', 'MD/PhD]'). A chunk of nothing but punctuation is no
1591 chunk."""
1592 if "." not in text.strip("."):
1593 return None
1594 i, j = 0, len(text)
1595 while i < j and text[i] in string.punctuation:
1596 i += 1
1597 while j > i and text[j - 1] in string.punctuation:
1598 j -= 1
1599 if any(c not in ".;&" for c in text[:i] + text[j:]):
1600 return None
1601 chunks = [b for c in text[i:j].split(".")
1602 if (b := c.strip(string.punctuation))]
1603 return chunks or None
1606def _holds_a_member(text: str, lexicon: Lexicon) -> bool:
1607 """Whether a chunk of TEXT cut at its periods (`_period_chunks`) is
1608 a member of the ambiguous class that does not count as a suffix as
1609 written ('Ed' in 'Msc.Ed')."""
1610 return any(_member(c, lexicon)
1611 and not suffix_as_written(_normalize(c), c, lexicon)
1612 for c in _period_chunks(text) or ())
1615def period_member(word: str, lexicon: Lexicon) -> bool:
1616 """Whether WORD, a word with interior periods that is no acronym
1617 listed once its periods are removed, is
1618 a period-joined credential (`period_joined_credential`) holding a
1619 member of the ambiguous class ('Msc.Ed'): `joined_member`'s
1620 question for a word the periods join (rules/S3, #1131)."""
1621 bare = word.strip(COMMA_STRIP)
1622 folded = _normalize(bare)
1623 return ("." in folded
1624 and folded.replace(".", "") not in lexicon._sets.suffix_acronyms
1625 and period_joined_credential(bare, lexicon)
1626 and _holds_a_member(bare, lexicon))
1629# rules/H5: "a word with interior periods every one of whose chunks is
1630# a title chains without a final period" -- 'Lt.Gov', where the
1631# periods join nothing but titles (decisions/H5, #1131)
1632def period_joined_title(text: str, lexicon: Lexicon) -> bool:
1633 """Whether TEXT, cut at its periods, is two or more chunks every
1634 one of which is title vocabulary ('Lt.Gov', 'Lt.Col'; not
1635 'Dr.PhD' or 'St.John')."""
1636 chunks = [_normalize(c) for c in text.split(".") if c]
1637 return len(chunks) >= 2 and all(
1638 c in lexicon._sets.titles for c in chunks)
1641def _is_suffix_strict_n(n: str, text: str, lexicon: Lexicon,
1642 one_case: bool | None = None) -> bool:
1643 # rules/T4: "in a name written wholly in one case, which says
1644 # nothing about any letter in it, a bare letter is an initial
1645 # wherever its capital would be" -- so the veto takes 'v' where it
1646 # takes 'V' (#727). Lower case first, at C level: a capital is
1647 # is_initial's already, and only a caller with the fact passes it
1648 if is_initial(text) or (
1649 one_case and len(text) == 1 and text.islower()
1650 and _INITIAL.fullmatch(text.upper()) is not None):
1651 # period-written ambiguous acronyms are exempt from the veto
1652 return _dotted(text) and \
1653 n.replace(".", "") in lexicon._sets.suffix_acronyms_ambiguous
1654 return suffix_as_written(n, text, lexicon)
1657def is_suffix_strict(text: str, lexicon: Lexicon,
1658 one_case: bool | None = None) -> bool:
1659 """v1's is_suffix: suffix_as_written with the initial veto ('V.' in
1660 'John V. Smith' is a middle initial, not roman five).
1662 `one_case` is whether the name's own words are written in one
1663 case (ParseState.one_case): where they are, a bare lowercase
1664 letter is the initial its capital is (rules/T4) and is vetoed as
1665 its capital is. None or False is the veto by the writing alone,
1666 which is v1's."""
1667 return _is_suffix_strict_n(_normalize(text), text, lexicon, one_case)
1670def is_suffix_lenient(text: str, lexicon: Lexicon) -> bool:
1671 """v1's is_suffix_lenient: suffix_words accepted unconditionally,
1672 bypassing the initial veto -- only safe in unambiguous positions
1673 (after a comma)."""
1674 n = _normalize(text)
1675 return (n in lexicon._sets.suffix_words
1676 and n not in lexicon._sets.suffix_words_ambiguous) \
1677 or _is_suffix_strict_n(n, text, lexicon)
1680#: The suffix delimiters every policy has, beside the comma. rules/C10:
1681#: "The semicolon, the hyphen and the slash are delimiters by default"
1682#: -- "no policy removes them, as none removes the comma", so a
1683#: policy's extra_suffix_delimiters add to them. decisions/C10 (D15,
1684#: #889).
1685BUILTIN_SUFFIX_DELIMITERS = frozenset({";", "-", "/"})
1687#: The built-in delimiters read glued to a word or inside one. rules/C10:
1688#: "a built-in hyphen or slash parts only standing alone" -- glued, it
1689#: is the word's own ('al-', 'MD/PhD'). decisions/C10 (#889).
1690BUILTIN_GLUED_DELIMITERS = frozenset({";"})
1693def delimiter_cores(policy_delimiters: frozenset[str]) -> frozenset[str]:
1694 """The built-in suffix delimiters with the configured ones,
1695 surrounding whitespace stripped: ' - ' -> '-', the cores that part
1696 a trailing suffix part STANDING ALONE. Whitespace-padded
1697 delimiters surface as standalone tokens; the stripped core is what
1698 tokenize produced."""
1699 if not policy_delimiters:
1700 return BUILTIN_SUFFIX_DELIMITERS
1701 return BUILTIN_SUFFIX_DELIMITERS | frozenset(
1702 d.strip() for d in policy_delimiters if d.strip())
1705def glued_cores(policy_delimiters: frozenset[str]) -> frozenset[str]:
1706 """The cores read glued to one end of a word or inside one: the
1707 semicolon and every configured core, never a built-in hyphen or
1708 slash that the caller did not declare (BUILTIN_GLUED_DELIMITERS).
1709 A configured core that is itself a word (' and ') is in the set:
1710 `glued_stem`'s callers ask `glues` before reading it glued, while
1711 `is_wholly_suffix`'s in-word split does not ('MDandPhD')."""
1712 if not policy_delimiters:
1713 return BUILTIN_GLUED_DELIMITERS
1714 return BUILTIN_GLUED_DELIMITERS | frozenset(
1715 d.strip() for d in policy_delimiters if d.strip())
1718# rules/C10: "the word keeps the delimiter in its text and reads as it
1719# would without it" -- the word, for every site that reads one: the
1720# tags classify writes, the comma decision's shape tests, C2's flag
1721# and the run test below (decisions/C10).
1722def glued_stem(text: str, cores: frozenset[str]) -> str | None:
1723 """The word a delimiter core glued to ONE end of `text` leaves
1724 ('MD;' -> 'MD', ';PhD' -> 'PhD', 'MD-PhD;' -> 'MD-PhD' under '-'),
1725 or None: no core at an end of the text, a core standing alone, one
1726 inside a word only ('MD;PhD', which splits_into_suffixes reads), or
1727 a core that is itself a word (see glues). Every core is tried, in
1728 sorted order, so the answer never depends on a set's iteration
1729 order (the second review of #871: 'MD/PhD; MBA' under '/' read two
1730 ways by hash seed). Callers test `core in text` inline first, so a
1731 word with no core pays no frame."""
1732 for core in sorted(cores):
1733 if (core in text and glues(core) and text.count(core) == 1
1734 and len(text) > len(core)):
1735 if text.endswith(core):
1736 return text[:-len(core)]
1737 if text.startswith(core):
1738 return text[len(core):]
1739 return None
1742def glues(core: str) -> bool:
1743 """Whether a delimiter core can be GLUED to a word: one written
1744 without a letter or digit (';', '/', '-'). A core that is a word
1745 (' and ' under a declared policy) is found inside other words by
1746 the substring test ('Stand'), so it parts only standing alone
1747 (rules/C10's glued bullet)."""
1748 return not any(ch.isalnum() for ch in core)
1751def splits_into_suffixes(text: str, cores: frozenset[str],
1752 lexicon: Lexicon) -> bool:
1753 """v1 expand_suffix_delimiter parity for delimiters WITHOUT
1754 whitespace ('RN/CRNA' with '/'): the token counts as a suffix when
1755 some core splits it into >=2 non-empty parts that are all suffixes.
1756 A core glued to ONE word ('MD;') is is_wholly_suffix's to read, by
1757 the test it gives that word alone (D15). The token text is never
1758 rewritten (anti-#100): it takes Role.SUFFIX whole, which renders
1759 'RN/CRNA' where v1 rendered 'RN, CRNA' -- the documented
1760 divergence, release-log classified."""
1761 for core in cores:
1762 if core in text:
1763 parts = [part for part in text.split(core) if part]
1764 if len(parts) >= 2 and all(
1765 is_suffix_lenient(part, lexicon) for part in parts):
1766 return True
1767 return False
1770# rules/S3: "a word with interior periods reads as a suffix when
1771# every one of its period-separated chunks counts as a suffix as
1772# written" -- "every chunk the vocabulary matches is a single ASCII
1773# character, the roman numerals and the lone digit the vocabulary
1774# lists, which are about generations rather than credentials". The
1775# chunk rule and its single-character exception (decisions/S3,
1776# #1131).
1777def period_joined_vocab(
1778 text: str, lexicon: Lexicon,
1779) -> Literal["title", "suffix", "shape"] | None:
1780 """v1's parse_pieces derivation for interior-period tokens
1781 ('Lt.Gov.', 'Msc.Ed.', and by the ANY rule 'Mr.Smith'): ANY title
1782 chunk makes the token a title (checked first, v1's continue); else
1783 the token is a suffix where EVERY chunk counts
1784 (`period_joined_credential`, #1131: 'Msc.Ed.', whose member 'ed'
1785 'msc' settles, but not 'Bob.PhD', which the shape verdict below
1786 reads by position). A suffix word that is also a name is a member
1787 there, settled by a credential beside it ('M.Filho' is no suffix,
1788 rules/S11). Returns "title", "suffix", "shape", or None.
1790 "shape" is the third verdict and it is the absence of a claim
1791 worth acting on (#516): two or more chunks, EVERY one of them
1792 alphabetic, that the every-chunk test does not read as a suffix
1793 ('Bob.PhD', 'BM.BS', #1131), chunks that are a SINGLE ASCII
1794 CHARACTER among them. That single-character exception is the
1795 roman-numeral accident retired narrowly: measured 2026-09-15 the
1796 one-character suffix vocabulary is {'2', 'i', 'v'} plus the glued
1797 CJK honorific tails, so 'John Smith R.A.I.' was reading as a
1798 generational suffix off the chunk 'i'. CHARACTER rather than
1799 LETTER because '2' is a digit and is in the set; ASCII because
1800 '씨' is the single character that must KEEP its claim ('J.씨'). A
1801 word whose every chunk counts still reads as a suffix, so 'Msc.Ed.'
1802 and 'JD.CPA' are untouched -- the WIDE retirement, where
1803 any chunk match yields to the shape, was measured and rejected: it
1804 costs 'Doe, John Msc.Ed.' a real credential and re-routes the
1805 honorific peel of '김민준씨, J.씨' for nothing this design wants
1806 (decisions/S3).
1808 The ALPHABETIC requirement is a SEPARATE, narrower gate on the
1809 shape verdict alone (#516 review round, decided by the
1810 orchestrator): a bare digit chunk is not an acronym letter by any
1811 reading, so 'Smith, 1.4' and 'John Smith 1.4' do not join the
1812 class by shape -- the only PROTECTED digit control this design
1813 had was the delimited 'Bridge (1.4)', and 'Smith, 1.4' and its
1814 no-comma twin were the gap that control did not cover, since
1815 delimited content is excluded by a different mechanism (extract's
1816 escape) and digits reach THIS detector, never that one.
1817 '.isalpha()' is asked of EVERY chunk, not just the matched ones,
1818 so a mixed token like 'X.Y.2.' is not shape-admitted either -- one
1819 non-letter chunk is enough to say the writing is not spelling an
1820 acronym. Chunks the vocabulary itself matches are already
1821 alphabetic in the shipped lexicon, so this narrows only the
1822 previously-unclaimed "shape" answer, never the "suffix" one.
1824 The INITIALLESS-SCRIPT guard is a second, independent narrowing of
1825 the same verdict, on the same reasoning `is_title_shaped` already
1826 gives its own period-abbreviation inference (#323): a script with
1827 no period abbreviations at all has nothing for interior periods to
1828 ABBREVIATE, so a CJK word glued into period-separated single
1829 characters is not spelling an acronym either -- 'John Smith
1830 田.中.' and '김 민준 이.박.' stayed family at every release before
1831 this gate existed, and would otherwise have joined the ambiguous
1832 class by shape for the first time (measured regression, #516
1833 review round).
1835 What a "shape" verdict MEANS is the caller's question, not this
1836 one's: classify writes the tag, and Policy.unlisted_dotted_suffixes
1837 decides whether it is class membership.
1838 """
1839 if not _PERIOD_NOT_AT_END.match(text):
1840 return None
1841 # rules/T8: "a saint particle glued to a name is one name word" --
1842 # no title chunk claims it and no dotted shape reads it
1843 # (decisions/T8 (#1071))
1844 if glued_saint_cut(text, lexicon) is not None:
1845 return None
1846 chunks = [_normalize(c) for c in text.split(".") if c]
1847 if any(c in lexicon._sets.titles for c in chunks):
1848 return "title"
1849 if period_joined_credential(text, lexicon):
1850 return "suffix"
1851 if (len(chunks) >= 2 and all(c.isalpha() for c in chunks)
1852 and (text.isascii() or not in_initialless_script(text))):
1853 return "shape"
1854 return None
1857def ambiguous_class_member(text: str, lexicon: Lexicon) -> bool:
1858 """Whether TEXT is a member of the LISTED ambiguous credential
1859 class, ignoring case and policy entirely (#289/#516).
1861 The case-INDEPENDENT half of the comma form's own candidate test
1862 (`ambiguous_class_candidate`, below, which ALSO admits a by-shape
1863 member where Policy allows it) and of `is_wholly_suffix`'s
1864 credential-lean disjunct: membership by vocabulary never needs the
1865 case fact, only the lean does. That split is what lets the comma
1866 form's lazy gate ask membership FIRST and pay for `one_case` only
1867 where this says yes (measured regression, #289/#516: forcing the
1868 fact first ran `own_words` -> `tag_marker_runs` for every
1869 single-token-after-the-comma name, none of them able to reach the
1870 class at all).
1872 Membership is the listed set, bare: a whole-token vocabulary match
1873 is not in this class at all, being settled ('M.A.', 'Ph.D.',
1874 'A.B.C.').
1876 Cheaper than `suffix_as_written(n, text, lexicon) or ...` would be
1877 here, and provably the same answer for UNDOTTED text: the Lexicon
1878 invariant that an ambiguous acronym is never also a suffix WORD
1879 (`__post_init__`'s gate_bypassed check) and is never counted
1880 without periods once it IS one (`suffix_as_written`'s own
1881 exclusion) together kill that predicate's two disjuncts once text
1882 is known to hold no period. One frame (`_normalize`) on the common
1883 path that never reaches the class, where the general predicate
1884 cost at least two (mechanisms.md#ONE-PREDICATE-PER-QUESTION's cost
1885 clause).
1887 The '.' gate here is DELIBERATELY STRICTER than S2's own
1888 dotted-form test (`_dotted`, a period after EACH letter or after
1889 each of two or more letter chunks): '.'
1890 anywhere excludes membership, so a single TRAILING period ('MA.',
1891 'Ed.') is excluded here even though the LEAN still reads it as
1892 the bare acronym's case ('Smith, MA.' -> suffix 'MA.', measured).
1893 Not a contradiction: the two questions this answers -- the
1894 comma-form CANDIDATE test below and the credential-lean disjunct
1895 -- are never asked of those shapes, the TAG path
1896 (`vocab:suffix-ambiguous`, read directly by the trailing peel and
1897 the post-comma slot) and H2's leading-title shape test already
1898 carrying them where they need to go.
1899 """
1900 if "." in text:
1901 return False
1902 n = _normalize(text)
1903 return (n in lexicon._sets.suffix_acronyms_ambiguous
1904 or n in lexicon._sets.suffix_words_ambiguous)
1907# #516's all-caps half, ONE PREDICATE for the shape test and its
1908# WHOLE-VOCABULARY exclusion, shared by the three sites that each
1909# needed the identical question answered (classify's tag emission,
1910# the comma decision's caps test -- `_segment.py`'s run test until #613
1911# moved the decision to `_comma.decide` -- and this module's own unit
1912# tests), where it had been spelled three times over (quality-review
1913# finding). The usual objection to sharing -- a call costing every
1914# default-policy parse a frame it cannot use -- does not apply: the
1915# trailing position is behind `CapsSuffixes.EVERYWHERE` (classify's
1916# first conjunct), and the comma decision, which IS on by default,
1917# asks C-level `isalpha()`/`isupper()` and the suffix tag first, so a
1918# comma name with no unlisted all-caps word never reaches it.
1919def caps_shape_candidate(text: str, lexicon: Lexicon, policy: Policy,
1920 one_case: bool | None) -> bool:
1921 """Whether TEXT is an UNLISTED all-caps credential candidate: two
1922 or more alphabetic characters, or letter chunks joined by '-' or
1923 '/' (`_joined_capitals`, #752), no period (a period fails
1924 `isalpha()` and the joined pattern outright, which is what excludes
1925 the dotted spelling here for free -- no separate '.' test needed),
1926 written in a name
1927 `one_case` says is mixed (`one_case is False`; `None`, "not
1928 established", declines exactly as if the name were one case).
1930 UNLISTED means in NO wordlist at all, not merely "no whole-token
1931 suffix vocabulary", and the roster is `_lexicon._VOCAB_FIELDS`
1932 itself rather than a list written out here -- checked directly
1933 against the lexicon rather than through a caller's tags (classify
1934 is deciding the very tag a caller would read, and `_comma.decide`
1935 asks of words classify left untagged). A hand-written roster is a second place to remember,
1936 and it had already gone wrong: it named eleven of the thirteen
1937 fields, leaving `surnames` and `honorific_tails` out, so
1938 a `Lexicon` whose `surnames` listed "dupont" still read
1939 `Jean Pierre DUPONT` as suffix `DUPONT` -- a caller listing a word
1940 as a SURNAME and getting it read as a credential is this switch's
1941 own worst failure, arriving through the one wordlist that says
1942 "this is a family name" (review round, #289/#516;
1943 `honorific_tails` was already excluded transitively, being a
1944 subset of `suffix_words` by Lexicon invariant, and joins the
1945 roster for completeness rather than for a behavior change).
1946 Measured, #516 review rounds: particles, ambiguous particles,
1947 conjunctions, bound-given heads, a maiden marker ('NEE'/'GEB') and
1948 a title all join the shape by capitalization alone if their field
1949 is left unchecked.
1951 `suffix_acronyms_ambiguous` is in the roster, which also makes
1952 this predicate stand in for `ambiguous_class_member` wherever a
1953 caller needs "and not already a LISTED member" (undotted text's
1954 only path into that function is the identical membership test);
1955 its callers still test the listed set first, in C (classify's
1956 `n not in suffix_acronyms_ambiguous`, `_comma.decide`'s
1957 AMBIGUOUS_ACRONYM_TAG test), so a listed member never pays for
1958 the call.
1959 """
1960 if (policy.unlisted_caps_suffixes is CapsSuffixes.OFF
1961 or one_case is not False):
1962 return False
1963 if not text.isascii():
1964 text = unicodedata.normalize("NFC", text)
1965 if not (len(text) >= 2 and text.isalpha() and text.isupper()
1966 or _joined_capitals(text)):
1967 return False
1968 return not in_any_wordlist(_normalize(text), lexicon)
1971#: A word of letter chunks joined by '-' or '/' ('PU-PH', 'MD/MBA').
1972_JOINED_LETTERS = re.compile(
1973 r'(?:' + _LETTER + r')+(?:[-/](?:' + _LETTER + r')+)+')
1976# rules/S9: "an unlisted word of capitals whose letter chunks are joined
1977# by a hyphen or a slash, one of them of two or more letters, is that
1978# shape too" -- the caps shape's joined spelling (decisions/S9 (#752))
1979def _joined_capitals(text: str) -> bool:
1980 """Whether TEXT is capitals in letter chunks joined by '-' or '/',
1981 one chunk of two or more letters ('PU-PH', 'MS-OMPT', not 'A-B',
1982 'MD/' or 'PU-PH2'). Its callers test `isupper()` and a joiner in C
1983 first, so a word holding neither pays no frame."""
1984 if not (text.isupper() and _JOINED_LETTERS.fullmatch(text)):
1985 return False
1986 for chunk in _JOINERS.split(text):
1987 if composed_len(chunk) >= 2:
1988 return True
1989 return False
1992def is_caps_word(text: str) -> bool:
1993 """Whether TEXT is alphabetic capitals once NFC-composed: `isalpha()`
1994 is False for a combining mark, so the decomposed spelling of
1995 'ŽŠČ' would otherwise not be a caps word (#652). Callers test the
1996 text as written inline first, in C, and call this only for text
1997 that is not NFC (`unicodedata.is_normalized`, also C), so composed
1998 text pays no frame."""
1999 composed = unicodedata.normalize("NFC", text)
2000 return composed.isalpha() and composed.isupper()
2003def composed(text: str) -> str:
2004 """TEXT NFC-composed where it is not ASCII, so a letter typed as its
2005 base and a combining mark is one letter, as the vocabulary lookup
2006 reads it (#665). The per-token predicates (`is_initial_shaped`,
2007 `composed_len`) fold inline instead, a call per token being the
2008 frame budget's scaling cost."""
2009 return text if text.isascii() else unicodedata.normalize("NFC", text)
2012def composed_len(text: str) -> int:
2013 """The length of TEXT in letters as the writer sees them: code
2014 points once NFC-composed. A caps word admitted decomposed (#652)
2015 is longer than its composed twin, and the two-letter pair test
2016 (`John Smith, MÁ`) counts letters."""
2017 return len(text if text.isascii() else unicodedata.normalize("NFC", text))
2020#: The marks a given-name key keeps: the tilde on n and the cedilla on
2021#: c, which spell other letters in Spanish and Catalan (Peña, Llorenç),
2022#: where an acute or grave accent or a diaeresis is how the same name
2023#: is spelled with or without them (José, Jose).
2024_KEPT_MARKS = frozenset({("n", "\u0303"), ("c", "\u0327")})
2027def given_name_key(text: str) -> str:
2028 """`text` folded as `Lexicon.given_names` is matched: the
2029 `_normalize` fold, with the accents and the diaeresis dropped and
2030 the tilde on n and the cedilla on c kept. tools/es_given_names.py
2031 writes the list with this same fold."""
2032 out: list[str] = []
2033 for ch in unicodedata.normalize("NFD", _normalize(text)):
2034 if unicodedata.combining(ch) and not (
2035 out and (out[-1], ch) in _KEPT_MARKS):
2036 continue
2037 out.append(ch)
2038 return unicodedata.normalize("NFC", "".join(out))
2041def is_listed_given_name(text: str, lexicon: Lexicon) -> bool:
2042 """Whether `text` is in `lexicon.given_names`, as written or by
2043 `given_name_key`, so a caller's entry "José" matches as written
2044 and a list written without accents matches "José" too (rules/O6's
2045 veto, #959)."""
2046 names = lexicon._sets.given_names
2047 return bool(names) and (_normalize(text) in names
2048 or given_name_key(text) in names)
2051def in_any_wordlist(n: str, lexicon: Lexicon) -> bool:
2052 """Whether the folded word `n` is in ANY of the lexicon's wordlists
2053 (`_lexicon._VOCAB_FIELDS`, the whole roster but four): the caps
2054 shape's "unlisted" (`caps_shape_candidate`), where a word a caller
2055 listed even as a SURNAME must never become a credential.
2057 The four left out: `title_phrases`, whose entries are two words and
2058 so never one; `titles_ambiguous`, a title that may be a given
2059 name, which outside a title phrase is read as the unlisted word it
2060 is there (rules/H9: "and elsewhere a name word") -- a phrase is two
2061 words and a comma part read by this shape is one; and
2062 `given_name_abbreviations`, a given name written short (#789),
2063 whose one reader is H2's shape, so anywhere else its word reads as
2064 the unlisted word it is ('John Smith, JAS' keeps suffix 'JAS');
2065 and `given_names`, whose one reader is O6 (#959), for the same
2066 reason ('Ana Pérez, LUIS' reads as with the list empty)."""
2067 sets = lexicon._sets
2068 for field in _LISTED_FIELDS:
2069 if n in getattr(sets, field):
2070 return True
2071 return False
2074_LISTED_FIELDS = tuple(f for f in _VOCAB_FIELDS
2075 if f not in ("title_phrases", "titles_ambiguous",
2076 "given_name_abbreviations",
2077 "given_names"))
2080#: The wordlists that claim a word as something OTHER than name text.
2081#: `surnames` and `bound_given_names` are left out: they say a word IS
2082#: a name word ('Kim', 'Abdul'), the opposite claim; and so is
2083#: `titles_ambiguous`, a title that may be a given name (rules/H9), with
2084#: `title_phrases`, whose two-word entries never match one word; and so
2085#: is `given_name_abbreviations`, a given name written short (#789), left
2086#: out of `_LISTED_FIELDS` above, with `given_names` (#959).
2087_NON_NAME_FIELDS = tuple(f for f in _LISTED_FIELDS
2088 if f not in ("surnames", "bound_given_names"))
2091def written_as_a_name(text: str) -> bool:
2092 """Whether TEXT is written the way a name is written in mixed case:
2093 it holds a capital (or titlecase letter) and its last LETTER is
2094 lowercase -- #564's test for the name's case contrast (Derek).
2095 'Smith', 'DiCaprio', 'IJzerman', 'al-Rashid', "d'Estaing",
2096 'McDonald', 'Džokić' pass. A surname written in capitals fails
2097 whatever is glued in front of it ("d'ESTAING", 'al-ASSAD',
2098 'McDONALD', 'FitzGERALD', 'DeVITO', 'St-PIERRE'), and so do a lone
2099 capital and a lowercase-only word.
2101 The letter read is the last one that carries case evidence, not the
2102 last character. The word is composed first (NFC), so a name typed
2103 with decomposed accents reads as its composed spelling ('André'
2104 ends in 'é', not in the combining accent). Passed over, as no
2105 evidence of case: every non-letter ("Jones'", 'Smith2', 'Smith)'),
2106 every caseless letter ("Jonesʼ" with U+02BC, 'Wafāʾ', 'Smith李'),
2107 and a lowercase letter whose capital is not a single character
2108 ('ß' -> 'SS', 'ĸ' with none): 'WEIß' and 'KAĸ' are written in
2109 capitals, 'Weiß' is not. One call and no generator, so a word costs
2110 one frame."""
2111 word = unicodedata.normalize("NFC", text)
2112 i = len(word)
2113 while i:
2114 i -= 1
2115 ch = word[i]
2116 if not (ch.isupper() or ch.islower() or ch.istitle()):
2117 continue # a non-letter or a caseless letter
2118 upper = ch.upper()
2119 if ch.islower() and (upper == ch or len(upper) != 1):
2120 continue
2121 # the capital is asked here, of the word that has a letter to
2122 # read; one with none falls through below ('李', '2', 'ß')
2123 return ch.islower() and text != text.lower()
2124 return False
2127def claimed_as_non_name(n: str, lexicon: Lexicon) -> bool:
2128 """Whether a wordlist claims the folded word `n` as a title,
2129 particle, connective, credential, generation, maiden marker or
2130 honorific -- not name text. #564's `_comma._name_contrast`
2131 asks it: such a word does not supply the name's case contrast. A
2132 different question from `in_any_wordlist`'s, which a surname list
2133 must also answer yes to."""
2134 sets = lexicon._sets
2135 for field in _NON_NAME_FIELDS:
2136 if n in getattr(sets, field):
2137 return True
2138 return False
2141# The candidate test for a comma part read before classify
2142# (rules/C4, decisions/S9): segment's C2 flag on a tail part.
2143def ambiguous_class_candidate(text: str, lexicon: Lexicon,
2144 policy: Policy) -> bool:
2145 """Whether TEXT is a CANDIDATE for the ambiguous credential class
2146 as segment's C2 flag asks it of a part past the second comma,
2147 before classify has tagged anything (`_segment.py`'s `class_run`;
2148 the comma's own decision, `_comma.decide`, reads classify's tags
2149 instead, #613): the LISTED half (`ambiguous_class_member`, case-free) OR, where Policy
2150 admits it, the SHAPE an unlisted dotted token wears
2151 (`period_joined_vocab`'s third verdict, #516).
2153 Case-free throughout, and the CAPS half of the class is
2154 deliberately not here: its real site is `_comma.decide`, which
2155 calls `caps_shape_candidate` directly because the caps shape needs
2156 the name's contrast. A second route through here existed briefly,
2157 behind an optional `one_case` parameter no production caller ever
2158 passed -- segment had nothing to hand in at the single-token
2159 test it then made -- so the branch
2160 answered False for every name the library parsed and only the
2161 unit tests reached it (review-round finding, #289/#516). A dead
2162 second route is a place for the two to disagree, not a
2163 convenience.
2165 A period anywhere is the gate for even ASKING the shape question,
2166 checked before the shape's own two calls: `period_joined_vocab`
2167 already declines a period-free text, but a Python-level call is
2168 not free and a comma name's post-comma part usually has no period
2169 ("Smith, John") -- measured regression, #516 review round, fixed
2170 by hoisting the same cheap substring test `ambiguous_class_member`
2171 already makes for its own reason. Where a period IS present,
2172 `suffix_as_written` runs only after `period_joined_vocab` says
2173 "shape": a dotted whole-token match with no single-chunk
2174 vocabulary hit of its own ('A.B.C.', via 'abc') would otherwise
2175 read "shape" from this function's chunk-level view alone,
2176 oblivious to the WHOLE-token match `suffix_as_written` already
2177 settled -- the same precedence classify's own tag order gives it
2178 (`vocab:suffix` is set before `period_joined_vocab` is even
2179 consulted).
2181 A LISTED member spelled with its periods is excluded from the
2182 shape branch by the same test classify's own shape branch makes:
2183 a caller who puts a dotted entry in `suffix_acronyms_ambiguous`
2184 has said the word is a listed member of this class, and reading
2185 it by shape instead loses the case lean the listing asks for
2186 (review-round finding, #289/#516 -- `Jack A.B.` with 'a.b' listed).
2187 It cannot change the answer for the shipped vocabulary, whose
2188 ambiguous entries carry no period at all.
2190 This function and classify's tag emission (`_tags_for`'s
2191 `derived == "shape"` branch) still ask the SAME question twice, of
2192 necessity: segment's C2 flag runs before `classify` and has no
2193 tags to read yet, so the two stages cannot share the call. Kept from
2194 drifting by
2195 `test_classify.test_ambiguous_class_candidate_agrees_with_the_tag`,
2196 which asks both of the same texts, rather than by a sentence
2197 alone.
2198 """
2199 if ambiguous_class_member(text, lexicon):
2200 return True
2201 if "." in text:
2202 # `_normalize` stays behind the shape verdict, where it always
2203 # was: it is a call, and a dotted post-comma token that is not
2204 # acronym-shaped at all ('Jr.') must not pay for it.
2205 if not (policy.unlisted_dotted_suffixes
2206 and period_joined_vocab(text, lexicon) == "shape"):
2207 return False
2208 n = _normalize(text)
2209 return (n not in lexicon._sets.suffix_acronyms_ambiguous
2210 and not suffix_as_written(n, text, lexicon))
2211 return False
2214# mechanisms.md#UNIT-PARTITION: "A rule that counts name words counts
2215# those units, and takes each whole or not at all" -- C3's count before
2216# a comma being the stated exception (`SURNAME_UNIT_TAGS`). The walk is shared
2217# by readers that hold the facts in different forms: post_rules and
2218# `_comma._whole_name` read classify's TAGS, while `name_word_count`
2219# builds the same tag names from the vocabulary
2220# (`surname_unit_tags`). One walk over tag sets, so the two cannot
2221# disagree about where a unit ends (mechanisms.md#ONE-PREDICATE-PER-
2222# QUESTION) -- only, at worst, about a token's facts, which
2223# test_classify's agreement test pins.
2224def unit_ends(tags: Sequence[Set[str]], chain: bool = True) -> list[int]:
2225 """The END (one past the last index) of each unit of `tags`, in
2226 order, partitioning `range(len(tags))`: one name word each, except
2227 where another rule has already made several words one name -- a
2228 particle and the words it chains (P2), a conjunction-joined run
2229 (P3), and a bound given-name word with the word it completes (P5).
2230 Each element is the tag set of one token, read for "particle",
2231 "conjunction", "vocab:bound-given" and "vocab:suffix".
2233 `chain` picks how far a particle reaches. True is P2's chain: the
2234 particle run and every word after it up to the next stop ("van der
2235 Berg Smith" is one unit). False is P1's fold reach: the particle
2236 run and the ONE unit it attaches to, so "de Mesnil Jean" is two --
2237 the count rules/C3 needs before a comma, where under a
2238 family-first order that part is family 'de Mesnil', given 'Jean'
2239 (#575).
2241 RECURSIVE, and that is the whole point: what a conjunction or a
2242 bound given-name word joins is the next UNIT, not the next word.
2243 Absorbing a single index instead strands a particle at the end of
2244 the unit, severed from the words it chains -- "de la Vega y la
2245 Vega" cut between `la` and `Vega`, reporting family
2246 "de la Vega y la", which is the same defect as a bare particle
2247 opening the given name, mirrored."""
2248 n = len(tags)
2250 def end_of(i: int) -> int:
2251 if "particle" in tags[i] and not chain:
2252 j = i + 1
2253 while j < n and "particle" in tags[j]:
2254 j += 1
2255 end = (end_of(j) if j < n and "conjunction" not in tags[j]
2256 and "vocab:suffix" not in tags[j] else j)
2257 elif "particle" in tags[i]:
2258 j = i
2259 while j + 1 < n and "particle" in tags[j + 1]:
2260 j += 1
2261 # ... then the words it joins, stopping where the next
2262 # particle starts a group of its own, at a suffix word (the
2263 # stop _group's chain uses), or at a conjunction, which the
2264 # shared loop below joins to the whole unit after it rather
2265 # than to the one word after it.
2266 while (j + 1 < n
2267 and "particle" not in tags[j + 1]
2268 and "conjunction" not in tags[j + 1]
2269 and "vocab:suffix" not in tags[j + 1]):
2270 j += 1
2271 end = j + 1
2272 else:
2273 end = i + 1
2274 if "vocab:bound-given" in tags[i] and end < n:
2275 end = end_of(end)
2276 while end + 1 < n and "conjunction" in tags[end]:
2277 end = end_of(end + 1)
2278 return end
2280 ends: list[int] = []
2281 i = 0
2282 while i < n:
2283 i = end_of(i)
2284 ends.append(i)
2285 return ends
2288#: The facts rules/C3's count before a comma reads (#575): a
2289#: particle starts a surname unit, and a suffix word stops it -- any
2290#: suffix word, a wider stop than group's particle chain makes, which
2291#: a suffix word ends only where a suffix run would start at it
2292#: (decisions/P2, #1177); #1259 left this count as it was. A word
2293#: that is ALSO title vocabulary ('Freiherr', 'St') LEADING the part is
2294#: not a particle here: in front of a name it reads as the title it
2295#: also is (rules/H1), so 'Freiherr von Berg, PhD' keeps its title,
2296#: while inside a surname it chains as P1 and P2 chain it ('de St
2297#: Pierre' is one surname). A bound given-name head that is also a
2298#: particle ('Abu') stays a particle: 'Abu Bakar, Ed' is a surname
2299#: before the comma, as 2.0 through 2.3 read it. Neither of the other
2300#: two joins `unit_ends` knows is read. A bound
2301#: given-name pair builds a GIVEN name, and P5 gives up a family word
2302#: where the name has no other ('abdul Salam' alone is given 'abdul',
2303#: family 'Salam'), so before a comma it is not one surname. Whether
2304#: P3 joins a single-letter connective depends on the words of the
2305#: whole name -- 'Carod i Rovira, Josep' joins as four words where
2306#: 'Ortega y Gasset' alone, three words, does not -- and the count is
2307#: part of deciding what the whole name is, so a count taken before
2308#: group could reach P3's answer only by copying P3. script_segment
2309#: and `name_word_count` build these facts from the vocabulary
2310#: (`surname_unit_tags`); `_comma._whole_name` derives them from
2311#: classify's tags (`surname_unit_facts`), so the counts read one set
2312#: of facts.
2313SURNAME_UNIT_TAGS = frozenset({"particle", "vocab:suffix"})
2314_PARTICLE_ONLY = frozenset({"particle"})
2315_SUFFIX_ONLY = frozenset({"vocab:suffix"})
2316_NO_TAGS: frozenset[str] = frozenset()
2317def surname_unit_facts(tags: Set[str], leading: bool) -> frozenset[str]:
2318 """`SURNAME_UNIT_TAGS` for one token, from classify's tags: the
2319 reading `_comma._whole_name`'s count before a comma takes.
2320 `leading` is whether
2321 the token opens the part."""
2322 particle = ("particle" in tags
2323 and not (leading and "vocab:title" in tags))
2324 if "vocab:suffix" in tags:
2325 return SURNAME_UNIT_TAGS if particle else _SUFFIX_ONLY
2326 return _PARTICLE_ONLY if particle else _NO_TAGS
2329def surname_unit_tags(text: str, lexicon: Lexicon,
2330 leading: bool) -> frozenset[str]:
2331 """`surname_unit_facts` for one token from the vocabulary alone --
2332 the vocabulary's view of classify's tags, for a count taken before
2333 classify runs (script_segment's) or without its tags
2334 (`name_word_count`), with
2335 classify's own tests: particle and title membership,
2336 `suffix_as_written`, and the period-joined derivation.
2337 Kept from drifting by
2338 test_classify.test_surname_unit_tags_agree_with_classify."""
2339 n = _normalize(text)
2340 # classify's whole-token test, then its period-joined derivation
2341 # for a token no whole-token title or suffix claimed ('JD.CPA'),
2342 # asked only of a token with a period, which it needs
2343 suffix = suffix_as_written(n, text, lexicon) or (
2344 "." in text and n not in lexicon._sets.titles
2345 and period_joined_vocab(text, lexicon) == "suffix")
2346 particle = n in lexicon._sets.particles and not (
2347 (leading and n in lexicon._sets.titles) or _spells_an_initial(n, text))
2348 if suffix:
2349 return SURNAME_UNIT_TAGS if particle else _SUFFIX_ONLY
2350 return _PARTICLE_ONLY if particle else _NO_TAGS
2353#: The connective after a title that may be a given name which makes
2354#: the two a title phrase, 'Dean of Students' (rules/H9). Coordinating
2355#: connectives are not: 'Dean and Mary Smith' is two people.
2356_PHRASE_LINKS = frozenset({"of"})
2357#: The connectives whose object a modifier after them is in, 'of the
2358#: United States', 'The Grand Master'; 'and' joins two titles instead.
2359_OBJECT_LINKS = frozenset({"of", "the"})
2361#: The article that is the phrase partner of a style that may be a
2362#: given name, written short (rules/H9, #999): `The Hon`.
2363_STYLE_ARTICLES = frozenset({"the"})
2366# rules/H9: "a word no listed phrase holds is a title only behind a
2367# style or a modifier, or before `of`" -- whether the word heads a title
2368# of itself, asked of the phrases `attaches` compiled rather than of a
2369# property of its own
2370def in_a_title_phrase(text: str, lexicon: Lexicon) -> bool:
2371 """Whether a phrase of `lexicon._sets.title_phrases` that classify can
2372 read holds `text`."""
2373 return _normalize(text) in _phrase_words(
2374 lexicon._sets.title_phrases, lexicon._sets.titles, lexicon._sets.titles_ambiguous)
2377@functools.lru_cache(maxsize=16)
2378def _phrase_words(phrases: frozenset[str], titles: frozenset[str],
2379 contested: frozenset[str]) -> frozenset[str]:
2380 """The words of the phrases classify can read: two words, each a
2381 title or a title that may be a given name, as its pair test asks.
2382 A phrase it never consults makes no word report."""
2383 listed = titles | contested
2384 pairs = (p.split(" ") for p in phrases)
2385 return frozenset(w for ws in pairs
2386 if len(ws) == 2 and all(w in listed for w in ws)
2387 for w in ws)
2390# rules/H9: "the word is a title only inside a title phrase, and
2391# elsewhere a name word" -- both halves, the partners of a title that may
2392# be a given name and of a modifier at the head of the name, each its
2393# bullet's (decisions/H9 #752)
2394def title_readings(texts: Sequence[str], folded: Sequence[str],
2395 lexicon: Lexicon, *, head: bool,
2396 titled: Sequence[bool] | None = None) -> list[bool]:
2397 """Which of these words, one part of the name in order, read as
2398 title words. Where none of them is a title that may be a given name
2399 or a modifier, the listed titles answer alone, which every caller
2400 asks first, in C, so that a name with neither pays no frame.
2402 A word of `titles_ambiguous` that is not also in `titles` (where a
2403 caller listing it makes it a title everywhere) is a title only
2404 beside a partner: a
2405 modifier, a style, or a title it stands behind in `title_phrases`,
2406 before it, and for a style of `title_abbreviations` the article
2407 'the' before it ('The Hon' under a Chinese choice); or 'of', or a title it stands in front of in
2408 `title_phrases`, after it. A salutation is never one, nor any other
2409 title before it ('Mr Dean Smith'), nor a title after it that it
2410 does not attach to ('SMITH, DEAN MR.', a given name and the
2411 salutation a donor list writes after it). Its neighbours are asked
2412 as LISTED, so 'Vice Dean' reads 'Vice' as the partner of 'Dean'
2413 whatever the modifier test below makes of 'Vice', save a word before
2414 it that is itself a title that may be a given name, which is a
2415 partner only where its own phrase made it a title ('Grace Dean
2416 Smith' reads 'Grace' as the given name, and 'Dean' with it).
2418 Where `head` is true -- the part before the first comma -- a word
2419 of `title_modifiers` is a title only with a head title behind it in
2420 its run of titles: any title that is not a modifier, or a word H2's
2421 shape reads as one, before 'of' ('Mother Superior of the Convent'),
2422 behind a title that attaches to it in `title_phrases` ('Princess
2423 Royal', #913), or in the object of 'of' or 'the' there (decision 10 of
2424 decisions/vocabulary-model). Only the leading run is the head: a
2425 modifier outside it is read as listed, one ending the name being
2426 H5's. A
2427 word in both sets takes the modifier test in every part, that run
2428 being its phrase ('Lau, Wing Kai' reads 'Wing' as the given name).
2429 After a comma any other modifier is read as listed: a part there
2430 that is a lone title word is the person's role ('John Smith,
2431 Resident'), and reading it as a name would reverse the name
2432 (decisions/H9).
2434 `texts` are the words as written, for the shape test; `folded`
2435 their `_normalize`d forms; `titled` which of them a caller already
2436 reads as titles, where that is more than `titles` lists (classify's
2437 period-joined `Lt.Gov.`), else membership in `titles`. Only a word
2438 of the two sets is ever read out of the title."""
2439 contested = lexicon._sets.titles_ambiguous
2440 modifiers = lexicon._sets.title_modifiers
2441 titles = lexicon._sets.titles
2442 # a word in titles is a title wherever it stands, whatever else it
2443 # is listed as; one of titles_ambiguous alone is listed as a title
2444 # to its neighbours and read by its phrase
2445 reading = [n in titles or n in contested for n in folded]
2446 if titled is not None:
2447 reading = [r or t for r, t in zip(reading, titled)]
2448 listed = list(reading)
2449 phrases = lexicon._sets.title_phrases
2450 last = len(folded) - 1
2451 for i, n in enumerate(folded):
2452 if n in titles or n not in contested or n in modifiers:
2453 continue
2454 prev = folded[i - 1] if i else None
2455 nxt = folded[i + 1] if i < last else None
2456 reading[i] = bool(
2457 # the word before as it was READ, left to right: a title that
2458 # may be a given name and stands in no phrase is no partner
2459 # ('Grace Dean Smith')
2460 (prev is not None and reading[i - 1]
2461 and (prev in modifiers or prev in lexicon._sets.title_styles
2462 or f"{prev} {n}" in phrases))
2463 # rules/H9: "a style that may be a given name, written short,
2464 # has its phrase partner in the article before it" -- 'The
2465 # Hon' under a Chinese choice (decisions/H9, #999)
2466 or (prev in _STYLE_ARTICLES and n in lexicon._sets.title_styles
2467 and n in lexicon._sets.title_abbreviations
2468 and prev in lexicon._sets.conjunctions)
2469 or (nxt is not None
2470 and ((nxt in _PHRASE_LINKS and nxt in lexicon._sets.conjunctions)
2471 or (listed[i + 1] and f"{n} {nxt}" in phrases))))
2472 # left to right, whether the run so far has passed 'of' or 'the': a
2473 # modifier after it is in the object of a phrase ('President of the
2474 # United States', 'the Queen's Bench'), which no head need follow. A
2475 # coordinating connective opens no object: 'Mr. and Mrs. Wing Lau'
2476 # is two salutations and a given name
2477 linked = [False] * len(folded)
2478 conjunctions = lexicon._sets.conjunctions
2479 for i in range(1, len(folded)):
2480 if reading[i - 1] or folded[i - 1] in conjunctions:
2481 linked[i] = linked[i - 1] or folded[i - 1] in _OBJECT_LINKS
2482 # the head of the name is its LEADING run of titles and connectives:
2483 # a modifier ending the name is H5's trailing run, read as listed,
2484 # save one that may be a given name, which the loop below reads
2485 # wherever it stands
2486 lead = 0
2487 if head:
2488 while lead < len(folded) and (reading[lead] or folded[lead] in conjunctions
2489 or is_title_shaped(texts[lead], lexicon)):
2490 lead += 1
2491 # right to left, so each modifier knows whether a head title stands
2492 # behind it before the run is broken; a connective carries the run
2493 behind = False
2494 for i in range(last, -1, -1):
2495 n = folded[i]
2496 if (n in modifiers and reading[i]
2497 and ((head and i < lead) or (n in contested and n not in titles))):
2498 reading[i] = behind or linked[i] or (
2499 i < last and folded[i + 1] in _PHRASE_LINKS
2500 and folded[i + 1] in conjunctions) or (
2501 # a title before it that attaches to it ('Princess Royal')
2502 i > 0 and reading[i - 1]
2503 and f"{folded[i - 1]} {n}" in phrases)
2504 behind = reading[i]
2505 elif reading[i] or (head and is_title_shaped(texts[i], lexicon)):
2506 behind = True
2507 elif n not in conjunctions:
2508 behind = False
2509 return reading
2512def name_word_count(texts: Sequence[str], lexicon: Lexicon,
2513 policy: Policy, letters_one_case: bool | None = None,
2514 members: Set[int] = frozenset()) -> int:
2515 """How many of these texts are NAME words -- not suffix
2516 vocabulary, not title vocabulary, nor a numeral the trailing fork
2517 reads as the generation ending the name words (rules/C3).
2519 rules/C4's count for the ambiguous class, and it is of names
2520 rather than of words because 'Smith Jr., MA' is two tokens and one
2521 name: counting tokens there flips the structure and hands the
2522 family to `given`, which no reading of that string wants
2523 (decisions/S6). The suffix half asks the POLICY-selected
2524 predicate, the same one is_wholly_suffix asks, so the two agree
2525 about what a suffix word is; the title half asks BOTH the listed
2526 lookup and H2's shape test (`is_title_shaped`) -- a bare
2527 `lexicon._sets.titles` lookup here read 'Xyz.' as a name word where the
2528 leading peel reads it as a title, and that divergence is recorded
2529 once, at `is_title_shaped` itself
2530 (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
2532 It counts UNITS holding a name word, not tokens (#575,
2533 mechanisms.md#UNIT-PARTITION): a particle chain (P2) builds one
2534 family name, so 'De La Cruz' is one name word and 'De La Cruz, Ed'
2535 reads as 'Royce, Ed' does. Why the other two joins are not read
2536 here: `SURNAME_UNIT_TAGS`.
2538 `letters_one_case` is `is_wholly_suffix`'s, for the same reason,
2539 and the case fact the generation fork reads (`ParseState.one_case`).
2540 `members` holds the positions of split credentials' members
2541 (classify's tag), each half a credential rather than a name, so
2542 one opening the part stays no name word, as `_comma._whole_name`
2543 reads it.
2544 """
2545 lenient = policy.lenient_comma_suffixes
2547 # One pass: each token's name-word verdict, and whether any token
2548 # is a particle. With none, every token is a unit of its own and
2549 # the count is a sum, so the commonest comma names never build the
2550 # units -- and pay no frame the token loop did not already pay.
2551 names: list[bool] = []
2552 suffixes: list[bool] = []
2553 has_particle = False
2554 lead: int | None = None
2555 lead_suffix = False
2556 # map, not a comprehension, which is a frame of its own on 3.11
2557 folded = list(map(_normalize, texts))
2558 # rules/H10: "the run reads as that title where a title the run
2559 # keeps stands behind it" -- the run read as the title it spells, so
2560 # the title test and H9's phrase test below see it. Unlike classify's
2561 # fold, which runs only where the name already holds a word H9
2562 # reads, this one always runs, so a run alone brings the phrase test
2563 # in. The set test first, C-level, so a dotted word that spells no
2564 # marked title pays no frame
2565 dotted: set[int] = set()
2566 for i, n in enumerate(folded):
2567 if ("." in n and n.replace(".", "") in lexicon._sets.dotted_titles
2568 and is_dotted_title(texts[i], n, lexicon)):
2569 folded[i] = n.replace(".", "")
2570 dotted.add(i)
2571 # rules/H9: "the word is a title only inside a title phrase" -- the
2572 # part before the comma is the head of the name, read as classify
2573 # reads it
2574 titled = (None if lexicon._sets.titles_ambiguous.isdisjoint(folded)
2575 and lexicon._sets.title_modifiers.isdisjoint(folded)
2576 else title_readings(texts, folded, lexicon, head=True))
2577 for i, n in enumerate(folded):
2578 text = texts[i]
2579 title = n in lexicon._sets.titles if titled is None else titled[i]
2580 if title and i in dotted:
2581 # a title only opening the part (behind any titles), and only
2582 # with a title, listed or by H2's shape, behind it and a word
2583 # of any kind behind that: H6 leaves the part's last word to
2584 # the family, so a title ending the part is not kept and the
2585 # run is given back ('U.S. Senator, Ed'); H3's floor is not
2586 # modelled here (decisions/H10)
2587 nxt = i + 1
2588 title = (lead is None and nxt < len(folded)
2589 and ((folded[nxt] in lexicon._sets.titles
2590 if titled is None else titled[nxt])
2591 or is_title_shaped(texts[nxt], lexicon))
2592 and nxt + 1 < len(folded))
2593 if n in lexicon._sets.particles:
2594 has_particle = True
2595 suffix = (is_suffix_lenient(text, lexicon) if lenient
2596 else is_suffix_strict(text, lexicon, letters_one_case))
2597 name = not (suffix or title or is_title_shaped(text, lexicon))
2598 # the opening word behind any titles; the shape test asked again
2599 # only of a suffix word there, so a run of them pays no frame
2600 if lead is None and (name or suffix and not title
2601 and not is_title_shaped(text, lexicon)):
2602 lead = i
2603 lead_suffix = not name and i not in members
2604 names.append(name)
2605 suffixes.append(suffix)
2606 # rules/C3: "A suffix word opening the part is a name word to the
2607 # count where a name word follows it" -- opening it behind its
2608 # titles, which stay no name word, this count being of names
2609 # (rules/C4)
2610 if lead_suffix and lead is not None and any(names[lead + 1:]):
2611 names[lead] = True
2612 # rules/C3: "a numeral ending the part, or standing before the
2613 # listed suffix words it ends with, that the trailing fork reads as the
2614 # generation is no name word", as a listed one is not
2615 # -- the suffix words stepped over as `_comma._whole_name` steps
2616 # over them, a dual of the title and suffix vocabulary included,
2617 # and a title that is no suffix word stopping both
2618 e = len(texts) - 1
2619 while e > 0 and suffixes[e]:
2620 e -= 1
2621 # the numeral's shape asked in C first, as group's count asks it, so
2622 # a part ending in a plain word pays no frame
2623 if (e > 0 and names[e]
2624 and _ROMAN.match(texts[e].rstrip(".")) is not None
2625 and ends_in_a_generation(texts[e - 1:e + 1], letters_one_case)):
2626 names[e] = False
2627 if not has_particle:
2628 return sum(names)
2629 count = 0
2630 start = 0
2631 for end in unit_ends([surname_unit_tags(t, lexicon, i == 0)
2632 for i, t in enumerate(texts)],
2633 chain=False):
2634 if any(names[start:end]):
2635 count += 1
2636 start = end
2637 return count
2640def is_wholly_suffix(texts: Sequence[str], lexicon: Lexicon,
2641 policy: Policy, one_case: bool | None = None,
2642 letters_one_case: bool | None = None,
2643 strict: bool = False) -> bool:
2644 """Every token in a RUN counts as a suffix -- C1's suffix test of
2645 a run, shared by `_comma.decide`, segment's C2 flag on a tail part
2646 and the peel (#319).
2648 NOT the plural of _script_segment._is_post_nominal, which asks
2649 is_suffix_strict per token. This asks the POLICY-selected predicate
2650 (lenient by default), plus period_joined_vocab, delimiter
2651 transparency and the Ph./D. merge. 'V.' is the input that tells
2652 them apart: it satisfies this predicate but is not a post-nominal
2653 -- and reading one for the other IS the #319 bug.
2655 An EMPTY run is False, not vacuously True: v1's suffix-comma
2656 detection fails on an empty parts[1] ('John Smith,, MD' is a
2657 family-comma parse). The 'wholly' idiom agrees -- _script_matcher's
2658 whole=True requires non-empty too -- which is why the name is that
2659 one rather than all_suffixes, where Python's all([]) would promise
2660 the opposite.
2662 An adjacent Ph./D. pair counts as ONE unit (v1's fix_phd extracted
2663 the credential pre-parse, so 'Smith, Ph. D.' read as suffix-comma):
2664 `split_pair`'s "suffix" answer, which group's merge and
2665 `_comma._pieces` read too. Its "split" answer ('Pharm D') does not
2666 count here: that pair is a member of the ambiguous class, which no
2667 run test takes by itself, as it takes no bare 'Ma' (rules/C4's
2668 count reads it instead).
2670 `one_case` is ParseState.one_case, and it admits the LEAN: a bare
2671 ambiguous acronym written in capitals inside a mixed-case name is
2672 a credential here, so 'Steven Hardman, MD, DO, DDS' reads its
2673 third segment as the credential run it is (#289). None -- the
2674 default, and what every caller with no state to ask has -- reads
2675 as no lean and is this predicate's behavior in every release
2676 before 2.4.
2678 `letters_one_case` is the same fact asked a different question,
2679 and the two are kept apart because a caller that wants no lean
2680 still wants the letters read by T4: under
2681 `lenient_comma_suffixes=False` the strict test vetoes an
2682 initial, and in a name written in one case a bare lowercase
2683 letter is one ('john smith, v' reads as 'JOHN SMITH, V', #727).
2684 The lenient test takes a suffix word whatever its shape, so the
2685 default never reads it.
2687 Neither Policy.unlisted_dotted_suffixes NOR
2688 Policy.unlisted_caps_suffixes reaches this predicate: admitting a
2689 by-shape member here unconditionally (an earlier version of this
2690 docstring described exactly that, for the dotted half alone)
2691 bypassed both the lean AND the NAME-word count, and combined with
2692 C1's then TOKEN-count disjunct in segment (deleted in #613, when
2693 the decision moved to `_comma.decide`) it flipped
2694 'Smith Jr., A.B.' to given 'Smith', suffix 'Jr., A.B.' with a
2695 self-contradicting report ("holds 1 name words, so it is read as
2696 a credential run") -- proved by mutation testing to be otherwise
2697 unreached: nothing but this predicate's own two unit tests
2698 depended on it, and 'John Smith, X.Y.Z.' flipped correctly through
2699 the NAME-word count alone (#516 review round). Neither by-shape
2700 class, dotted or caps, reaches the comma form through this
2701 predicate at all -- only through classify's shape tags and
2702 `_comma.decide`'s caps test (#613).
2704 A delimiter core glued to one word ('MD;', ';PhD') leaves the word
2705 read by the test it would get without the core (D15, rules/C10),
2706 so 'MD;' counts where 'MD' does. `strict` asks the strict token
2707 test whatever the policy says: `_comma.decide` passes it behind a
2708 head that is not a whole name, where C8 reads 'Smith, V MD' with
2709 the initial veto, so that 'Smith, V; MD' reads as its space twin.
2710 """
2711 if not texts:
2712 return False
2713 lenient = policy.lenient_comma_suffixes and not strict
2714 # v1 expand_suffix_delimiter parity (#206): a configured delimiter
2715 # is TRANSPARENT in the all-suffix tests -- v1 split the part string
2716 # on the delimiter before checking, so the delimiter never counted;
2717 # the built-in cores are transparent the same way (rules/C10)
2718 cores = delimiter_cores(policy.extra_suffix_delimiters)
2719 inner = glued_cores(policy.extra_suffix_delimiters)
2721 def counts_as_suffix(text: str) -> bool:
2722 if (one_case is not None
2723 and ambiguous_class_member(text, lexicon)
2724 and ambiguous_lean(text, one_case, lexicon_cue(
2725 _normalize(text), lexicon)) == "credential"):
2726 return True
2727 if ((is_suffix_lenient(text, lexicon) if lenient
2728 else is_suffix_strict(text, lexicon, letters_one_case))
2729 or period_joined_vocab(text, lexicon) == "suffix"):
2730 return True
2731 # the split only where a core stands in the word: the built-in
2732 # ';' makes `inner` non-empty for every policy, and a call per
2733 # word would be a frame on every name word a run test asks
2734 for core in inner:
2735 if core in text:
2736 stem = glued_stem(text, inner)
2737 if stem is not None:
2738 return counts_as_suffix(stem)
2739 return splits_into_suffixes(text, inner, lexicon)
2740 return False
2742 merged = list(texts)
2743 k = 0
2744 while k < len(merged) - 1:
2745 if (SPLIT_PAIR_OPENS.fullmatch(merged[k])
2746 and split_pair(merged[k], merged[k + 1], lexicon)
2747 == "suffix"):
2748 merged[k:k + 2] = ["phd"]
2749 else:
2750 k += 1
2751 return all(t in cores or counts_as_suffix(t) for t in merged)
2754# The two derived views of a marker vocabulary, cached per-vocabulary
2755# on _script_segment._longest_entry's precedent -- same function shape
2756# (a scalar derived from a vocabulary frozenset), same key space, and
2757# its reasoning for maxsize=16 carries over verbatim: a process holds
2758# the default vocabulary plus one per constructed pack parser, so 16
2759# bounds many-lexicon churn without ever evicting in normal use.
2760# (NOT _extract._delimiter_chars' precedent, which these once cited:
2761# that one is consulted once per parse, so it says nothing about a
2762# lookup on the per-token path.) Keying on the frozenset costs a
2763# cached hash, not a sweep of its contents.
2764@functools.lru_cache(maxsize=16)
2765def _longest_marker(markers: frozenset[str]) -> int:
2766 """How many words the longest entry of `markers` spans -- the
2767 lookahead bound, computed from the vocabulary rather than fixed at a
2768 literal so a caller's four-word entry works and an all-single-word
2769 set leaves the common path at one lookup. Entries are stored
2770 space-joined with single separators, so the space count IS the word
2771 count."""
2772 return max((entry.count(" ") + 1 for entry in markers), default=0)
2775@functools.lru_cache(maxsize=16)
2776def _marker_heads(markers: frozenset[str]) -> frozenset[str]:
2777 """The first word of every entry -- the set a run can possibly open
2778 with. Entries are already stored per-word folded, so an entry's
2779 first word is the same fold the lookup builds."""
2780 return frozenset(entry.split(" ", 1)[0] for entry in markers)
2783def maiden_marker_head(n: str, markers: frozenset[str]) -> bool:
2784 """Could a maiden marker run START here? `n` is _normalize(word),
2785 passed in so callers fold once -- suffix_as_written's shape exactly
2786 ("`n` is `_normalize(text)`, passed in so callers normalize once"),
2787 and with no raw-text sibling for the same reason it has none: every
2788 caller is on the per-token path and has the fold in hand already.
2790 A SUPERSET test. True means only that some entry opens with this
2791 word, never that a run matches -- maiden_marker_run is the answer,
2792 and it calls this function, so the two cannot drift. Exported
2793 because a caller scanning a whole token stream needs to know
2794 whether assembling a candidate sequence is worth doing at all, and
2795 for almost every token it is not: _classify's pass would otherwise
2796 walk structural boundaries once per token to build a lookahead the
2797 predicate discards on this very test.
2798 """
2799 return n in _marker_heads(markers)
2802def maiden_marker_run(words: Sequence[str], markers: frozenset[str]) -> int:
2803 """How many of `words` a maiden marker claims, longest first; 0 for
2804 none.
2806 Phrases are stored space-joined and per-word normalized (_title_key's
2807 storage rule), so the key is rebuilt the same way here -- normalizing
2808 the joined phrase instead would leave interior periods.
2810 `words` must be words that stand TOGETHER -- one clause's, or one
2811 segment's. This answers only what the vocabulary says about the
2812 sequence it is handed; whether a sequence is a sequence is the
2813 caller's, and classify's contiguity rule is where that is decided
2814 for the token stream.
2816 Longest first, so a caller configuring both 'geb' and 'geb von' gets
2817 the phrase where it matches and the bare word everywhere else. The
2818 one answer to "does a marker start here, and where does it end": the
2819 stages that can call it do (classify over token texts, extract over
2820 a clause's whitespace words), and the stage that runs after classify
2821 reads the tags classify recorded instead
2822 (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
2823 """
2824 # Fast path first: no entry opens with this word, so no length can
2825 # match. It cannot hide a match -- every key the loop builds opens
2826 # with _normalize(words[0]) unless that word folds away, and a
2827 # folded-away first word fails the n-word test below for every
2828 # n > 1 and is not a stored entry for n == 1.
2829 if not words:
2830 return 0
2831 head = _normalize(words[0])
2832 if not maiden_marker_head(head, markers):
2833 return 0
2834 cap = min(len(words), _longest_marker(markers))
2835 # Fold each word ONCE, then key from a prefix. _title_key(words[:n])
2836 # per candidate length re-folds the whole prefix every time, which
2837 # is quadratic in cap and makes a one-word hit cost more than a
2838 # two-word one -- the longest key is always built and discarded
2839 # first, and all but one shipped entry is a single word. The join
2840 # below IS _title_key's body over pre-folded words (per-word fold,
2841 # empties dropped, space-joined); test_vocab pins that the two
2842 # agree, since this is a copy of a fold whose definition lives in
2843 # _lexicon.
2844 folded = [head] + [_normalize(w) for w in words[1:cap]]
2845 for n in range(cap, 0, -1):
2846 key = " ".join(filter(None, folded[:n]))
2847 # a word that folds away is DROPPED from the key, so 'née'
2848 # followed by a lone '.' would key as 'née' and a one-word
2849 # marker would claim the period as part of its run. n words in,
2850 # n words out: anything else is not this key's n-word phrase.
2851 if key.count(" ") == n - 1 and key in markers:
2852 return n
2853 return 0
2857# rules/M3: "a marker followed by a comma is no marker"
2858def comma_follows_marker(word: str) -> bool:
2859 """Whether `word`, the one standing after a marker run inside a
2860 delimited clause, opens with a comma -- standing alone (' , ') or
2861 glued to the name after it (' ,Jones'). A comma glued to the marker
2862 itself ('née,') needs no test: that word is no marker to begin with,
2863 the vocabulary storing no comma. The one predicate for extract's M3
2864 test over whitespace words and group's M1 drop over tokens, which
2865 since #653 hold the same comma
2866 (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
2868 The comma need not be the word's first character, only the first
2869 that is a comma or a letter or digit. A whitespace word can open
2870 with what the tokenizer drops or divides at -- a bidi mark, an
2871 emoji, a nakaguro -- so '\u200f, Jones' is a comma-led WORD only
2872 past its mark, while the token group asks about is ',' already;
2873 reading past the rest makes the two sites agree by construction
2874 (#653's second review)."""
2875 for ch in word:
2876 if ch in COMMA_CHARS:
2877 return True
2878 if ch.isalnum():
2879 return False
2880 return False
2883def tag_marker_runs(tokens: Sequence[WorkToken], comma_offsets: Sequence[int],
2884 markers: frozenset[str],
2885 folded: Sequence[str] | None = None) -> dict[int, str]:
2886 """Which tokens are maiden marker runs: index -> "vocab:maiden-marker"
2887 for a run's head, "vocab:maiden-marker-cont" for the rest.
2889 Returns the decision rather than rewriting the tokens: a caller
2890 that writes tags into the one pass that builds them (classify)
2891 consults this map rather than deciding a run twice, so a marker
2892 token is never replaced twice; a caller that never writes tags at
2893 all (own_words) reads the same map for its span instead.
2895 Moved here from classify (#289/#516) so a caller outside classify
2896 -- _pieces.own_words, when it has no marker map yet -- can build
2897 the SAME map classify would, rather than approximating it with a
2898 text-only head test. `_pieces.py` may import only _state and
2899 _vocab (tests/v2/test_layering.py), so the one shared function
2900 both sites call has to live here, in the layer both can reach.
2902 `folded` is _normalize per token; classify has already built it
2903 for the vocabulary-tag pass and hands it over so this pass costs
2904 no second fold, and a caller with only tokens (own_words' self-
2905 built path) omits it and pays the fold here instead -- paid only
2906 on that path, never on classify's, so the reference name's frame
2907 count is unchanged (#289/#516).
2909 The one sequence pass in this stage, and it has to be one: a marker
2910 entry may be a PHRASE whose words are not markers individually
2911 ('z', 'domu'), so no per-token membership test can find it.
2912 Left to right, longest first at each position, then skip past what
2913 the run claimed -- a second marker cannot start inside the first.
2915 This is where the tag is DECIDED for the stages that read it
2916 afterwards. group runs later and asks its questions of these tags
2917 rather than re-deriving the run (the recorded-answer half of
2918 mechanisms.md#ONE-PREDICATE-PER-QUESTION); extract runs EARLIER,
2919 before tokens exist, so it calls the predicate itself over the
2920 clause's whitespace words.
2922 A tagged run is structurally contiguous, and the test is
2923 one-directional: a role change IS a clause edge, so no run spans
2924 one, but not every clause edge is a role change -- two ADJACENT
2925 clauses of the same role are indistinguishable here, and
2926 'Jane (z) (domu) Jones' does tag a run across them. Both consumers
2927 refuse that run for reasons of their own (the piece walk never sees
2928 role-bearing tokens at all; the clause drop is scoped to one
2929 clause's span), so no reading depends on it today, and the claim
2930 this pass can honestly make is the weaker one. What it does
2931 guarantee is what _group._marker_run_pieces needs: a run inside the
2932 MAIN stream stays inside one segment. Without it this pass walked
2933 the whole span-sorted stream while group walked one segment --
2934 _segment keeps only role-less tokens and buckets them by the commas
2935 before them -- so a run half inside a bracketed clause was tagged
2936 whole and consumed as a proper PREFIX of itself, and
2937 'Anna z (domu) Nowak' read family 'Anna', maiden 'Nowak': the bare
2938 preposition eating the name, which is the exact damage the phrase
2939 entry exists to prevent. Refusing to tag such a run is the fix;
2940 truncating it instead would hand M2 the same wrong prefix one word
2941 shorter.
2942 """
2943 # the lookahead the vocabulary actually needs; 0 for an empty set,
2944 # which skips the pass entirely
2945 cap = _longest_marker(markers)
2946 if not cap:
2947 return {}
2948 if folded is None:
2949 folded = [_normalize(t.text) for t in tokens]
2950 n_tokens = len(tokens)
2951 # Deferred, not computed up front: only the contiguity walk reads
2952 # it, only a phrase vocabulary runs that walk, and only at a token
2953 # that opens an entry -- so a single-word vocabulary, and a
2954 # phrase vocabulary over a name holding no marker, never pay the
2955 # sweep at all.
2956 buckets: list[int] | None = None
2957 tags: dict[int, str] = {}
2958 i = 0
2959 while i < n_tokens:
2960 # The predicate's own head test first, over the fold the caller
2961 # already has: almost no token opens any entry, and for those
2962 # there is nothing to assemble. Same function maiden_marker_run
2963 # consults, so a token skipped here is one it would refuse.
2964 if not maiden_marker_head(folded[i], markers):
2965 i += 1
2966 continue
2967 # Bound the lookahead at the first structural boundary, so the
2968 # predicate is asked over the words that could form one run and
2969 # answers longest-first WITHIN them -- a two-word entry refused
2970 # at a clause edge still leaves a one-word entry starting there
2971 # free to match.
2972 limit = 1
2973 if cap > 1:
2974 if buckets is None:
2975 buckets = [comma_bucket(t.span.start, comma_offsets)
2976 for t in tokens]
2977 role, bucket = tokens[i].role, buckets[i]
2978 while (limit < cap and i + limit < n_tokens
2979 and tokens[i + limit].role is role
2980 and buckets[i + limit] == bucket):
2981 limit += 1
2982 run = maiden_marker_run(
2983 [tokens[k].text for k in range(i, i + limit)], markers)
2984 if not run:
2985 i += 1
2986 continue
2987 tags[i] = "vocab:maiden-marker"
2988 for k in range(i + 1, i + run):
2989 tags[k] = "vocab:maiden-marker-cont"
2990 i += run
2991 return tags
2994def _normalized_for_script(text: str) -> str | None:
2995 """The guard AND the two normalizations single_script and
2996 effective_script's license path both need, single-sourced so they
2997 cannot drift: trailing full stops are dropped (FULL_STOPS, #323),
2998 then None for the two shapes neither ever classifies (nothing
2999 left, and the common all-ASCII Latin token -- skipped before
3000 normalizing, since ASCII is already NFC and every _SCRIPT_RANGES
3001 entry is non-ASCII regardless), else an NFC-normalized copy.
3003 Trailing stops, not raw: a period glued to a script-written token
3004 ('양.', '太郎.') is not a character of any script, so classifying
3005 raw text handed the token no script at all, and three readers
3006 spent that None -- the surname site stepped past the family name
3007 onto the given name ('양. 지훈' cut 지훈 in half), the order rule
3008 fell back to positional ('양 지훈.' lost family-first), and the
3009 segmenter's neighbour precondition missed a writer-drawn boundary
3010 ('山田太郎 田中.' consulted the segmenter on 山田太郎 as if it stood
3011 alone). The scripts this classifies -- every _SCRIPT_RANGES entry,
3012 which today coincide with _policy._NO_INITIALS (#320), a
3013 coincidence _policy says a new member must not inherit -- have no
3014 initials and no period abbreviations, so a stop on such a token
3015 carries no information about the word; ASCII text is stripped too,
3016 but the guard below returns None for it regardless, so 'Smith.'
3017 never classifies. TRAILING only, matching the surname site's own
3018 rstrip in _script_segment (the same arithmetic, not a shared
3019 gate -- this fold decides only whether the surname site, the order
3020 rule and the segmenter ever see the token): a leading stop is not
3021 a shape any script writes before a name word, and HIDING such a
3022 token from those three readers -- no script, so no surname site,
3023 which is what the tree before #323 did -- is safer than admitting
3024 it. Admitted, '.김민준' classifies as hangul, becomes a surname
3025 site, is declined by the head match (which rstrips) and falls
3026 through to a configured segmenter, which answering offset 1
3027 divides it into the stop and the name. The peel is not gated by
3028 this fold; its own rstrip carries a leading-stop token, see
3029 _script_segment. The vocabulary fold alone reads BOTH edges
3030 (_lexicon._normalize): '.씨' is still the honorific, and a lookup
3031 divides nothing.
3033 NFC, not raw: NFD input decomposes precomposed katakana onto a
3034 base character plus a COMBINING mark (U+3099/U+309A, which sit in
3035 the HIRAGANA block, not katakana's), so classifying raw NFD text
3036 can hand a pure-katakana token the kana license by accident.
3037 Decomposed hangul is conjoining jamo, which HANGUL holds, so the
3038 fold no longer decides a hangul token (decisions/nfd-shapes,
3039 #1087).
3041 Voicing marks, last: NFC composes a mark only where a precomposed
3042 kana exists (カ + U+3099 is ガ), so ア + U+3099, ン + U+3099 and
3043 every spacing mark (U+309B/U+309C) survive it, and all four marks
3044 sit in the HIRAGANA block. A mark voices the kana it follows and
3045 belongs to neither syllabary, so each one standing after a kana is
3046 deleted from the copy (_drop_kana_voicing_marks) and the token
3047 takes its base's script: ア゙イ is katakana, not kana-licensed
3048 Japanese (#596). A mark after anything else stays, and stays in
3049 the hiragana block, as it always has. Classification-only and
3050 read-only, like the rest: the
3051 returned copy is never what gets tokenized, so token text and
3052 spans stay exactly what the caller wrote.
3054 Vocabulary MATCHING composes NFC too, since #322
3055 (_lexicon._normalize folds every lookup and every stored entry the
3056 same way), so an NFD suffix word reaches its NFC entry. What stays
3057 raw is SEGMENTATION -- the surname site's direct membership test
3058 and the peel's tail slice index the token's own text -- where NFD
3059 degrades to no-split, never to a wrong split (decisions/W1,
3060 the 2026-07-29 ja amendment).
3061 """
3062 text = text.rstrip(FULL_STOPS)
3063 if not text or text.isascii():
3064 return None
3065 composed = unicodedata.normalize("NFC", text)
3066 if _VOICING_MARKS.isdisjoint(composed):
3067 return composed
3068 return _drop_kana_voicing_marks(composed)
3071def _classify(normalized: str) -> Script | None:
3072 """The FIRST _SCRIPT_MATCHERS entry covering all of `normalized`
3073 (already NFC, via _normalized_for_script), else None. Shared by
3074 both public classifiers so each of them normalizes exactly once."""
3075 for script, matcher in _SCRIPT_MATCHERS.items():
3076 if matcher(normalized):
3077 return script
3078 return None
3081def single_script(text: str) -> Script | None:
3082 """The one Script whose ranges cover EVERY char of `text`, else
3083 None (mixed-script text has no well-defined convention to apply;
3084 the caller falls back to the positional default). Classifies an
3085 NFC-normalized copy of `text` -- see _normalized_for_script.
3086 Callers wanting the kana-mixed license (a kanji+kana composite
3087 resolving to HIRAGANA) want effective_script, not this function."""
3088 normalized = _normalized_for_script(text)
3089 if normalized is None:
3090 return None
3091 return _classify(normalized)
3094def effective_script(text: str) -> Script | None:
3095 """single_script, extended by the kana license (#272 amendment):
3096 a MIXED token wholly within Han∪hiragana∪katakana is Japanese --
3097 it necessarily contains kana (pure Han is not mixed), cannot be
3098 Chinese, and is not a foreign transcription (those are
3099 katakana-only: マイケル has no kanji, but さくらエミ -- hiragana
3100 plus katakana -- is kana-only AND licensed) -- and resolves to the
3101 HIRAGANA carrier entry. Pure-katakana stays KATAKANA
3102 (single_script's answer): a lone katakana token may be a
3103 transcribed foreign name or a Japanese reading, and the script
3104 cannot say which, so nothing defaults on it. Halfwidth kana is
3105 katakana by the table (#594), so 山田ﾀﾛｳ is licensed too."""
3106 # None for both shapes _wholly_ja could never match anyway (empty
3107 # text, or all-ASCII text): real work, not a leftover "if text"
3108 # guard, since the ASCII case is one a bare emptiness check would
3109 # let through. The single normalized copy then serves both the
3110 # single-script answer and the license below.
3111 normalized = _normalized_for_script(text)
3112 if normalized is None:
3113 return None
3114 script = _classify(normalized)
3115 if script is not None:
3116 return script
3117 if _wholly_ja(normalized):
3118 return Script.HIRAGANA
3119 return None
3122def resolve_script_set(scripts: Iterable[Script]) -> Script | None:
3123 """Generalizes effective_script's kana license from one token's
3124 CHARACTERS to a whole name's PIECES (#272): `scripts` is the
3125 effective_script of every name token, already resolved
3126 individually -- '高橋' (Han) and 'みなみ' (Hiragana) are two
3127 separately single-script pieces (split by a space, not mixed
3128 within one token), but together are exactly the repertoire
3129 effective_script licenses inside a single token (高橋みなみ). A
3130 single distinct script is returned as-is (the ordinary case,
3131 including a lone wholly-katakana name, which callers key with no
3132 table entry); more than one collapses to the HIRAGANA carrier
3133 when confined to Han/Hiragana/Katakana, the same set
3134 effective_script's license tests; any other mix (Han+Hangul, or
3135 no scripts at all -- an empty `scripts`) returns None -- the
3136 caller's cue to fall back to the positional default, exactly like
3137 effective_script's own None. A non-None result reports what was
3138 FOUND, not that a license fired: callers wanting to know whether
3139 the kana license specifically was the reason must compare the
3140 result against a specific Script (e.g. `is Script.HIRAGANA`), not
3141 just its truthiness -- a lone wholly-Han name also returns
3142 non-None here, licensing nothing."""
3143 found = frozenset(scripts)
3144 if len(found) <= 1:
3145 return next(iter(found), None)
3146 if found.issubset(_JA_SCRIPTS):
3147 return Script.HIRAGANA
3148 return None
3151# -- the reading a decision did not take (Ambiguity.alternative, #752
3152# phase 2m). Each report names it from what its own site knows: the
3153# word's tags say which reading the vocabulary or a shape gave it, and
3154# the emitter says which field (decisions/vocabulary-model, 2026-10-26)
3156#: Tags that say the vocabulary declares the word a name as well as a
3157#: word of another category: the derived markers (decisions/
3158#: vocabulary-model, 2026-10-25), and a given name written short. The
3159#: ambiguous class's tag is also written by SHAPE (rules/S9), which
3160#: `name_reading` reads off the shape tag beside it.
3161_DECLARED_NAME_TAGS = frozenset({
3162 AMBIGUOUS_ACRONYM_TAG, AMBIGUOUS_WORD_TAG, "vocab:particle-ambiguous",
3163 TITLE_AMBIGUOUS_TAG, GIVEN_ABBREVIATION_TAG})
3166def suffix_reading(token: WorkToken) -> PendingReading:
3167 """The suffix reading of a word a decision read as a name: a
3168 generation or agnomen by its listed word, a numeral by its shape
3169 unless listed, another listed suffix word by its vocabulary kind,
3170 and otherwise a credential, listed or shaped."""
3171 kind, source = suffix_kind(token.text, token.tags)
3172 return PendingReading(kind, source, Role.SUFFIX)
3175def suffix_kind(text: str,
3176 tags: Set[str]) -> tuple[ReadingKind, ReadingSource]:
3177 """`suffix_reading`'s kind and source, which assemble also asks of
3178 a word the parse read as a suffix (`Token.reading`), so the
3179 reading taken and the reading not taken are named by one
3180 predicate."""
3181 if AGNOMEN_TAG in tags:
3182 return ReadingKind.AGNOMEN, ReadingSource.VOCABULARY
3183 if CREDENTIAL_WORD_TAG in tags:
3184 return ReadingKind.CREDENTIAL, ReadingSource.VOCABULARY
3185 if AMBIGUOUS_WORD_TAG in tags or GENERATION_WORD_TAG in tags:
3186 return ReadingKind.GENERATION, ReadingSource.VOCABULARY
3187 if _ROMAN.match(text.rstrip(".")) is not None:
3188 listed = "vocab:suffix" in tags or "vocab:suffix-word" in tags
3189 return (ReadingKind.GENERATION,
3190 ReadingSource.VOCABULARY if listed else ReadingSource.SHAPE)
3191 if "vocab:suffix-word" in tags:
3192 return (ReadingKind.HONORIFIC if HONORIFIC_WORD_TAG in tags
3193 else ReadingKind.POST_NOMINAL), ReadingSource.VOCABULARY
3194 return (ReadingKind.CREDENTIAL,
3195 ReadingSource.SHAPE if SHAPE_ACRONYM_TAG in tags
3196 else ReadingSource.VOCABULARY)
3199def name_reading(token: WorkToken, role: Role | Deferred) -> PendingReading:
3200 """The name reading of a word a decision read as something else,
3201 landing in `role`: an initial where the word is one letter or
3202 initial-shaped, otherwise a name word, declared by the vocabulary
3203 where a derived marker says so and standing by position where
3204 nothing does."""
3205 text = token.text
3206 # C-level for an ASCII word, so a report pays no frame for the test
3207 if (len(text.rstrip(".")) == 1 or _INITIAL.fullmatch(text)
3208 if text.isascii()
3209 else composed_len(text.rstrip(".")) == 1
3210 or is_initial_shaped(text)):
3211 return PendingReading(ReadingKind.INITIAL, ReadingSource.SHAPE, role)
3212 return PendingReading(ReadingKind.NAME, name_source(token.tags), role)
3215def name_source(tags: Set[str]) -> ReadingSource:
3216 """Why a word reads as a name word: declared by the vocabulary
3217 where a derived marker says so, and by position where nothing
3218 does. `name_reading`'s source, which assemble also asks of a word
3219 the parse read as a name (`Token.reading`)."""
3220 # classify writes the ambiguous class's tag on a word admitted by
3221 # its shape too, so the shape tag says no list declared it
3222 if (not tags.isdisjoint(_DECLARED_NAME_TAGS)
3223 and SHAPE_ACRONYM_TAG not in tags):
3224 return ReadingSource.DECLARED
3225 return ReadingSource.POSITION
3228#: The particle reading of a word read as a name: a particle joins the
3229#: words after it, the last group reading as the family (rules/P2).
3230PARTICLE_READING = PendingReading(ReadingKind.PARTICLE,
3231 ReadingSource.VOCABULARY, Role.FAMILY)
3232#: The same for a patronymic particle (rules/R9): its particle reading
3233#: is a patronymic, joining the parent's name after it.
3234PATRONYMIC_READING = PendingReading(ReadingKind.PATRONYMIC,
3235 ReadingSource.VOCABULARY, Role.FAMILY)
3238def particle_reading(tags: frozenset[str]) -> PendingReading:
3239 """The particle reading of a word read as something else, by the
3240 kind classify tagged it with: a patronymic particle's is
3241 `PATRONYMIC_READING`, any other's `PARTICLE_READING`."""
3242 return PATRONYMIC_READING if PATRONYMIC_TAG in tags else PARTICLE_READING
3245#: The title reading of a word read as a name.
3246TITLE_READING = PendingReading(ReadingKind.TITLE, ReadingSource.VOCABULARY,
3247 Role.TITLE)