Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Shared vocabulary predicates for pipeline stages.
3Text-level tests used by more than one stage; piece-level ones live
4in _pieces, the sibling layer over tokens-plus-tags. All take normalized-or-raw text
5explicitly -- no state.
7is_wholly_suffix departs from that shape twice, deliberately. It is
8RUN-level rather than text-level, because the question it answers is
9genuinely about a run: the Ph./D. merge spans two tokens, so no
10per-token predicate composed with all() can express it. And it takes
11the Policy OBJECT, where delimiter_cores takes a pre-extracted
12frozenset so its caller hands in one field rather than the config --
13is_wholly_suffix needs TWO policy fields (lenient_comma_suffixes and
14extra_suffix_delimiters), and threading both past every caller costs
15more than the config parameter saves. Still no state: Policy is frozen
16config, not pipeline state.
18maiden_marker_run is run-level for the first of those reasons and not
19the second: a maiden marker may be a PHRASE ('z domu'), so how far one
20reaches is a question about a run of words that no per-word membership
21test can answer, and the vocabulary reaches it as a plain frozenset
22field like every other predicate here.
24Layering: imports _lexicon, _types, and _policy only.
25"""
26from __future__ import annotations
28import functools
29import re
30import unicodedata
31from collections.abc import Callable, Iterable, Sequence
33from nameparser._lexicon import FULL_STOPS, Lexicon, _normalize
34from nameparser._policy import (Policy, Script, _JA_SCRIPTS, _NO_INITIALS,
35 _SCRIPT_RANGES, _script_matcher)
37# Ported verbatim from v1 (nameparser/config/regexes.py "initial") minus
38# its empty-string alternative -- WorkToken text is never empty. Kept in
39# sync by hand; layering forbids importing the config package here.
40# "Verbatim" is a promise about the PATTERN, not about the predicate:
41# since #320 is_initial is this SHAPE test ANDed with a repertoire test
42# (in_initialless_script, below), so _INITIAL.fullmatch(text) and
43# is_initial(text) are no longer the same question -- '씨.' answers yes
44# to the first and no to the second. Call is_initial; the bare pattern
45# is not the thing to ask. The narrowing lives in the predicate
46# precisely so this copy can stay exactly as verbatim as it ever was
47# -- REGEXES["initial"] is public v1 API and cannot narrow, and the
48# only difference between the two remains the empty alternative noted
49# above (config's `?`), which test_regex_sync splices back in.
50_INITIAL = re.compile(r"^(\w\.|[A-Z])$")
52# Ported verbatim from v1 (nameparser/config/regexes.py
53# "period_not_at_end") -- layering forbids the config import; keep in
54# sync by hand.
55_PERIOD_NOT_AT_END = re.compile(r".*\..+$", re.I)
57# The fix_phd credential pair ('Ph.' + 'D.' as adjacent tokens), shared
58# by is_wholly_suffix below and group's merge (v1 extracted the
59# credential pre-parse; the predicate and the stage must agree on the
60# pattern).
61PH = re.compile(r"^ph\.?$", re.IGNORECASE)
62D = re.compile(r"^d\.?$", re.IGNORECASE)
64# The codepoint table lives in _policy beside Script -- one copy
65# importable from the pipeline and the locale packs alike; everything
66# here DERIVES from it. (single_script's sweep was first written
67# per-char on the _EMOJI_RANGES precedent in _tokenize.py, on the
68# theory that a range test needs no regex; measured at token scale the
69# compiled regex wins by 3-9x, and by roughly 70x on long tokens.)
70# Derived, never hand-written -- even the class construction goes
71# through the shared factory: one wholly-of predicate per script, in
72# the table's key order -- the FIRST-covering-entry rule _classify
73# documents.
74_SCRIPT_MATCHERS: dict[Script, Callable[[str], bool]] = {
75 script: _script_matcher(script, whole=True)
76 for script in _SCRIPT_RANGES
77}
79# The whole-token matcher over _policy's _JA_SCRIPTS union, backing
80# effective_script's kana license.
81_wholly_ja = _script_matcher(*_JA_SCRIPTS, whole=True)
83# The repertoire half of is_initial (_policy._NO_INITIALS), kept apart
84# from _INITIAL's SHAPE half so the pattern itself stays v1-verbatim
85# and its three copies stay pinned by tests/v2/test_regex_sync.py.
86# contains-any, not whole=True: the shape half has already admitted the
87# trailing period, so the text reaching here is '씨.' rather than '씨'
88# and a wholly-of match would be False for every case this exists for.
89# The second caller, _pieces.is_leading_title, admits two or more
90# characters, so contains-any there means one CJK character anywhere
91# vetoes the whole word -- 'Kim김.' is refused as a title along with
92# '田中.' -- and that is deliberate: a word carrying a script with no
93# abbreviations is not wearing an abbreviation's period.
94in_initialless_script = _script_matcher(*_NO_INITIALS, whole=False)
97def is_initial_shaped(text: str) -> bool:
98 """v1's is_an_initial verbatim: the SHAPE half alone -- one word
99 character plus a period, or a bare ASCII capital.
101 Callers asking whether a token is STRUCTURALLY part of an initial
102 run want this; callers asking whether it can really stand in for a
103 name want is_initial (#320). The two answers differ only inside
104 _NO_INITIALS scripts, where '씨.' is initial-SHAPED but is not an
105 initial -- see assign's roman-numeral fork, the shape caller, for
106 what picking the wrong one costs."""
107 return bool(_INITIAL.fullmatch(text))
110# v1 regexes.py "roman_numeral", pinned by tests/v2/test_regex_sync.py.
111_ROMAN = re.compile(r'^(X|IX|IV|V?I{0,3})$', re.I)
114# rules.md#S2: "a trailing word of the suffix vocabulary reads as a
115# suffix" -- the roman-numeral half of that rule, which the initial
116# veto would otherwise take: V, I and X are suffix vocabulary AND bare
117# capitals, and a bare capital inside a name is a middle initial.
118def is_trailing_numeral_suffix(text: str, preceding: str) -> bool:
119 """assign's roman-numeral fork, shared with group's bound-given
120 reserve (#401): a FINAL single-token piece that is a roman numeral
121 reads as the suffix when the piece before it does not look like
122 part of an initial run. `preceding` is that piece's first token;
123 the callers establish that `text` is last and that a name piece
124 precedes it.
126 is_initial_shaped, not is_initial: this asks whether the preceding
127 piece looks like part of an initial run, which is a question
128 about layout, and #320 narrowed the tag to initials that can
129 really stand in for a name. Reading the tag here made '씨.' stop
130 suppressing the fork and cost 'John 씨. V' its family name."""
131 return (_ROMAN.match(text) is not None
132 and not is_initial_shaped(preceding))
135def is_initial(text: str) -> bool:
136 """'A.' / 'j.' / bare capital -- v1's is_an_initial, narrowed to
137 scripts that HAVE initials (#320). v1's \\w is Unicode-aware and
138 matched CJK too, which made period-written CJK honorifics ('씨.')
139 fail is_suffix_strict -- the veto in _is_suffix_strict_n, NOT the
140 vocabulary: suffix_as_written has no veto, so classify tagged '씨.'
141 'vocab:suffix' either way, and is_suffix_lenient took it either way
142 too. Downstream of that one strict-test No, the glued honorific in
143 a name carrying such a token went unpeeled ('田中さん 様.')."""
144 return is_initial_shaped(text) and not in_initialless_script(text)
147def is_one_case(texts: Sequence[str]) -> bool:
148 """Whether a name is written wholly in ONE case -- all upper or all
149 lower alike -- and so carries no case EVIDENCE about any letter in
150 it (rules.md#P3, #383/#479). The caller passes the name's OWN
151 words: a maiden marker's clause and any delimited (nickname)
152 content are not among them, and appending one must not flip the
153 reading of words that did not change.
155 Mirrors the SHAPE of the comparison the R5 gate in
156 `_render.capitalized` makes, not its SPAN: R5 joins every token,
157 nickname and maiden content included, while classify's caller hands
158 in only the name's own words (rules.md#P3's own-words doctrine, see
159 above) -- so a clause-bearing name can be one-case to this function
160 and mixed to R5 (measured: `'JUAN GARCIA Y LOPEZ née Jones'` is one
161 case here, mixed there). Not shared by import today -- render is a
162 layer this module does not reach into, and #492 is where the two
163 spans are reconciled if they ever need to be.
165 `Sequence`, not `Iterable`: the caller passes a list it already
166 built rather than a fresh generator, so `is_one_case` costs one
167 profiler frame per parse rather than one per token (#475).
169 A CASELESS script answers True, harmlessly: `'محمد و علي'.upper()`
170 is the string itself, so the comparison holds, and the only caller
171 also requires a token whose own `upper()` and `lower()` differ --
172 which a caseless letter's never do. So a caseless name never
173 reaches the decision this gates, and "one case" is the honest
174 verdict for text that has only one.
175 """
176 joined = " ".join(texts)
177 return joined in (joined.upper(), joined.lower())
180_DOTTED = re.compile(r"(?:[^\W\d_]\.)+")
183def _dotted(text: str) -> bool:
184 """Written with its periods: one after each letter ('M.A.',
185 'J.D.'), the acronym's own spelling. A single trailing period
186 ('Ma.', 'Ed.', 'Ms.') is the abbreviation shape any word can wear
187 -- the honorific's, a name's -- and is not the gate's "written
188 with periods" (rules.md#S2). Until #296's review the gate was
189 "any period", and 'Smith, Ms.' passed it as the degree."""
190 return _DOTTED.fullmatch(text) is not None
193def suffix_as_written(n: str, text: str, lexicon: Lexicon) -> bool:
194 """Counts as a suffix as written, with NO initial veto (the veto
195 differs by caller): unambiguous suffix vocabulary, or an ambiguous
196 acronym written with periods ('M.A.' yes, 'Ma' no). `n` is
197 _normalize(text), passed in so callers normalize once.
199 Single source for classify's "vocab:suffix" tag and the segment/
200 assign predicates. The ambiguous subset is EXCLUDED from the plain
201 membership test: in the real data suffix_acronyms_ambiguous is a
202 subset of suffix_acronyms, and without the exclusion the period
203 gate is dead code (bare 'Ed'/'Jd' would silently become suffixes).
204 """
205 # acronyms may be written with periods ('M.B.A.'): the ACRONYM
206 # membership alone uses the period-free form (v1's is_suffix
207 # removed periods only for the suffix_acronyms test); suffix WORDS
208 # match on the plain normalized form
209 a = n.replace(".", "")
210 if a in lexicon.suffix_acronyms_ambiguous and _dotted(text):
211 return True
212 return (a in lexicon.suffix_acronyms
213 and a not in lexicon.suffix_acronyms_ambiguous) \
214 or n in lexicon.suffix_words
217def _is_suffix_strict_n(n: str, text: str, lexicon: Lexicon) -> bool:
218 if is_initial(text):
219 # period-written ambiguous acronyms are exempt from the veto
220 return _dotted(text) and \
221 n.replace(".", "") in lexicon.suffix_acronyms_ambiguous
222 return suffix_as_written(n, text, lexicon)
225def is_suffix_strict(text: str, lexicon: Lexicon) -> bool:
226 """v1's is_suffix: suffix_as_written with the initial veto ('V.' in
227 'John V. Smith' is a middle initial, not roman five)."""
228 return _is_suffix_strict_n(_normalize(text), text, lexicon)
231def is_suffix_lenient(text: str, lexicon: Lexicon) -> bool:
232 """v1's is_suffix_lenient: suffix_words accepted unconditionally,
233 bypassing the initial veto -- only safe in unambiguous positions
234 (after a comma)."""
235 n = _normalize(text)
236 return n in lexicon.suffix_words \
237 or _is_suffix_strict_n(n, text, lexicon)
240def delimiter_cores(policy_delimiters: frozenset[str]) -> frozenset[str]:
241 """Configured suffix delimiters with surrounding whitespace
242 stripped: ' - ' -> '-'. Whitespace-padded delimiters surface as
243 standalone tokens; the stripped core is what tokenize produced."""
244 return frozenset(d.strip() for d in policy_delimiters if d.strip())
247def splits_into_suffixes(text: str, cores: frozenset[str],
248 lexicon: Lexicon) -> bool:
249 """v1 expand_suffix_delimiter parity for delimiters WITHOUT
250 whitespace ('RN/CRNA' with '/'): the token counts as a suffix when
251 some core splits it into >=2 non-empty parts that are all suffixes.
252 The token text is never rewritten (anti-#100): it takes Role.SUFFIX
253 whole, which renders 'RN/CRNA' where v1 rendered 'RN, CRNA' -- the
254 documented divergence, release-log classified."""
255 for core in cores:
256 if core in text:
257 parts = [part for part in text.split(core) if part]
258 if len(parts) >= 2 and all(
259 is_suffix_lenient(part, lexicon) for part in parts):
260 return True
261 return False
264# rules.md#S3: "a word with interior periods reads as a suffix when
265# any of its period-separated chunks is suffix vocabulary"
266def period_joined_vocab(text: str, lexicon: Lexicon) -> str | None:
267 """v1's parse_pieces derivation for interior-period tokens
268 ('Lt.Gov.', 'Msc.Ed.', and by the ANY rule 'Mr.Smith'): ANY title
269 chunk makes the token a title (checked first, v1's continue); else
270 ANY suffix chunk makes it a suffix. Chunk-level suffix membership
271 is v1's is_suffix: bare ambiguous acronyms COUNT ('Msc.Ed.'
272 derives via 'ed') -- the ambiguous period-gate applies to whole
273 tokens only. Returns "title", "suffix", or None."""
274 if not _PERIOD_NOT_AT_END.match(text):
275 return None
276 chunks = [_normalize(c) for c in text.split(".") if c]
277 if any(c in lexicon.titles for c in chunks):
278 return "title"
279 if any(c in lexicon.suffix_acronyms or c in lexicon.suffix_words
280 for c in chunks):
281 return "suffix"
282 return None
285def is_wholly_suffix(texts: Sequence[str], lexicon: Lexicon,
286 policy: Policy) -> bool:
287 """Every token in a RUN counts as a suffix -- segment's
288 suffix-comma test, lifted out of it so the peel can ask the same
289 question (#319).
291 NOT the plural of _script_segment._is_post_nominal, which asks
292 is_suffix_strict per token. This asks the POLICY-selected predicate
293 (lenient by default), plus period_joined_vocab, delimiter
294 transparency and the Ph./D. merge. 'V.' is the input that tells
295 them apart: it satisfies this predicate but is not a post-nominal
296 -- and reading one for the other IS the #319 bug.
298 An EMPTY run is False, not vacuously True: v1's suffix-comma
299 detection fails on an empty parts[1] ('John Smith,, MD' is a
300 family-comma parse). The 'wholly' idiom agrees -- _script_matcher's
301 whole=True requires non-empty too -- which is why the name is that
302 one rather than all_suffixes, where Python's all([]) would promise
303 the opposite.
305 An adjacent Ph./D. pair counts as ONE unit (v1's fix_phd extracted
306 the credential pre-parse, so 'Smith, Ph. D.' read as suffix-comma);
307 keep in sync with group's _PH/_D merge.
308 """
309 if not texts:
310 return False
311 predicate = (is_suffix_lenient if policy.lenient_comma_suffixes
312 else is_suffix_strict)
313 # v1 expand_suffix_delimiter parity (#206): a configured delimiter
314 # is TRANSPARENT in the all-suffix tests -- v1 split the part string
315 # on the delimiter before checking, so the delimiter never counted
316 cores = delimiter_cores(policy.extra_suffix_delimiters)
318 def counts_as_suffix(text: str) -> bool:
319 if text in cores:
320 return True
321 return (predicate(text, lexicon)
322 or period_joined_vocab(text, lexicon) == "suffix"
323 or (bool(cores)
324 and splits_into_suffixes(text, cores, lexicon)))
326 merged = list(texts)
327 k = 0
328 while k < len(merged) - 1:
329 if PH.fullmatch(merged[k]) and D.fullmatch(merged[k + 1]):
330 merged[k:k + 2] = ["phd"]
331 else:
332 k += 1
333 return all(counts_as_suffix(t) for t in merged)
336# The two derived views of a marker vocabulary, cached per-vocabulary
337# on _script_segment._longest_entry's precedent -- same function shape
338# (a scalar derived from a vocabulary frozenset), same key space, and
339# its reasoning for maxsize=16 carries over verbatim: a process holds
340# the default vocabulary plus one per constructed pack parser, so 16
341# bounds many-lexicon churn without ever evicting in normal use.
342# (NOT _extract._delimiter_chars' precedent, which these once cited:
343# that one is consulted once per parse, so it says nothing about a
344# lookup on the per-token path.) Keying on the frozenset costs a
345# cached hash, not a sweep of its contents.
346@functools.lru_cache(maxsize=16)
347def _longest_marker(markers: frozenset[str]) -> int:
348 """How many words the longest entry of `markers` spans -- the
349 lookahead bound, computed from the vocabulary rather than fixed at a
350 literal so a caller's four-word entry works and an all-single-word
351 set leaves the common path at one lookup. Entries are stored
352 space-joined with single separators, so the space count IS the word
353 count."""
354 return max((entry.count(" ") + 1 for entry in markers), default=0)
357@functools.lru_cache(maxsize=16)
358def _marker_heads(markers: frozenset[str]) -> frozenset[str]:
359 """The first word of every entry -- the set a run can possibly open
360 with. Entries are already stored per-word folded, so an entry's
361 first word is the same fold the lookup builds."""
362 return frozenset(entry.split(" ", 1)[0] for entry in markers)
365def maiden_marker_head(n: str, markers: frozenset[str]) -> bool:
366 """Could a maiden marker run START here? `n` is _normalize(word),
367 passed in so callers fold once -- suffix_as_written's shape exactly
368 ("`n` is `_normalize(text)`, passed in so callers normalize once"),
369 and with no raw-text sibling for the same reason it has none: every
370 caller is on the per-token path and has the fold in hand already.
372 A SUPERSET test. True means only that some entry opens with this
373 word, never that a run matches -- maiden_marker_run is the answer,
374 and it calls this function, so the two cannot drift. Exported
375 because a caller scanning a whole token stream needs to know
376 whether assembling a candidate sequence is worth doing at all, and
377 for almost every token it is not: _classify's pass would otherwise
378 walk structural boundaries once per token to build a lookahead the
379 predicate discards on this very test.
380 """
381 return n in _marker_heads(markers)
384def maiden_marker_run(words: Sequence[str], markers: frozenset[str]) -> int:
385 """How many of `words` a maiden marker claims, longest first; 0 for
386 none.
388 Phrases are stored space-joined and per-word normalized (_title_key's
389 storage rule), so the key is rebuilt the same way here -- normalizing
390 the joined phrase instead would leave interior periods.
392 `words` must be words that stand TOGETHER -- one clause's, or one
393 segment's. This answers only what the vocabulary says about the
394 sequence it is handed; whether a sequence is a sequence is the
395 caller's, and classify's contiguity rule is where that is decided
396 for the token stream.
398 Longest first, so a caller configuring both 'geb' and 'geb von' gets
399 the phrase where it matches and the bare word everywhere else. The
400 one answer to "does a marker start here, and where does it end": the
401 stages that can call it do (classify over token texts, extract over
402 a clause's whitespace words), and the stage that runs after classify
403 reads the tags classify recorded instead
404 (mechanisms.md#ONE-PREDICATE-PER-QUESTION).
405 """
406 # Fast path first: no entry opens with this word, so no length can
407 # match. It cannot hide a match -- every key the loop builds opens
408 # with _normalize(words[0]) unless that word folds away, and a
409 # folded-away first word fails the n-word test below for every
410 # n > 1 and is not a stored entry for n == 1.
411 if not words:
412 return 0
413 head = _normalize(words[0])
414 if not maiden_marker_head(head, markers):
415 return 0
416 cap = min(len(words), _longest_marker(markers))
417 # Fold each word ONCE, then key from a prefix. _title_key(words[:n])
418 # per candidate length re-folds the whole prefix every time, which
419 # is quadratic in cap and makes a one-word hit cost more than a
420 # two-word one -- the longest key is always built and discarded
421 # first, and all but one shipped entry is a single word. The join
422 # below IS _title_key's body over pre-folded words (per-word fold,
423 # empties dropped, space-joined); test_vocab pins that the two
424 # agree, since this is a copy of a fold whose definition lives in
425 # _lexicon.
426 folded = [head] + [_normalize(w) for w in words[1:cap]]
427 for n in range(cap, 0, -1):
428 key = " ".join(filter(None, folded[:n]))
429 # a word that folds away is DROPPED from the key, so 'née'
430 # followed by a lone '.' would key as 'née' and a one-word
431 # marker would claim the period as part of its run. n words in,
432 # n words out: anything else is not this key's n-word phrase.
433 if key.count(" ") == n - 1 and key in markers:
434 return n
435 return 0
438def _normalized_for_script(text: str) -> str | None:
439 """The guard AND the two normalizations single_script and
440 effective_script's license path both need, single-sourced so they
441 cannot drift: trailing full stops are dropped (FULL_STOPS, #323),
442 then None for the two shapes neither ever classifies (nothing
443 left, and the common all-ASCII Latin token -- skipped before
444 normalizing, since ASCII is already NFC and every _SCRIPT_RANGES
445 entry is non-ASCII regardless), else an NFC-normalized copy.
447 Trailing stops, not raw: a period glued to a script-written token
448 ('양.', '太郎.') is not a character of any script, so classifying
449 raw text handed the token no script at all, and three readers
450 spent that None -- the surname site stepped past the family name
451 onto the given name ('양. 지훈' cut 지훈 in half), the order rule
452 fell back to positional ('양 지훈.' lost family-first), and the
453 segmenter's neighbour precondition missed a writer-drawn boundary
454 ('山田太郎 田中.' consulted the segmenter on 山田太郎 as if it stood
455 alone). The scripts this classifies -- every _SCRIPT_RANGES entry,
456 which today coincide with _policy._NO_INITIALS (#320), a
457 coincidence _policy says a new member must not inherit -- have no
458 initials and no period abbreviations, so a stop on such a token
459 carries no information about the word; ASCII text is stripped too,
460 but the guard below returns None for it regardless, so 'Smith.'
461 never classifies. TRAILING only, matching the surname site's own
462 rstrip in _script_segment (the same arithmetic, not a shared
463 gate -- this fold decides only whether the surname site, the order
464 rule and the segmenter ever see the token): a leading stop is not
465 a shape any script writes before a name word, and HIDING such a
466 token from those three readers -- no script, so no surname site,
467 which is what the tree before #323 did -- is safer than admitting
468 it. Admitted, '.김민준' classifies as hangul, becomes a surname
469 site, is declined by the head match (which rstrips) and falls
470 through to a configured segmenter, which answering offset 1
471 divides it into the stop and the name. The peel is not gated by
472 this fold; its own rstrip carries a leading-stop token, see
473 _script_segment. The vocabulary fold alone reads BOTH edges
474 (_lexicon._normalize): '.씨' is still the honorific, and a lookup
475 divides nothing.
477 NFC, not raw: NFD input decomposes precomposed katakana onto a
478 base character plus a COMBINING mark (U+3099/U+309A, which sit in
479 the HIRAGANA block, not katakana's), so classifying raw NFD text
480 can hand a pure-katakana token the kana license by accident; NFD
481 also decomposes Hangul syllables onto bare jamo (U+1100-U+11FF),
482 entirely outside the HANGUL range, so raw NFD Korean input misses
483 the shipped family-first order rule rather than merely misfiring.
484 Normalizing first fixes both. Classification-only and read-only:
485 the returned copy is never what gets tokenized, so token text and
486 spans stay exactly what the caller wrote.
488 Vocabulary MATCHING composes NFC too, since #322
489 (_lexicon._normalize folds every lookup and every stored entry the
490 same way), so an NFD suffix word reaches its NFC entry. What stays
491 raw is SEGMENTATION -- the surname site's direct membership test
492 and the peel's tail slice index the token's own text -- where NFD
493 degrades to no-split, never to a wrong split (decisions.md#W1,
494 the 2026-07-29 ja amendment).
495 """
496 text = text.rstrip(FULL_STOPS)
497 if not text or text.isascii():
498 return None
499 return unicodedata.normalize("NFC", text)
502def _classify(normalized: str) -> Script | None:
503 """The FIRST _SCRIPT_MATCHERS entry covering all of `normalized`
504 (already NFC, via _normalized_for_script), else None. Shared by
505 both public classifiers so each of them normalizes exactly once."""
506 for script, matcher in _SCRIPT_MATCHERS.items():
507 if matcher(normalized):
508 return script
509 return None
512def single_script(text: str) -> Script | None:
513 """The one Script whose ranges cover EVERY char of `text`, else
514 None (mixed-script text has no well-defined convention to apply;
515 the caller falls back to the positional default). Classifies an
516 NFC-normalized copy of `text` -- see _normalized_for_script.
517 Callers wanting the kana-mixed license (a kanji+kana composite
518 resolving to HIRAGANA) want effective_script, not this function."""
519 normalized = _normalized_for_script(text)
520 if normalized is None:
521 return None
522 return _classify(normalized)
525def effective_script(text: str) -> Script | None:
526 """single_script, extended by the kana license (#272 amendment):
527 a MIXED token wholly within Han∪hiragana∪katakana is Japanese --
528 it necessarily contains kana (pure Han is not mixed), cannot be
529 Chinese, and is not a foreign transcription (those are
530 katakana-only: マイケル has no kanji, but さくらエミ -- hiragana
531 plus katakana -- is kana-only AND licensed) -- and resolves to the
532 HIRAGANA carrier entry. Pure-katakana stays KATAKANA
533 (single_script's answer): a lone katakana token is predominantly a
534 transcribed foreign name, so nothing defaults on it."""
535 # None for both shapes _wholly_ja could never match anyway (empty
536 # text, or all-ASCII text): real work, not a leftover "if text"
537 # guard, since the ASCII case is one a bare emptiness check would
538 # let through. The single normalized copy then serves both the
539 # single-script answer and the license below.
540 normalized = _normalized_for_script(text)
541 if normalized is None:
542 return None
543 script = _classify(normalized)
544 if script is not None:
545 return script
546 if _wholly_ja(normalized):
547 return Script.HIRAGANA
548 return None
551def resolve_script_set(scripts: Iterable[Script]) -> Script | None:
552 """Generalizes effective_script's kana license from one token's
553 CHARACTERS to a whole name's PIECES (#272): `scripts` is the
554 effective_script of every name token, already resolved
555 individually -- '高橋' (Han) and 'みなみ' (Hiragana) are two
556 separately single-script pieces (split by a space, not mixed
557 within one token), but together are exactly the repertoire
558 effective_script licenses inside a single token (高橋みなみ). A
559 single distinct script is returned as-is (the ordinary case,
560 including a lone wholly-katakana name, which callers key with no
561 table entry); more than one collapses to the HIRAGANA carrier
562 when confined to Han/Hiragana/Katakana, the same set
563 effective_script's license tests; any other mix (Han+Hangul, or
564 no scripts at all -- an empty `scripts`) returns None -- the
565 caller's cue to fall back to the positional default, exactly like
566 effective_script's own None. A non-None result reports what was
567 FOUND, not that a license fired: callers wanting to know whether
568 the kana license specifically was the reason must compare the
569 result against a specific Script (e.g. `is Script.HIRAGANA`), not
570 just its truthiness -- a lone wholly-Han name also returns
571 non-None here, licensing nothing."""
572 found = frozenset(scripts)
573 if len(found) <= 1:
574 return next(iter(found), None)
575 if found.issubset(_JA_SCRIPTS):
576 return Script.HIRAGANA
577 return None