Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/nameparser/_parser.py: 38%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Parser and the module-level parse() for the 2.0 API.
3Layering: sits on _types/_lexicon/_policy/_locale/_pipeline; never
4imports _render or the v1 facade (enforced by tests/v2/test_layering.py).
6_default_parser is THE one sanctioned module-level global: a
7functools.cache'd frozen Parser over default config.
8"""
9from __future__ import annotations
11import dataclasses
12import functools
13import warnings
14from dataclasses import dataclass, field
16from nameparser._lexicon import Lexicon
17from nameparser._locale import Locale
18from nameparser._pipeline import run
19from nameparser._pipeline._assemble import assemble
20from nameparser._pipeline._post_rules import suffix_entries
21from nameparser._pipeline._state import ParseState
22from nameparser._pipeline._vocab import _SCRIPT_MATCHERS
23from nameparser._policy import UNSET, Policy, PolicyPatch, _Unset, apply_patch
24from nameparser._types import (
25 FOLDED_TAG, ParsedName, Role, Segmenter, Token, _guarded_getstate,
26 _guarded_setstate, _validated_field_strings,
27)
30@dataclass(frozen=True, slots=True)
31class Parser:
32 """A configured name parser: a :class:`Lexicon` (vocabulary) plus
33 a :class:`Policy` (behavior), both defaulted when omitted. Build
34 one when you need non-default configuration, build it once, and
35 call :meth:`parse` many times -- it is immutable and thread-safe.
37 An optional keyword-only ``segmenter`` (a :data:`~nameparser.Segmenter`)
38 plugs in outside knowledge of where an unspaced CJK token divides --
39 Japanese kanji names, which no bundled list can settle. It is
40 consulted only for a token the segmentation stage gates in and the
41 vocabulary DECLINES, so a locale pack's surnames always win where
42 they match; returning None declines in turn and the token stays
43 whole. Two promises narrow when one is supplied (the first is
44 rules.md#A1's Accepted clause):
45 parse-totality gains its one exception -- an exception raised by
46 the segmenter propagates, because a user-supplied callable's own
47 error is a user-code error, not a content error -- and this Parser
48 pickles only if its segmenter does (a module-level function
49 pickles; a lambda or closure does not). With no segmenter, both
50 promises hold unconditionally: all validity checking happens at
51 construction, so a Parser that constructs successfully cannot fail
52 at parse time on any str content.
54 (The None field defaults resolve in __post_init__; after
55 construction lexicon and policy are always non-None -- the
56 annotations state the steady-state truth, hence the assignment
57 ignores on the defaults.)"""
59 lexicon: Lexicon = None # type: ignore[assignment] # None -> default()
60 policy: Policy = None # type: ignore[assignment] # None -> Policy()
61 #: An optional hook supplying outside knowledge of where an unspaced
62 #: token divides -- see the class docstring; None leaves such tokens
63 #: whole. Keyword-only, so the reserved growth stays additive:
64 #: positional construction keeps its two-argument shape.
65 segmenter: Segmenter | None = field(default=None, kw_only=True)
67 # in the class body so @dataclass(slots=True) keeps them
68 __getstate__ = _guarded_getstate
69 __setstate__ = _guarded_setstate
71 def __post_init__(self) -> None:
72 if self.lexicon is None:
73 object.__setattr__(self, "lexicon", Lexicon.default())
74 elif not isinstance(self.lexicon, Lexicon):
75 raise TypeError(
76 f"lexicon must be a Lexicon or None, got {self.lexicon!r}")
77 if self.policy is None:
78 object.__setattr__(self, "policy", Policy())
79 elif not isinstance(self.policy, Policy):
80 raise TypeError(
81 f"policy must be a Policy or None, got {self.policy!r}")
82 if self.segmenter is not None and not callable(self.segmenter):
83 raise TypeError(
84 f"segmenter must be callable or None, got {self.segmenter!r}")
85 # A configuration gap that used to be silent (#272's API, made
86 # loud before 2.1.0): segment_scripts can activate a script
87 # that neither the vocabulary nor a segmenter can ever divide
88 # -- the JA pack's whole shape, when its segmenter is
89 # forgotten. The parser then behaves identically to a working
90 # one minus the feature, which reads as "not working" with no
91 # signal why. Statically decidable here, so say it here; a
92 # warning rather than an error because the inert pack is a
93 # pinned, deliberate property (a JA registration must be safe
94 # without the extra), and warnings are filterable by the rare
95 # caller who wants exactly that.
96 if self.segmenter is None:
97 uncovered = sorted(
98 script.value
99 for script in self.policy.segment_scripts
100 if not any(_SCRIPT_MATCHERS[script](entry)
101 for entry in self.lexicon.surnames))
102 if uncovered:
103 names = ", ".join(uncovered)
104 one = len(uncovered) == 1
105 # the ja hint only where a Japanese script is among the
106 # dead ones -- a hangul-only gap (a from-scratch
107 # lexicon under the default policy) has different
108 # remedies, and pointing it at ja_segmenter would be a
109 # non sequitur
110 ja_hint = (
111 " For Japanese, pass "
112 "segmenter=locales.ja_segmenter() (install with: "
113 "pip install 'nameparser[ja]')."
114 if {"han", "hiragana", "katakana"} & set(uncovered)
115 else "")
116 warnings.warn(
117 f"Policy.segment_scripts activates {names} but the "
118 f"vocabulary has no surnames in "
119 f"{'that script' if one else 'those scripts'} "
120 f"and no segmenter is configured: unspaced names "
121 f"written in {'it' if one else 'them'} will never "
122 f"divide. Supply covering surnames, pass a "
123 f"segmenter, or deactivate with "
124 f"Policy(segment_scripts=frozenset()).{ja_hint}",
125 UserWarning, stacklevel=3)
127 def __repr__(self) -> str:
128 # composes the two bounded component reprs; the
129 # segmenter shows by name, and only when one is set, so the
130 # default Parser's repr is unchanged
131 seg = ""
132 if self.segmenter is not None:
133 # never repr() the callable itself: a partial reprs its
134 # bound arguments and a callable instance its address, both
135 # unbounded -- the class name is the bounded fallback
136 name = (getattr(self.segmenter, "__qualname__", None)
137 or type(self.segmenter).__name__)
138 seg = f", segmenter={name}"
139 return f"Parser({self.lexicon!r}, {self.policy!r}{seg})"
141 def parse(self, text: str) -> ParsedName:
142 """Parse one name string into a :class:`ParsedName`. Never
143 raises on string content (unparseable input yields empty
144 fields plus ambiguities); non-str raises TypeError eagerly,
145 with a decode hint for bytes (bytes support ended with 1.x).
146 The one exception to that totality is a configured
147 ``segmenter``, whose own exceptions propagate (see the class
148 docstring)."""
149 if isinstance(text, bytes):
150 raise TypeError(
151 "parse() takes str, not bytes -- decode first, e.g. "
152 "raw.decode('utf-8')")
153 if not isinstance(text, str):
154 raise TypeError(f"parse() takes str, got {text!r}")
155 state = ParseState(original=text, lexicon=self.lexicon,
156 policy=self.policy, segmenter=self.segmenter)
157 return assemble(run(state))
159 # -- editing ----------------------------------------------------------
161 def revise(self, name: ParsedName, **fields: str) -> ParsedName:
162 """:meth:`ParsedName.replace` with this parser's vocabulary:
163 each value is tokenized and classified by a full sub-parse, so
164 the stable tags survive and the tag-driven views
165 (family_particles, initials(), the suffix join, and since #407
166 capitalized()) behave as if the text had been parsed. The
167 value is classified ON ITS OWN, though -- a word whose reading
168 depends on surrounding context may classify differently than
169 it would in place, and a glued CJK honorific the whole name
170 kept on an initial may peel in the value. The sub-parse's role
171 choices and ambiguities are discarded -- every harvested token
172 takes the named field's role -- and its structural behavior
173 applies: delimiter characters do not become tokens, and a
174 maiden marker is consumed as in parsing -- mid-value always,
175 and leading a DELIMITED value under a policy routing that pair
176 to maiden, where "(née Jones)" revises to "Jones" while the
177 bare "née Jones" keeps its marker, a leading marker in an
178 undelimited value being no marker at all (#329).
179 A suffix value's ENTRY structure is derived from the value's
180 own commas after the role is forced, by the rule a whole name
181 uses: a comma parts two credentials and a space joins them, so
182 ``revise(n, suffix="MD PhD")`` is one entry and a name's
183 rendered suffix revises back to itself wherever the value's
184 words read as the whole name read them -- the honorific peel
185 above is the one corpus exception of 368 suffix-bearing names,
186 2026-09-06 (#511). A delimiter the policy names through
187 ``extra_suffix_delimiters`` parts a value only where the
188 value's own words, read as a name, give it a tail segment for
189 the core to be dropped on; a run of post-nominals has none,
190 with or without a comma of its own (``"MD PhD - FACS"`` and
191 ``"MD, PhD - FACS"`` both keep the dash as a word), so write a
192 comma at the boundary you want rather than the delimiter.
193 Tokens are synthetic (span=None); original is unchanged; a
194 value with no name content (empty, whitespace, or punctuation
195 only) clears the field; ambiguities referencing replaced
196 tokens are dropped."""
197 if not isinstance(name, ParsedName):
198 raise TypeError(f"revise() takes a ParsedName, got {name!r}")
199 replaced = _validated_field_strings(fields)
200 harvested: dict[Role, tuple[Token, ...]] = {}
201 for role, value in replaced.items():
202 # the same construction parse() makes, spelled twice
203 # rather than through a helper: routing parse() through
204 # one cost a frame per parse on the hot path (py3.11,
205 # 2026-09-06: 415 calls/name against 414 without it, in a
206 # 402-418 band), the same trade _mark_suffix_entries
207 # refused, one row over
208 state = run(ParseState(original=value, lexicon=self.lexicon,
209 policy=self.policy,
210 segmenter=self.segmenter))
211 dropped = set(state.dropped)
212 # Force the role BEFORE the entry pass: it keys on
213 # Role.SUFFIX, and a bare value's sub-parse reads its words
214 # as a name ('MD PhD' is a title and a family there), so
215 # inside the sub-parse it joined nothing. The sub-parse's
216 # tags are KEPT, "joined" included: a within-piece mark
217 # (the Ph. D. merge) is role-blind and right for every
218 # role, and a clear was measured and backed out --
219 # decisions.md#C1 (2026-09-06 #511) carries the rest: the
220 # between-piece mark that rides onto a non-suffix role, and
221 # which views read it. Dropped tokens keep their role: the
222 # pass filters them by index and assemble omits them.
223 forced = tuple(
224 tok if i in dropped
225 else dataclasses.replace(tok, role=role)
226 for i, tok in enumerate(state.tokens))
227 # rules.md#R1: "a run of post-nominals written with spaces
228 # renders with spaces, and one written with commas keeps
229 # them" -- the same pass post_rules runs last, over the
230 # value's own comma offsets, so the harvest carries exactly
231 # the entry structure the value's commas describe. Run for
232 # every role: for a non-suffix role it is a no-op, kept
233 # unconditional as the simpler contract, revise() being off
234 # the call-count band.
235 entried = suffix_entries(
236 dataclasses.replace(state, tokens=forced))
237 harvested[role] = tuple(
238 Token(t.text, None, role, t.tags - {FOLDED_TAG})
239 for t in assemble(entried).tokens)
240 return name._with_field_tokens(harvested)
242 # -- comparison -------------------------------------------------------
244 def matches(self, a: str | ParsedName, b: str | ParsedName) -> bool:
245 """Component-wise case-insensitive comparison of two names,
246 parsing str arguments with THIS parser.
247 :meth:`ParsedName.matches` parses its str argument with the
248 DEFAULT parser instead -- for names parsed with a custom
249 Parser, use this method."""
250 if isinstance(a, str):
251 a = self.parse(a)
252 elif not isinstance(a, ParsedName):
253 raise TypeError(f"matches() takes str or ParsedName, got {a!r}")
254 if isinstance(b, str):
255 b = self.parse(b)
256 elif not isinstance(b, ParsedName):
257 raise TypeError(f"matches() takes str or ParsedName, got {b!r}")
258 return a.comparison_key() == b.comparison_key()
260 # -- rendering delegates ----------------------------------------------
262 def capitalized(self, name: ParsedName, *,
263 force: bool = False) -> ParsedName:
264 """:meth:`ParsedName.capitalized` under THIS parser's lexicon.
265 The no-argument form of that method uses the DEFAULT lexicon --
266 for names parsed with a custom Parser, use this method."""
267 if not isinstance(name, ParsedName):
268 raise TypeError(f"capitalized() takes a ParsedName, got {name!r}")
269 return name.capitalized(self.lexicon, force=force)
272@functools.cache
273def _default_parser() -> Parser:
274 return Parser()
277def parse(text: str) -> ParsedName:
278 """Parse a name with the default configuration and return a
279 :class:`ParsedName`. Equivalent to ``Parser().parse(text)``; build
280 your own :class:`Parser` (or use :func:`parser_for`) for custom
281 vocabulary or behavior. Never raises on string content."""
282 return _default_parser().parse(text)
285def parser_for(*locales: Locale, base: Parser | None = None,
286 segmenter: Segmenter | None | _Unset = UNSET) -> Parser:
287 """Lexicon fragments unioned left-to-right onto base's; policy
288 patches applied left-to-right (later wins; set-valued fields union
289 per the patch metadata). Validation errors raised while applying a
290 pack are wrapped with that pack's identity (rule D2) --
291 PolicyPatch validates lazily, so with stacked packs the raw error
292 would otherwise point at nothing. Two packs setting the same SCALAR
293 field is a declared conflict: UserWarning, later wins.
295 A ``segmenter`` is passed straight through to the built Parser --
296 ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` is how
297 a pack and a segmenter combine, since packs are pure data and
298 cannot supply one. The argument has THREE states, the same
299 :data:`~nameparser.UNSET` spelling a PolicyPatch field uses, because
300 None is a meaningful value here and not an absence: omitted (UNSET)
301 carries base's segmenter through unchanged; a callable OVERRIDES
302 base's (later wins, the rule scalar policy fields follow); and an
303 explicit ``None`` CLEARS base's, which is how you derive an
304 unsegmented parser from a segmented one without rebuilding its
305 lexicon and policy by hand."""
306 if base is not None and not isinstance(base, Parser):
307 raise TypeError(f"base must be a Parser or None, got {base!r}")
308 for loc in locales:
309 if not isinstance(loc, Locale):
310 raise TypeError(f"parser_for() takes Locale packs, got {loc!r}")
311 lexicon = base.lexicon if base is not None else Lexicon.default()
312 policy = base.policy if base is not None else Policy()
313 # Resolved here rather than at the return because the return builds
314 # a FRESH Parser: any field not listed there silently takes its
315 # default, and a dropped segmenter would be invisible. UNSET, not
316 # None, is what "not given" means -- None is the CLEAR request, and
317 # collapsing the two would make an explicit
318 # parser_for(..., segmenter=None) silently inherit the very
319 # segmenter it was asked to drop.
320 if segmenter is UNSET:
321 segmenter = base.segmenter if base is not None else None
322 scalar_setters: dict[str, str] = {}
323 for loc in locales:
324 for f in dataclasses.fields(PolicyPatch):
325 if f.metadata.get("compose") == "union":
326 continue
327 if getattr(loc.policy, f.name) is UNSET:
328 continue
329 if f.name in scalar_setters and scalar_setters[f.name] != loc.code:
330 warnings.warn(
331 f"locale {loc.code!r} overrides scalar policy field "
332 f"{f.name!r} already set by locale "
333 f"{scalar_setters[f.name]!r}; later wins",
334 UserWarning, stacklevel=2)
335 scalar_setters[f.name] = loc.code
336 try:
337 lexicon = lexicon | loc.lexicon
338 policy = apply_patch(policy, loc.policy)
339 except (TypeError, ValueError) as exc:
340 # safe: every raise in the apply path (Policy.__post_init__,
341 # Lexicon.__or__, apply_patch) is a PLAIN TypeError/ValueError --
342 # a subclass with extra mandatory args would break this rewrap
343 raise type(exc)(
344 f"while applying locale {loc.code!r}: {exc}") from exc
345 # rules.md#D1: "constructing a parser that activates division for
346 # scripts with no covering surnames and no segmenter warns at
347 # construction, naming the dead scripts and each way out"
348 # (history: decisions.md#D1)
349 # rules.md#D2: "applying a locale pack wraps any such error with
350 # the locale's code, so a stacked configuration names which layer
351 # broke" (history: decisions.md#D2)
352 # Construction warnings (the segmenterless-activation check in
353 # Parser.__post_init__) re-emit from THIS frame: its stacklevel is
354 # sized for direct Parser(...) construction, and through this
355 # function's extra frame the default single-line rendering would
356 # point into the library instead of at the caller -- the exact
357 # call the message tells them to change.
358 with warnings.catch_warnings(record=True) as caught:
359 warnings.simplefilter("always")
360 built = Parser(lexicon=lexicon, policy=policy, segmenter=segmenter)
361 for w in caught:
362 warnings.warn(w.message, stacklevel=2)
363 return built