Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/wcwidth/grapheme.py: 24%

1""" 

2Grapheme cluster segmentation following Unicode Standard Annex #29. 

3 

4This module provides pure-Python implementation of the grapheme cluster boundary algorithm as 

5defined in UAX #29: Unicode Text Segmentation. 

6 

7https://www.unicode.org/reports/tr29/ 

8""" 

9 

10from __future__ import annotations 

11 

12# std imports 

13from enum import IntEnum 

14from functools import lru_cache 

15 

16from typing import TYPE_CHECKING, Optional, NamedTuple 

17 

18__lazy_modules__ = [ 

19 "wcwidth.bisearch", 

20 "wcwidth.table_grapheme", 

21] 

22# local 

23from .bisearch import bisearch as _bisearch 

24from .table_grapheme import (GRAPHEME_L, 

25 GRAPHEME_T, 

26 GRAPHEME_V, 

27 GRAPHEME_LV, 

28 INCB_EXTEND, 

29 INCB_LINKER, 

30 GRAPHEME_LVT, 

31 INCB_CONSONANT, 

32 GRAPHEME_EXTEND, 

33 GRAPHEME_CONTROL, 

34 GRAPHEME_PREPEND, 

35 GRAPHEME_SPACINGMARK, 

36 EXTENDED_PICTOGRAPHIC, 

37 GRAPHEME_REGIONAL_INDICATOR) 

38 

39if TYPE_CHECKING: # pragma: no cover 

40 # std imports 

41 from collections.abc import Iterator 

42 

43# Maximum backward scan distance when finding grapheme cluster boundaries. 

44# Covers all known Unicode grapheme clusters with margin; longer sequences are pathological. 

45MAX_GRAPHEME_SCAN = 32 

46 

47 

48class GCB(IntEnum): 

49 """Grapheme Cluster Break property values.""" 

50 

51 OTHER = 0 

52 CR = 1 

53 LF = 2 

54 CONTROL = 3 

55 EXTEND = 4 

56 ZWJ = 5 

57 REGIONAL_INDICATOR = 6 

58 PREPEND = 7 

59 SPACING_MARK = 8 

60 L = 9 

61 V = 10 

62 T = 11 

63 LV = 12 

64 LVT = 13 

65 

66 

67# All lru_cache sizes in this file use maxsize=1024, chosen by benchmarking UDHR data (500+ 

68# languages) and considering typical process-long sessions: western scripts need ~64 unique 

69# codepoints, and CJK can reach ~2000. 

70@lru_cache(maxsize=1024) 

71def _grapheme_cluster_break(ucs: int) -> GCB: 

72 # pylint: disable=too-many-branches,too-complex 

73 """Return the Grapheme_Cluster_Break property for a codepoint.""" 

74 # Single codepoint matches 

75 if ucs == 0x000d: 

76 return GCB.CR 

77 if ucs == 0x000a: 

78 return GCB.LF 

79 if ucs == 0x200d: 

80 return GCB.ZWJ 

81 # Matching by codepoint ranges, requiring binary search 

82 if _bisearch(ucs, GRAPHEME_CONTROL): 

83 return GCB.CONTROL 

84 if _bisearch(ucs, GRAPHEME_EXTEND): 

85 return GCB.EXTEND 

86 if _bisearch(ucs, GRAPHEME_REGIONAL_INDICATOR): 

87 return GCB.REGIONAL_INDICATOR 

88 if _bisearch(ucs, GRAPHEME_PREPEND): 

89 return GCB.PREPEND 

90 if _bisearch(ucs, GRAPHEME_SPACINGMARK): 

91 return GCB.SPACING_MARK 

92 if _bisearch(ucs, GRAPHEME_L): 

93 return GCB.L 

94 if _bisearch(ucs, GRAPHEME_V): 

95 return GCB.V 

96 if _bisearch(ucs, GRAPHEME_T): 

97 return GCB.T 

98 if _bisearch(ucs, GRAPHEME_LV): 

99 return GCB.LV 

100 if _bisearch(ucs, GRAPHEME_LVT): 

101 return GCB.LVT 

102 return GCB.OTHER 

103 

104 

105@lru_cache(maxsize=1024) 

106def _is_extended_pictographic(ucs: int) -> bool: 

107 """Check if codepoint has Extended_Pictographic property.""" 

108 return bool(_bisearch(ucs, EXTENDED_PICTOGRAPHIC)) 

109 

110 

111@lru_cache(maxsize=1024) 

112def _is_incb_linker(ucs: int) -> bool: 

113 """Check if codepoint has InCB=Linker property.""" 

114 return bool(_bisearch(ucs, INCB_LINKER)) 

115 

116 

117@lru_cache(maxsize=1024) 

118def _is_incb_consonant(ucs: int) -> bool: 

119 """Check if codepoint has InCB=Consonant property.""" 

120 return bool(_bisearch(ucs, INCB_CONSONANT)) 

121 

122 

123@lru_cache(maxsize=1024) 

124def _is_incb_extend(ucs: int) -> bool: 

125 """Check if codepoint has InCB=Extend property.""" 

126 return bool(_bisearch(ucs, INCB_EXTEND)) 

127 

128 

129class BreakResult(NamedTuple): 

130 """Result of grapheme cluster break decision.""" 

131 

132 should_break: bool 

133 ri_count: int 

134 

135 

136@lru_cache(maxsize=1024) 

137def _simple_break_check(prev_gcb: GCB, curr_gcb: GCB) -> Optional[BreakResult]: 

138 """ 

139 Check simple GCB-pair-based break rules (cacheable). 

140 

141 Returns BreakResult for rules that can be determined from GCB properties alone, or None if 

142 complex lookback rules (GB9c, GB11) need to be checked. 

143 """ 

144 # GB3: CR x LF 

145 if prev_gcb == GCB.CR and curr_gcb == GCB.LF: 

146 return BreakResult(should_break=False, ri_count=0) 

147 

148 # GB4: (Control|CR|LF) ÷ 

149 if prev_gcb in (GCB.CONTROL, GCB.CR, GCB.LF): 

150 return BreakResult(should_break=True, ri_count=0) 

151 

152 # GB5: ÷ (Control|CR|LF) 

153 if curr_gcb in (GCB.CONTROL, GCB.CR, GCB.LF): 

154 return BreakResult(should_break=True, ri_count=0) 

155 

156 # GB6: L x (L|V|LV|LVT) 

157 if prev_gcb == GCB.L and curr_gcb in (GCB.L, GCB.V, GCB.LV, GCB.LVT): 

158 return BreakResult(should_break=False, ri_count=0) 

159 

160 # GB7: (LV|V) x (V|T) 

161 if prev_gcb in (GCB.LV, GCB.V) and curr_gcb in (GCB.V, GCB.T): 

162 return BreakResult(should_break=False, ri_count=0) 

163 

164 # GB8: (LVT|T) x T 

165 if prev_gcb in (GCB.LVT, GCB.T) and curr_gcb == GCB.T: 

166 return BreakResult(should_break=False, ri_count=0) 

167 

168 # GB9: x (Extend|ZWJ) - but ZWJ needs GB11 check, so only handle Extend here 

169 if curr_gcb == GCB.EXTEND: 

170 return BreakResult(should_break=False, ri_count=0) 

171 

172 # GB9a: x SpacingMark 

173 if curr_gcb == GCB.SPACING_MARK: 

174 return BreakResult(should_break=False, ri_count=0) 

175 

176 # GB9b: Prepend x 

177 if prev_gcb == GCB.PREPEND: 

178 return BreakResult(should_break=False, ri_count=0) 

179 

180 # GB9c and GB11 need lookback - return None to signal complex check needed 

181 # GB12/13 (RI pairs) need ri_count state - also handled in main function 

182 return None 

183 

184 

185def _should_break( 

186 prev_gcb: GCB, 

187 curr_gcb: GCB, 

188 text: str, 

189 curr_idx: int, 

190 ri_count: int, 

191) -> BreakResult: 

192 # pylint: disable=too-many-branches,too-complex 

193 """ 

194 Determine if there should be a grapheme cluster break between prev and curr. 

195 

196 Implements UAX #29 grapheme cluster boundary rules. 

197 """ 

198 # Try cached simple rules first 

199 result = _simple_break_check(prev_gcb, curr_gcb) 

200 if result is not None: 

201 return result 

202 

203 # GB9: x ZWJ (not cached because GB11 needs lookback when prev is ZWJ) 

204 if curr_gcb == GCB.ZWJ: 

205 return BreakResult(should_break=False, ri_count=0) 

206 

207 # GB9c: Indic conjunct cluster 

208 # \p{InCB=Linker} \p{InCB=Extend}* x \p{InCB=Consonant} 

209 curr_ucs = ord(text[curr_idx]) 

210 if _is_incb_consonant(curr_ucs): 

211 i = curr_idx - 1 

212 while i >= 0 and _is_incb_extend(ord(text[i])): 

213 i -= 1 

214 if i >= 0 and _is_incb_linker(ord(text[i])): 

215 return BreakResult(should_break=False, ri_count=0) 

216 

217 # GB11: ExtPict Extend* ZWJ x ExtPict 

218 if prev_gcb == GCB.ZWJ and _is_extended_pictographic(curr_ucs): 

219 i = curr_idx - 2 # Skip the ZWJ at curr_idx - 1 

220 while i >= 0: 

221 prev_ucs = ord(text[i]) 

222 prev_prop = _grapheme_cluster_break(prev_ucs) 

223 if prev_prop == GCB.EXTEND: 

224 i -= 1 

225 elif _is_extended_pictographic(prev_ucs): 

226 return BreakResult(should_break=False, ri_count=0) 

227 else: 

228 break 

229 

230 # GB12/GB13: RI x RI (pair matching) 

231 if prev_gcb == GCB.REGIONAL_INDICATOR and curr_gcb == GCB.REGIONAL_INDICATOR: 

232 if ri_count % 2 == 1: 

233 return BreakResult(should_break=False, ri_count=ri_count + 1) 

234 return BreakResult(should_break=True, ri_count=1) 

235 

236 # GB999: Any ÷ Any 

237 ri_count = 1 if curr_gcb == GCB.REGIONAL_INDICATOR else 0 

238 return BreakResult(should_break=True, ri_count=ri_count) 

239 

240 

241def _iter_graphemes_python( 

242 unistr: str, 

243 start: int = 0, 

244 end: int | None = None, 

245) -> Iterator[str]: 

246 r""" 

247 Iterate over grapheme clusters by UAX #29 extended grapheme cluster rules. 

248 

249 Grapheme clusters are "user-perceived characters" - what a user would 

250 consider a single character, which may consist of multiple Unicode 

251 codepoints (e.g., a base character with combining marks, emoji sequences). 

252 

253 :param unistr: The Unicode string to segment. 

254 :param start: Starting index (default 0). 

255 :param end: Ending index (default len(unistr)). 

256 :yields: Grapheme cluster substrings. 

257 

258 Example:: 

259 

260 >>> list(iter_graphemes('cafe\u0301')) 

261 ['c', 'a', 'f', 'é'] 

262 >>> list(iter_graphemes('ok\U0001F468\u200D\U0001F469\u200D\U0001F467')) 

263 ['o', 'k', '👨\u200d👩\u200d👧'] 

264 >>> list(iter_graphemes('ok\U0001F1FA\U0001F1F8')) 

265 ['o', 'k', '🇺🇸'] 

266 

267 .. versionadded:: 0.3.0 

268 """ 

269 if not unistr: 

270 return 

271 

272 length = len(unistr) 

273 

274 if end is None: 

275 end = length 

276 

277 if start >= end or start >= length: 

278 return 

279 

280 end = min(end, length) 

281 

282 # Track state for grapheme cluster boundaries 

283 cluster_start = start 

284 ri_count = 0 

285 

286 # Get GCB for first character 

287 prev_gcb = _grapheme_cluster_break(ord(unistr[start])) 

288 

289 # Handle Regional Indicator count initialization 

290 if prev_gcb == GCB.REGIONAL_INDICATOR: 

291 ri_count = 1 

292 

293 for idx in range(start + 1, end): 

294 curr_gcb = _grapheme_cluster_break(ord(unistr[idx])) 

295 

296 result = _should_break(prev_gcb, curr_gcb, unistr, idx, ri_count) 

297 ri_count = result.ri_count 

298 

299 if result.should_break: 

300 yield unistr[cluster_start:idx] 

301 cluster_start = idx 

302 

303 prev_gcb = curr_gcb 

304 

305 # Yield the final cluster 

306 yield unistr[cluster_start:end] 

307 

308 

309def _find_cluster_start(text: str, pos: int) -> int: 

310 """ 

311 Find the start of the grapheme cluster containing the character before pos. 

312 

313 Scans backwards from pos to find a safe starting point, then iterates forward using standard 

314 break rules to find the actual cluster boundary. 

315 

316 :param text: The Unicode string. 

317 :param pos: Position to search before (exclusive). 

318 :returns: Start position of the grapheme cluster. 

319 """ 

320 target_cp = ord(text[pos - 1]) 

321 

322 # GB3: CR x LF - LF after CR is part of same cluster 

323 if target_cp == 0x0A and pos >= 2 and text[pos - 2] == '\r': 

324 return pos - 2 

325 

326 # Fast path: ASCII (except LF) starts its own cluster 

327 if target_cp < 0x80: 

328 # GB9b: Check for preceding PREPEND (rare: Arabic/Brahmic) 

329 if pos >= 2 and target_cp >= 0x20: 

330 prev_cp = ord(text[pos - 2]) 

331 if prev_cp >= 0x80 and _grapheme_cluster_break(prev_cp) == GCB.PREPEND: 

332 return _find_cluster_start(text, pos - 1) 

333 return pos - 1 

334 

335 # Scan backward to find a safe starting point 

336 safe_start = pos - 1 

337 while safe_start > 0 and (pos - safe_start) < MAX_GRAPHEME_SCAN: 

338 cp = ord(text[safe_start]) 

339 if 0x20 <= cp < 0x80: # ASCII always starts a cluster 

340 break 

341 if _grapheme_cluster_break(cp) == GCB.CONTROL: # GB4 

342 break 

343 safe_start -= 1 

344 

345 # Verify forward to find the actual cluster boundary 

346 cluster_start = safe_start 

347 left_gcb = _grapheme_cluster_break(ord(text[safe_start])) 

348 ri_count = 1 if left_gcb == GCB.REGIONAL_INDICATOR else 0 

349 

350 for i in range(safe_start + 1, pos): 

351 right_gcb = _grapheme_cluster_break(ord(text[i])) 

352 result = _should_break(left_gcb, right_gcb, text, i, ri_count) 

353 ri_count = result.ri_count 

354 if result.should_break: 

355 cluster_start = i 

356 left_gcb = right_gcb 

357 

358 return cluster_start 

359 

360 

361def grapheme_boundary_before(unistr: str, pos: int) -> int: 

362 r""" 

363 Find the grapheme cluster boundary immediately before a position. 

364 

365 :param unistr: The Unicode string to search. 

366 :param pos: Position in the string (0 < pos <= len(unistr)). 

367 :returns: Start index of the grapheme cluster containing the character at pos-1. 

368 

369 Example:: 

370 

371 >>> grapheme_boundary_before('Hello \U0001F44B\U0001F3FB', 8) 

372 6 

373 >>> grapheme_boundary_before('a\r\nb', 3) 

374 1 

375 

376 .. versionadded:: 0.3.6 

377 """ 

378 if pos <= 0: 

379 return 0 

380 return _find_cluster_start(unistr, min(pos, len(unistr))) 

381 

382 

383def iter_graphemes_reverse( 

384 unistr: str, 

385 start: int = 0, 

386 end: Optional[int] = None, 

387) -> Iterator[str]: 

388 r""" 

389 Iterate over grapheme clusters in reverse order (last to first). 

390 

391 :param unistr: The Unicode string to segment. 

392 :param start: Starting index (default 0). 

393 :param end: Ending index (default len(unistr)). 

394 :yields: Grapheme cluster substrings in reverse order. 

395 

396 Example:: 

397 

398 >>> list(iter_graphemes_reverse('cafe\u0301')) 

399 ['e\u0301', 'f', 'a', 'c'] 

400 

401 .. versionadded:: 0.3.6 

402 """ 

403 if not unistr: 

404 return 

405 

406 length = len(unistr) 

407 

408 end = length if end is None else min(end, length) 

409 start = max(start, 0) 

410 

411 if start >= end or start >= length: 

412 return 

413 

414 pos = end 

415 while pos > start: 

416 cluster_start = _find_cluster_start(unistr, pos) 

417 # Don't yield partial graphemes that extend before start 

418 if cluster_start < start: 

419 break 

420 yield unistr[cluster_start:pos] 

421 pos = cluster_start 

422 

423 

424# Bind iter_graphemes at module level to avoid per-call dispatch overhead. 

425iter_graphemes = _iter_graphemes_python