Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/wcwidth/grapheme.py: 23%

1""" 

2Grapheme cluster segmentation following Unicode Standard Annex #29. 

3 

4This module provides pure-Python implementation of the grapheme cluster boundary algorithm as 

5defined in UAX #29: Unicode Text Segmentation. 

6 

7https://www.unicode.org/reports/tr29/ 

8""" 

9 

10from __future__ import annotations 

11 

12# std imports 

13from enum import IntEnum 

14from functools import lru_cache 

15 

16from typing import TYPE_CHECKING, Optional, NamedTuple 

17 

18__lazy_modules__ = [ 

19 "wcwidth.bisearch", 

20 "wcwidth.table_grapheme", 

21] 

22# local 

23from .bisearch import bisearch as _bisearch 

24from .table_grapheme import (GRAPHEME_L, 

25 GRAPHEME_T, 

26 GRAPHEME_V, 

27 GRAPHEME_LV, 

28 INCB_EXTEND, 

29 INCB_LINKER, 

30 GRAPHEME_LVT, 

31 INCB_CONSONANT, 

32 GRAPHEME_EXTEND, 

33 GRAPHEME_CONTROL, 

34 GRAPHEME_PREPEND, 

35 GRAPHEME_SPACINGMARK, 

36 EXTENDED_PICTOGRAPHIC, 

37 GRAPHEME_REGIONAL_INDICATOR) 

38 

39if TYPE_CHECKING: # pragma: no cover 

40 # std imports 

41 from collections.abc import Iterator 

42 

43# Maximum backward scan distance before falling back to scanning to the start. 

44# Covers all known Unicode grapheme clusters with margin; longer sequences are pathological. 

45MAX_GRAPHEME_SCAN = 16 

46 

47 

48class GCB(IntEnum): 

49 """Grapheme Cluster Break property values.""" 

50 

51 OTHER = 0 

52 CR = 1 

53 LF = 2 

54 CONTROL = 3 

55 EXTEND = 4 

56 ZWJ = 5 

57 REGIONAL_INDICATOR = 6 

58 PREPEND = 7 

59 SPACING_MARK = 8 

60 L = 9 

61 V = 10 

62 T = 11 

63 LV = 12 

64 LVT = 13 

65 

66 

67# All lru_cache sizes in this file use maxsize=1024, chosen by benchmarking UDHR data (500+ 

68# languages) and considering typical process-long sessions: western scripts need ~64 unique 

69# codepoints, and CJK can reach ~2000. 

70@lru_cache(maxsize=1024) 

71def _grapheme_cluster_break(ucs: int) -> GCB: 

72 # pylint: disable=too-many-branches,too-complex 

73 """Return the Grapheme_Cluster_Break property for a codepoint.""" 

74 # Single codepoint matches 

75 if ucs == 0x000d: 

76 return GCB.CR 

77 if ucs == 0x000a: 

78 return GCB.LF 

79 if ucs == 0x200d: 

80 return GCB.ZWJ 

81 # Matching by codepoint ranges, requiring binary search 

82 if _bisearch(ucs, GRAPHEME_CONTROL): 

83 return GCB.CONTROL 

84 if _bisearch(ucs, GRAPHEME_EXTEND): 

85 return GCB.EXTEND 

86 if _bisearch(ucs, GRAPHEME_REGIONAL_INDICATOR): 

87 return GCB.REGIONAL_INDICATOR 

88 if _bisearch(ucs, GRAPHEME_PREPEND): 

89 return GCB.PREPEND 

90 if _bisearch(ucs, GRAPHEME_SPACINGMARK): 

91 return GCB.SPACING_MARK 

92 if _bisearch(ucs, GRAPHEME_L): 

93 return GCB.L 

94 if _bisearch(ucs, GRAPHEME_V): 

95 return GCB.V 

96 if _bisearch(ucs, GRAPHEME_T): 

97 return GCB.T 

98 if _bisearch(ucs, GRAPHEME_LV): 

99 return GCB.LV 

100 if _bisearch(ucs, GRAPHEME_LVT): 

101 return GCB.LVT 

102 return GCB.OTHER 

103 

104 

105@lru_cache(maxsize=1024) 

106def _is_extended_pictographic(ucs: int) -> bool: 

107 """Check if codepoint has Extended_Pictographic property.""" 

108 return bool(_bisearch(ucs, EXTENDED_PICTOGRAPHIC)) 

109 

110 

111@lru_cache(maxsize=1024) 

112def _is_incb_linker(ucs: int) -> bool: 

113 """Check if codepoint has InCB=Linker property.""" 

114 return bool(_bisearch(ucs, INCB_LINKER)) 

115 

116 

117@lru_cache(maxsize=1024) 

118def _is_incb_consonant(ucs: int) -> bool: 

119 """Check if codepoint has InCB=Consonant property.""" 

120 return bool(_bisearch(ucs, INCB_CONSONANT)) 

121 

122 

123@lru_cache(maxsize=1024) 

124def _is_incb_extend(ucs: int) -> bool: 

125 """Check if codepoint has InCB=Extend property.""" 

126 return bool(_bisearch(ucs, INCB_EXTEND)) 

127 

128 

129class BreakResult(NamedTuple): 

130 """Result of grapheme cluster break decision.""" 

131 

132 should_break: bool 

133 ri_count: int 

134 

135 

136@lru_cache(maxsize=1024) 

137def _simple_break_check(prev_gcb: GCB, curr_gcb: GCB) -> Optional[BreakResult]: 

138 """ 

139 Check simple GCB-pair-based break rules (cacheable). 

140 

141 Returns BreakResult for rules that can be determined from GCB properties alone, or None if 

142 complex lookback rules (GB9c, GB11) need to be checked. 

143 """ 

144 # GB3: CR x LF 

145 if prev_gcb == GCB.CR and curr_gcb == GCB.LF: 

146 return BreakResult(should_break=False, ri_count=0) 

147 

148 # GB4: (Control|CR|LF) ÷ 

149 if prev_gcb in (GCB.CONTROL, GCB.CR, GCB.LF): 

150 return BreakResult(should_break=True, 

151 ri_count=1 if curr_gcb == GCB.REGIONAL_INDICATOR else 0) 

152 

153 # GB5: ÷ (Control|CR|LF) 

154 if curr_gcb in (GCB.CONTROL, GCB.CR, GCB.LF): 

155 return BreakResult(should_break=True, ri_count=0) 

156 

157 # GB6: L x (L|V|LV|LVT) 

158 if prev_gcb == GCB.L and curr_gcb in (GCB.L, GCB.V, GCB.LV, GCB.LVT): 

159 return BreakResult(should_break=False, ri_count=0) 

160 

161 # GB7: (LV|V) x (V|T) 

162 if prev_gcb in (GCB.LV, GCB.V) and curr_gcb in (GCB.V, GCB.T): 

163 return BreakResult(should_break=False, ri_count=0) 

164 

165 # GB8: (LVT|T) x T 

166 if prev_gcb in (GCB.LVT, GCB.T) and curr_gcb == GCB.T: 

167 return BreakResult(should_break=False, ri_count=0) 

168 

169 # GB9: x (Extend|ZWJ) - but ZWJ needs GB11 check, so only handle Extend here 

170 if curr_gcb == GCB.EXTEND: 

171 return BreakResult(should_break=False, ri_count=0) 

172 

173 # GB9a: x SpacingMark 

174 if curr_gcb == GCB.SPACING_MARK: 

175 return BreakResult(should_break=False, ri_count=0) 

176 

177 # GB9b: Prepend x 

178 if prev_gcb == GCB.PREPEND: 

179 return BreakResult(should_break=False, 

180 ri_count=1 if curr_gcb == GCB.REGIONAL_INDICATOR else 0) 

181 

182 # GB9c and GB11 need lookback - return None to signal complex check needed 

183 # GB12/13 (RI pairs) need ri_count state - also handled in main function 

184 return None 

185 

186 

187def _should_break( 

188 prev_gcb: GCB, 

189 curr_gcb: GCB, 

190 text: str, 

191 curr_idx: int, 

192 ri_count: int, 

193 *, 

194 start: int = 0, 

195) -> BreakResult: 

196 # pylint: disable=too-many-branches,too-complex 

197 """ 

198 Determine if there should be a grapheme cluster break between prev and curr. 

199 

200 Implements UAX #29 grapheme cluster boundary rules. 

201 """ 

202 # Try cached simple rules first 

203 result = _simple_break_check(prev_gcb, curr_gcb) 

204 if result is not None: 

205 return result 

206 

207 # GB9: x ZWJ (not cached because GB11 needs lookback when prev is ZWJ) 

208 if curr_gcb == GCB.ZWJ: 

209 return BreakResult(should_break=False, ri_count=0) 

210 

211 # GB9c: Indic conjunct cluster 

212 # \p{InCB=Linker} \p{InCB=Extend}* x \p{InCB=Consonant} 

213 curr_ucs = ord(text[curr_idx]) 

214 if _is_incb_consonant(curr_ucs): 

215 i = curr_idx - 1 

216 while i >= start and _is_incb_extend(ord(text[i])): 

217 i -= 1 

218 if i >= start and _is_incb_linker(ord(text[i])): 

219 return BreakResult(should_break=False, ri_count=0) 

220 

221 # GB11: ExtPict Extend* ZWJ x ExtPict 

222 if prev_gcb == GCB.ZWJ and _is_extended_pictographic(curr_ucs): 

223 i = curr_idx - 2 # Skip the ZWJ at curr_idx - 1 

224 while i >= start: 

225 prev_ucs = ord(text[i]) 

226 prev_prop = _grapheme_cluster_break(prev_ucs) 

227 if prev_prop == GCB.EXTEND: 

228 i -= 1 

229 elif _is_extended_pictographic(prev_ucs): 

230 return BreakResult(should_break=False, ri_count=0) 

231 else: 

232 break 

233 

234 # GB12/GB13: RI x RI (pair matching) 

235 if prev_gcb == GCB.REGIONAL_INDICATOR and curr_gcb == GCB.REGIONAL_INDICATOR: 

236 if ri_count % 2 == 1: 

237 return BreakResult(should_break=False, ri_count=ri_count + 1) 

238 return BreakResult(should_break=True, ri_count=1) 

239 

240 # GB999: Any ÷ Any 

241 ri_count = 1 if curr_gcb == GCB.REGIONAL_INDICATOR else 0 

242 return BreakResult(should_break=True, ri_count=ri_count) 

243 

244 

245def _iter_graphemes_python( 

246 unistr: str, 

247 start: int = 0, 

248 end: int | None = None, 

249) -> Iterator[str]: 

250 r""" 

251 Iterate over grapheme clusters by UAX #29 extended grapheme cluster rules. 

252 

253 Grapheme clusters are "user-perceived characters" - what a user would 

254 consider a single character, which may consist of multiple Unicode 

255 codepoints (e.g., a base character with combining marks, emoji sequences). 

256 

257 :param unistr: The Unicode string to segment. 

258 :param start: Starting index (default 0). 

259 :param end: Ending index (default len(unistr)). 

260 :yields: Grapheme cluster substrings. 

261 

262 Example:: 

263 

264 >>> list(iter_graphemes('cafe\u0301')) 

265 ['c', 'a', 'f', 'é'] 

266 >>> list(iter_graphemes('ok\U0001F468\u200D\U0001F469\u200D\U0001F467')) 

267 ['o', 'k', '👨\u200d👩\u200d👧'] 

268 >>> list(iter_graphemes('ok\U0001F1FA\U0001F1F8')) 

269 ['o', 'k', '🇺🇸'] 

270 

271 .. versionadded:: 0.3.0 

272 """ 

273 if not unistr: 

274 return 

275 

276 length = len(unistr) 

277 

278 start = max(start, 0) 

279 

280 if end is None: 

281 end = length 

282 

283 if start >= end or start >= length: 

284 return 

285 

286 end = min(end, length) 

287 

288 # Track state for grapheme cluster boundaries 

289 cluster_start = start 

290 ri_count = 0 

291 

292 # Get GCB for first character 

293 prev_gcb = _grapheme_cluster_break(ord(unistr[start])) 

294 

295 # Handle Regional Indicator count initialization 

296 if prev_gcb == GCB.REGIONAL_INDICATOR: 

297 ri_count = 1 

298 

299 for idx in range(start + 1, end): 

300 curr_gcb = _grapheme_cluster_break(ord(unistr[idx])) 

301 

302 result = _should_break(prev_gcb, curr_gcb, unistr, idx, ri_count, start=start) 

303 ri_count = result.ri_count 

304 

305 if result.should_break: 

306 yield unistr[cluster_start:idx] 

307 cluster_start = idx 

308 

309 prev_gcb = curr_gcb 

310 

311 # Yield the final cluster 

312 yield unistr[cluster_start:end] 

313 

314 

315def _find_cluster_start(text: str, pos: int) -> int: 

316 """ 

317 Find the start of the grapheme cluster containing the character before pos. 

318 

319 Scans backwards from pos to find a safe starting point, then iterates forward using standard 

320 break rules to find the actual cluster boundary. 

321 

322 :param text: The Unicode string. 

323 :param pos: Position to search before (exclusive). 

324 :returns: Start position of the grapheme cluster. 

325 """ 

326 target_cp = ord(text[pos - 1]) 

327 

328 # GB3: CR x LF - LF after CR is part of same cluster 

329 if target_cp == 0x0A and pos >= 2 and text[pos - 2] == '\r': 

330 return pos - 2 

331 

332 # Fast path: ASCII (except LF) starts its own cluster 

333 if target_cp < 0x80: 

334 # GB9b: Check for preceding PREPEND (rare: Arabic/Brahmic) 

335 if pos >= 2 and target_cp >= 0x20: 

336 prev_cp = ord(text[pos - 2]) 

337 if prev_cp >= 0x80 and _grapheme_cluster_break(prev_cp) == GCB.PREPEND: 

338 return _find_cluster_start(text, pos - 1) 

339 return pos - 1 

340 

341 # Scan backward to find a safe starting point 

342 safe_start = pos - 1 

343 while safe_start > 0: 

344 cp = ord(text[safe_start]) 

345 if 0x20 <= cp < 0x80: # ASCII always starts a cluster 

346 break 

347 if _grapheme_cluster_break(cp) == GCB.CONTROL: # GB4 

348 break 

349 if pos - safe_start >= MAX_GRAPHEME_SCAN: 

350 safe_start = 0 

351 break 

352 safe_start -= 1 

353 

354 # Verify forward to find the actual cluster boundary 

355 cluster_start = safe_start 

356 left_gcb = _grapheme_cluster_break(ord(text[safe_start])) 

357 ri_count = 1 if left_gcb == GCB.REGIONAL_INDICATOR else 0 

358 

359 for i in range(safe_start + 1, pos): 

360 right_gcb = _grapheme_cluster_break(ord(text[i])) 

361 result = _should_break(left_gcb, right_gcb, text, i, ri_count) 

362 ri_count = result.ri_count 

363 if result.should_break: 

364 cluster_start = i 

365 left_gcb = right_gcb 

366 

367 return cluster_start 

368 

369 

370def grapheme_boundary_before(unistr: str, pos: int) -> int: 

371 r""" 

372 Find the grapheme cluster boundary immediately before a position. 

373 

374 :param unistr: The Unicode string to search. 

375 :param pos: Position in the string (0 < pos <= len(unistr)). 

376 :returns: Start index of the grapheme cluster containing the character at pos-1. 

377 

378 Example:: 

379 

380 >>> grapheme_boundary_before('Hello \U0001F44B\U0001F3FB', 8) 

381 6 

382 >>> grapheme_boundary_before('a\r\nb', 3) 

383 1 

384 

385 .. versionadded:: 0.3.6 

386 """ 

387 if pos <= 0 or not unistr: 

388 return 0 

389 return _find_cluster_start(unistr, min(pos, len(unistr))) 

390 

391 

392def iter_graphemes_reverse( 

393 unistr: str, 

394 start: int = 0, 

395 end: Optional[int] = None, 

396) -> Iterator[str]: 

397 r""" 

398 Iterate over grapheme clusters in reverse order (last to first). 

399 

400 :param unistr: The Unicode string to segment. 

401 :param start: Starting index (default 0). 

402 :param end: Ending index (default len(unistr)). 

403 :yields: Grapheme cluster substrings in reverse order. 

404 

405 Example:: 

406 

407 >>> list(iter_graphemes_reverse('cafe\u0301')) 

408 ['e\u0301', 'f', 'a', 'c'] 

409 

410 .. versionadded:: 0.3.6 

411 """ 

412 if not unistr: 

413 return 

414 

415 length = len(unistr) 

416 

417 end = length if end is None else min(end, length) 

418 start = max(start, 0) 

419 

420 if start >= end or start >= length: 

421 return 

422 

423 pos = end 

424 while pos > start: 

425 cluster_start = _find_cluster_start(unistr, pos) 

426 # Don't yield partial graphemes that extend before start 

427 if cluster_start < start: 

428 break 

429 yield unistr[cluster_start:pos] 

430 pos = cluster_start 

431 

432 

433# Bind iter_graphemes at module level to avoid per-call dispatch overhead. 

434iter_graphemes = _iter_graphemes_python