Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/boltons/strutils.py: 19%

1# Copyright (c) 2013, Mahmoud Hashemi 

2# 

3# Redistribution and use in source and binary forms, with or without 

4# modification, are permitted provided that the following conditions are 

5# met: 

6# 

7# * Redistributions of source code must retain the above copyright 

8# notice, this list of conditions and the following disclaimer. 

9# 

10# * Redistributions in binary form must reproduce the above 

11# copyright notice, this list of conditions and the following 

12# disclaimer in the documentation and/or other materials provided 

13# with the distribution. 

14# 

15# * The names of the contributors may not be used to endorse or 

16# promote products derived from this software without specific 

17# prior written permission. 

18# 

19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 

20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 

21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 

22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 

23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 

24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 

25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 

26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 

27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 

28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 

29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 

30 

31"""So much practical programming involves string manipulation, which 

32Python readily accommodates. Still, there are dozens of basic and 

33common capabilities missing from the standard library, several of them 

34provided by ``strutils``. 

35""" 

36 

37 

38import builtins 

39import collections 

40import re 

41import string 

42import sys 

43import typing 

44import unicodedata 

45import uuid 

46import zlib 

47from collections.abc import Mapping 

48from gzip import GzipFile 

49from html import entities as htmlentitydefs 

50from html.parser import HTMLParser 

51from io import BytesIO as StringIO 

52 

53__all__ = ['camel2under', 'under2camel', 'slugify', 'split_punct_ws', 

54 'unit_len', 'ordinalize', 'cardinalize', 'pluralize', 'singularize', 

55 'asciify', 'is_ascii', 'is_uuid', 'html2text', 'strip_ansi', 

56 'bytes2human', 'find_hashtags', 'a10n', 'gzip_bytes', 'gunzip_bytes', 

57 'iter_splitlines', 'indent', 'escape_shell_args', 

58 'args2cmd', 'args2sh', 'parse_int_list', 'format_int_list', 

59 'complement_int_list', 'int_ranges_from_int_list', 'MultiReplace', 

60 'multi_replace', 'unwrap_text', 'removeprefix', 

61 'human_readable_list', 'ellipsize'] 

62 

63 

64_punct_ws_str = string.punctuation + string.whitespace 

65_punct_re = re.compile('[' + _punct_ws_str + ']+') 

66_camel2under_re = re.compile('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))') 

67 

68 

69def camel2under(camel_string): 

70 """Converts a camelcased string to underscores. Useful for turning a 

71 class name into a function name. 

72 

73 >>> camel2under('BasicParseTest') 

74 'basic_parse_test' 

75 """ 

76 return _camel2under_re.sub(r'_\1', camel_string).lower() 

77 

78 

79def under2camel(under_string): 

80 """Converts an underscored string to camelcased. Useful for turning a 

81 function name into a class name. 

82 

83 >>> under2camel('complex_tokenizer') 

84 'ComplexTokenizer' 

85 """ 

86 return ''.join(w.capitalize() or '_' for w in under_string.split('_')) 

87 

88 

89def slugify(text, delim='_', lower=True, ascii=False): 

90 """ 

91 A basic function that turns text full of scary characters 

92 (i.e., punctuation and whitespace), into a relatively safe 

93 lowercased string separated only by the delimiter specified 

94 by *delim*, which defaults to ``_``. 

95 

96 The *ascii* convenience flag will :func:`asciify` the slug if 

97 you require ascii-only slugs. 

98 

99 >>> slugify('First post! Hi!!!!~1 ') 

100 'first_post_hi_1' 

101 

102 >>> slugify("Kurt Gödel's pretty cool.", ascii=True) == \ 

103 b'kurt_goedel_s_pretty_cool' 

104 True 

105 

106 """ 

107 ret = delim.join(split_punct_ws(text)) or delim if text else '' 

108 if ascii: 

109 ret = asciify(ret) 

110 if lower: 

111 ret = ret.lower() 

112 return ret 

113 

114 

115def split_punct_ws(text): 

116 """While :meth:`str.split` will split on whitespace, 

117 :func:`split_punct_ws` will split on punctuation and 

118 whitespace. This used internally by :func:`slugify`, above. 

119 

120 >>> split_punct_ws('First post! Hi!!!!~1 ') 

121 ['First', 'post', 'Hi', '1'] 

122 """ 

123 return [w for w in _punct_re.split(text) if w] 

124 

125 

126def unit_len(sized_iterable, unit_noun='item'): # TODO: len_units()/unitize()? 

127 """Returns a plain-English description of an iterable's 

128 :func:`len()`, conditionally pluralized with :func:`cardinalize`, 

129 detailed below. 

130 

131 >>> print(unit_len(range(10), 'number')) 

132 10 numbers 

133 >>> print(unit_len('aeiou', 'vowel')) 

134 5 vowels 

135 >>> print(unit_len([], 'worry')) 

136 No worries 

137 """ 

138 count = len(sized_iterable) 

139 units = cardinalize(unit_noun, count) 

140 if count: 

141 return f'{count} {units}' 

142 return f'No {units}' 

143 

144 

145_ORDINAL_MAP = {'1': 'st', 

146 '2': 'nd', 

147 '3': 'rd'} # 'th' is the default 

148 

149 

150def ordinalize(number, ext_only=False): 

151 """Turns *number* into its cardinal form, i.e., 1st, 2nd, 

152 3rd, 4th, etc. If the last character isn't a digit, it returns the 

153 string value unchanged. 

154 

155 Args: 

156 number (int or str): Number to be cardinalized. 

157 ext_only (bool): Whether to return only the suffix. Default ``False``. 

158 

159 >>> print(ordinalize(1)) 

160 1st 

161 >>> print(ordinalize(3694839230)) 

162 3694839230th 

163 >>> print(ordinalize('hi')) 

164 hi 

165 >>> print(ordinalize(1515)) 

166 1515th 

167 """ 

168 numstr, ext = str(number), '' 

169 if numstr and numstr[-1] in string.digits: 

170 try: 

171 # first check for teens 

172 if numstr[-2] == '1': 

173 ext = 'th' 

174 else: 

175 # all other cases 

176 ext = _ORDINAL_MAP.get(numstr[-1], 'th') 

177 except IndexError: 

178 # single digit numbers (will reach here based on [-2] above) 

179 ext = _ORDINAL_MAP.get(numstr[-1], 'th') 

180 if ext_only: 

181 return ext 

182 else: 

183 return numstr + ext 

184 

185 

186def cardinalize(unit_noun, count): 

187 """Conditionally pluralizes a singular word *unit_noun* if 

188 *count* is not one, preserving case when possible. 

189 

190 >>> vowels = 'aeiou' 

191 >>> print(len(vowels), cardinalize('vowel', len(vowels))) 

192 5 vowels 

193 >>> print(3, cardinalize('Wish', 3)) 

194 3 Wishes 

195 """ 

196 if count == 1: 

197 return unit_noun 

198 return pluralize(unit_noun) 

199 

200 

201def singularize(word): 

202 """Semi-intelligently converts an English plural *word* to its 

203 singular form, preserving case pattern. 

204 

205 >>> singularize('chances') 

206 'chance' 

207 >>> singularize('Activities') 

208 'Activity' 

209 >>> singularize('Glasses') 

210 'Glass' 

211 >>> singularize('FEET') 

212 'FOOT' 

213 

214 """ 

215 orig_word, word = word, word.strip().lower() 

216 if not word or word in _IRR_S2P: 

217 return orig_word 

218 

219 irr_singular = _IRR_P2S.get(word) 

220 if irr_singular: 

221 singular = irr_singular 

222 elif not word.endswith('s'): 

223 return orig_word 

224 elif len(word) == 2: 

225 singular = word[:-1] # or just return word? 

226 elif word.endswith('ies') and word[-4:-3] not in 'aeiou': 

227 singular = word[:-3] + 'y' 

228 elif word.endswith('es') and word[-3] == 's': 

229 singular = word[:-2] 

230 elif word.endswith('ss'): 

231 # Words ending in a double 's' (glass, boss, kiss) are already 

232 # singular; their plurals end in 'sses' and are handled above. Do 

233 # not blindly strip the trailing 's', which would produce 'glas', 

234 # 'bos', 'kis' and break idempotency (singularize('Glasses') == 

235 # 'Glass', but 'Glass' must stay 'Glass'). 

236 return orig_word 

237 else: 

238 singular = word[:-1] 

239 return _match_case(orig_word, singular) 

240 

241 

242def pluralize(word): 

243 """Semi-intelligently converts an English *word* from singular form to 

244 plural, preserving case pattern. 

245 

246 >>> pluralize('friend') 

247 'friends' 

248 >>> pluralize('enemy') 

249 'enemies' 

250 >>> pluralize('Sheep') 

251 'Sheep' 

252 """ 

253 orig_word, word = word, word.strip().lower() 

254 if not word or word in _IRR_P2S: 

255 return orig_word 

256 irr_plural = _IRR_S2P.get(word) 

257 if irr_plural: 

258 plural = irr_plural 

259 elif word.endswith('y') and word[-2:-1] not in 'aeiou': 

260 plural = word[:-1] + 'ies' 

261 elif word[-1] in 'sx' or word.endswith('ch') or word.endswith('sh'): 

262 plural = word if word.endswith('es') else word + 'es' 

263 else: 

264 plural = word + 's' 

265 return _match_case(orig_word, plural) 

266 

267 

268def _match_case(master, disciple): 

269 if not master.strip(): 

270 return disciple 

271 if master.lower() == master: 

272 return disciple.lower() 

273 elif master.upper() == master: 

274 return disciple.upper() 

275 elif master.title() == master: 

276 return disciple.title() 

277 return disciple 

278 

279 

280# Singular to plural map of irregular pluralizations 

281_IRR_S2P = {'addendum': 'addenda', 'alga': 'algae', 'alumna': 'alumnae', 

282 'alumnus': 'alumni', 'analysis': 'analyses', 'antenna': 'antennae', 

283 'appendix': 'appendices', 'axis': 'axes', 'bacillus': 'bacilli', 

284 'bacterium': 'bacteria', 'basis': 'bases', 'beau': 'beaux', 

285 'bison': 'bison', 'bureau': 'bureaus', 'cactus': 'cacti', 

286 'calf': 'calves', 'child': 'children', 'corps': 'corps', 

287 'corpus': 'corpora', 'crisis': 'crises', 'criterion': 'criteria', 

288 'curriculum': 'curricula', 'datum': 'data', 'deer': 'deer', 

289 'diagnosis': 'diagnoses', 'die': 'dice', 'dwarf': 'dwarves', 

290 'echo': 'echoes', 'elf': 'elves', 'ellipsis': 'ellipses', 

291 'embargo': 'embargoes', 'emphasis': 'emphases', 'erratum': 'errata', 

292 'fireman': 'firemen', 'fish': 'fish', 'focus': 'foci', 

293 'foot': 'feet', 'formula': 'formulae', 'formula': 'formulas', 

294 'fungus': 'fungi', 'genus': 'genera', 'goose': 'geese', 

295 'half': 'halves', 'hero': 'heroes', 'hippopotamus': 'hippopotami', 

296 'hoof': 'hooves', 'hypothesis': 'hypotheses', 'index': 'indices', 

297 'knife': 'knives', 'leaf': 'leaves', 'life': 'lives', 

298 'loaf': 'loaves', 'louse': 'lice', 'man': 'men', 

299 'matrix': 'matrices', 'means': 'means', 'medium': 'media', 

300 'memorandum': 'memoranda', 'millennium': 'milennia', 'moose': 'moose', 

301 'mosquito': 'mosquitoes', 'mouse': 'mice', 'nebula': 'nebulae', 

302 'neurosis': 'neuroses', 'nucleus': 'nuclei', 'oasis': 'oases', 

303 'octopus': 'octopi', 'offspring': 'offspring', 'ovum': 'ova', 

304 'ox': 'oxen', 'paralysis': 'paralyses', 'parenthesis': 'parentheses', 

305 'person': 'people', 'phenomenon': 'phenomena', 'potato': 'potatoes', 

306 'radius': 'radii', 'scarf': 'scarves', 'scissors': 'scissors', 

307 'self': 'selves', 'sense': 'senses', 'series': 'series', 'sheep': 

308 'sheep', 'shelf': 'shelves', 'species': 'species', 'stimulus': 

309 'stimuli', 'stratum': 'strata', 'syllabus': 'syllabi', 'symposium': 

310 'symposia', 'synopsis': 'synopses', 'synthesis': 'syntheses', 

311 'tableau': 'tableaux', 'that': 'those', 'thesis': 'theses', 

312 'thief': 'thieves', 'this': 'these', 'tomato': 'tomatoes', 'tooth': 

313 'teeth', 'torpedo': 'torpedoes', 'vertebra': 'vertebrae', 'veto': 

314 'vetoes', 'vita': 'vitae', 'watch': 'watches', 'wife': 'wives', 

315 'wolf': 'wolves', 'woman': 'women'} 

316 

317 

318# Reverse index of the above 

319_IRR_P2S = {v: k for k, v in _IRR_S2P.items()} 

320 

321HASHTAG_RE = re.compile(r"(?:^|\s)[＃#]{1}(\w+)", re.UNICODE) 

322 

323 

324def find_hashtags(string): 

325 """Finds and returns all hashtags in a string, with the hashmark 

326 removed. Supports full-width hashmarks for Asian languages and 

327 does not false-positive on URL anchors. 

328 

329 >>> find_hashtags('#atag http://asite/#ananchor') 

330 ['atag'] 

331 

332 ``find_hashtags`` also works with unicode hashtags. 

333 """ 

334 

335 # the following works, doctest just struggles with it 

336 # >>> find_hashtags(u"can't get enough of that dignity chicken #肯德基 woo") 

337 # [u'\u80af\u5fb7\u57fa'] 

338 return HASHTAG_RE.findall(string) 

339 

340 

341def a10n(string): 

342 """That thing where "internationalization" becomes "i18n", what's it 

343 called? Abbreviation? Oh wait, no: ``a10n``. (It's actually a form 

344 of `numeronym`_.) 

345 

346 >>> a10n('abbreviation') 

347 'a10n' 

348 >>> a10n('internationalization') 

349 'i18n' 

350 >>> a10n('') 

351 '' 

352 

353 .. _numeronym: http://en.wikipedia.org/wiki/Numeronym 

354 """ 

355 if len(string) < 3: 

356 return string 

357 return f'{string[0]}{len(string[1:-1])}{string[-1]}' 

358 

359 

360# Based on https://en.wikipedia.org/wiki/ANSI_escape_code#Escape_sequences 

361ANSI_SEQUENCES = re.compile(r''' 

362 \x1B # Sequence starts with ESC, i.e. hex 0x1B 

363 (?: 

364 [@-Z\\-_] # Second byte: 

365 # all 0x40–0x5F range but CSI char, i.e ASCII @A–Z\]^_ 

366 | # Or 

367 \[ # CSI sequences, starting with [ 

368 [0-?]* # Parameter bytes: 

369 # range 0x30–0x3F, ASCII 0–9:;<=>? 

370 [ -/]* # Intermediate bytes: 

371 # range 0x20–0x2F, ASCII space and !"#$%&'()*+,-./ 

372 [@-~] # Final byte 

373 # range 0x40–0x7E, ASCII @A–Z[\]^_`a–z{|}~ 

374 ) 

375''', re.VERBOSE) 

376 

377 

378def strip_ansi(text): 

379 """Strips ANSI escape codes from *text*. Useful for the occasional 

380 time when a log or redirected output accidentally captures console 

381 color codes and the like. 

382 

383 >>> strip_ansi('\x1b[0m\x1b[1;36mart\x1b[46;34m') 

384 'art' 

385 

386 Supports str, bytes and bytearray content as input. Returns the 

387 same type as the input. 

388 

389 There's a lot of ANSI art available for testing on `sixteencolors.net`_. 

390 This function does not interpret or render ANSI art, but you can do so with 

391 `ansi2img`_ or `escapes.js`_. 

392 

393 .. _sixteencolors.net: http://sixteencolors.net 

394 .. _ansi2img: http://www.bedroomlan.org/projects/ansi2img 

395 .. _escapes.js: https://github.com/atdt/escapes.js 

396 """ 

397 # TODO: move to cliutils.py 

398 

399 # Transform any ASCII-like content to unicode to allow regex to match, and 

400 # save input type for later. 

401 target_type = None 

402 # Unicode type aliased to str is code-smell for Boltons in Python 3 env. 

403 if isinstance(text, (bytes, bytearray)): 

404 target_type = type(text) 

405 text = text.decode('utf-8') 

406 

407 cleaned = ANSI_SEQUENCES.sub('', text) 

408 

409 # Transform back the result to the same bytearray type provided by the user. 

410 if target_type and target_type != type(cleaned): 

411 cleaned = target_type(cleaned, 'utf-8') 

412 

413 return cleaned 

414 

415 

416def asciify(text, ignore=False): 

417 """Converts a unicode or bytestring, *text*, into a bytestring with 

418 just ascii characters. Performs basic deaccenting for all you 

419 Europhiles out there. 

420 

421 Also, a gentle reminder that this is a **utility**, primarily meant 

422 for slugification. Whenever possible, make your application work 

423 **with** unicode, not against it. 

424 

425 Args: 

426 text (str): The string to be asciified. 

427 ignore (bool): Configures final encoding to ignore remaining 

428 unasciified string instead of replacing it. 

429 

430 >>> asciify('Beyoncé') == b'Beyonce' 

431 True 

432 """ 

433 try: 

434 try: 

435 return text.encode('ascii') 

436 except UnicodeDecodeError: 

437 # this usually means you passed in a non-unicode string 

438 text = text.decode('utf-8') 

439 return text.encode('ascii') 

440 except UnicodeEncodeError: 

441 mode = 'replace' 

442 if ignore: 

443 mode = 'ignore' 

444 transd = unicodedata.normalize('NFKD', text.translate(DEACCENT_MAP)) 

445 ret = transd.encode('ascii', mode) 

446 return ret 

447 

448 

449def is_ascii(text): 

450 """Check if a string or bytestring, *text*, is composed of ascii 

451 characters only. Raises :exc:`ValueError` if argument is not text. 

452 

453 Args: 

454 text (str): The string to be checked. 

455 

456 >>> is_ascii('Beyoncé') 

457 False 

458 >>> is_ascii('Beyonce') 

459 True 

460 """ 

461 if isinstance(text, str): 

462 try: 

463 text.encode('ascii') 

464 except UnicodeEncodeError: 

465 return False 

466 elif isinstance(text, bytes): 

467 try: 

468 text.decode('ascii') 

469 except UnicodeDecodeError: 

470 return False 

471 else: 

472 raise ValueError('expected text or bytes, not %r' % type(text)) 

473 return True 

474 

475 

476class DeaccenterDict(dict): 

477 "A small caching dictionary for deaccenting." 

478 def __missing__(self, key): 

479 ch = self.get(key) 

480 if ch is not None: 

481 return ch 

482 try: 

483 de = unicodedata.decomposition(chr(key)) 

484 p1, _, p2 = de.rpartition(' ') 

485 if int(p2, 16) == 0x308: 

486 ch = self.get(key) 

487 else: 

488 ch = int(p1, 16) 

489 except (IndexError, ValueError): 

490 ch = self.get(key, key) 

491 self[key] = ch 

492 return ch 

493 

494 

495# http://chmullig.com/2009/12/python-unicode-ascii-ifier/ 

496# For something more complete, investigate the unidecode 

497# or isounidecode packages, which are capable of performing 

498# crude transliteration. 

499_BASE_DEACCENT_MAP = { 

500 0xc6: "AE", # Æ LATIN CAPITAL LETTER AE 

501 0xd0: "D", # Ð LATIN CAPITAL LETTER ETH 

502 0xd8: "OE", # Ø LATIN CAPITAL LETTER O WITH STROKE 

503 0xde: "Th", # Þ LATIN CAPITAL LETTER THORN 

504 0xc4: 'Ae', # Ä LATIN CAPITAL LETTER A WITH DIAERESIS 

505 0xd6: 'Oe', # Ö LATIN CAPITAL LETTER O WITH DIAERESIS 

506 0xdc: 'Ue', # Ü LATIN CAPITAL LETTER U WITH DIAERESIS 

507 0xc0: "A", # À LATIN CAPITAL LETTER A WITH GRAVE 

508 0xc1: "A", # Á LATIN CAPITAL LETTER A WITH ACUTE 

509 0xc3: "A", # Ã LATIN CAPITAL LETTER A WITH TILDE 

510 0xc7: "C", # Ç LATIN CAPITAL LETTER C WITH CEDILLA 

511 0xc8: "E", # È LATIN CAPITAL LETTER E WITH GRAVE 

512 0xc9: "E", # É LATIN CAPITAL LETTER E WITH ACUTE 

513 0xca: "E", # Ê LATIN CAPITAL LETTER E WITH CIRCUMFLEX 

514 0xcc: "I", # Ì LATIN CAPITAL LETTER I WITH GRAVE 

515 0xcd: "I", # Í LATIN CAPITAL LETTER I WITH ACUTE 

516 0xd2: "O", # Ò LATIN CAPITAL LETTER O WITH GRAVE 

517 0xd3: "O", # Ó LATIN CAPITAL LETTER O WITH ACUTE 

518 0xd5: "O", # Õ LATIN CAPITAL LETTER O WITH TILDE 

519 0xd9: "U", # Ù LATIN CAPITAL LETTER U WITH GRAVE 

520 0xda: "U", # Ú LATIN CAPITAL LETTER U WITH ACUTE 

521 0xdf: "ss", # ß LATIN SMALL LETTER SHARP S 

522 0xe6: "ae", # æ LATIN SMALL LETTER AE 

523 0xf0: "d", # ð LATIN SMALL LETTER ETH 

524 0xf8: "oe", # ø LATIN SMALL LETTER O WITH STROKE 

525 0xfe: "th", # þ LATIN SMALL LETTER THORN, 

526 0xe4: 'ae', # ä LATIN SMALL LETTER A WITH DIAERESIS 

527 0xf6: 'oe', # ö LATIN SMALL LETTER O WITH DIAERESIS 

528 0xfc: 'ue', # ü LATIN SMALL LETTER U WITH DIAERESIS 

529 0xe0: "a", # à LATIN SMALL LETTER A WITH GRAVE 

530 0xe1: "a", # á LATIN SMALL LETTER A WITH ACUTE 

531 0xe3: "a", # ã LATIN SMALL LETTER A WITH TILDE 

532 0xe7: "c", # ç LATIN SMALL LETTER C WITH CEDILLA 

533 0xe8: "e", # è LATIN SMALL LETTER E WITH GRAVE 

534 0xe9: "e", # é LATIN SMALL LETTER E WITH ACUTE 

535 0xea: "e", # ê LATIN SMALL LETTER E WITH CIRCUMFLEX 

536 0xec: "i", # ì LATIN SMALL LETTER I WITH GRAVE 

537 0xed: "i", # í LATIN SMALL LETTER I WITH ACUTE 

538 0xf2: "o", # ò LATIN SMALL LETTER O WITH GRAVE 

539 0xf3: "o", # ó LATIN SMALL LETTER O WITH ACUTE 

540 0xf5: "o", # õ LATIN SMALL LETTER O WITH TILDE 

541 0xf9: "u", # ù LATIN SMALL LETTER U WITH GRAVE 

542 0xfa: "u", # ú LATIN SMALL LETTER U WITH ACUTE 

543 0x2018: "'", # ‘ LEFT SINGLE QUOTATION MARK 

544 0x2019: "'", # ’ RIGHT SINGLE QUOTATION MARK 

545 0x201c: '"', # “ LEFT DOUBLE QUOTATION MARK 

546 0x201d: '"', # ” RIGHT DOUBLE QUOTATION MARK 

547 } 

548 

549 

550DEACCENT_MAP = DeaccenterDict(_BASE_DEACCENT_MAP) 

551 

552 

553_SIZE_SYMBOLS = ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') 

554_SIZE_BOUNDS = [(1024 ** i, sym) for i, sym in enumerate(_SIZE_SYMBOLS)] 

555_SIZE_BOUNDS.append((float('inf'), '')) 

556_SIZE_RANGES = list(zip(_SIZE_BOUNDS, _SIZE_BOUNDS[1:])) 

557 

558 

559def bytes2human(nbytes, ndigits=0): 

560 """Turns an integer value of *nbytes* into a human readable format. Set 

561 *ndigits* to control how many digits after the decimal point 

562 should be shown (default ``0``). 

563 

564 >>> bytes2human(128991) 

565 '126K' 

566 >>> bytes2human(100001221) 

567 '95M' 

568 >>> bytes2human(0, 2) 

569 '0.00B' 

570 >>> bytes2human(1024) 

571 '1K' 

572 >>> bytes2human(1024 ** 8) 

573 '1Y' 

574 """ 

575 abs_bytes = abs(nbytes) 

576 for (size, symbol), (next_size, next_symbol) in _SIZE_RANGES: 

577 if abs_bytes < next_size: 

578 break 

579 hnbytes = float(nbytes) / size 

580 return '{hnbytes:.{ndigits}f}{symbol}'.format(hnbytes=hnbytes, 

581 ndigits=ndigits, 

582 symbol=symbol) 

583 

584 

585class HTMLTextExtractor(HTMLParser): 

586 def __init__(self) -> None: 

587 super().__init__(convert_charrefs=True) 

588 self.result: list[str] = [] 

589 

590 def handle_data(self, d): 

591 self.result.append(d) 

592 

593 def handle_charref(self, number): 

594 if number[0] == 'x' or number[0] == 'X': 

595 codepoint = int(number[1:], 16) 

596 else: 

597 codepoint = int(number) 

598 self.result.append(chr(codepoint)) 

599 

600 def handle_entityref(self, name): 

601 try: 

602 codepoint = htmlentitydefs.name2codepoint[name] 

603 except KeyError: 

604 self.result.append('&' + name + ';') 

605 else: 

606 self.result.append(chr(codepoint)) 

607 

608 def get_text(self): 

609 return ''.join(self.result) 

610 

611 

612def html2text(html): 

613 """Strips tags from HTML text, returning markup-free text. Also, does 

614 a best effort replacement of entities like "&nbsp;" 

615 

616 >>> r = html2text(u'<a href="#">Test &amp;<em>(\u0394&#x03b7;&#956;&#x03CE;)</em></a>') 

617 >>> r == u'Test &(\u0394\u03b7\u03bc\u03ce)' 

618 True 

619 """ 

620 # based on answers to http://stackoverflow.com/questions/753052/ 

621 s = HTMLTextExtractor() 

622 s.feed(html) 

623 return s.get_text() 

624 

625 

626_EMPTY_GZIP_BYTES = b'\x1f\x8b\x08\x089\xf3\xb9U\x00\x03empty\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00' 

627_NON_EMPTY_GZIP_BYTES = b'\x1f\x8b\x08\x08\xbc\xf7\xb9U\x00\x03not_empty\x00K\xaa,I-N\xcc\xc8\xafT\xe4\x02\x00\xf3nb\xbf\x0b\x00\x00\x00' 

628 

629 

630def gunzip_bytes(bytestring): 

631 """The :mod:`gzip` module is great if you have a file or file-like 

632 object, but what if you just have bytes. StringIO is one 

633 possibility, but it's often faster, easier, and simpler to just 

634 use this one-liner. Use this tried-and-true utility function to 

635 decompress gzip from bytes. 

636 

637 >>> gunzip_bytes(_EMPTY_GZIP_BYTES) == b'' 

638 True 

639 >>> gunzip_bytes(_NON_EMPTY_GZIP_BYTES).rstrip() == b'bytesahoy!' 

640 True 

641 """ 

642 return zlib.decompress(bytestring, 16 + zlib.MAX_WBITS) 

643 

644 

645def gzip_bytes(bytestring, level=6): 

646 """Turn some bytes into some compressed bytes. 

647 

648 >>> len(gzip_bytes(b'a' * 10000)) 

649 46 

650 

651 Args: 

652 bytestring (bytes): Bytes to be compressed 

653 level (int): An integer, 1-9, controlling the 

654 speed/compression. 1 is fastest, least compressed, 9 is 

655 slowest, but most compressed. 

656 

657 Note that all levels of gzip are pretty fast these days, though 

658 it's not really a competitor in compression, at any level. 

659 """ 

660 out = StringIO() 

661 f = GzipFile(fileobj=out, mode='wb', compresslevel=level) 

662 f.write(bytestring) 

663 f.close() 

664 return out.getvalue() 

665 

666 

667 

668_line_ending_re = re.compile(r'(\r\n|\n|\x0b|\f|\r|\x85|\u2028|\u2029)', 

669 re.UNICODE) 

670 

671 

672def iter_splitlines(text): 

673 r"""Like :meth:`str.splitlines`, but returns an iterator of lines 

674 instead of a list. Also similar to :meth:`file.next`, as that also 

675 lazily reads and yields lines from a file. 

676 

677 This function works with a variety of line endings, but as always, 

678 be careful when mixing line endings within a file. 

679 

680 >>> list(iter_splitlines('\nhi\nbye\n')) 

681 ['', 'hi', 'bye', ''] 

682 >>> list(iter_splitlines('\r\nhi\rbye\r\n')) 

683 ['', 'hi', 'bye', ''] 

684 >>> list(iter_splitlines('')) 

685 [] 

686 """ 

687 prev_end, len_text = 0, len(text) 

688 # print('last: %r' % last_idx) 

689 # start, end = None, None 

690 for match in _line_ending_re.finditer(text): 

691 start, end = match.start(1), match.end(1) 

692 # print(start, end) 

693 if prev_end <= start: 

694 yield text[prev_end:start] 

695 if end == len_text: 

696 yield '' 

697 prev_end = end 

698 tail = text[prev_end:] 

699 if tail: 

700 yield tail 

701 return 

702 

703 

704def indent(text, margin, newline='\n', key=bool): 

705 """The missing counterpart to the built-in :func:`textwrap.dedent`. 

706 

707 Args: 

708 text (str): The text to indent. 

709 margin (str): The string to prepend to each line. 

710 newline (str): The newline used to rejoin the lines (default: ``\\n``) 

711 key (callable): Called on each line to determine whether to 

712 indent it. Default: :class:`bool`, to ensure that empty lines do 

713 not get whitespace added. 

714 """ 

715 indented_lines = [(margin + line if key(line) else line) 

716 for line in iter_splitlines(text)] 

717 return newline.join(indented_lines) 

718 

719 

720def is_uuid(obj, version=4): 

721 """Check the argument is either a valid UUID object or string. 

722 

723 Args: 

724 obj (object): The test target. Strings and UUID objects supported. 

725 version (int): The target UUID version, set to 0 to skip version check. 

726 

727 >>> is_uuid('e682ccca-5a4c-4ef2-9711-73f9ad1e15ea') 

728 True 

729 >>> is_uuid('0221f0d9-d4b9-11e5-a478-10ddb1c2feb9') 

730 False 

731 >>> is_uuid('0221f0d9-d4b9-11e5-a478-10ddb1c2feb9', version=1) 

732 True 

733 """ 

734 if not isinstance(obj, uuid.UUID): 

735 try: 

736 obj = uuid.UUID(obj) 

737 except (TypeError, ValueError, AttributeError): 

738 return False 

739 if version and obj.version != int(version): 

740 return False 

741 return True 

742 

743 

744def escape_shell_args(args, sep=' ', style=None): 

745 """Returns an escaped version of each string in *args*, according to 

746 *style*. 

747 

748 Args: 

749 args (list): A list of arguments to escape and join together 

750 sep (str): The separator used to join the escaped arguments. 

751 style (str): The style of escaping to use. Can be one of 

752 ``cmd`` or ``sh``, geared toward Windows and Linux/BSD/etc., 

753 respectively. If *style* is ``None``, then it is picked 

754 according to the system platform. 

755 

756 See :func:`args2cmd` and :func:`args2sh` for details and example 

757 output for each style. 

758 """ 

759 if not style: 

760 style = 'cmd' if sys.platform == 'win32' else 'sh' 

761 

762 if style == 'sh': 

763 return args2sh(args, sep=sep) 

764 elif style == 'cmd': 

765 return args2cmd(args, sep=sep) 

766 

767 raise ValueError("style expected one of 'cmd' or 'sh', not %r" % style) 

768 

769 

770_find_sh_unsafe = re.compile(r'[^a-zA-Z0-9_@%+=:,./-]').search 

771 

772 

773def args2sh(args, sep=' '): 

774 """Return a shell-escaped string version of *args*, separated by 

775 *sep*, based on the rules of sh, bash, and other shells in the 

776 Linux/BSD/MacOS ecosystem. 

777 

778 >>> print(args2sh(['aa', '[bb]', "cc'cc", 'dd"dd'])) 

779 aa '[bb]' 'cc'"'"'cc' 'dd"dd' 

780 

781 As you can see, arguments with no special characters are not 

782 escaped, arguments with special characters are quoted with single 

783 quotes, and single quotes themselves are quoted with double 

784 quotes. Double quotes are handled like any other special 

785 character. 

786 

787 Based on code from the :mod:`pipes`/:mod:`shlex` modules. Also 

788 note that :mod:`shlex` and :mod:`argparse` have functions to split 

789 and parse strings escaped in this manner. 

790 """ 

791 ret_list = [] 

792 

793 for arg in args: 

794 if not arg: 

795 ret_list.append("''") 

796 continue 

797 if _find_sh_unsafe(arg) is None: 

798 ret_list.append(arg) 

799 continue 

800 # use single quotes, and put single quotes into double quotes 

801 # the string $'b is then quoted as '$'"'"'b' 

802 ret_list.append("'" + arg.replace("'", "'\"'\"'") + "'") 

803 

804 return sep.join(ret_list) 

805 

806 

807def args2cmd(args, sep=' '): 

808 r"""Return a shell-escaped string version of *args*, separated by 

809 *sep*, using the same rules as the Microsoft C runtime. 

810 

811 >>> print(args2cmd(['aa', '[bb]', "cc'cc", 'dd"dd'])) 

812 aa [bb] cc'cc dd\"dd 

813 

814 As you can see, escaping is through backslashing and not quoting, 

815 and double quotes are the only special character. See the comment 

816 in the code for more details. Based on internal code from the 

817 :mod:`subprocess` module. 

818 

819 """ 

820 # technique description from subprocess below 

821 """ 

822 1) Arguments are delimited by white space, which is either a 

823 space or a tab. 

824 

825 2) A string surrounded by double quotation marks is 

826 interpreted as a single argument, regardless of white space 

827 contained within. A quoted string can be embedded in an 

828 argument. 

829 

830 3) A double quotation mark preceded by a backslash is 

831 interpreted as a literal double quotation mark. 

832 

833 4) Backslashes are interpreted literally, unless they 

834 immediately precede a double quotation mark. 

835 

836 5) If backslashes immediately precede a double quotation mark, 

837 every pair of backslashes is interpreted as a literal 

838 backslash. If the number of backslashes is odd, the last 

839 backslash escapes the next double quotation mark as 

840 described in rule 3. 

841 

842 See http://msdn.microsoft.com/en-us/library/17w5ykft.aspx 

843 or search http://msdn.microsoft.com for 

844 "Parsing C++ Command-Line Arguments" 

845 """ 

846 result = [] 

847 needquote = False 

848 for arg in args: 

849 bs_buf = [] 

850 

851 # Add the separator between this argument and the others 

852 if result: 

853 result.append(sep) 

854 

855 needquote = (" " in arg) or ("\t" in arg) or not arg 

856 if needquote: 

857 result.append('"') 

858 

859 for c in arg: 

860 if c == '\\': 

861 # Don't know if we need to double yet. 

862 bs_buf.append(c) 

863 elif c == '"': 

864 # Double backslashes. 

865 result.append('\\' * len(bs_buf)*2) 

866 bs_buf = [] 

867 result.append('\\"') 

868 else: 

869 # Normal char 

870 if bs_buf: 

871 result.extend(bs_buf) 

872 bs_buf = [] 

873 result.append(c) 

874 

875 # Add remaining backslashes, if any. 

876 if bs_buf: 

877 result.extend(bs_buf) 

878 

879 if needquote: 

880 result.extend(bs_buf) 

881 result.append('"') 

882 

883 return ''.join(result) 

884 

885 

886def parse_int_list(range_string, delim=',', range_delim='-'): 

887 """Returns a sorted list of positive integers based on 

888 *range_string*. Reverse of :func:`format_int_list`. 

889 

890 Args: 

891 range_string (str): String of comma separated positive 

892 integers or ranges (e.g. '1,2,4-6,8'). Typical of a custom 

893 page range string used in printer dialogs. 

894 delim (char): Defaults to ','. Separates integers and 

895 contiguous ranges of integers. 

896 range_delim (char): Defaults to '-'. Indicates a contiguous 

897 range of integers. 

898 

899 >>> parse_int_list('1,3,5-8,10-11,15') 

900 [1, 3, 5, 6, 7, 8, 10, 11, 15] 

901 

902 """ 

903 output = [] 

904 

905 for x in range_string.strip().split(delim): 

906 

907 # Range 

908 if range_delim in x: 

909 range_limits = list(map(int, x.split(range_delim))) 

910 output += list(range(min(range_limits), max(range_limits)+1)) 

911 

912 # Empty String 

913 elif not x: 

914 continue 

915 

916 # Integer 

917 else: 

918 output.append(int(x)) 

919 

920 return sorted(output) 

921 

922 

923def format_int_list(int_list, delim=',', range_delim='-', delim_space=False): 

924 """Returns a sorted range string from a list of positive integers 

925 (*int_list*). Contiguous ranges of integers are collapsed to min 

926 and max values. Reverse of :func:`parse_int_list`. 

927 

928 Args: 

929 int_list (list): List of positive integers to be converted 

930 into a range string (e.g. [1,2,4,5,6,8]). 

931 delim (char): Defaults to ','. Separates integers and 

932 contiguous ranges of integers. 

933 range_delim (char): Defaults to '-'. Indicates a contiguous 

934 range of integers. 

935 delim_space (bool): Defaults to ``False``. If ``True``, adds a 

936 space after all *delim* characters. 

937 

938 >>> format_int_list([1,3,5,6,7,8,10,11,15]) 

939 '1,3,5-8,10-11,15' 

940 

941 """ 

942 output = [] 

943 contig_range = collections.deque() 

944 

945 for x in sorted(int_list): 

946 

947 # Handle current (and first) value. 

948 if len(contig_range) < 1: 

949 contig_range.append(x) 

950 

951 # Handle current value, given multiple previous values are contiguous. 

952 elif len(contig_range) > 1: