Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/boltons/strutils.py: 20%

1# Copyright (c) 2013, Mahmoud Hashemi 

2# 

3# Redistribution and use in source and binary forms, with or without 

4# modification, are permitted provided that the following conditions are 

5# met: 

6# 

7# * Redistributions of source code must retain the above copyright 

8# notice, this list of conditions and the following disclaimer. 

9# 

10# * Redistributions in binary form must reproduce the above 

11# copyright notice, this list of conditions and the following 

12# disclaimer in the documentation and/or other materials provided 

13# with the distribution. 

14# 

15# * The names of the contributors may not be used to endorse or 

16# promote products derived from this software without specific 

17# prior written permission. 

18# 

19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 

20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 

21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 

22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 

23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 

24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 

25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 

26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 

27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 

28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 

29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 

30 

31"""So much practical programming involves string manipulation, which 

32Python readily accommodates. Still, there are dozens of basic and 

33common capabilities missing from the standard library, several of them 

34provided by ``strutils``. 

35""" 

36 

37 

38import builtins 

39import collections 

40import re 

41import string 

42import sys 

43import typing 

44import unicodedata 

45import uuid 

46import zlib 

47from collections.abc import Mapping 

48from gzip import GzipFile 

49from html import entities as htmlentitydefs 

50from html.parser import HTMLParser 

51from io import BytesIO as StringIO 

52 

53__all__ = ['camel2under', 'under2camel', 'slugify', 'split_punct_ws', 

54 'unit_len', 'ordinalize', 'cardinalize', 'pluralize', 'singularize', 

55 'asciify', 'is_ascii', 'is_uuid', 'html2text', 'strip_ansi', 

56 'bytes2human', 'find_hashtags', 'a10n', 'gzip_bytes', 'gunzip_bytes', 

57 'iter_splitlines', 'indent', 'escape_shell_args', 

58 'args2cmd', 'args2sh', 'parse_int_list', 'format_int_list', 

59 'complement_int_list', 'int_ranges_from_int_list', 'MultiReplace', 

60 'multi_replace', 'unwrap_text', 'removeprefix', 

61 'human_readable_list'] 

62 

63 

64_punct_ws_str = string.punctuation + string.whitespace 

65_punct_re = re.compile('[' + _punct_ws_str + ']+') 

66_camel2under_re = re.compile('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))') 

67 

68 

69def camel2under(camel_string): 

70 """Converts a camelcased string to underscores. Useful for turning a 

71 class name into a function name. 

72 

73 >>> camel2under('BasicParseTest') 

74 'basic_parse_test' 

75 """ 

76 return _camel2under_re.sub(r'_\1', camel_string).lower() 

77 

78 

79def under2camel(under_string): 

80 """Converts an underscored string to camelcased. Useful for turning a 

81 function name into a class name. 

82 

83 >>> under2camel('complex_tokenizer') 

84 'ComplexTokenizer' 

85 """ 

86 return ''.join(w.capitalize() or '_' for w in under_string.split('_')) 

87 

88 

89def slugify(text, delim='_', lower=True, ascii=False): 

90 """ 

91 A basic function that turns text full of scary characters 

92 (i.e., punctuation and whitespace), into a relatively safe 

93 lowercased string separated only by the delimiter specified 

94 by *delim*, which defaults to ``_``. 

95 

96 The *ascii* convenience flag will :func:`asciify` the slug if 

97 you require ascii-only slugs. 

98 

99 >>> slugify('First post! Hi!!!!~1 ') 

100 'first_post_hi_1' 

101 

102 >>> slugify("Kurt Gödel's pretty cool.", ascii=True) == \ 

103 b'kurt_goedel_s_pretty_cool' 

104 True 

105 

106 """ 

107 ret = delim.join(split_punct_ws(text)) or delim if text else '' 

108 if ascii: 

109 ret = asciify(ret) 

110 if lower: 

111 ret = ret.lower() 

112 return ret 

113 

114 

115def split_punct_ws(text): 

116 """While :meth:`str.split` will split on whitespace, 

117 :func:`split_punct_ws` will split on punctuation and 

118 whitespace. This used internally by :func:`slugify`, above. 

119 

120 >>> split_punct_ws('First post! Hi!!!!~1 ') 

121 ['First', 'post', 'Hi', '1'] 

122 """ 

123 return [w for w in _punct_re.split(text) if w] 

124 

125 

126def unit_len(sized_iterable, unit_noun='item'): # TODO: len_units()/unitize()? 

127 """Returns a plain-English description of an iterable's 

128 :func:`len()`, conditionally pluralized with :func:`cardinalize`, 

129 detailed below. 

130 

131 >>> print(unit_len(range(10), 'number')) 

132 10 numbers 

133 >>> print(unit_len('aeiou', 'vowel')) 

134 5 vowels 

135 >>> print(unit_len([], 'worry')) 

136 No worries 

137 """ 

138 count = len(sized_iterable) 

139 units = cardinalize(unit_noun, count) 

140 if count: 

141 return f'{count} {units}' 

142 return f'No {units}' 

143 

144 

145_ORDINAL_MAP = {'1': 'st', 

146 '2': 'nd', 

147 '3': 'rd'} # 'th' is the default 

148 

149 

150def ordinalize(number, ext_only=False): 

151 """Turns *number* into its cardinal form, i.e., 1st, 2nd, 

152 3rd, 4th, etc. If the last character isn't a digit, it returns the 

153 string value unchanged. 

154 

155 Args: 

156 number (int or str): Number to be cardinalized. 

157 ext_only (bool): Whether to return only the suffix. Default ``False``. 

158 

159 >>> print(ordinalize(1)) 

160 1st 

161 >>> print(ordinalize(3694839230)) 

162 3694839230th 

163 >>> print(ordinalize('hi')) 

164 hi 

165 >>> print(ordinalize(1515)) 

166 1515th 

167 """ 

168 numstr, ext = str(number), '' 

169 if numstr and numstr[-1] in string.digits: 

170 try: 

171 # first check for teens 

172 if numstr[-2] == '1': 

173 ext = 'th' 

174 else: 

175 # all other cases 

176 ext = _ORDINAL_MAP.get(numstr[-1], 'th') 

177 except IndexError: 

178 # single digit numbers (will reach here based on [-2] above) 

179 ext = _ORDINAL_MAP.get(numstr[-1], 'th') 

180 if ext_only: 

181 return ext 

182 else: 

183 return numstr + ext 

184 

185 

186def cardinalize(unit_noun, count): 

187 """Conditionally pluralizes a singular word *unit_noun* if 

188 *count* is not one, preserving case when possible. 

189 

190 >>> vowels = 'aeiou' 

191 >>> print(len(vowels), cardinalize('vowel', len(vowels))) 

192 5 vowels 

193 >>> print(3, cardinalize('Wish', 3)) 

194 3 Wishes 

195 """ 

196 if count == 1: 

197 return unit_noun 

198 return pluralize(unit_noun) 

199 

200 

201def singularize(word): 

202 """Semi-intelligently converts an English plural *word* to its 

203 singular form, preserving case pattern. 

204 

205 >>> singularize('chances') 

206 'chance' 

207 >>> singularize('Activities') 

208 'Activity' 

209 >>> singularize('Glasses') 

210 'Glass' 

211 >>> singularize('FEET') 

212 'FOOT' 

213 

214 """ 

215 orig_word, word = word, word.strip().lower() 

216 if not word or word in _IRR_S2P: 

217 return orig_word 

218 

219 irr_singular = _IRR_P2S.get(word) 

220 if irr_singular: 

221 singular = irr_singular 

222 elif not word.endswith('s'): 

223 return orig_word 

224 elif len(word) == 2: 

225 singular = word[:-1] # or just return word? 

226 elif word.endswith('ies') and word[-4:-3] not in 'aeiou': 

227 singular = word[:-3] + 'y' 

228 elif word.endswith('es') and word[-3] == 's': 

229 singular = word[:-2] 

230 elif word.endswith('ss'): 

231 # Words ending in a double 's' (glass, boss, kiss) are already 

232 # singular; their plurals end in 'sses' and are handled above. Do 

233 # not blindly strip the trailing 's', which would produce 'glas', 

234 # 'bos', 'kis' and break idempotency (singularize('Glasses') == 

235 # 'Glass', but 'Glass' must stay 'Glass'). 

236 return orig_word 

237 else: 

238 singular = word[:-1] 

239 return _match_case(orig_word, singular) 

240 

241 

242def pluralize(word): 

243 """Semi-intelligently converts an English *word* from singular form to 

244 plural, preserving case pattern. 

245 

246 >>> pluralize('friend') 

247 'friends' 

248 >>> pluralize('enemy') 

249 'enemies' 

250 >>> pluralize('Sheep') 

251 'Sheep' 

252 """ 

253 orig_word, word = word, word.strip().lower() 

254 if not word or word in _IRR_P2S: 

255 return orig_word 

256 irr_plural = _IRR_S2P.get(word) 

257 if irr_plural: 

258 plural = irr_plural 

259 elif word.endswith('y') and word[-2:-1] not in 'aeiou': 

260 plural = word[:-1] + 'ies' 

261 elif word[-1] == 's' or word.endswith('ch') or word.endswith('sh'): 

262 plural = word if word.endswith('es') else word + 'es' 

263 else: 

264 plural = word + 's' 

265 return _match_case(orig_word, plural) 

266 

267 

268def _match_case(master, disciple): 

269 if not master.strip(): 

270 return disciple 

271 if master.lower() == master: 

272 return disciple.lower() 

273 elif master.upper() == master: 

274 return disciple.upper() 

275 elif master.title() == master: 

276 return disciple.title() 

277 return disciple 

278 

279 

280# Singular to plural map of irregular pluralizations 

281_IRR_S2P = {'addendum': 'addenda', 'alga': 'algae', 'alumna': 'alumnae', 

282 'alumnus': 'alumni', 'analysis': 'analyses', 'antenna': 'antennae', 

283 'appendix': 'appendices', 'axis': 'axes', 'bacillus': 'bacilli', 

284 'bacterium': 'bacteria', 'basis': 'bases', 'beau': 'beaux', 

285 'bison': 'bison', 'bureau': 'bureaus', 'cactus': 'cacti', 

286 'calf': 'calves', 'child': 'children', 'corps': 'corps', 

287 'corpus': 'corpora', 'crisis': 'crises', 'criterion': 'criteria', 

288 'curriculum': 'curricula', 'datum': 'data', 'deer': 'deer', 

289 'diagnosis': 'diagnoses', 'die': 'dice', 'dwarf': 'dwarves', 

290 'echo': 'echoes', 'elf': 'elves', 'ellipsis': 'ellipses', 

291 'embargo': 'embargoes', 'emphasis': 'emphases', 'erratum': 'errata', 

292 'fireman': 'firemen', 'fish': 'fish', 'focus': 'foci', 

293 'foot': 'feet', 'formula': 'formulae', 'formula': 'formulas', 

294 'fungus': 'fungi', 'genus': 'genera', 'goose': 'geese', 

295 'half': 'halves', 'hero': 'heroes', 'hippopotamus': 'hippopotami', 

296 'hoof': 'hooves', 'hypothesis': 'hypotheses', 'index': 'indices', 

297 'knife': 'knives', 'leaf': 'leaves', 'life': 'lives', 

298 'loaf': 'loaves', 'louse': 'lice', 'man': 'men', 

299 'matrix': 'matrices', 'means': 'means', 'medium': 'media', 

300 'memorandum': 'memoranda', 'millennium': 'milennia', 'moose': 'moose', 

301 'mosquito': 'mosquitoes', 'mouse': 'mice', 'nebula': 'nebulae', 

302 'neurosis': 'neuroses', 'nucleus': 'nuclei', 'oasis': 'oases', 

303 'octopus': 'octopi', 'offspring': 'offspring', 'ovum': 'ova', 

304 'ox': 'oxen', 'paralysis': 'paralyses', 'parenthesis': 'parentheses', 

305 'person': 'people', 'phenomenon': 'phenomena', 'potato': 'potatoes', 

306 'radius': 'radii', 'scarf': 'scarves', 'scissors': 'scissors', 

307 'self': 'selves', 'sense': 'senses', 'series': 'series', 'sheep': 

308 'sheep', 'shelf': 'shelves', 'species': 'species', 'stimulus': 

309 'stimuli', 'stratum': 'strata', 'syllabus': 'syllabi', 'symposium': 

310 'symposia', 'synopsis': 'synopses', 'synthesis': 'syntheses', 

311 'tableau': 'tableaux', 'that': 'those', 'thesis': 'theses', 

312 'thief': 'thieves', 'this': 'these', 'tomato': 'tomatoes', 'tooth': 

313 'teeth', 'torpedo': 'torpedoes', 'vertebra': 'vertebrae', 'veto': 

314 'vetoes', 'vita': 'vitae', 'watch': 'watches', 'wife': 'wives', 

315 'wolf': 'wolves', 'woman': 'women'} 

316 

317 

318# Reverse index of the above 

319_IRR_P2S = {v: k for k, v in _IRR_S2P.items()} 

320 

321HASHTAG_RE = re.compile(r"(?:^|\s)[＃#]{1}(\w+)", re.UNICODE) 

322 

323 

324def find_hashtags(string): 

325 """Finds and returns all hashtags in a string, with the hashmark 

326 removed. Supports full-width hashmarks for Asian languages and 

327 does not false-positive on URL anchors. 

328 

329 >>> find_hashtags('#atag http://asite/#ananchor') 

330 ['atag'] 

331 

332 ``find_hashtags`` also works with unicode hashtags. 

333 """ 

334 

335 # the following works, doctest just struggles with it 

336 # >>> find_hashtags(u"can't get enough of that dignity chicken #肯德基 woo") 

337 # [u'\u80af\u5fb7\u57fa'] 

338 return HASHTAG_RE.findall(string) 

339 

340 

341def a10n(string): 

342 """That thing where "internationalization" becomes "i18n", what's it 

343 called? Abbreviation? Oh wait, no: ``a10n``. (It's actually a form 

344 of `numeronym`_.) 

345 

346 >>> a10n('abbreviation') 

347 'a10n' 

348 >>> a10n('internationalization') 

349 'i18n' 

350 >>> a10n('') 

351 '' 

352 

353 .. _numeronym: http://en.wikipedia.org/wiki/Numeronym 

354 """ 

355 if len(string) < 3: 

356 return string 

357 return f'{string[0]}{len(string[1:-1])}{string[-1]}' 

358 

359 

360# Based on https://en.wikipedia.org/wiki/ANSI_escape_code#Escape_sequences 

361ANSI_SEQUENCES = re.compile(r''' 

362 \x1B # Sequence starts with ESC, i.e. hex 0x1B 

363 (?: 

364 [@-Z\\-_] # Second byte: 

365 # all 0x40–0x5F range but CSI char, i.e ASCII @A–Z\]^_ 

366 | # Or 

367 \[ # CSI sequences, starting with [ 

368 [0-?]* # Parameter bytes: 

369 # range 0x30–0x3F, ASCII 0–9:;<=>? 

370 [ -/]* # Intermediate bytes: 

371 # range 0x20–0x2F, ASCII space and !"#$%&'()*+,-./ 

372 [@-~] # Final byte 

373 # range 0x40–0x7E, ASCII @A–Z[\]^_`a–z{|}~ 

374 ) 

375''', re.VERBOSE) 

376 

377 

378def strip_ansi(text): 

379 """Strips ANSI escape codes from *text*. Useful for the occasional 

380 time when a log or redirected output accidentally captures console 

381 color codes and the like. 

382 

383 >>> strip_ansi('\x1b[0m\x1b[1;36mart\x1b[46;34m') 

384 'art' 

385 

386 Supports str, bytes and bytearray content as input. Returns the 

387 same type as the input. 

388 

389 There's a lot of ANSI art available for testing on `sixteencolors.net`_. 

390 This function does not interpret or render ANSI art, but you can do so with 

391 `ansi2img`_ or `escapes.js`_. 

392 

393 .. _sixteencolors.net: http://sixteencolors.net 

394 .. _ansi2img: http://www.bedroomlan.org/projects/ansi2img 

395 .. _escapes.js: https://github.com/atdt/escapes.js 

396 """ 

397 # TODO: move to cliutils.py 

398 

399 # Transform any ASCII-like content to unicode to allow regex to match, and 

400 # save input type for later. 

401 target_type = None 

402 # Unicode type aliased to str is code-smell for Boltons in Python 3 env. 

403 if isinstance(text, (bytes, bytearray)): 

404 target_type = type(text) 

405 text = text.decode('utf-8') 

406 

407 cleaned = ANSI_SEQUENCES.sub('', text) 

408 

409 # Transform back the result to the same bytearray type provided by the user. 

410 if target_type and target_type != type(cleaned): 

411 cleaned = target_type(cleaned, 'utf-8') 

412 

413 return cleaned 

414 

415 

416def asciify(text, ignore=False): 

417 """Converts a unicode or bytestring, *text*, into a bytestring with 

418 just ascii characters. Performs basic deaccenting for all you 

419 Europhiles out there. 

420 

421 Also, a gentle reminder that this is a **utility**, primarily meant 

422 for slugification. Whenever possible, make your application work 

423 **with** unicode, not against it. 

424 

425 Args: 

426 text (str): The string to be asciified. 

427 ignore (bool): Configures final encoding to ignore remaining 

428 unasciified string instead of replacing it. 

429 

430 >>> asciify('Beyoncé') == b'Beyonce' 

431 True 

432 """ 

433 try: 

434 try: 

435 return text.encode('ascii') 

436 except UnicodeDecodeError: 

437 # this usually means you passed in a non-unicode string 

438 text = text.decode('utf-8') 

439 return text.encode('ascii') 

440 except UnicodeEncodeError: 

441 mode = 'replace' 

442 if ignore: 

443 mode = 'ignore' 

444 transd = unicodedata.normalize('NFKD', text.translate(DEACCENT_MAP)) 

445 ret = transd.encode('ascii', mode) 

446 return ret 

447 

448 

449def is_ascii(text): 

450 """Check if a string or bytestring, *text*, is composed of ascii 

451 characters only. Raises :exc:`ValueError` if argument is not text. 

452 

453 Args: 

454 text (str): The string to be checked. 

455 

456 >>> is_ascii('Beyoncé') 

457 False 

458 >>> is_ascii('Beyonce') 

459 True 

460 """ 

461 if isinstance(text, str): 

462 try: 

463 text.encode('ascii') 

464 except UnicodeEncodeError: 

465 return False 

466 elif isinstance(text, bytes): 

467 try: 

468 text.decode('ascii') 

469 except UnicodeDecodeError: 

470 return False 

471 else: 

472 raise ValueError('expected text or bytes, not %r' % type(text)) 

473 return True 

474 

475 

476class DeaccenterDict(dict): 

477 "A small caching dictionary for deaccenting." 

478 def __missing__(self, key): 

479 ch = self.get(key) 

480 if ch is not None: 

481 return ch 

482 try: 

483 de = unicodedata.decomposition(chr(key)) 

484 p1, _, p2 = de.rpartition(' ') 

485 if int(p2, 16) == 0x308: 

486 ch = self.get(key) 

487 else: 

488 ch = int(p1, 16) 

489 except (IndexError, ValueError): 

490 ch = self.get(key, key) 

491 self[key] = ch 

492 return ch 

493 

494 

495# http://chmullig.com/2009/12/python-unicode-ascii-ifier/ 

496# For something more complete, investigate the unidecode 

497# or isounidecode packages, which are capable of performing 

498# crude transliteration. 

499_BASE_DEACCENT_MAP = { 

500 0xc6: "AE", # Æ LATIN CAPITAL LETTER AE 

501 0xd0: "D", # Ð LATIN CAPITAL LETTER ETH 

502 0xd8: "OE", # Ø LATIN CAPITAL LETTER O WITH STROKE 

503 0xde: "Th", # Þ LATIN CAPITAL LETTER THORN 

504 0xc4: 'Ae', # Ä LATIN CAPITAL LETTER A WITH DIAERESIS 

505 0xd6: 'Oe', # Ö LATIN CAPITAL LETTER O WITH DIAERESIS 

506 0xdc: 'Ue', # Ü LATIN CAPITAL LETTER U WITH DIAERESIS 

507 0xc0: "A", # À LATIN CAPITAL LETTER A WITH GRAVE 

508 0xc1: "A", # Á LATIN CAPITAL LETTER A WITH ACUTE 

509 0xc3: "A", # Ã LATIN CAPITAL LETTER A WITH TILDE 

510 0xc7: "C", # Ç LATIN CAPITAL LETTER C WITH CEDILLA 

511 0xc8: "E", # È LATIN CAPITAL LETTER E WITH GRAVE 

512 0xc9: "E", # É LATIN CAPITAL LETTER E WITH ACUTE 

513 0xca: "E", # Ê LATIN CAPITAL LETTER E WITH CIRCUMFLEX 

514 0xcc: "I", # Ì LATIN CAPITAL LETTER I WITH GRAVE 

515 0xcd: "I", # Í LATIN CAPITAL LETTER I WITH ACUTE 

516 0xd2: "O", # Ò LATIN CAPITAL LETTER O WITH GRAVE 

517 0xd3: "O", # Ó LATIN CAPITAL LETTER O WITH ACUTE 

518 0xd5: "O", # Õ LATIN CAPITAL LETTER O WITH TILDE 

519 0xd9: "U", # Ù LATIN CAPITAL LETTER U WITH GRAVE 

520 0xda: "U", # Ú LATIN CAPITAL LETTER U WITH ACUTE 

521 0xdf: "ss", # ß LATIN SMALL LETTER SHARP S 

522 0xe6: "ae", # æ LATIN SMALL LETTER AE 

523 0xf0: "d", # ð LATIN SMALL LETTER ETH 

524 0xf8: "oe", # ø LATIN SMALL LETTER O WITH STROKE 

525 0xfe: "th", # þ LATIN SMALL LETTER THORN, 

526 0xe4: 'ae', # ä LATIN SMALL LETTER A WITH DIAERESIS 

527 0xf6: 'oe', # ö LATIN SMALL LETTER O WITH DIAERESIS 

528 0xfc: 'ue', # ü LATIN SMALL LETTER U WITH DIAERESIS 

529 0xe0: "a", # à LATIN SMALL LETTER A WITH GRAVE 

530 0xe1: "a", # á LATIN SMALL LETTER A WITH ACUTE 

531 0xe3: "a", # ã LATIN SMALL LETTER A WITH TILDE 

532 0xe7: "c", # ç LATIN SMALL LETTER C WITH CEDILLA 

533 0xe8: "e", # è LATIN SMALL LETTER E WITH GRAVE 

534 0xe9: "e", # é LATIN SMALL LETTER E WITH ACUTE 

535 0xea: "e", # ê LATIN SMALL LETTER E WITH CIRCUMFLEX 

536 0xec: "i", # ì LATIN SMALL LETTER I WITH GRAVE 

537 0xed: "i", # í LATIN SMALL LETTER I WITH ACUTE 

538 0xf2: "o", # ò LATIN SMALL LETTER O WITH GRAVE 

539 0xf3: "o", # ó LATIN SMALL LETTER O WITH ACUTE 

540 0xf5: "o", # õ LATIN SMALL LETTER O WITH TILDE 

541 0xf9: "u", # ù LATIN SMALL LETTER U WITH GRAVE 

542 0xfa: "u", # ú LATIN SMALL LETTER U WITH ACUTE 

543 0x2018: "'", # ‘ LEFT SINGLE QUOTATION MARK 

544 0x2019: "'", # ’ RIGHT SINGLE QUOTATION MARK 

545 0x201c: '"', # “ LEFT DOUBLE QUOTATION MARK 

546 0x201d: '"', # ” RIGHT DOUBLE QUOTATION MARK 

547 } 

548 

549 

550DEACCENT_MAP = DeaccenterDict(_BASE_DEACCENT_MAP) 

551 

552 

553_SIZE_SYMBOLS = ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') 

554_SIZE_BOUNDS = [(1024 ** i, sym) for i, sym in enumerate(_SIZE_SYMBOLS)] 

555_SIZE_RANGES = list(zip(_SIZE_BOUNDS, _SIZE_BOUNDS[1:])) 

556 

557 

558def bytes2human(nbytes, ndigits=0): 

559 """Turns an integer value of *nbytes* into a human readable format. Set 

560 *ndigits* to control how many digits after the decimal point 

561 should be shown (default ``0``). 

562 

563 >>> bytes2human(128991) 

564 '126K' 

565 >>> bytes2human(100001221) 

566 '95M' 

567 >>> bytes2human(0, 2) 

568 '0.00B' 

569 >>> bytes2human(1024) 

570 '1K' 

571 """ 

572 abs_bytes = abs(nbytes) 

573 for (size, symbol), (next_size, next_symbol) in _SIZE_RANGES: 

574 if abs_bytes < next_size: 

575 break 

576 hnbytes = float(nbytes) / size 

577 return '{hnbytes:.{ndigits}f}{symbol}'.format(hnbytes=hnbytes, 

578 ndigits=ndigits, 

579 symbol=symbol) 

580 

581 

582class HTMLTextExtractor(HTMLParser): 

583 def __init__(self) -> None: 

584 super().__init__(convert_charrefs=True) 

585 self.result: list[str] = [] 

586 

587 def handle_data(self, d): 

588 self.result.append(d) 

589 

590 def handle_charref(self, number): 

591 if number[0] == 'x' or number[0] == 'X': 

592 codepoint = int(number[1:], 16) 

593 else: 

594 codepoint = int(number) 

595 self.result.append(chr(codepoint)) 

596 

597 def handle_entityref(self, name): 

598 try: 

599 codepoint = htmlentitydefs.name2codepoint[name] 

600 except KeyError: 

601 self.result.append('&' + name + ';') 

602 else: 

603 self.result.append(chr(codepoint)) 

604 

605 def get_text(self): 

606 return ''.join(self.result) 

607 

608 

609def html2text(html): 

610 """Strips tags from HTML text, returning markup-free text. Also, does 

611 a best effort replacement of entities like "&nbsp;" 

612 

613 >>> r = html2text(u'<a href="#">Test &amp;<em>(\u0394&#x03b7;&#956;&#x03CE;)</em></a>') 

614 >>> r == u'Test &(\u0394\u03b7\u03bc\u03ce)' 

615 True 

616 """ 

617 # based on answers to http://stackoverflow.com/questions/753052/ 

618 s = HTMLTextExtractor() 

619 s.feed(html) 

620 return s.get_text() 

621 

622 

623_EMPTY_GZIP_BYTES = b'\x1f\x8b\x08\x089\xf3\xb9U\x00\x03empty\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00' 

624_NON_EMPTY_GZIP_BYTES = b'\x1f\x8b\x08\x08\xbc\xf7\xb9U\x00\x03not_empty\x00K\xaa,I-N\xcc\xc8\xafT\xe4\x02\x00\xf3nb\xbf\x0b\x00\x00\x00' 

625 

626 

627def gunzip_bytes(bytestring): 

628 """The :mod:`gzip` module is great if you have a file or file-like 

629 object, but what if you just have bytes. StringIO is one 

630 possibility, but it's often faster, easier, and simpler to just 

631 use this one-liner. Use this tried-and-true utility function to 

632 decompress gzip from bytes. 

633 

634 >>> gunzip_bytes(_EMPTY_GZIP_BYTES) == b'' 

635 True 

636 >>> gunzip_bytes(_NON_EMPTY_GZIP_BYTES).rstrip() == b'bytesahoy!' 

637 True 

638 """ 

639 return zlib.decompress(bytestring, 16 + zlib.MAX_WBITS) 

640 

641 

642def gzip_bytes(bytestring, level=6): 

643 """Turn some bytes into some compressed bytes. 

644 

645 >>> len(gzip_bytes(b'a' * 10000)) 

646 46 

647 

648 Args: 

649 bytestring (bytes): Bytes to be compressed 

650 level (int): An integer, 1-9, controlling the 

651 speed/compression. 1 is fastest, least compressed, 9 is 

652 slowest, but most compressed. 

653 

654 Note that all levels of gzip are pretty fast these days, though 

655 it's not really a competitor in compression, at any level. 

656 """ 

657 out = StringIO() 

658 f = GzipFile(fileobj=out, mode='wb', compresslevel=level) 

659 f.write(bytestring) 

660 f.close() 

661 return out.getvalue() 

662 

663 

664 

665_line_ending_re = re.compile(r'(\r\n|\n|\x0b|\f|\r|\x85|\x2028|\x2029)', 

666 re.UNICODE) 

667 

668 

669def iter_splitlines(text): 

670 r"""Like :meth:`str.splitlines`, but returns an iterator of lines 

671 instead of a list. Also similar to :meth:`file.next`, as that also 

672 lazily reads and yields lines from a file. 

673 

674 This function works with a variety of line endings, but as always, 

675 be careful when mixing line endings within a file. 

676 

677 >>> list(iter_splitlines('\nhi\nbye\n')) 

678 ['', 'hi', 'bye', ''] 

679 >>> list(iter_splitlines('\r\nhi\rbye\r\n')) 

680 ['', 'hi', 'bye', ''] 

681 >>> list(iter_splitlines('')) 

682 [] 

683 """ 

684 prev_end, len_text = 0, len(text) 

685 # print('last: %r' % last_idx) 

686 # start, end = None, None 

687 for match in _line_ending_re.finditer(text): 

688 start, end = match.start(1), match.end(1) 

689 # print(start, end) 

690 if prev_end <= start: 

691 yield text[prev_end:start] 

692 if end == len_text: 

693 yield '' 

694 prev_end = end 

695 tail = text[prev_end:] 

696 if tail: 

697 yield tail 

698 return 

699 

700 

701def indent(text, margin, newline='\n', key=bool): 

702 """The missing counterpart to the built-in :func:`textwrap.dedent`. 

703 

704 Args: 

705 text (str): The text to indent. 

706 margin (str): The string to prepend to each line. 

707 newline (str): The newline used to rejoin the lines (default: ``\\n``) 

708 key (callable): Called on each line to determine whether to 

709 indent it. Default: :class:`bool`, to ensure that empty lines do 

710 not get whitespace added. 

711 """ 

712 indented_lines = [(margin + line if key(line) else line) 

713 for line in iter_splitlines(text)] 

714 return newline.join(indented_lines) 

715 

716 

717def is_uuid(obj, version=4): 

718 """Check the argument is either a valid UUID object or string. 

719 

720 Args: 

721 obj (object): The test target. Strings and UUID objects supported. 

722 version (int): The target UUID version, set to 0 to skip version check. 

723 

724 >>> is_uuid('e682ccca-5a4c-4ef2-9711-73f9ad1e15ea') 

725 True 

726 >>> is_uuid('0221f0d9-d4b9-11e5-a478-10ddb1c2feb9') 

727 False 

728 >>> is_uuid('0221f0d9-d4b9-11e5-a478-10ddb1c2feb9', version=1) 

729 True 

730 """ 

731 if not isinstance(obj, uuid.UUID): 

732 try: 

733 obj = uuid.UUID(obj) 

734 except (TypeError, ValueError, AttributeError): 

735 return False 

736 if version and obj.version != int(version): 

737 return False 

738 return True 

739 

740 

741def escape_shell_args(args, sep=' ', style=None): 

742 """Returns an escaped version of each string in *args*, according to 

743 *style*. 

744 

745 Args: 

746 args (list): A list of arguments to escape and join together 

747 sep (str): The separator used to join the escaped arguments. 

748 style (str): The style of escaping to use. Can be one of 

749 ``cmd`` or ``sh``, geared toward Windows and Linux/BSD/etc., 

750 respectively. If *style* is ``None``, then it is picked 

751 according to the system platform. 

752 

753 See :func:`args2cmd` and :func:`args2sh` for details and example 

754 output for each style. 

755 """ 

756 if not style: 

757 style = 'cmd' if sys.platform == 'win32' else 'sh' 

758 

759 if style == 'sh': 

760 return args2sh(args, sep=sep) 

761 elif style == 'cmd': 

762 return args2cmd(args, sep=sep) 

763 

764 raise ValueError("style expected one of 'cmd' or 'sh', not %r" % style) 

765 

766 

767_find_sh_unsafe = re.compile(r'[^a-zA-Z0-9_@%+=:,./-]').search 

768 

769 

770def args2sh(args, sep=' '): 

771 """Return a shell-escaped string version of *args*, separated by 

772 *sep*, based on the rules of sh, bash, and other shells in the 

773 Linux/BSD/MacOS ecosystem. 

774 

775 >>> print(args2sh(['aa', '[bb]', "cc'cc", 'dd"dd'])) 

776 aa '[bb]' 'cc'"'"'cc' 'dd"dd' 

777 

778 As you can see, arguments with no special characters are not 

779 escaped, arguments with special characters are quoted with single 

780 quotes, and single quotes themselves are quoted with double 

781 quotes. Double quotes are handled like any other special 

782 character. 

783 

784 Based on code from the :mod:`pipes`/:mod:`shlex` modules. Also 

785 note that :mod:`shlex` and :mod:`argparse` have functions to split 

786 and parse strings escaped in this manner. 

787 """ 

788 ret_list = [] 

789 

790 for arg in args: 

791 if not arg: 

792 ret_list.append("''") 

793 continue 

794 if _find_sh_unsafe(arg) is None: 

795 ret_list.append(arg) 

796 continue 

797 # use single quotes, and put single quotes into double quotes 

798 # the string $'b is then quoted as '$'"'"'b' 

799 ret_list.append("'" + arg.replace("'", "'\"'\"'") + "'") 

800 

801 return ' '.join(ret_list) 

802 

803 

804def args2cmd(args, sep=' '): 

805 r"""Return a shell-escaped string version of *args*, separated by 

806 *sep*, using the same rules as the Microsoft C runtime. 

807 

808 >>> print(args2cmd(['aa', '[bb]', "cc'cc", 'dd"dd'])) 

809 aa [bb] cc'cc dd\"dd 

810 

811 As you can see, escaping is through backslashing and not quoting, 

812 and double quotes are the only special character. See the comment 

813 in the code for more details. Based on internal code from the 

814 :mod:`subprocess` module. 

815 

816 """ 

817 # technique description from subprocess below 

818 """ 

819 1) Arguments are delimited by white space, which is either a 

820 space or a tab. 

821 

822 2) A string surrounded by double quotation marks is 

823 interpreted as a single argument, regardless of white space 

824 contained within. A quoted string can be embedded in an 

825 argument. 

826 

827 3) A double quotation mark preceded by a backslash is 

828 interpreted as a literal double quotation mark. 

829 

830 4) Backslashes are interpreted literally, unless they 

831 immediately precede a double quotation mark. 

832 

833 5) If backslashes immediately precede a double quotation mark, 

834 every pair of backslashes is interpreted as a literal 

835 backslash. If the number of backslashes is odd, the last 

836 backslash escapes the next double quotation mark as 

837 described in rule 3. 

838 

839 See http://msdn.microsoft.com/en-us/library/17w5ykft.aspx 

840 or search http://msdn.microsoft.com for 

841 "Parsing C++ Command-Line Arguments" 

842 """ 

843 result = [] 

844 needquote = False 

845 for arg in args: 

846 bs_buf = [] 

847 

848 # Add a space to separate this argument from the others 

849 if result: 

850 result.append(' ') 

851 

852 needquote = (" " in arg) or ("\t" in arg) or not arg 

853 if needquote: 

854 result.append('"') 

855 

856 for c in arg: 

857 if c == '\\': 

858 # Don't know if we need to double yet. 

859 bs_buf.append(c) 

860 elif c == '"': 

861 # Double backslashes. 

862 result.append('\\' * len(bs_buf)*2) 

863 bs_buf = [] 

864 result.append('\\"') 

865 else: 

866 # Normal char 

867 if bs_buf: 

868 result.extend(bs_buf) 

869 bs_buf = [] 

870 result.append(c) 

871 

872 # Add remaining backslashes, if any. 

873 if bs_buf: 

874 result.extend(bs_buf) 

875 

876 if needquote: 

877 result.extend(bs_buf) 

878 result.append('"') 

879 

880 return ''.join(result) 

881 

882 

883def parse_int_list(range_string, delim=',', range_delim='-'): 

884 """Returns a sorted list of positive integers based on 

885 *range_string*. Reverse of :func:`format_int_list`. 

886 

887 Args: 

888 range_string (str): String of comma separated positive 

889 integers or ranges (e.g. '1,2,4-6,8'). Typical of a custom 

890 page range string used in printer dialogs. 

891 delim (char): Defaults to ','. Separates integers and 

892 contiguous ranges of integers. 

893 range_delim (char): Defaults to '-'. Indicates a contiguous 

894 range of integers. 

895 

896 >>> parse_int_list('1,3,5-8,10-11,15') 

897 [1, 3, 5, 6, 7, 8, 10, 11, 15] 

898 

899 """ 

900 output = [] 

901 

902 for x in range_string.strip().split(delim): 

903 

904 # Range 

905 if range_delim in x: 

906 range_limits = list(map(int, x.split(range_delim))) 

907 output += list(range(min(range_limits), max(range_limits)+1)) 

908 

909 # Empty String 

910 elif not x: 

911 continue 

912 

913 # Integer 

914 else: 

915 output.append(int(x)) 

916 

917 return sorted(output) 

918 

919 

920def format_int_list(int_list, delim=',', range_delim='-', delim_space=False): 

921 """Returns a sorted range string from a list of positive integers 

922 (*int_list*). Contiguous ranges of integers are collapsed to min 

923 and max values. Reverse of :func:`parse_int_list`. 

924 

925 Args: 

926 int_list (list): List of positive integers to be converted 

927 into a range string (e.g. [1,2,4,5,6,8]). 

928 delim (char): Defaults to ','. Separates integers and 

929 contiguous ranges of integers. 

930 range_delim (char): Defaults to '-'. Indicates a contiguous 

931 range of integers. 

932 delim_space (bool): Defaults to ``False``. If ``True``, adds a 

933 space after all *delim* characters. 

934 

935 >>> format_int_list([1,3,5,6,7,8,10,11,15]) 

936 '1,3,5-8,10-11,15' 

937 

938 """ 

939 output = [] 

940 contig_range = collections.deque() 

941 

942 for x in sorted(int_list): 

943 

944 # Handle current (and first) value. 

945 if len(contig_range) < 1: 

946 contig_range.append(x) 

947 

948 # Handle current value, given multiple previous values are contiguous. 

949 elif len(contig_range) > 1: 

950 delta = x - contig_range[-1] 

951 

952 # Current value is contiguous.