Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/distlib/util.py: 21%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1315 statements  

1# 

2# Copyright (C) 2012-2026 The Python Software Foundation. 

3# See LICENSE.txt and CONTRIBUTORS.txt. 

4# 

5import codecs 

6from collections import deque 

7import contextlib 

8import csv 

9from glob import iglob as std_iglob 

10import io 

11import json 

12import logging 

13import os 

14import py_compile 

15import re 

16import socket 

17try: 

18 import ssl 

19except ImportError: # pragma: no cover 

20 ssl = None 

21import subprocess 

22import sys 

23import tarfile 

24import tempfile 

25import textwrap 

26 

27try: 

28 import threading 

29except ImportError: # pragma: no cover 

30 import dummy_threading as threading 

31import time 

32 

33from . import DistlibException 

34from .compat import (string_types, text_type, shutil, raw_input, StringIO, cache_from_source, urlopen, urljoin, httplib, 

35 xmlrpclib, HTTPHandler, BaseConfigurator, valid_ident, Container, configparser, URLError, ZipFile, 

36 fsdecode, unquote, urlparse, text_open) 

37 

38logger = logging.getLogger(__name__) 

39 

40# 

41# Requirement parsing code as per PEP 508 

42# 

43 

44IDENTIFIER = re.compile(r'^([\w\.-]+)\s*') 

45VERSION_IDENTIFIER = re.compile(r'^([\w\.*+!-]+)\s*') 

46COMPARE_OP = re.compile(r'^(<=?|>=?|={2,3}|[~!]=)\s*') 

47MARKER_OP = re.compile(r'^((<=?)|(>=?)|={2,3}|[~!]=|in|not\s+in)\s*') 

48OR = re.compile(r'^or\b\s*') 

49AND = re.compile(r'^and\b\s*') 

50NON_SPACE = re.compile(r'(\S+)\s*') 

51STRING_CHUNK = re.compile(r'([\s\w\.{}()*+#:;,/?!~`@$%^&=|<>\[\]-]+)') 

52 

53 

54def parse_marker(marker_string): 

55 """ 

56 Parse a marker string and return a dictionary containing a marker expression. 

57 

58 The dictionary will contain keys "op", "lhs" and "rhs" for non-terminals in 

59 the expression grammar, or strings. A string contained in quotes is to be 

60 interpreted as a literal string, and a string not contained in quotes is a 

61 variable (such as os_name). 

62 """ 

63 

64 def marker_var(remaining): 

65 # either identifier, or literal string 

66 m = IDENTIFIER.match(remaining) 

67 if m: 

68 result = m.groups()[0] 

69 remaining = remaining[m.end():] 

70 elif not remaining: 

71 raise SyntaxError('unexpected end of input') 

72 else: 

73 q = remaining[0] 

74 if q not in '\'"': 

75 raise SyntaxError('invalid expression: %s' % remaining) 

76 oq = '\'"'.replace(q, '') 

77 remaining = remaining[1:] 

78 parts = [q] 

79 while remaining: 

80 # either a string chunk, or oq, or q to terminate 

81 if remaining[0] == q: 

82 break 

83 elif remaining[0] == oq: 

84 parts.append(oq) 

85 remaining = remaining[1:] 

86 else: 

87 m = STRING_CHUNK.match(remaining) 

88 if not m: 

89 raise SyntaxError('error in string literal: %s' % remaining) 

90 parts.append(m.groups()[0]) 

91 remaining = remaining[m.end():] 

92 else: 

93 s = ''.join(parts) 

94 raise SyntaxError('unterminated string: %s' % s) 

95 parts.append(q) 

96 result = ''.join(parts) 

97 remaining = remaining[1:].lstrip() # skip past closing quote 

98 return result, remaining 

99 

100 def marker_expr(remaining): 

101 if remaining and remaining[0] == '(': 

102 result, remaining = marker(remaining[1:].lstrip()) 

103 if not remaining or remaining[0] != ')': 

104 raise SyntaxError('unterminated parenthesis: %s' % remaining) 

105 remaining = remaining[1:].lstrip() 

106 else: 

107 lhs, remaining = marker_var(remaining) 

108 while remaining: 

109 m = MARKER_OP.match(remaining) 

110 if not m: 

111 break 

112 op = m.groups()[0] 

113 remaining = remaining[m.end():] 

114 rhs, remaining = marker_var(remaining) 

115 lhs = {'op': op, 'lhs': lhs, 'rhs': rhs} 

116 result = lhs 

117 return result, remaining 

118 

119 def marker_and(remaining): 

120 lhs, remaining = marker_expr(remaining) 

121 while remaining: 

122 m = AND.match(remaining) 

123 if not m: 

124 break 

125 remaining = remaining[m.end():] 

126 rhs, remaining = marker_expr(remaining) 

127 lhs = {'op': 'and', 'lhs': lhs, 'rhs': rhs} 

128 return lhs, remaining 

129 

130 def marker(remaining): 

131 lhs, remaining = marker_and(remaining) 

132 while remaining: 

133 m = OR.match(remaining) 

134 if not m: 

135 break 

136 remaining = remaining[m.end():] 

137 rhs, remaining = marker_and(remaining) 

138 lhs = {'op': 'or', 'lhs': lhs, 'rhs': rhs} 

139 return lhs, remaining 

140 

141 return marker(marker_string) 

142 

143 

144def parse_requirement(req): 

145 """ 

146 Parse a requirement passed in as a string. Return a Container 

147 whose attributes contain the various parts of the requirement. 

148 """ 

149 remaining = req.strip() 

150 if not remaining or remaining.startswith('#'): 

151 return None 

152 m = IDENTIFIER.match(remaining) 

153 if not m: 

154 raise SyntaxError('name expected: %s' % remaining) 

155 distname = m.groups()[0] 

156 remaining = remaining[m.end():] 

157 extras = mark_expr = versions = uri = None 

158 if remaining and remaining[0] == '[': 

159 i = remaining.find(']', 1) 

160 if i < 0: 

161 raise SyntaxError('unterminated extra: %s' % remaining) 

162 s = remaining[1:i] 

163 remaining = remaining[i + 1:].lstrip() 

164 extras = [] 

165 while s: 

166 m = IDENTIFIER.match(s) 

167 if not m: 

168 raise SyntaxError('malformed extra: %s' % s) 

169 extras.append(m.groups()[0]) 

170 s = s[m.end():] 

171 if not s: 

172 break 

173 if s[0] != ',': 

174 raise SyntaxError('comma expected in extras: %s' % s) 

175 s = s[1:].lstrip() 

176 if not extras: 

177 extras = None 

178 if remaining: 

179 if remaining[0] == '@': 

180 # it's a URI 

181 remaining = remaining[1:].lstrip() 

182 m = NON_SPACE.match(remaining) 

183 if not m: 

184 raise SyntaxError('invalid URI: %s' % remaining) 

185 uri = m.groups()[0] 

186 t = urlparse(uri) 

187 # there are issues with Python and URL parsing, so this test 

188 # is a bit crude. See bpo-20271, bpo-23505. Python doesn't 

189 # always parse invalid URLs correctly - it should raise 

190 # exceptions for malformed URLs 

191 if not (t.scheme and t.netloc): 

192 raise SyntaxError('Invalid URL: %s' % uri) 

193 remaining = remaining[m.end():].lstrip() 

194 else: 

195 

196 def get_versions(ver_remaining): 

197 """ 

198 Return a list of operator, version tuples if any are 

199 specified, else None. 

200 """ 

201 m = COMPARE_OP.match(ver_remaining) 

202 versions = None 

203 if m: 

204 versions = [] 

205 while True: 

206 op = m.groups()[0] 

207 ver_remaining = ver_remaining[m.end():] 

208 m = VERSION_IDENTIFIER.match(ver_remaining) 

209 if not m: 

210 raise SyntaxError('invalid version: %s' % ver_remaining) 

211 v = m.groups()[0] 

212 versions.append((op, v)) 

213 ver_remaining = ver_remaining[m.end():] 

214 if not ver_remaining or ver_remaining[0] != ',': 

215 break 

216 ver_remaining = ver_remaining[1:].lstrip() 

217 # Some packages have a trailing comma which would break things 

218 # See issue #148 

219 if not ver_remaining: 

220 break 

221 m = COMPARE_OP.match(ver_remaining) 

222 if not m: 

223 raise SyntaxError('invalid constraint: %s' % ver_remaining) 

224 if not versions: 

225 versions = None 

226 return versions, ver_remaining 

227 

228 if remaining[0] != '(': 

229 versions, remaining = get_versions(remaining) 

230 else: 

231 i = remaining.find(')', 1) 

232 if i < 0: 

233 raise SyntaxError('unterminated parenthesis: %s' % remaining) 

234 s = remaining[1:i] 

235 remaining = remaining[i + 1:].lstrip() 

236 # As a special diversion from PEP 508, allow a version number 

237 # a.b.c in parentheses as a synonym for ~= a.b.c (because this 

238 # is allowed in earlier PEPs) 

239 if COMPARE_OP.match(s): 

240 versions, _ = get_versions(s) 

241 else: 

242 m = VERSION_IDENTIFIER.match(s) 

243 if not m: 

244 raise SyntaxError('invalid constraint: %s' % s) 

245 v = m.groups()[0] 

246 s = s[m.end():].lstrip() 

247 if s: 

248 raise SyntaxError('invalid constraint: %s' % s) 

249 versions = [('~=', v)] 

250 

251 if remaining: 

252 if remaining[0] != ';': 

253 raise SyntaxError('invalid requirement: %s' % remaining) 

254 remaining = remaining[1:].lstrip() 

255 

256 mark_expr, remaining = parse_marker(remaining) 

257 

258 if remaining and remaining[0] != '#': 

259 raise SyntaxError('unexpected trailing data: %s' % remaining) 

260 

261 if not versions: 

262 rs = distname 

263 else: 

264 rs = '%s %s' % (distname, ', '.join(['%s %s' % con for con in versions])) 

265 return Container(name=distname, extras=extras, constraints=versions, marker=mark_expr, url=uri, requirement=rs) 

266 

267 

268def get_resources_dests(resources_root, rules): 

269 """Find destinations for resources files""" 

270 

271 def get_rel_path(root, path): 

272 # normalizes and returns a lstripped-/-separated path 

273 root = root.replace(os.path.sep, '/') 

274 path = path.replace(os.path.sep, '/') 

275 assert path.startswith(root) 

276 return path[len(root):].lstrip('/') 

277 

278 destinations = {} 

279 for base, suffix, dest in rules: 

280 prefix = os.path.join(resources_root, base) 

281 for abs_base in iglob(prefix): 

282 abs_glob = os.path.join(abs_base, suffix) 

283 for abs_path in iglob(abs_glob): 

284 resource_file = get_rel_path(resources_root, abs_path) 

285 if dest is None: # remove the entry if it was here 

286 destinations.pop(resource_file, None) 

287 else: 

288 rel_path = get_rel_path(abs_base, abs_path) 

289 rel_dest = dest.replace(os.path.sep, '/').rstrip('/') 

290 destinations[resource_file] = rel_dest + '/' + rel_path 

291 return destinations 

292 

293 

294def in_venv(): 

295 if hasattr(sys, 'real_prefix'): 

296 # virtualenv venvs 

297 result = True 

298 else: 

299 # PEP 405 venvs 

300 result = sys.prefix != getattr(sys, 'base_prefix', sys.prefix) 

301 return result 

302 

303 

304def get_executable(): 

305 # The __PYVENV_LAUNCHER__ dance is apparently no longer needed, as 

306 # changes to the stub launcher mean that sys.executable always points 

307 # to the stub on OS X 

308 # if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__' 

309 # in os.environ): 

310 # result = os.environ['__PYVENV_LAUNCHER__'] 

311 # else: 

312 # result = sys.executable 

313 # return result 

314 # Avoid normcasing: see issue #143 

315 # result = os.path.normcase(sys.executable) 

316 result = sys.executable 

317 if not isinstance(result, text_type): 

318 result = fsdecode(result) 

319 return result 

320 

321 

322def proceed(prompt, allowed_chars, error_prompt=None, default=None): 

323 p = prompt 

324 while True: 

325 s = raw_input(p) 

326 p = prompt 

327 if not s and default: 

328 s = default 

329 if s: 

330 c = s[0].lower() 

331 if c in allowed_chars: 

332 break 

333 if error_prompt: 

334 p = '%c: %s\n%s' % (c, error_prompt, prompt) 

335 return c 

336 

337 

338def extract_by_key(d, keys): 

339 if isinstance(keys, string_types): 

340 keys = keys.split() 

341 result = {} 

342 for key in keys: 

343 if key in d: 

344 result[key] = d[key] 

345 return result 

346 

347 

348def read_exports(stream): 

349 if sys.version_info[0] >= 3: 

350 # needs to be a text stream 

351 stream = codecs.getreader('utf-8')(stream) 

352 # Try to load as JSON, falling back on legacy format 

353 data = stream.read() 

354 stream = StringIO(data) 

355 try: 

356 jdata = json.load(stream) 

357 result = jdata['extensions']['python.exports']['exports'] 

358 for group, entries in result.items(): 

359 for k, v in entries.items(): 

360 s = '%s = %s' % (k, v) 

361 entry = get_export_entry(s) 

362 assert entry is not None 

363 entries[k] = entry 

364 return result 

365 except Exception: 

366 stream.seek(0, 0) 

367 

368 def read_stream(cp, stream): 

369 if hasattr(cp, 'read_file'): 

370 cp.read_file(stream) 

371 else: 

372 cp.readfp(stream) 

373 

374 cp = configparser.ConfigParser() 

375 try: 

376 read_stream(cp, stream) 

377 except configparser.MissingSectionHeaderError: 

378 stream.close() 

379 data = textwrap.dedent(data) 

380 stream = StringIO(data) 

381 read_stream(cp, stream) 

382 

383 result = {} 

384 for key in cp.sections(): 

385 result[key] = entries = {} 

386 for name, value in cp.items(key): 

387 s = '%s = %s' % (name, value) 

388 entry = get_export_entry(s) 

389 assert entry is not None 

390 # entry.dist = self 

391 entries[name] = entry 

392 return result 

393 

394 

395def write_exports(exports, stream): 

396 if sys.version_info[0] >= 3: 

397 # needs to be a text stream 

398 stream = codecs.getwriter('utf-8')(stream) 

399 cp = configparser.ConfigParser() 

400 for k, v in exports.items(): 

401 # TODO check k, v for valid values 

402 cp.add_section(k) 

403 for entry in v.values(): 

404 if entry.suffix is None: 

405 s = entry.prefix 

406 else: 

407 s = '%s:%s' % (entry.prefix, entry.suffix) 

408 if entry.flags: 

409 s = '%s [%s]' % (s, ', '.join(entry.flags)) 

410 cp.set(k, entry.name, s) 

411 cp.write(stream) 

412 

413 

414@contextlib.contextmanager 

415def tempdir(): 

416 td = tempfile.mkdtemp() 

417 try: 

418 yield td 

419 finally: 

420 shutil.rmtree(td) 

421 

422 

423@contextlib.contextmanager 

424def chdir(d): 

425 cwd = os.getcwd() 

426 try: 

427 os.chdir(d) 

428 yield 

429 finally: 

430 os.chdir(cwd) 

431 

432 

433@contextlib.contextmanager 

434def socket_timeout(seconds=15): 

435 cto = socket.getdefaulttimeout() 

436 try: 

437 socket.setdefaulttimeout(seconds) 

438 yield 

439 finally: 

440 socket.setdefaulttimeout(cto) 

441 

442 

443class cached_property(object): 

444 

445 def __init__(self, func): 

446 self.func = func 

447 # for attr in ('__name__', '__module__', '__doc__'): 

448 # setattr(self, attr, getattr(func, attr, None)) 

449 

450 def __get__(self, obj, cls=None): 

451 if obj is None: 

452 return self 

453 value = self.func(obj) 

454 object.__setattr__(obj, self.func.__name__, value) 

455 # obj.__dict__[self.func.__name__] = value = self.func(obj) 

456 return value 

457 

458 

459def convert_path(pathname): 

460 """Return 'pathname' as a name that will work on the native filesystem. 

461 

462 The path is split on '/' and put back together again using the current 

463 directory separator. Needed because filenames in the setup script are 

464 always supplied in Unix style, and have to be converted to the local 

465 convention before we can actually use them in the filesystem. Raises 

466 ValueError on non-Unix-ish systems if 'pathname' either starts or 

467 ends with a slash. 

468 """ 

469 if os.sep == '/': 

470 return pathname 

471 if not pathname: 

472 return pathname 

473 if pathname[0] == '/': 

474 raise ValueError("path '%s' cannot be absolute" % pathname) 

475 if pathname[-1] == '/': 

476 raise ValueError("path '%s' cannot end with '/'" % pathname) 

477 

478 paths = pathname.split('/') 

479 while os.curdir in paths: 

480 paths.remove(os.curdir) 

481 if not paths: 

482 return os.curdir 

483 return os.path.join(*paths) 

484 

485 

486class FileOperator(object): 

487 

488 def __init__(self, dry_run=False): 

489 self.dry_run = dry_run 

490 self.ensured = set() 

491 self._init_record() 

492 

493 def _init_record(self): 

494 self.record = False 

495 self.files_written = set() 

496 self.dirs_created = set() 

497 

498 def record_as_written(self, path): 

499 if self.record: 

500 self.files_written.add(path) 

501 

502 def newer(self, source, target): 

503 """Tell if the target is newer than the source. 

504 

505 Returns true if 'source' exists and is more recently modified than 

506 'target', or if 'source' exists and 'target' doesn't. 

507 

508 Returns false if both exist and 'target' is the same age or younger 

509 than 'source'. Raise PackagingFileError if 'source' does not exist. 

510 

511 Note that this test is not very accurate: files created in the same 

512 second will have the same "age". 

513 """ 

514 if not os.path.exists(source): 

515 raise DistlibException("file '%r' does not exist" % os.path.abspath(source)) 

516 if not os.path.exists(target): 

517 return True 

518 

519 return os.stat(source).st_mtime > os.stat(target).st_mtime 

520 

521 def copy_file(self, infile, outfile, check=True): 

522 """Copy a file respecting dry-run and force flags. 

523 """ 

524 self.ensure_dir(os.path.dirname(outfile)) 

525 logger.info('Copying %s to %s', infile, outfile) 

526 if not self.dry_run: 

527 msg = None 

528 if check: 

529 if os.path.islink(outfile): 

530 msg = '%s is a symlink' % outfile 

531 elif os.path.exists(outfile) and not os.path.isfile(outfile): 

532 msg = '%s is a non-regular file' % outfile 

533 if msg: 

534 raise ValueError(msg + ' which would be overwritten') 

535 shutil.copyfile(infile, outfile) 

536 self.record_as_written(outfile) 

537 

538 def copy_stream(self, instream, outfile, encoding=None): 

539 assert not os.path.isdir(outfile) 

540 self.ensure_dir(os.path.dirname(outfile)) 

541 logger.info('Copying stream %s to %s', instream, outfile) 

542 if not self.dry_run: 

543 if encoding is None: 

544 outstream = open(outfile, 'wb') 

545 else: 

546 outstream = text_open(outfile, 'w', encoding=encoding) 

547 try: 

548 shutil.copyfileobj(instream, outstream) 

549 finally: 

550 outstream.close() 

551 self.record_as_written(outfile) 

552 

553 def write_binary_file(self, path, data): 

554 self.ensure_dir(os.path.dirname(path)) 

555 if not self.dry_run: 

556 if os.path.exists(path): 

557 os.remove(path) 

558 with open(path, 'wb') as f: 

559 f.write(data) 

560 self.record_as_written(path) 

561 

562 def write_text_file(self, path, data, encoding): 

563 self.write_binary_file(path, data.encode(encoding)) 

564 

565 def set_mode(self, bits, mask, files): 

566 if os.name == 'posix' or (os.name == 'java' and os._name == 'posix'): 

567 # Set the executable bits (owner, group, and world) on 

568 # all the files specified. 

569 for f in files: 

570 if self.dry_run: 

571 logger.info("changing mode of %s", f) 

572 else: 

573 mode = (os.stat(f).st_mode | bits) & mask 

574 logger.info("changing mode of %s to %o", f, mode) 

575 os.chmod(f, mode) 

576 

577 set_executable_mode = lambda s, f: s.set_mode(0o555, 0o7777, f) 

578 

579 def ensure_dir(self, path): 

580 path = os.path.abspath(path) 

581 if path not in self.ensured and not os.path.exists(path): 

582 self.ensured.add(path) 

583 d, f = os.path.split(path) 

584 self.ensure_dir(d) 

585 logger.info('Creating %s' % path) 

586 if not self.dry_run: 

587 os.mkdir(path) 

588 if self.record: 

589 self.dirs_created.add(path) 

590 

591 def byte_compile(self, path, optimize=False, force=False, prefix=None, hashed_invalidation=False): 

592 if not optimize: 

593 optimization = '' 

594 else: 

595 optimization = '1' 

596 dpath = cache_from_source(path, optimization=optimization) 

597 logger.info('Byte-compiling %s to %s', path, dpath) 

598 if not self.dry_run: 

599 if force or self.newer(path, dpath): 

600 if not prefix: 

601 diagpath = None 

602 else: 

603 assert path.startswith(prefix) 

604 diagpath = path[len(prefix):] 

605 compile_kwargs = {} 

606 if sys.version_info[0] >= 3: 

607 compile_kwargs['optimize'] = int(bool(optimize)) 

608 if hashed_invalidation and hasattr(py_compile, 'PycInvalidationMode'): 

609 if not isinstance(hashed_invalidation, py_compile.PycInvalidationMode): 

610 hashed_invalidation = py_compile.PycInvalidationMode.CHECKED_HASH 

611 compile_kwargs['invalidation_mode'] = hashed_invalidation 

612 py_compile.compile(path, dpath, diagpath, True, **compile_kwargs) # raise error 

613 self.record_as_written(dpath) 

614 return dpath 

615 

616 def ensure_removed(self, path): 

617 if os.path.lexists(path): 

618 if os.path.isdir(path) and not os.path.islink(path): 

619 logger.debug('Removing directory tree at %s', path) 

620 if not self.dry_run: 

621 shutil.rmtree(path) 

622 if self.record: 

623 if path in self.dirs_created: 

624 self.dirs_created.remove(path) 

625 else: 

626 if os.path.islink(path): 

627 s = 'link' 

628 else: 

629 s = 'file' 

630 logger.debug('Removing %s %s', s, path) 

631 if not self.dry_run: 

632 os.remove(path) 

633 if self.record: 

634 if path in self.files_written: 

635 self.files_written.remove(path) 

636 

637 def is_writable(self, path): 

638 result = False 

639 while not result: 

640 if os.path.exists(path): 

641 result = os.access(path, os.W_OK) 

642 break 

643 parent = os.path.dirname(path) 

644 if parent == path: 

645 break 

646 path = parent 

647 return result 

648 

649 def commit(self): 

650 """ 

651 Commit recorded changes, turn off recording, return 

652 changes. 

653 """ 

654 assert self.record 

655 result = self.files_written, self.dirs_created 

656 self._init_record() 

657 return result 

658 

659 def rollback(self): 

660 if not self.dry_run: 

661 for f in list(self.files_written): 

662 if os.path.exists(f): 

663 os.remove(f) 

664 # dirs should all be empty now, except perhaps for 

665 # __pycache__ subdirs 

666 # reverse so that subdirs appear before their parents 

667 dirs = sorted(self.dirs_created, reverse=True) 

668 for d in dirs: 

669 flist = os.listdir(d) 

670 if flist: 

671 assert flist == ['__pycache__'] 

672 sd = os.path.join(d, flist[0]) 

673 os.rmdir(sd) 

674 os.rmdir(d) # should fail if non-empty 

675 self._init_record() 

676 

677 

678def resolve(module_name, dotted_path): 

679 if module_name in sys.modules: 

680 mod = sys.modules[module_name] 

681 else: 

682 mod = __import__(module_name) 

683 if dotted_path is None: 

684 result = mod 

685 else: 

686 parts = dotted_path.split('.') 

687 result = getattr(mod, parts.pop(0)) 

688 for p in parts: 

689 result = getattr(result, p) 

690 return result 

691 

692 

693class ExportEntry(object): 

694 

695 def __init__(self, name, prefix, suffix, flags): 

696 self.name = name 

697 self.prefix = prefix 

698 self.suffix = suffix 

699 self.flags = flags 

700 

701 @cached_property 

702 def value(self): 

703 return resolve(self.prefix, self.suffix) 

704 

705 def __repr__(self): # pragma: no cover 

706 return '<ExportEntry %s = %s:%s %s>' % (self.name, self.prefix, self.suffix, self.flags) 

707 

708 def __eq__(self, other): 

709 if not isinstance(other, ExportEntry): 

710 result = False 

711 else: 

712 result = (self.name == other.name and self.prefix == other.prefix and self.suffix == other.suffix 

713 and self.flags == other.flags) 

714 return result 

715 

716 __hash__ = object.__hash__ 

717 

718 

719ENTRY_RE = re.compile( 

720 r'''(?P<name>([^\[]\S*)) 

721 \s*=\s*(?P<callable>(\w+)([:\.]\w+)*) 

722 \s*(\[\s*(?P<flags>[\w-]+(=\w+)?(,\s*\w+(=\w+)?)*)\s*\])? 

723 ''', re.VERBOSE) 

724 

725# ENTRY_RE, but only starting at the first or last character of a whitespace 

726# run. See _search_export_entry. 

727_ENTRY_RE_AT_SPACE = re.compile(r'(?:(?<!\s)(?=\s)|(?=\s\S))' + 

728 ENTRY_RE.pattern, re.VERBOSE) 

729 

730 

731def _search_export_entry(specification): 

732 """ 

733 Return ENTRY_RE.search(specification), in linear rather than quadratic 

734 time. 

735 

736 search() tries every start position, and each attempt can rescan the rest 

737 of a long name. Most positions cannot hold the leftmost match: a match that 

738 starts inside a run of non-space characters implies one starting earlier in 

739 the run (or at the whitespace just before it, since a name may start with 

740 whitespace), and a match starting inside a whitespace run, other than at 

741 its last character, implies one at its first. So the leftmost match starts 

742 at the first character that is not '[' (a name cannot start with '['), or 

743 at the first or last character of a whitespace run. 

744 """ 

745 start = len(specification) - len(specification.lstrip('[')) 

746 m = ENTRY_RE.match(specification, start) 

747 if not m: 

748 m = _ENTRY_RE_AT_SPACE.search(specification, start) 

749 return m 

750 

751 

752def get_export_entry(specification): 

753 m = _search_export_entry(specification) 

754 if not m: 

755 result = None 

756 if '[' in specification or ']' in specification: 

757 raise DistlibException("Invalid specification " 

758 "'%s'" % specification) 

759 else: 

760 d = m.groupdict() 

761 name = d['name'] 

762 path = d['callable'] 

763 colons = path.count(':') 

764 if colons == 0: 

765 prefix, suffix = path, None 

766 else: 

767 if colons != 1: 

768 raise DistlibException("Invalid specification " 

769 "'%s'" % specification) 

770 prefix, suffix = path.split(':') 

771 flags = d['flags'] 

772 if flags is None: 

773 if '[' in specification or ']' in specification: 

774 raise DistlibException("Invalid specification " 

775 "'%s'" % specification) 

776 flags = [] 

777 else: 

778 flags = [f.strip() for f in flags.split(',')] 

779 result = ExportEntry(name, prefix, suffix, flags) 

780 return result 

781 

782 

783def get_cache_base(suffix=None): 

784 """ 

785 Return the default base location for distlib caches. If the directory does 

786 not exist, it is created. Use the suffix provided for the base directory, 

787 and default to '.distlib' if it isn't provided. 

788 

789 On Windows, if LOCALAPPDATA is defined in the environment, then it is 

790 assumed to be a directory, and will be the parent directory of the result. 

791 On POSIX, and on Windows if LOCALAPPDATA is not defined, the user's home 

792 directory - using os.expanduser('~') - will be the parent directory of 

793 the result. 

794 

795 The result is just the directory '.distlib' in the parent directory as 

796 determined above, or with the name specified with ``suffix``. 

797 """ 

798 if suffix is None: 

799 suffix = '.distlib' 

800 if os.name == 'nt' and 'LOCALAPPDATA' in os.environ: 

801 result = os.path.expandvars('$localappdata') 

802 else: 

803 # Assume posix, or old Windows 

804 result = os.path.expanduser('~') 

805 # we use 'isdir' instead of 'exists', because we want to 

806 # fail if there's a file with that name 

807 if os.path.isdir(result): 

808 usable = os.access(result, os.W_OK) 

809 if not usable: 

810 logger.warning('Directory exists but is not writable: %s', result) 

811 else: 

812 try: 

813 os.makedirs(result) 

814 usable = True 

815 except OSError: 

816 logger.warning('Unable to create %s', result, exc_info=True) 

817 usable = False 

818 if not usable: 

819 result = tempfile.mkdtemp() 

820 logger.warning('Default location unusable, using %s', result) 

821 return os.path.join(result, suffix) 

822 

823 

824def path_to_cache_dir(path, use_abspath=True): 

825 """ 

826 Convert an absolute path to a directory name for use in a cache. 

827 

828 The algorithm used is: 

829 

830 #. On Windows, any ``':'`` in the drive is replaced with ``'---'``. 

831 #. Any occurrence of ``os.sep`` is replaced with ``'--'``. 

832 #. ``'.cache'`` is appended. 

833 """ 

834 d, p = os.path.splitdrive(os.path.abspath(path) if use_abspath else path) 

835 if d: 

836 d = d.replace(':', '---') 

837 p = p.replace(os.sep, '--') 

838 return d + p + '.cache' 

839 

840 

841def ensure_slash(s): 

842 if not s.endswith('/'): 

843 return s + '/' 

844 return s 

845 

846 

847def parse_credentials(netloc): 

848 username = password = None 

849 if '@' in netloc: 

850 prefix, netloc = netloc.rsplit('@', 1) 

851 if ':' not in prefix: 

852 username = prefix 

853 else: 

854 username, password = prefix.split(':', 1) 

855 if username: 

856 username = unquote(username) 

857 if password: 

858 password = unquote(password) 

859 return username, password, netloc 

860 

861 

862def get_process_umask(): 

863 result = os.umask(0o22) 

864 os.umask(result) 

865 return result 

866 

867 

868def is_string_sequence(seq): 

869 result = True 

870 i = None 

871 for i, s in enumerate(seq): 

872 if not isinstance(s, string_types): 

873 result = False 

874 break 

875 assert i is not None 

876 return result 

877 

878 

879PROJECT_NAME_AND_VERSION = re.compile('([a-z0-9_]+([.-][a-z_][a-z0-9_]*)*)-' 

880 '([a-z0-9_.+-]+)', re.I) 

881PYTHON_VERSION = re.compile(r'-py(\d\.?\d?)') 

882 

883 

884def split_filename(filename, project_name=None): 

885 """ 

886 Extract name, version, python version from a filename (no extension) 

887 

888 Return name, version, pyver or None 

889 """ 

890 result = None 

891 pyver = None 

892 filename = unquote(filename).replace(' ', '-') 

893 m = PYTHON_VERSION.search(filename) 

894 if m: 

895 pyver = m.group(1) 

896 filename = filename[:m.start()] 

897 if project_name and len(filename) > len(project_name) + 1: 

898 m = re.match(re.escape(project_name) + r'\b', filename) 

899 if m: 

900 n = m.end() 

901 result = filename[:n], filename[n + 1:], pyver 

902 if result is None: 

903 m = PROJECT_NAME_AND_VERSION.match(filename) 

904 if m: 

905 result = m.group(1), m.group(3), pyver 

906 return result 

907 

908 

909# Allow spaces in name because of legacy dists like "Twisted Core" 

910NAME_VERSION_RE = re.compile(r'(?P<name>[\w .-]+)\s*' 

911 r'\(\s*(?P<ver>[^\s)]+)\)$') 

912 

913 

914def parse_name_and_version(p): 

915 """ 

916 A utility method used to get name and version from a string. 

917 

918 From e.g. a Provides-Dist value. 

919 

920 :param p: A value in a form 'foo (1.0)' 

921 :return: The name and version as a tuple. 

922 """ 

923 m = NAME_VERSION_RE.match(p) 

924 if not m: 

925 raise DistlibException('Ill-formed name/version string: \'%s\'' % p) 

926 d = m.groupdict() 

927 return d['name'].strip().lower(), d['ver'] 

928 

929 

930def get_extras(requested, available): 

931 result = set() 

932 requested = set(requested or []) 

933 available = set(available or []) 

934 if '*' in requested: 

935 requested.remove('*') 

936 result |= available 

937 for r in requested: 

938 if r == '-': 

939 result.add(r) 

940 elif r.startswith('-'): 

941 unwanted = r[1:] 

942 if unwanted not in available: 

943 logger.warning('undeclared extra: %s' % unwanted) 

944 if unwanted in result: 

945 result.remove(unwanted) 

946 else: 

947 if r not in available: 

948 logger.warning('undeclared extra: %s' % r) 

949 result.add(r) 

950 return result 

951 

952 

953# 

954# Extended metadata functionality 

955# 

956 

957 

958def _get_external_data(url): 

959 result = {} 

960 try: 

961 # urlopen might fail if it runs into redirections, 

962 # because of Python issue #13696. Fixed in locators 

963 # using a custom redirect handler. 

964 resp = urlopen(url) 

965 headers = resp.info() 

966 ct = headers.get('Content-Type') 

967 if not ct.startswith('application/json'): 

968 logger.debug('Unexpected response for JSON request: %s', ct) 

969 else: 

970 reader = codecs.getreader('utf-8')(resp) 

971 # data = reader.read().decode('utf-8') 

972 # result = json.loads(data) 

973 result = json.load(reader) 

974 except Exception as e: 

975 logger.exception('Failed to get external data for %s: %s', url, e) 

976 return result 

977 

978 

979_external_data_base_url = 'https://www.red-dove.com/pypi/projects/' 

980 

981 

982def get_project_data(name): 

983 url = '%s/%s/project.json' % (name[0].upper(), name) 

984 url = urljoin(_external_data_base_url, url) 

985 result = _get_external_data(url) 

986 return result 

987 

988 

989def get_package_data(name, version): 

990 url = '%s/%s/package-%s.json' % (name[0].upper(), name, version) 

991 url = urljoin(_external_data_base_url, url) 

992 return _get_external_data(url) 

993 

994 

995class Cache(object): 

996 """ 

997 A class implementing a cache for resources that need to live in the file system 

998 e.g. shared libraries. This class was moved from resources to here because it 

999 could be used by other modules, e.g. the wheel module. 

1000 """ 

1001 

1002 def __init__(self, base): 

1003 """ 

1004 Initialise an instance. 

1005 

1006 :param base: The base directory where the cache should be located. 

1007 """ 

1008 # we use 'isdir' instead of 'exists', because we want to 

1009 # fail if there's a file with that name 

1010 if not os.path.isdir(base): # pragma: no cover 

1011 os.makedirs(base) 

1012 if (os.stat(base).st_mode & 0o77) != 0: 

1013 logger.warning('Directory \'%s\' is not private', base) 

1014 self.base = os.path.abspath(os.path.normpath(base)) 

1015 

1016 def prefix_to_dir(self, prefix, use_abspath=True): 

1017 """ 

1018 Converts a resource prefix to a directory name in the cache. 

1019 """ 

1020 return path_to_cache_dir(prefix, use_abspath=use_abspath) 

1021 

1022 def clear(self): 

1023 """ 

1024 Clear the cache. 

1025 """ 

1026 not_removed = [] 

1027 for fn in os.listdir(self.base): 

1028 fn = os.path.join(self.base, fn) 

1029 try: 

1030 if os.path.islink(fn) or os.path.isfile(fn): 

1031 os.remove(fn) 

1032 elif os.path.isdir(fn): 

1033 shutil.rmtree(fn) 

1034 except Exception: 

1035 not_removed.append(fn) 

1036 return not_removed 

1037 

1038 

1039class EventMixin(object): 

1040 """ 

1041 A very simple publish/subscribe system. 

1042 """ 

1043 

1044 def __init__(self): 

1045 self._subscribers = {} 

1046 

1047 def add(self, event, subscriber, append=True): 

1048 """ 

1049 Add a subscriber for an event. 

1050 

1051 :param event: The name of an event. 

1052 :param subscriber: The subscriber to be added (and called when the 

1053 event is published). 

1054 :param append: Whether to append or prepend the subscriber to an 

1055 existing subscriber list for the event. 

1056 """ 

1057 subs = self._subscribers 

1058 if event not in subs: 

1059 subs[event] = deque([subscriber]) 

1060 else: 

1061 sq = subs[event] 

1062 if append: 

1063 sq.append(subscriber) 

1064 else: 

1065 sq.appendleft(subscriber) 

1066 

1067 def remove(self, event, subscriber): 

1068 """ 

1069 Remove a subscriber for an event. 

1070 

1071 :param event: The name of an event. 

1072 :param subscriber: The subscriber to be removed. 

1073 """ 

1074 subs = self._subscribers 

1075 if event not in subs: 

1076 raise ValueError('No subscribers: %r' % event) 

1077 subs[event].remove(subscriber) 

1078 

1079 def get_subscribers(self, event): 

1080 """ 

1081 Return an iterator for the subscribers for an event. 

1082 :param event: The event to return subscribers for. 

1083 """ 

1084 return iter(self._subscribers.get(event, ())) 

1085 

1086 def publish(self, event, *args, **kwargs): 

1087 """ 

1088 Publish a event and return a list of values returned by its 

1089 subscribers. 

1090 

1091 :param event: The event to publish. 

1092 :param args: The positional arguments to pass to the event's 

1093 subscribers. 

1094 :param kwargs: The keyword arguments to pass to the event's 

1095 subscribers. 

1096 """ 

1097 result = [] 

1098 for subscriber in self.get_subscribers(event): 

1099 try: 

1100 value = subscriber(event, *args, **kwargs) 

1101 except Exception: 

1102 logger.exception('Exception during event publication') 

1103 value = None 

1104 result.append(value) 

1105 logger.debug('publish %s: args = %s, kwargs = %s, result = %s', event, args, kwargs, result) 

1106 return result 

1107 

1108 

1109# 

1110# Simple sequencing 

1111# 

1112class Sequencer(object): 

1113 

1114 def __init__(self): 

1115 self._preds = {} 

1116 self._succs = {} 

1117 self._nodes = set() # nodes with no preds/succs 

1118 

1119 def add_node(self, node): 

1120 self._nodes.add(node) 

1121 

1122 def remove_node(self, node, edges=False): 

1123 if node in self._nodes: 

1124 self._nodes.remove(node) 

1125 if edges: 

1126 for p in set(self._preds.get(node, ())): 

1127 self.remove(p, node) 

1128 for s in set(self._succs.get(node, ())): 

1129 self.remove(node, s) 

1130 # Remove empties 

1131 for k, v in list(self._preds.items()): 

1132 if not v: 

1133 del self._preds[k] 

1134 for k, v in list(self._succs.items()): 

1135 if not v: 

1136 del self._succs[k] 

1137 

1138 def add(self, pred, succ): 

1139 assert pred != succ 

1140 self._preds.setdefault(succ, set()).add(pred) 

1141 self._succs.setdefault(pred, set()).add(succ) 

1142 

1143 def remove(self, pred, succ): 

1144 assert pred != succ 

1145 try: 

1146 preds = self._preds[succ] 

1147 succs = self._succs[pred] 

1148 except KeyError: # pragma: no cover 

1149 raise ValueError('%r not a successor of anything' % succ) 

1150 try: 

1151 preds.remove(pred) 

1152 succs.remove(succ) 

1153 except KeyError: # pragma: no cover 

1154 raise ValueError('%r not a successor of %r' % (succ, pred)) 

1155 

1156 def is_step(self, step): 

1157 return (step in self._preds or step in self._succs or step in self._nodes) 

1158 

1159 def get_steps(self, final): 

1160 if not self.is_step(final): 

1161 raise ValueError('Unknown: %r' % final) 

1162 result = [] 

1163 todo = [] 

1164 seen = set() 

1165 todo.append(final) 

1166 while todo: 

1167 step = todo.pop(0) 

1168 if step in seen: 

1169 # if a step was already seen, 

1170 # move it to the end (so it will appear earlier 

1171 # when reversed on return) ... but not for the 

1172 # final step, as that would be confusing for 

1173 # users 

1174 if step != final: 

1175 result.remove(step) 

1176 result.append(step) 

1177 else: 

1178 seen.add(step) 

1179 result.append(step) 

1180 preds = self._preds.get(step, ()) 

1181 todo.extend(preds) 

1182 return reversed(result) 

1183 

1184 @property 

1185 def strong_connections(self): 

1186 # http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm 

1187 index_counter = [0] 

1188 stack = [] 

1189 lowlinks = {} 

1190 index = {} 

1191 result = [] 

1192 

1193 graph = self._succs 

1194 

1195 def strongconnect(node): 

1196 # set the depth index for this node to the smallest unused index 

1197 index[node] = index_counter[0] 

1198 lowlinks[node] = index_counter[0] 

1199 index_counter[0] += 1 

1200 stack.append(node) 

1201 

1202 # Consider successors 

1203 try: 

1204 successors = graph[node] 

1205 except Exception: 

1206 successors = [] 

1207 for successor in successors: 

1208 if successor not in lowlinks: 

1209 # Successor has not yet been visited 

1210 strongconnect(successor) 

1211 lowlinks[node] = min(lowlinks[node], lowlinks[successor]) 

1212 elif successor in stack: 

1213 # the successor is in the stack and hence in the current 

1214 # strongly connected component (SCC) 

1215 lowlinks[node] = min(lowlinks[node], index[successor]) 

1216 

1217 # If `node` is a root node, pop the stack and generate an SCC 

1218 if lowlinks[node] == index[node]: 

1219 connected_component = [] 

1220 

1221 while True: 

1222 successor = stack.pop() 

1223 connected_component.append(successor) 

1224 if successor == node: 

1225 break 

1226 component = tuple(connected_component) 

1227 # storing the result 

1228 result.append(component) 

1229 

1230 for node in graph: 

1231 if node not in lowlinks: 

1232 strongconnect(node) 

1233 

1234 return result 

1235 

1236 @property 

1237 def dot(self): 

1238 result = ['digraph G {'] 

1239 for succ in self._preds: 

1240 preds = self._preds[succ] 

1241 for pred in preds: 

1242 result.append(' %s -> %s;' % (pred, succ)) 

1243 for node in self._nodes: 

1244 result.append(' %s;' % node) 

1245 result.append('}') 

1246 return '\n'.join(result) 

1247 

1248 

1249# 

1250# Unarchiving functionality for zip, tar, tgz, tbz, whl 

1251# 

1252 

1253ARCHIVE_EXTENSIONS = ('.tar.gz', '.tar.bz2', '.tar', '.zip', '.tgz', '.tbz', '.whl') 

1254 

1255 

1256def unarchive(archive_filename, dest_dir, format=None, check=True): 

1257 

1258 # Symlinks seen so far in the archive, mapping the link's resolved path 

1259 # to its resolved target. Earlier members don't exist on disk while we 

1260 # check, so later member paths are resolved through this map lexically. 

1261 links = {} 

1262 

1263 def resolve(path): 

1264 cur = dest_dir 

1265 if not (path == dest_dir or path.startswith(dest_dir + os.sep)): 

1266 return os.path.realpath(path) 

1267 for part in path[len(dest_dir):].split(os.sep): 

1268 if part in ('', '.'): 

1269 continue 

1270 if part == '..': 

1271 cur = os.path.dirname(cur) 

1272 else: 

1273 cur = links.get(os.path.join(cur, part), os.path.join(cur, part)) 

1274 return os.path.realpath(cur) 

1275 

1276 def check_path(path, base=None): 

1277 if not isinstance(path, text_type): 

1278 path = path.decode('utf-8') 

1279 if base is None: 

1280 base = dest_dir 

1281 p = resolve(os.path.join(base, path)) 

1282 if not (p == dest_dir or p.startswith(dest_dir + os.sep)): 

1283 raise ValueError('path outside destination: %r' % p) 

1284 return p 

1285 

1286 def check_link(member): 

1287 # A symlink/hardlink member's name is validated like any other 

1288 # member, but its target (linkname) is not covered by extractall's 

1289 # name-based handling. An unchecked target lets a later member be 

1290 # written through the link to a location outside dest_dir. Validate 

1291 # the resolved target stays within dest_dir. Symlink targets are 

1292 # relative to the member's own directory; hardlink targets are 

1293 # relative to the archive root (i.e. dest_dir). 

1294 if not (member.issym() or member.islnk()): 

1295 return 

1296 if member.islnk(): 

1297 check_path(member.linkname) 

1298 return 

1299 name = member.name.rstrip('/') 

1300 parent = check_path(os.path.dirname(name) or '.') 

1301 links[os.path.join(parent, os.path.basename(name))] = check_path(member.linkname, base=parent) 

1302 

1303 dest_dir = os.path.realpath(dest_dir) 

1304 archive = None 

1305 if format is None: 

1306 if archive_filename.endswith(('.zip', '.whl')): 

1307 format = 'zip' 

1308 elif archive_filename.endswith(('.tar.gz', '.tgz')): 

1309 format = 'tgz' 

1310 mode = 'r:gz' 

1311 elif archive_filename.endswith(('.tar.bz2', '.tbz')): 

1312 format = 'tbz' 

1313 mode = 'r:bz2' 

1314 elif archive_filename.endswith('.tar'): 

1315 format = 'tar' 

1316 mode = 'r' 

1317 else: # pragma: no cover 

1318 raise ValueError('Unknown format for %r' % archive_filename) 

1319 try: 

1320 if format == 'zip': 

1321 archive = ZipFile(archive_filename, 'r') 

1322 if check: 

1323 names = archive.namelist() 

1324 for name in names: 

1325 check_path(name) 

1326 else: 

1327 archive = tarfile.open(archive_filename, mode) 

1328 if check: 

1329 for member in archive.getmembers(): 

1330 check_path(member.name) 

1331 check_link(member) 

1332 if format != 'zip' and sys.version_info[0] < 3: 

1333 # See Python issue 17153. If the dest path contains Unicode, 

1334 # tarfile extraction fails on Python 2.x if a member path name 

1335 # contains non-ASCII characters - it leads to an implicit 

1336 # bytes -> unicode conversion using ASCII to decode. 

1337 for tarinfo in archive.getmembers(): 

1338 if not isinstance(tarinfo.name, text_type): 

1339 tarinfo.name = tarinfo.name.decode('utf-8') 

1340 

1341 # Limit extraction of dangerous items, if this Python 

1342 # allows it easily. If not, just trust the input. 

1343 # See: https://docs.python.org/3/library/tarfile.html#extraction-filters 

1344 # data_filter is the stdlib default from Python 3.14 onwards; using the 

1345 # laxer tar_filter here would downgrade that secure default. 

1346 def extraction_filter(member, path): 

1347 """Run tarfile.data_filter, but raise the expected ValueError""" 

1348 # This is only called if the current Python has tarfile filters 

1349 try: 

1350 return tarfile.data_filter(member, path) 

1351 except tarfile.FilterError as exc: 

1352 raise ValueError(str(exc)) 

1353 

1354 archive.extraction_filter = extraction_filter 

1355 

1356 archive.extractall(dest_dir) 

1357 

1358 finally: 

1359 if archive: 

1360 archive.close() 

1361 

1362 

1363def zip_dir(directory): 

1364 """zip a directory tree into a BytesIO object""" 

1365 result = io.BytesIO() 

1366 dlen = len(directory) 

1367 with ZipFile(result, "w") as zf: 

1368 for root, dirs, files in os.walk(directory): 

1369 for name in files: 

1370 full = os.path.join(root, name) 

1371 rel = root[dlen:] 

1372 dest = os.path.join(rel, name) 

1373 zf.write(full, dest) 

1374 return result 

1375 

1376 

1377# 

1378# Simple progress bar 

1379# 

1380 

1381UNITS = ('', 'K', 'M', 'G', 'T', 'P') 

1382 

1383 

1384class Progress(object): 

1385 unknown = 'UNKNOWN' 

1386 

1387 def __init__(self, minval=0, maxval=100): 

1388 assert maxval is None or maxval >= minval 

1389 self.min = self.cur = minval 

1390 self.max = maxval 

1391 self.started = None 

1392 self.elapsed = 0 

1393 self.done = False 

1394 

1395 def update(self, curval): 

1396 assert self.min <= curval 

1397 assert self.max is None or curval <= self.max 

1398 self.cur = curval 

1399 now = time.time() 

1400 if self.started is None: 

1401 self.started = now 

1402 else: 

1403 self.elapsed = now - self.started 

1404 

1405 def increment(self, incr): 

1406 assert incr >= 0 

1407 self.update(self.cur + incr) 

1408 

1409 def start(self): 

1410 self.update(self.min) 

1411 return self 

1412 

1413 def stop(self): 

1414 if self.max is not None: 

1415 self.update(self.max) 

1416 self.done = True 

1417 

1418 @property 

1419 def maximum(self): 

1420 return self.unknown if self.max is None else self.max 

1421 

1422 @property 

1423 def percentage(self): 

1424 if self.done: 

1425 result = '100 %' 

1426 elif self.max is None: 

1427 result = ' ?? %' 

1428 else: 

1429 v = 100.0 * (self.cur - self.min) / (self.max - self.min) 

1430 result = '%3d %%' % v 

1431 return result 

1432 

1433 def format_duration(self, duration): 

1434 if (duration <= 0) and self.max is None or self.cur == self.min: 

1435 result = '??:??:??' 

1436 # elif duration < 1: 

1437 # result = '--:--:--' 

1438 else: 

1439 result = time.strftime('%H:%M:%S', time.gmtime(duration)) 

1440 return result 

1441 

1442 @property 

1443 def ETA(self): 

1444 if self.done: 

1445 prefix = 'Done' 

1446 t = self.elapsed 

1447 # import pdb; pdb.set_trace() 

1448 else: 

1449 prefix = 'ETA ' 

1450 if self.max is None: 

1451 t = -1 

1452 elif self.elapsed == 0 or (self.cur == self.min): 

1453 t = 0 

1454 else: 

1455 # import pdb; pdb.set_trace() 

1456 t = float(self.max - self.min) 

1457 t /= self.cur - self.min 

1458 t = (t - 1) * self.elapsed 

1459 return '%s: %s' % (prefix, self.format_duration(t)) 

1460 

1461 @property 

1462 def speed(self): 

1463 if self.elapsed == 0: 

1464 result = 0.0 

1465 else: 

1466 result = (self.cur - self.min) / self.elapsed 

1467 for unit in UNITS: 

1468 if result < 1000: 

1469 break 

1470 result /= 1000.0 

1471 return '%d %sB/s' % (result, unit) 

1472 

1473 

1474# 

1475# Glob functionality 

1476# 

1477 

1478RICH_GLOB = re.compile(r'\{([^}]*)\}') 

1479_CHECK_RECURSIVE_GLOB = re.compile(r'[^/\\,{]\*\*|\*\*[^/\\,}]') 

1480_CHECK_MISMATCH_SET = re.compile(r'^[^{]*\}|\{[^}]*$') 

1481 

1482 

1483def iglob(path_glob): 

1484 """Extended globbing function that supports ** and {opt1,opt2,opt3}.""" 

1485 if _CHECK_RECURSIVE_GLOB.search(path_glob): 

1486 msg = """invalid glob %r: recursive glob "**" must be used alone""" 

1487 raise ValueError(msg % path_glob) 

1488 if _CHECK_MISMATCH_SET.search(path_glob): 

1489 msg = """invalid glob %r: mismatching set marker '{' or '}'""" 

1490 raise ValueError(msg % path_glob) 

1491 return _iglob(path_glob) 

1492 

1493 

1494def _iglob(path_glob): 

1495 rich_path_glob = RICH_GLOB.split(path_glob, 1) 

1496 if len(rich_path_glob) > 1: 

1497 assert len(rich_path_glob) == 3, rich_path_glob 

1498 prefix, set, suffix = rich_path_glob 

1499 for item in set.split(','): 

1500 for path in _iglob(''.join((prefix, item, suffix))): 

1501 yield path 

1502 else: 

1503 if '**' not in path_glob: 

1504 for item in std_iglob(path_glob): 

1505 yield item 

1506 else: 

1507 prefix, radical = path_glob.split('**', 1) 

1508 if prefix == '': 

1509 prefix = '.' 

1510 if radical == '': 

1511 radical = '*' 

1512 else: 

1513 # we support both 

1514 radical = radical.lstrip('/') 

1515 radical = radical.lstrip('\\') 

1516 for path, dir, files in os.walk(prefix): 

1517 path = os.path.normpath(path) 

1518 for fn in _iglob(os.path.join(path, radical)): 

1519 yield fn 

1520 

1521 

1522if ssl: 

1523 from .compat import (HTTPSHandler as BaseHTTPSHandler, match_hostname, CertificateError) 

1524 

1525 # 

1526 # HTTPSConnection which verifies certificates/matches domains 

1527 # 

1528 

1529 

1530 class HTTPSConnection(httplib.HTTPSConnection): 

1531 ca_certs = None # set this to the path to the certs file (.pem) 

1532 check_domain = True # only used if ca_certs is not None 

1533 

1534 # noinspection PyPropertyAccess 

1535 def connect(self): 

1536 sock = socket.create_connection((self.host, self.port), self.timeout) 

1537 if getattr(self, '_tunnel_host', False): 

1538 self.sock = sock 

1539 self._tunnel() 

1540 

1541 context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) 

1542 if hasattr(ssl, 'OP_NO_SSLv2'): 

1543 context.options |= ssl.OP_NO_SSLv2 

1544 if getattr(self, 'cert_file', None): 

1545 context.load_cert_chain(self.cert_file, self.key_file) 

1546 kwargs = {} 

1547 if self.ca_certs: 

1548 context.verify_mode = ssl.CERT_REQUIRED 

1549 context.load_verify_locations(cafile=self.ca_certs) 

1550 if getattr(ssl, 'HAS_SNI', False): 

1551 kwargs['server_hostname'] = self.host 

1552 

1553 self.sock = context.wrap_socket(sock, **kwargs) 

1554 if self.ca_certs and self.check_domain: 

1555 try: 

1556 match_hostname(self.sock.getpeercert(), self.host) 

1557 logger.debug('Host verified: %s', self.host) 

1558 except CertificateError: # pragma: no cover 

1559 self.sock.shutdown(socket.SHUT_RDWR) 

1560 self.sock.close() 

1561 raise 

1562 

1563 class HTTPSHandler(BaseHTTPSHandler): 

1564 

1565 def __init__(self, ca_certs, check_domain=True): 

1566 BaseHTTPSHandler.__init__(self) 

1567 self.ca_certs = ca_certs 

1568 self.check_domain = check_domain 

1569 

1570 def _conn_maker(self, *args, **kwargs): 

1571 """ 

1572 This is called to create a connection instance. Normally you'd 

1573 pass a connection class to do_open, but it doesn't actually check for 

1574 a class, and just expects a callable. As long as we behave just as a 

1575 constructor would have, we should be OK. If it ever changes so that 

1576 we *must* pass a class, we'll create an UnsafeHTTPSConnection class 

1577 which just sets check_domain to False in the class definition, and 

1578 choose which one to pass to do_open. 

1579 """ 

1580 result = HTTPSConnection(*args, **kwargs) 

1581 if self.ca_certs: 

1582 result.ca_certs = self.ca_certs 

1583 result.check_domain = self.check_domain 

1584 return result 

1585 

1586 def https_open(self, req): 

1587 try: 

1588 return self.do_open(self._conn_maker, req) 

1589 except URLError as e: 

1590 if 'certificate verify failed' in str(e.reason): 

1591 raise CertificateError('Unable to verify server certificate ' 

1592 'for %s' % req.host) 

1593 else: 

1594 raise 

1595 

1596 # 

1597 # To prevent against mixing HTTP traffic with HTTPS (examples: A Man-In-The- 

1598 # Middle proxy using HTTP listens on port 443, or an index mistakenly serves 

1599 # HTML containing a http://xyz link when it should be https://xyz), 

1600 # you can use the following handler class, which does not allow HTTP traffic. 

1601 # 

1602 # It works by inheriting from HTTPHandler - so build_opener won't add a 

1603 # handler for HTTP itself. 

1604 # 

1605 class HTTPSOnlyHandler(HTTPSHandler, HTTPHandler): 

1606 

1607 def http_open(self, req): 

1608 raise URLError('Unexpected HTTP request on what should be a secure ' 

1609 'connection: %s' % req) 

1610 

1611 

1612# 

1613# XML-RPC with timeouts 

1614# 

1615class Transport(xmlrpclib.Transport): 

1616 

1617 def __init__(self, timeout, use_datetime=0): 

1618 self.timeout = timeout 

1619 xmlrpclib.Transport.__init__(self, use_datetime) 

1620 

1621 def make_connection(self, host): 

1622 h, eh, x509 = self.get_host_info(host) 

1623 if not self._connection or host != self._connection[0]: 

1624 self._extra_headers = eh 

1625 self._connection = host, httplib.HTTPConnection(h) 

1626 return self._connection[1] 

1627 

1628 

1629if ssl: 

1630 

1631 class SafeTransport(xmlrpclib.SafeTransport): 

1632 

1633 def __init__(self, timeout, use_datetime=0): 

1634 self.timeout = timeout 

1635 xmlrpclib.SafeTransport.__init__(self, use_datetime) 

1636 

1637 def make_connection(self, host): 

1638 h, eh, kwargs = self.get_host_info(host) 

1639 if not kwargs: 

1640 kwargs = {} 

1641 kwargs['timeout'] = self.timeout 

1642 if not self._connection or host != self._connection[0]: 

1643 self._extra_headers = eh 

1644 self._connection = host, httplib.HTTPSConnection(h, None, **kwargs) 

1645 return self._connection[1] 

1646 

1647 

1648class ServerProxy(xmlrpclib.ServerProxy): 

1649 

1650 def __init__(self, uri, **kwargs): 

1651 self.timeout = timeout = kwargs.pop('timeout', None) 

1652 # The above classes only come into play if a timeout 

1653 # is specified 

1654 if timeout is not None: 

1655 # scheme = splittype(uri) # deprecated as of Python 3.8 

1656 scheme = urlparse(uri)[0] 

1657 use_datetime = kwargs.get('use_datetime', 0) 

1658 if scheme == 'https': 

1659 tcls = SafeTransport 

1660 else: 

1661 tcls = Transport 

1662 kwargs['transport'] = t = tcls(timeout, use_datetime=use_datetime) 

1663 self.transport = t 

1664 xmlrpclib.ServerProxy.__init__(self, uri, **kwargs) 

1665 

1666 

1667# 

1668# CSV functionality. This is provided because on 2.x, the csv module can't 

1669# handle Unicode. However, we need to deal with Unicode in e.g. RECORD files. 

1670# 

1671 

1672 

1673def _csv_open(fn, mode, **kwargs): 

1674 if sys.version_info[0] < 3: 

1675 mode += 'b' 

1676 else: 

1677 kwargs['newline'] = '' 

1678 # Python 3 determines encoding from locale. Force 'utf-8' 

1679 # file encoding to match other forced utf-8 encoding 

1680 kwargs['encoding'] = 'utf-8' 

1681 return open(fn, mode, **kwargs) 

1682 

1683 

1684class CSVBase(object): 

1685 defaults = { 

1686 'delimiter': str(','), # The strs are used because we need native 

1687 'quotechar': str('"'), # str in the csv API (2.x won't take 

1688 'lineterminator': str('\n') # Unicode) 

1689 } 

1690 

1691 def __enter__(self): 

1692 return self 

1693 

1694 def __exit__(self, *exc_info): 

1695 self.stream.close() 

1696 

1697 

1698class CSVReader(CSVBase): 

1699 

1700 def __init__(self, **kwargs): 

1701 if 'stream' in kwargs: 

1702 stream = kwargs['stream'] 

1703 if sys.version_info[0] >= 3: 

1704 # needs to be a text stream 

1705 stream = codecs.getreader('utf-8')(stream) 

1706 self.stream = stream 

1707 else: 

1708 self.stream = _csv_open(kwargs['path'], 'r') 

1709 self.reader = csv.reader(self.stream, **self.defaults) 

1710 

1711 def __iter__(self): 

1712 return self 

1713 

1714 def next(self): 

1715 result = next(self.reader) 

1716 if sys.version_info[0] < 3: 

1717 for i, item in enumerate(result): 

1718 if not isinstance(item, text_type): 

1719 result[i] = item.decode('utf-8') 

1720 return result 

1721 

1722 __next__ = next 

1723 

1724 

1725class CSVWriter(CSVBase): 

1726 

1727 def __init__(self, fn, **kwargs): 

1728 self.stream = _csv_open(fn, 'w') 

1729 self.writer = csv.writer(self.stream, **self.defaults) 

1730 

1731 def writerow(self, row): 

1732 if sys.version_info[0] < 3: 

1733 r = [] 

1734 for item in row: 

1735 if isinstance(item, text_type): 

1736 item = item.encode('utf-8') 

1737 r.append(item) 

1738 row = r 

1739 self.writer.writerow(row) 

1740 

1741 

1742# 

1743# Configurator functionality 

1744# 

1745 

1746 

1747class Configurator(BaseConfigurator): 

1748 

1749 value_converters = dict(BaseConfigurator.value_converters) 

1750 value_converters['inc'] = 'inc_convert' 

1751 

1752 def __init__(self, config, base=None): 

1753 super(Configurator, self).__init__(config) 

1754 self.base = base or os.getcwd() 

1755 

1756 def configure_custom(self, config): 

1757 

1758 def convert(o): 

1759 if isinstance(o, (list, tuple)): 

1760 result = type(o)([convert(i) for i in o]) 

1761 elif isinstance(o, dict): 

1762 if '()' in o: 

1763 result = self.configure_custom(o) 

1764 else: 

1765 result = {} 

1766 for k in o: 

1767 result[k] = convert(o[k]) 

1768 else: 

1769 result = self.convert(o) 

1770 return result 

1771 

1772 c = config.pop('()') 

1773 if not callable(c): 

1774 c = self.resolve(c) 

1775 props = config.pop('.', None) 

1776 # Check for valid identifiers 

1777 args = config.pop('[]', ()) 

1778 if args: 

1779 args = tuple([convert(o) for o in args]) 

1780 items = [(k, convert(config[k])) for k in config if valid_ident(k)] 

1781 kwargs = dict(items) 

1782 result = c(*args, **kwargs) 

1783 if props: 

1784 for n, v in props.items(): 

1785 setattr(result, n, convert(v)) 

1786 return result 

1787 

1788 def __getitem__(self, key): 

1789 result = self.config[key] 

1790 if isinstance(result, dict) and '()' in result: 

1791 self.config[key] = result = self.configure_custom(result) 

1792 return result 

1793 

1794 def inc_convert(self, value): 

1795 """Default converter for the inc:// protocol.""" 

1796 if not os.path.isabs(value): 

1797 value = os.path.join(self.base, value) 

1798 with text_open(value, 'r', encoding='utf-8') as f: 

1799 result = json.load(f) 

1800 return result 

1801 

1802 

1803class SubprocessMixin(object): 

1804 """ 

1805 Mixin for running subprocesses and capturing their output 

1806 """ 

1807 

1808 def __init__(self, verbose=False, progress=None): 

1809 self.verbose = verbose 

1810 self.progress = progress 

1811 

1812 def reader(self, stream, context): 

1813 """ 

1814 Read lines from a subprocess' output stream and either pass to a progress 

1815 callable (if specified) or write progress information to sys.stderr. 

1816 """ 

1817 progress = self.progress 

1818 verbose = self.verbose 

1819 while True: 

1820 s = stream.readline() 

1821 if not s: 

1822 break 

1823 if progress is not None: 

1824 progress(s, context) 

1825 else: 

1826 if not verbose: 

1827 sys.stderr.write('.') 

1828 else: 

1829 sys.stderr.write(s.decode('utf-8')) 

1830 sys.stderr.flush() 

1831 stream.close() 

1832 

1833 def run_command(self, cmd, **kwargs): 

1834 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) 

1835 t1 = threading.Thread(target=self.reader, args=(p.stdout, 'stdout')) 

1836 t1.start() 

1837 t2 = threading.Thread(target=self.reader, args=(p.stderr, 'stderr')) 

1838 t2.start() 

1839 p.wait() 

1840 t1.join() 

1841 t2.join() 

1842 if self.progress is not None: 

1843 self.progress('done.', 'main') 

1844 elif self.verbose: 

1845 sys.stderr.write('done.\n') 

1846 return p 

1847 

1848 

1849def normalize_name(name): 

1850 """Normalize a python package name a la PEP 503""" 

1851 # https://peps.python.org/pep-0503/#normalized-names 

1852 # Emulates re.sub(r"[-_.]+", "-", name).lower() 

1853 # Much faster than re, and even faster than str.translate 

1854 value = name.lower().replace("_", "-").replace(".", "-") 

1855 while "--" in value: 

1856 value = value.replace("--", "-") 

1857 return value 

1858 

1859 

1860# def _get_pypirc_command(): 

1861# """ 

1862# Get the distutils command for interacting with PyPI configurations. 

1863# :return: the command. 

1864# """ 

1865# from distutils.core import Distribution 

1866# from distutils.config import PyPIRCCommand 

1867# d = Distribution() 

1868# return PyPIRCCommand(d) 

1869 

1870 

1871class PyPIRCFile(object): 

1872 

1873 DEFAULT_REPOSITORY = 'https://upload.pypi.org/legacy/' 

1874 DEFAULT_REALM = 'pypi' 

1875 

1876 def __init__(self, fn=None, url=None): 

1877 if fn is None: 

1878 fn = os.path.join(os.path.expanduser('~'), '.pypirc') 

1879 self.filename = fn 

1880 self.url = url 

1881 

1882 def read(self): 

1883 result = {} 

1884 

1885 if os.path.exists(self.filename): 

1886 repository = self.url or self.DEFAULT_REPOSITORY 

1887 

1888 config = configparser.RawConfigParser() 

1889 config.read(self.filename) 

1890 sections = config.sections() 

1891 if 'distutils' in sections: 

1892 # let's get the list of servers 

1893 index_servers = config.get('distutils', 'index-servers') 

1894 _servers = [server.strip() for server in index_servers.split('\n') if server.strip() != ''] 

1895 if _servers == []: 

1896 # nothing set, let's try to get the default pypi 

1897 if 'pypi' in sections: 

1898 _servers = ['pypi'] 

1899 else: 

1900 for server in _servers: 

1901 result = {'server': server} 

1902 result['username'] = config.get(server, 'username') 

1903 

1904 # optional params 

1905 for key, default in (('repository', self.DEFAULT_REPOSITORY), ('realm', self.DEFAULT_REALM), 

1906 ('password', None)): 

1907 if config.has_option(server, key): 

1908 result[key] = config.get(server, key) 

1909 else: 

1910 result[key] = default 

1911 

1912 # work around people having "repository" for the "pypi" 

1913 # section of their config set to the HTTP (rather than 

1914 # HTTPS) URL 

1915 if (server == 'pypi' and repository in (self.DEFAULT_REPOSITORY, 'pypi')): 

1916 result['repository'] = self.DEFAULT_REPOSITORY 

1917 elif (result['server'] != repository and result['repository'] != repository): 

1918 result = {} 

1919 elif 'server-login' in sections: 

1920 # old format 

1921 server = 'server-login' 

1922 if config.has_option(server, 'repository'): 

1923 repository = config.get(server, 'repository') 

1924 else: 

1925 repository = self.DEFAULT_REPOSITORY 

1926 result = { 

1927 'username': config.get(server, 'username'), 

1928 'password': config.get(server, 'password'), 

1929 'repository': repository, 

1930 'server': server, 

1931 'realm': self.DEFAULT_REALM 

1932 } 

1933 return result 

1934 

1935 def update(self, username, password): 

1936 # import pdb; pdb.set_trace() 

1937 config = configparser.RawConfigParser() 

1938 fn = self.filename 

1939 config.read(fn) 

1940 if not config.has_section('pypi'): 

1941 config.add_section('pypi') 

1942 config.set('pypi', 'username', username) 

1943 config.set('pypi', 'password', password) 

1944 with open(fn, 'w') as f: 

1945 config.write(f) 

1946 

1947 

1948def _load_pypirc(index): 

1949 """ 

1950 Read the PyPI access configuration as supported by distutils. 

1951 """ 

1952 return PyPIRCFile(url=index.url).read() 

1953 

1954 

1955def _store_pypirc(index): 

1956 PyPIRCFile().update(index.username, index.password) 

1957 

1958 

1959# 

1960# get_platform()/get_host_platform() copied from Python 3.10.a0 source, with some minor 

1961# tweaks 

1962# 

1963 

1964 

1965def get_host_platform(): 

1966 """Return a string that identifies the current platform. This is used mainly to 

1967 distinguish platform-specific build directories and platform-specific built 

1968 distributions. Typically includes the OS name and version and the 

1969 architecture (as supplied by 'os.uname()'), although the exact information 

1970 included depends on the OS; eg. on Linux, the kernel version isn't 

1971 particularly important. 

1972 

1973 Examples of returned values: 

1974 linux-i586 

1975 linux-alpha (?) 

1976 solaris-2.6-sun4u 

1977 

1978 Windows will return one of: 

1979 win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) 

1980 win32 (all others - specifically, sys.platform is returned) 

1981 

1982 For other non-POSIX platforms, currently just returns 'sys.platform'. 

1983 

1984 """ 

1985 if os.name == 'nt': 

1986 if 'amd64' in sys.version.lower(): 

1987 return 'win-amd64' 

1988 if '(arm)' in sys.version.lower(): 

1989 return 'win-arm32' 

1990 if '(arm64)' in sys.version.lower(): 

1991 return 'win-arm64' 

1992 return sys.platform 

1993 

1994 # Set for cross builds explicitly 

1995 if "_PYTHON_HOST_PLATFORM" in os.environ: 

1996 return os.environ["_PYTHON_HOST_PLATFORM"] 

1997 

1998 if os.name != 'posix' or not hasattr(os, 'uname'): 

1999 # XXX what about the architecture? NT is Intel or Alpha, 

2000 # Mac OS is M68k or PPC, etc. 

2001 return sys.platform 

2002 

2003 # Try to distinguish various flavours of Unix 

2004 

2005 (osname, host, release, version, machine) = os.uname() 

2006 

2007 # Convert the OS name to lowercase, remove '/' characters, and translate 

2008 # spaces (for "Power Macintosh") 

2009 osname = osname.lower().replace('/', '') 

2010 machine = machine.replace(' ', '_').replace('/', '-') 

2011 

2012 if osname[:5] == 'linux': 

2013 # At least on Linux/Intel, 'machine' is the processor -- 

2014 # i386, etc. 

2015 # XXX what about Alpha, SPARC, etc? 

2016 return "%s-%s" % (osname, machine) 

2017 

2018 elif osname[:5] == 'sunos': 

2019 if release[0] >= '5': # SunOS 5 == Solaris 2 

2020 osname = 'solaris' 

2021 release = '%d.%s' % (int(release[0]) - 3, release[2:]) 

2022 # We can't use 'platform.architecture()[0]' because a 

2023 # bootstrap problem. We use a dict to get an error 

2024 # if some suspicious happens. 

2025 bitness = {2147483647: '32bit', 9223372036854775807: '64bit'} 

2026 machine += '.%s' % bitness[sys.maxsize] 

2027 # fall through to standard osname-release-machine representation 

2028 elif osname[:3] == 'aix': 

2029 from _aix_support import aix_platform 

2030 return aix_platform() 

2031 elif osname[:6] == 'cygwin': 

2032 osname = 'cygwin' 

2033 rel_re = re.compile(r'[\d.]+', re.ASCII) 

2034 m = rel_re.match(release) 

2035 if m: 

2036 release = m.group() 

2037 elif osname[:6] == 'darwin': 

2038 import _osx_support 

2039 try: 

2040 from distutils import sysconfig 

2041 except ImportError: 

2042 import sysconfig 

2043 osname, release, machine = _osx_support.get_platform_osx(sysconfig.get_config_vars(), osname, release, machine) 

2044 

2045 return '%s-%s-%s' % (osname, release, machine) 

2046 

2047 

2048_TARGET_TO_PLAT = { 

2049 'x86': 'win32', 

2050 'x64': 'win-amd64', 

2051 'arm': 'win-arm32', 

2052} 

2053 

2054 

2055def get_platform(): 

2056 if os.name != 'nt': 

2057 return get_host_platform() 

2058 cross_compilation_target = os.environ.get('VSCMD_ARG_TGT_ARCH') 

2059 if cross_compilation_target not in _TARGET_TO_PLAT: 

2060 return get_host_platform() 

2061 return _TARGET_TO_PLAT[cross_compilation_target] 

2062 

2063 

2064def is_in_directory(path, target): 

2065 """ 

2066 Check if a path is inside a target directory. This doesn't check case (might be an issue on Windows) 

2067 """ 

2068 path = os.path.abspath(path) 

2069 target = os.path.abspath(target) 

2070 return path == target or path.startswith(target + os.sep)