Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/distlib/util.py: 21%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# Copyright (C) 2012-2026 The Python Software Foundation.
3# See LICENSE.txt and CONTRIBUTORS.txt.
4#
5import codecs
6from collections import deque
7import contextlib
8import csv
9from glob import iglob as std_iglob
10import io
11import json
12import logging
13import os
14import py_compile
15import re
16import socket
17try:
18 import ssl
19except ImportError: # pragma: no cover
20 ssl = None
21import subprocess
22import sys
23import tarfile
24import tempfile
25import textwrap
27try:
28 import threading
29except ImportError: # pragma: no cover
30 import dummy_threading as threading
31import time
33from . import DistlibException
34from .compat import (string_types, text_type, shutil, raw_input, StringIO, cache_from_source, urlopen, urljoin, httplib,
35 xmlrpclib, HTTPHandler, BaseConfigurator, valid_ident, Container, configparser, URLError, ZipFile,
36 fsdecode, unquote, urlparse, text_open)
38logger = logging.getLogger(__name__)
40#
41# Requirement parsing code as per PEP 508
42#
44IDENTIFIER = re.compile(r'^([\w\.-]+)\s*')
45VERSION_IDENTIFIER = re.compile(r'^([\w\.*+!-]+)\s*')
46COMPARE_OP = re.compile(r'^(<=?|>=?|={2,3}|[~!]=)\s*')
47MARKER_OP = re.compile(r'^((<=?)|(>=?)|={2,3}|[~!]=|in|not\s+in)\s*')
48OR = re.compile(r'^or\b\s*')
49AND = re.compile(r'^and\b\s*')
50NON_SPACE = re.compile(r'(\S+)\s*')
51STRING_CHUNK = re.compile(r'([\s\w\.{}()*+#:;,/?!~`@$%^&=|<>\[\]-]+)')
54def parse_marker(marker_string):
55 """
56 Parse a marker string and return a dictionary containing a marker expression.
58 The dictionary will contain keys "op", "lhs" and "rhs" for non-terminals in
59 the expression grammar, or strings. A string contained in quotes is to be
60 interpreted as a literal string, and a string not contained in quotes is a
61 variable (such as os_name).
62 """
64 def marker_var(remaining):
65 # either identifier, or literal string
66 m = IDENTIFIER.match(remaining)
67 if m:
68 result = m.groups()[0]
69 remaining = remaining[m.end():]
70 elif not remaining:
71 raise SyntaxError('unexpected end of input')
72 else:
73 q = remaining[0]
74 if q not in '\'"':
75 raise SyntaxError('invalid expression: %s' % remaining)
76 oq = '\'"'.replace(q, '')
77 remaining = remaining[1:]
78 parts = [q]
79 while remaining:
80 # either a string chunk, or oq, or q to terminate
81 if remaining[0] == q:
82 break
83 elif remaining[0] == oq:
84 parts.append(oq)
85 remaining = remaining[1:]
86 else:
87 m = STRING_CHUNK.match(remaining)
88 if not m:
89 raise SyntaxError('error in string literal: %s' % remaining)
90 parts.append(m.groups()[0])
91 remaining = remaining[m.end():]
92 else:
93 s = ''.join(parts)
94 raise SyntaxError('unterminated string: %s' % s)
95 parts.append(q)
96 result = ''.join(parts)
97 remaining = remaining[1:].lstrip() # skip past closing quote
98 return result, remaining
100 def marker_expr(remaining):
101 if remaining and remaining[0] == '(':
102 result, remaining = marker(remaining[1:].lstrip())
103 if not remaining or remaining[0] != ')':
104 raise SyntaxError('unterminated parenthesis: %s' % remaining)
105 remaining = remaining[1:].lstrip()
106 else:
107 lhs, remaining = marker_var(remaining)
108 while remaining:
109 m = MARKER_OP.match(remaining)
110 if not m:
111 break
112 op = m.groups()[0]
113 remaining = remaining[m.end():]
114 rhs, remaining = marker_var(remaining)
115 lhs = {'op': op, 'lhs': lhs, 'rhs': rhs}
116 result = lhs
117 return result, remaining
119 def marker_and(remaining):
120 lhs, remaining = marker_expr(remaining)
121 while remaining:
122 m = AND.match(remaining)
123 if not m:
124 break
125 remaining = remaining[m.end():]
126 rhs, remaining = marker_expr(remaining)
127 lhs = {'op': 'and', 'lhs': lhs, 'rhs': rhs}
128 return lhs, remaining
130 def marker(remaining):
131 lhs, remaining = marker_and(remaining)
132 while remaining:
133 m = OR.match(remaining)
134 if not m:
135 break
136 remaining = remaining[m.end():]
137 rhs, remaining = marker_and(remaining)
138 lhs = {'op': 'or', 'lhs': lhs, 'rhs': rhs}
139 return lhs, remaining
141 return marker(marker_string)
144def parse_requirement(req):
145 """
146 Parse a requirement passed in as a string. Return a Container
147 whose attributes contain the various parts of the requirement.
148 """
149 remaining = req.strip()
150 if not remaining or remaining.startswith('#'):
151 return None
152 m = IDENTIFIER.match(remaining)
153 if not m:
154 raise SyntaxError('name expected: %s' % remaining)
155 distname = m.groups()[0]
156 remaining = remaining[m.end():]
157 extras = mark_expr = versions = uri = None
158 if remaining and remaining[0] == '[':
159 i = remaining.find(']', 1)
160 if i < 0:
161 raise SyntaxError('unterminated extra: %s' % remaining)
162 s = remaining[1:i]
163 remaining = remaining[i + 1:].lstrip()
164 extras = []
165 while s:
166 m = IDENTIFIER.match(s)
167 if not m:
168 raise SyntaxError('malformed extra: %s' % s)
169 extras.append(m.groups()[0])
170 s = s[m.end():]
171 if not s:
172 break
173 if s[0] != ',':
174 raise SyntaxError('comma expected in extras: %s' % s)
175 s = s[1:].lstrip()
176 if not extras:
177 extras = None
178 if remaining:
179 if remaining[0] == '@':
180 # it's a URI
181 remaining = remaining[1:].lstrip()
182 m = NON_SPACE.match(remaining)
183 if not m:
184 raise SyntaxError('invalid URI: %s' % remaining)
185 uri = m.groups()[0]
186 t = urlparse(uri)
187 # there are issues with Python and URL parsing, so this test
188 # is a bit crude. See bpo-20271, bpo-23505. Python doesn't
189 # always parse invalid URLs correctly - it should raise
190 # exceptions for malformed URLs
191 if not (t.scheme and t.netloc):
192 raise SyntaxError('Invalid URL: %s' % uri)
193 remaining = remaining[m.end():].lstrip()
194 else:
196 def get_versions(ver_remaining):
197 """
198 Return a list of operator, version tuples if any are
199 specified, else None.
200 """
201 m = COMPARE_OP.match(ver_remaining)
202 versions = None
203 if m:
204 versions = []
205 while True:
206 op = m.groups()[0]
207 ver_remaining = ver_remaining[m.end():]
208 m = VERSION_IDENTIFIER.match(ver_remaining)
209 if not m:
210 raise SyntaxError('invalid version: %s' % ver_remaining)
211 v = m.groups()[0]
212 versions.append((op, v))
213 ver_remaining = ver_remaining[m.end():]
214 if not ver_remaining or ver_remaining[0] != ',':
215 break
216 ver_remaining = ver_remaining[1:].lstrip()
217 # Some packages have a trailing comma which would break things
218 # See issue #148
219 if not ver_remaining:
220 break
221 m = COMPARE_OP.match(ver_remaining)
222 if not m:
223 raise SyntaxError('invalid constraint: %s' % ver_remaining)
224 if not versions:
225 versions = None
226 return versions, ver_remaining
228 if remaining[0] != '(':
229 versions, remaining = get_versions(remaining)
230 else:
231 i = remaining.find(')', 1)
232 if i < 0:
233 raise SyntaxError('unterminated parenthesis: %s' % remaining)
234 s = remaining[1:i]
235 remaining = remaining[i + 1:].lstrip()
236 # As a special diversion from PEP 508, allow a version number
237 # a.b.c in parentheses as a synonym for ~= a.b.c (because this
238 # is allowed in earlier PEPs)
239 if COMPARE_OP.match(s):
240 versions, _ = get_versions(s)
241 else:
242 m = VERSION_IDENTIFIER.match(s)
243 if not m:
244 raise SyntaxError('invalid constraint: %s' % s)
245 v = m.groups()[0]
246 s = s[m.end():].lstrip()
247 if s:
248 raise SyntaxError('invalid constraint: %s' % s)
249 versions = [('~=', v)]
251 if remaining:
252 if remaining[0] != ';':
253 raise SyntaxError('invalid requirement: %s' % remaining)
254 remaining = remaining[1:].lstrip()
256 mark_expr, remaining = parse_marker(remaining)
258 if remaining and remaining[0] != '#':
259 raise SyntaxError('unexpected trailing data: %s' % remaining)
261 if not versions:
262 rs = distname
263 else:
264 rs = '%s %s' % (distname, ', '.join(['%s %s' % con for con in versions]))
265 return Container(name=distname, extras=extras, constraints=versions, marker=mark_expr, url=uri, requirement=rs)
268def get_resources_dests(resources_root, rules):
269 """Find destinations for resources files"""
271 def get_rel_path(root, path):
272 # normalizes and returns a lstripped-/-separated path
273 root = root.replace(os.path.sep, '/')
274 path = path.replace(os.path.sep, '/')
275 assert path.startswith(root)
276 return path[len(root):].lstrip('/')
278 destinations = {}
279 for base, suffix, dest in rules:
280 prefix = os.path.join(resources_root, base)
281 for abs_base in iglob(prefix):
282 abs_glob = os.path.join(abs_base, suffix)
283 for abs_path in iglob(abs_glob):
284 resource_file = get_rel_path(resources_root, abs_path)
285 if dest is None: # remove the entry if it was here
286 destinations.pop(resource_file, None)
287 else:
288 rel_path = get_rel_path(abs_base, abs_path)
289 rel_dest = dest.replace(os.path.sep, '/').rstrip('/')
290 destinations[resource_file] = rel_dest + '/' + rel_path
291 return destinations
294def in_venv():
295 if hasattr(sys, 'real_prefix'):
296 # virtualenv venvs
297 result = True
298 else:
299 # PEP 405 venvs
300 result = sys.prefix != getattr(sys, 'base_prefix', sys.prefix)
301 return result
304def get_executable():
305 # The __PYVENV_LAUNCHER__ dance is apparently no longer needed, as
306 # changes to the stub launcher mean that sys.executable always points
307 # to the stub on OS X
308 # if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__'
309 # in os.environ):
310 # result = os.environ['__PYVENV_LAUNCHER__']
311 # else:
312 # result = sys.executable
313 # return result
314 # Avoid normcasing: see issue #143
315 # result = os.path.normcase(sys.executable)
316 result = sys.executable
317 if not isinstance(result, text_type):
318 result = fsdecode(result)
319 return result
322def proceed(prompt, allowed_chars, error_prompt=None, default=None):
323 p = prompt
324 while True:
325 s = raw_input(p)
326 p = prompt
327 if not s and default:
328 s = default
329 if s:
330 c = s[0].lower()
331 if c in allowed_chars:
332 break
333 if error_prompt:
334 p = '%c: %s\n%s' % (c, error_prompt, prompt)
335 return c
338def extract_by_key(d, keys):
339 if isinstance(keys, string_types):
340 keys = keys.split()
341 result = {}
342 for key in keys:
343 if key in d:
344 result[key] = d[key]
345 return result
348def read_exports(stream):
349 if sys.version_info[0] >= 3:
350 # needs to be a text stream
351 stream = codecs.getreader('utf-8')(stream)
352 # Try to load as JSON, falling back on legacy format
353 data = stream.read()
354 stream = StringIO(data)
355 try:
356 jdata = json.load(stream)
357 result = jdata['extensions']['python.exports']['exports']
358 for group, entries in result.items():
359 for k, v in entries.items():
360 s = '%s = %s' % (k, v)
361 entry = get_export_entry(s)
362 assert entry is not None
363 entries[k] = entry
364 return result
365 except Exception:
366 stream.seek(0, 0)
368 def read_stream(cp, stream):
369 if hasattr(cp, 'read_file'):
370 cp.read_file(stream)
371 else:
372 cp.readfp(stream)
374 cp = configparser.ConfigParser()
375 try:
376 read_stream(cp, stream)
377 except configparser.MissingSectionHeaderError:
378 stream.close()
379 data = textwrap.dedent(data)
380 stream = StringIO(data)
381 read_stream(cp, stream)
383 result = {}
384 for key in cp.sections():
385 result[key] = entries = {}
386 for name, value in cp.items(key):
387 s = '%s = %s' % (name, value)
388 entry = get_export_entry(s)
389 assert entry is not None
390 # entry.dist = self
391 entries[name] = entry
392 return result
395def write_exports(exports, stream):
396 if sys.version_info[0] >= 3:
397 # needs to be a text stream
398 stream = codecs.getwriter('utf-8')(stream)
399 cp = configparser.ConfigParser()
400 for k, v in exports.items():
401 # TODO check k, v for valid values
402 cp.add_section(k)
403 for entry in v.values():
404 if entry.suffix is None:
405 s = entry.prefix
406 else:
407 s = '%s:%s' % (entry.prefix, entry.suffix)
408 if entry.flags:
409 s = '%s [%s]' % (s, ', '.join(entry.flags))
410 cp.set(k, entry.name, s)
411 cp.write(stream)
414@contextlib.contextmanager
415def tempdir():
416 td = tempfile.mkdtemp()
417 try:
418 yield td
419 finally:
420 shutil.rmtree(td)
423@contextlib.contextmanager
424def chdir(d):
425 cwd = os.getcwd()
426 try:
427 os.chdir(d)
428 yield
429 finally:
430 os.chdir(cwd)
433@contextlib.contextmanager
434def socket_timeout(seconds=15):
435 cto = socket.getdefaulttimeout()
436 try:
437 socket.setdefaulttimeout(seconds)
438 yield
439 finally:
440 socket.setdefaulttimeout(cto)
443class cached_property(object):
445 def __init__(self, func):
446 self.func = func
447 # for attr in ('__name__', '__module__', '__doc__'):
448 # setattr(self, attr, getattr(func, attr, None))
450 def __get__(self, obj, cls=None):
451 if obj is None:
452 return self
453 value = self.func(obj)
454 object.__setattr__(obj, self.func.__name__, value)
455 # obj.__dict__[self.func.__name__] = value = self.func(obj)
456 return value
459def convert_path(pathname):
460 """Return 'pathname' as a name that will work on the native filesystem.
462 The path is split on '/' and put back together again using the current
463 directory separator. Needed because filenames in the setup script are
464 always supplied in Unix style, and have to be converted to the local
465 convention before we can actually use them in the filesystem. Raises
466 ValueError on non-Unix-ish systems if 'pathname' either starts or
467 ends with a slash.
468 """
469 if os.sep == '/':
470 return pathname
471 if not pathname:
472 return pathname
473 if pathname[0] == '/':
474 raise ValueError("path '%s' cannot be absolute" % pathname)
475 if pathname[-1] == '/':
476 raise ValueError("path '%s' cannot end with '/'" % pathname)
478 paths = pathname.split('/')
479 while os.curdir in paths:
480 paths.remove(os.curdir)
481 if not paths:
482 return os.curdir
483 return os.path.join(*paths)
486class FileOperator(object):
488 def __init__(self, dry_run=False):
489 self.dry_run = dry_run
490 self.ensured = set()
491 self._init_record()
493 def _init_record(self):
494 self.record = False
495 self.files_written = set()
496 self.dirs_created = set()
498 def record_as_written(self, path):
499 if self.record:
500 self.files_written.add(path)
502 def newer(self, source, target):
503 """Tell if the target is newer than the source.
505 Returns true if 'source' exists and is more recently modified than
506 'target', or if 'source' exists and 'target' doesn't.
508 Returns false if both exist and 'target' is the same age or younger
509 than 'source'. Raise PackagingFileError if 'source' does not exist.
511 Note that this test is not very accurate: files created in the same
512 second will have the same "age".
513 """
514 if not os.path.exists(source):
515 raise DistlibException("file '%r' does not exist" % os.path.abspath(source))
516 if not os.path.exists(target):
517 return True
519 return os.stat(source).st_mtime > os.stat(target).st_mtime
521 def copy_file(self, infile, outfile, check=True):
522 """Copy a file respecting dry-run and force flags.
523 """
524 self.ensure_dir(os.path.dirname(outfile))
525 logger.info('Copying %s to %s', infile, outfile)
526 if not self.dry_run:
527 msg = None
528 if check:
529 if os.path.islink(outfile):
530 msg = '%s is a symlink' % outfile
531 elif os.path.exists(outfile) and not os.path.isfile(outfile):
532 msg = '%s is a non-regular file' % outfile
533 if msg:
534 raise ValueError(msg + ' which would be overwritten')
535 shutil.copyfile(infile, outfile)
536 self.record_as_written(outfile)
538 def copy_stream(self, instream, outfile, encoding=None):
539 assert not os.path.isdir(outfile)
540 self.ensure_dir(os.path.dirname(outfile))
541 logger.info('Copying stream %s to %s', instream, outfile)
542 if not self.dry_run:
543 if encoding is None:
544 outstream = open(outfile, 'wb')
545 else:
546 outstream = text_open(outfile, 'w', encoding=encoding)
547 try:
548 shutil.copyfileobj(instream, outstream)
549 finally:
550 outstream.close()
551 self.record_as_written(outfile)
553 def write_binary_file(self, path, data):
554 self.ensure_dir(os.path.dirname(path))
555 if not self.dry_run:
556 if os.path.exists(path):
557 os.remove(path)
558 with open(path, 'wb') as f:
559 f.write(data)
560 self.record_as_written(path)
562 def write_text_file(self, path, data, encoding):
563 self.write_binary_file(path, data.encode(encoding))
565 def set_mode(self, bits, mask, files):
566 if os.name == 'posix' or (os.name == 'java' and os._name == 'posix'):
567 # Set the executable bits (owner, group, and world) on
568 # all the files specified.
569 for f in files:
570 if self.dry_run:
571 logger.info("changing mode of %s", f)
572 else:
573 mode = (os.stat(f).st_mode | bits) & mask
574 logger.info("changing mode of %s to %o", f, mode)
575 os.chmod(f, mode)
577 set_executable_mode = lambda s, f: s.set_mode(0o555, 0o7777, f)
579 def ensure_dir(self, path):
580 path = os.path.abspath(path)
581 if path not in self.ensured and not os.path.exists(path):
582 self.ensured.add(path)
583 d, f = os.path.split(path)
584 self.ensure_dir(d)
585 logger.info('Creating %s' % path)
586 if not self.dry_run:
587 os.mkdir(path)
588 if self.record:
589 self.dirs_created.add(path)
591 def byte_compile(self, path, optimize=False, force=False, prefix=None, hashed_invalidation=False):
592 if not optimize:
593 optimization = ''
594 else:
595 optimization = '1'
596 dpath = cache_from_source(path, optimization=optimization)
597 logger.info('Byte-compiling %s to %s', path, dpath)
598 if not self.dry_run:
599 if force or self.newer(path, dpath):
600 if not prefix:
601 diagpath = None
602 else:
603 assert path.startswith(prefix)
604 diagpath = path[len(prefix):]
605 compile_kwargs = {}
606 if sys.version_info[0] >= 3:
607 compile_kwargs['optimize'] = int(bool(optimize))
608 if hashed_invalidation and hasattr(py_compile, 'PycInvalidationMode'):
609 if not isinstance(hashed_invalidation, py_compile.PycInvalidationMode):
610 hashed_invalidation = py_compile.PycInvalidationMode.CHECKED_HASH
611 compile_kwargs['invalidation_mode'] = hashed_invalidation
612 py_compile.compile(path, dpath, diagpath, True, **compile_kwargs) # raise error
613 self.record_as_written(dpath)
614 return dpath
616 def ensure_removed(self, path):
617 if os.path.lexists(path):
618 if os.path.isdir(path) and not os.path.islink(path):
619 logger.debug('Removing directory tree at %s', path)
620 if not self.dry_run:
621 shutil.rmtree(path)
622 if self.record:
623 if path in self.dirs_created:
624 self.dirs_created.remove(path)
625 else:
626 if os.path.islink(path):
627 s = 'link'
628 else:
629 s = 'file'
630 logger.debug('Removing %s %s', s, path)
631 if not self.dry_run:
632 os.remove(path)
633 if self.record:
634 if path in self.files_written:
635 self.files_written.remove(path)
637 def is_writable(self, path):
638 result = False
639 while not result:
640 if os.path.exists(path):
641 result = os.access(path, os.W_OK)
642 break
643 parent = os.path.dirname(path)
644 if parent == path:
645 break
646 path = parent
647 return result
649 def commit(self):
650 """
651 Commit recorded changes, turn off recording, return
652 changes.
653 """
654 assert self.record
655 result = self.files_written, self.dirs_created
656 self._init_record()
657 return result
659 def rollback(self):
660 if not self.dry_run:
661 for f in list(self.files_written):
662 if os.path.exists(f):
663 os.remove(f)
664 # dirs should all be empty now, except perhaps for
665 # __pycache__ subdirs
666 # reverse so that subdirs appear before their parents
667 dirs = sorted(self.dirs_created, reverse=True)
668 for d in dirs:
669 flist = os.listdir(d)
670 if flist:
671 assert flist == ['__pycache__']
672 sd = os.path.join(d, flist[0])
673 os.rmdir(sd)
674 os.rmdir(d) # should fail if non-empty
675 self._init_record()
678def resolve(module_name, dotted_path):
679 if module_name in sys.modules:
680 mod = sys.modules[module_name]
681 else:
682 mod = __import__(module_name)
683 if dotted_path is None:
684 result = mod
685 else:
686 parts = dotted_path.split('.')
687 result = getattr(mod, parts.pop(0))
688 for p in parts:
689 result = getattr(result, p)
690 return result
693class ExportEntry(object):
695 def __init__(self, name, prefix, suffix, flags):
696 self.name = name
697 self.prefix = prefix
698 self.suffix = suffix
699 self.flags = flags
701 @cached_property
702 def value(self):
703 return resolve(self.prefix, self.suffix)
705 def __repr__(self): # pragma: no cover
706 return '<ExportEntry %s = %s:%s %s>' % (self.name, self.prefix, self.suffix, self.flags)
708 def __eq__(self, other):
709 if not isinstance(other, ExportEntry):
710 result = False
711 else:
712 result = (self.name == other.name and self.prefix == other.prefix and self.suffix == other.suffix
713 and self.flags == other.flags)
714 return result
716 __hash__ = object.__hash__
719ENTRY_RE = re.compile(
720 r'''(?P<name>([^\[]\S*))
721 \s*=\s*(?P<callable>(\w+)([:\.]\w+)*)
722 \s*(\[\s*(?P<flags>[\w-]+(=\w+)?(,\s*\w+(=\w+)?)*)\s*\])?
723 ''', re.VERBOSE)
725# ENTRY_RE, but only starting at the first or last character of a whitespace
726# run. See _search_export_entry.
727_ENTRY_RE_AT_SPACE = re.compile(r'(?:(?<!\s)(?=\s)|(?=\s\S))' +
728 ENTRY_RE.pattern, re.VERBOSE)
731def _search_export_entry(specification):
732 """
733 Return ENTRY_RE.search(specification), in linear rather than quadratic
734 time.
736 search() tries every start position, and each attempt can rescan the rest
737 of a long name. Most positions cannot hold the leftmost match: a match that
738 starts inside a run of non-space characters implies one starting earlier in
739 the run (or at the whitespace just before it, since a name may start with
740 whitespace), and a match starting inside a whitespace run, other than at
741 its last character, implies one at its first. So the leftmost match starts
742 at the first character that is not '[' (a name cannot start with '['), or
743 at the first or last character of a whitespace run.
744 """
745 start = len(specification) - len(specification.lstrip('['))
746 m = ENTRY_RE.match(specification, start)
747 if not m:
748 m = _ENTRY_RE_AT_SPACE.search(specification, start)
749 return m
752def get_export_entry(specification):
753 m = _search_export_entry(specification)
754 if not m:
755 result = None
756 if '[' in specification or ']' in specification:
757 raise DistlibException("Invalid specification "
758 "'%s'" % specification)
759 else:
760 d = m.groupdict()
761 name = d['name']
762 path = d['callable']
763 colons = path.count(':')
764 if colons == 0:
765 prefix, suffix = path, None
766 else:
767 if colons != 1:
768 raise DistlibException("Invalid specification "
769 "'%s'" % specification)
770 prefix, suffix = path.split(':')
771 flags = d['flags']
772 if flags is None:
773 if '[' in specification or ']' in specification:
774 raise DistlibException("Invalid specification "
775 "'%s'" % specification)
776 flags = []
777 else:
778 flags = [f.strip() for f in flags.split(',')]
779 result = ExportEntry(name, prefix, suffix, flags)
780 return result
783def get_cache_base(suffix=None):
784 """
785 Return the default base location for distlib caches. If the directory does
786 not exist, it is created. Use the suffix provided for the base directory,
787 and default to '.distlib' if it isn't provided.
789 On Windows, if LOCALAPPDATA is defined in the environment, then it is
790 assumed to be a directory, and will be the parent directory of the result.
791 On POSIX, and on Windows if LOCALAPPDATA is not defined, the user's home
792 directory - using os.expanduser('~') - will be the parent directory of
793 the result.
795 The result is just the directory '.distlib' in the parent directory as
796 determined above, or with the name specified with ``suffix``.
797 """
798 if suffix is None:
799 suffix = '.distlib'
800 if os.name == 'nt' and 'LOCALAPPDATA' in os.environ:
801 result = os.path.expandvars('$localappdata')
802 else:
803 # Assume posix, or old Windows
804 result = os.path.expanduser('~')
805 # we use 'isdir' instead of 'exists', because we want to
806 # fail if there's a file with that name
807 if os.path.isdir(result):
808 usable = os.access(result, os.W_OK)
809 if not usable:
810 logger.warning('Directory exists but is not writable: %s', result)
811 else:
812 try:
813 os.makedirs(result)
814 usable = True
815 except OSError:
816 logger.warning('Unable to create %s', result, exc_info=True)
817 usable = False
818 if not usable:
819 result = tempfile.mkdtemp()
820 logger.warning('Default location unusable, using %s', result)
821 return os.path.join(result, suffix)
824def path_to_cache_dir(path, use_abspath=True):
825 """
826 Convert an absolute path to a directory name for use in a cache.
828 The algorithm used is:
830 #. On Windows, any ``':'`` in the drive is replaced with ``'---'``.
831 #. Any occurrence of ``os.sep`` is replaced with ``'--'``.
832 #. ``'.cache'`` is appended.
833 """
834 d, p = os.path.splitdrive(os.path.abspath(path) if use_abspath else path)
835 if d:
836 d = d.replace(':', '---')
837 p = p.replace(os.sep, '--')
838 return d + p + '.cache'
841def ensure_slash(s):
842 if not s.endswith('/'):
843 return s + '/'
844 return s
847def parse_credentials(netloc):
848 username = password = None
849 if '@' in netloc:
850 prefix, netloc = netloc.rsplit('@', 1)
851 if ':' not in prefix:
852 username = prefix
853 else:
854 username, password = prefix.split(':', 1)
855 if username:
856 username = unquote(username)
857 if password:
858 password = unquote(password)
859 return username, password, netloc
862def get_process_umask():
863 result = os.umask(0o22)
864 os.umask(result)
865 return result
868def is_string_sequence(seq):
869 result = True
870 i = None
871 for i, s in enumerate(seq):
872 if not isinstance(s, string_types):
873 result = False
874 break
875 assert i is not None
876 return result
879PROJECT_NAME_AND_VERSION = re.compile('([a-z0-9_]+([.-][a-z_][a-z0-9_]*)*)-'
880 '([a-z0-9_.+-]+)', re.I)
881PYTHON_VERSION = re.compile(r'-py(\d\.?\d?)')
884def split_filename(filename, project_name=None):
885 """
886 Extract name, version, python version from a filename (no extension)
888 Return name, version, pyver or None
889 """
890 result = None
891 pyver = None
892 filename = unquote(filename).replace(' ', '-')
893 m = PYTHON_VERSION.search(filename)
894 if m:
895 pyver = m.group(1)
896 filename = filename[:m.start()]
897 if project_name and len(filename) > len(project_name) + 1:
898 m = re.match(re.escape(project_name) + r'\b', filename)
899 if m:
900 n = m.end()
901 result = filename[:n], filename[n + 1:], pyver
902 if result is None:
903 m = PROJECT_NAME_AND_VERSION.match(filename)
904 if m:
905 result = m.group(1), m.group(3), pyver
906 return result
909# Allow spaces in name because of legacy dists like "Twisted Core"
910NAME_VERSION_RE = re.compile(r'(?P<name>[\w .-]+)\s*'
911 r'\(\s*(?P<ver>[^\s)]+)\)$')
914def parse_name_and_version(p):
915 """
916 A utility method used to get name and version from a string.
918 From e.g. a Provides-Dist value.
920 :param p: A value in a form 'foo (1.0)'
921 :return: The name and version as a tuple.
922 """
923 m = NAME_VERSION_RE.match(p)
924 if not m:
925 raise DistlibException('Ill-formed name/version string: \'%s\'' % p)
926 d = m.groupdict()
927 return d['name'].strip().lower(), d['ver']
930def get_extras(requested, available):
931 result = set()
932 requested = set(requested or [])
933 available = set(available or [])
934 if '*' in requested:
935 requested.remove('*')
936 result |= available
937 for r in requested:
938 if r == '-':
939 result.add(r)
940 elif r.startswith('-'):
941 unwanted = r[1:]
942 if unwanted not in available:
943 logger.warning('undeclared extra: %s' % unwanted)
944 if unwanted in result:
945 result.remove(unwanted)
946 else:
947 if r not in available:
948 logger.warning('undeclared extra: %s' % r)
949 result.add(r)
950 return result
953#
954# Extended metadata functionality
955#
958def _get_external_data(url):
959 result = {}
960 try:
961 # urlopen might fail if it runs into redirections,
962 # because of Python issue #13696. Fixed in locators
963 # using a custom redirect handler.
964 resp = urlopen(url)
965 headers = resp.info()
966 ct = headers.get('Content-Type')
967 if not ct.startswith('application/json'):
968 logger.debug('Unexpected response for JSON request: %s', ct)
969 else:
970 reader = codecs.getreader('utf-8')(resp)
971 # data = reader.read().decode('utf-8')
972 # result = json.loads(data)
973 result = json.load(reader)
974 except Exception as e:
975 logger.exception('Failed to get external data for %s: %s', url, e)
976 return result
979_external_data_base_url = 'https://www.red-dove.com/pypi/projects/'
982def get_project_data(name):
983 url = '%s/%s/project.json' % (name[0].upper(), name)
984 url = urljoin(_external_data_base_url, url)
985 result = _get_external_data(url)
986 return result
989def get_package_data(name, version):
990 url = '%s/%s/package-%s.json' % (name[0].upper(), name, version)
991 url = urljoin(_external_data_base_url, url)
992 return _get_external_data(url)
995class Cache(object):
996 """
997 A class implementing a cache for resources that need to live in the file system
998 e.g. shared libraries. This class was moved from resources to here because it
999 could be used by other modules, e.g. the wheel module.
1000 """
1002 def __init__(self, base):
1003 """
1004 Initialise an instance.
1006 :param base: The base directory where the cache should be located.
1007 """
1008 # we use 'isdir' instead of 'exists', because we want to
1009 # fail if there's a file with that name
1010 if not os.path.isdir(base): # pragma: no cover
1011 os.makedirs(base)
1012 if (os.stat(base).st_mode & 0o77) != 0:
1013 logger.warning('Directory \'%s\' is not private', base)
1014 self.base = os.path.abspath(os.path.normpath(base))
1016 def prefix_to_dir(self, prefix, use_abspath=True):
1017 """
1018 Converts a resource prefix to a directory name in the cache.
1019 """
1020 return path_to_cache_dir(prefix, use_abspath=use_abspath)
1022 def clear(self):
1023 """
1024 Clear the cache.
1025 """
1026 not_removed = []
1027 for fn in os.listdir(self.base):
1028 fn = os.path.join(self.base, fn)
1029 try:
1030 if os.path.islink(fn) or os.path.isfile(fn):
1031 os.remove(fn)
1032 elif os.path.isdir(fn):
1033 shutil.rmtree(fn)
1034 except Exception:
1035 not_removed.append(fn)
1036 return not_removed
1039class EventMixin(object):
1040 """
1041 A very simple publish/subscribe system.
1042 """
1044 def __init__(self):
1045 self._subscribers = {}
1047 def add(self, event, subscriber, append=True):
1048 """
1049 Add a subscriber for an event.
1051 :param event: The name of an event.
1052 :param subscriber: The subscriber to be added (and called when the
1053 event is published).
1054 :param append: Whether to append or prepend the subscriber to an
1055 existing subscriber list for the event.
1056 """
1057 subs = self._subscribers
1058 if event not in subs:
1059 subs[event] = deque([subscriber])
1060 else:
1061 sq = subs[event]
1062 if append:
1063 sq.append(subscriber)
1064 else:
1065 sq.appendleft(subscriber)
1067 def remove(self, event, subscriber):
1068 """
1069 Remove a subscriber for an event.
1071 :param event: The name of an event.
1072 :param subscriber: The subscriber to be removed.
1073 """
1074 subs = self._subscribers
1075 if event not in subs:
1076 raise ValueError('No subscribers: %r' % event)
1077 subs[event].remove(subscriber)
1079 def get_subscribers(self, event):
1080 """
1081 Return an iterator for the subscribers for an event.
1082 :param event: The event to return subscribers for.
1083 """
1084 return iter(self._subscribers.get(event, ()))
1086 def publish(self, event, *args, **kwargs):
1087 """
1088 Publish a event and return a list of values returned by its
1089 subscribers.
1091 :param event: The event to publish.
1092 :param args: The positional arguments to pass to the event's
1093 subscribers.
1094 :param kwargs: The keyword arguments to pass to the event's
1095 subscribers.
1096 """
1097 result = []
1098 for subscriber in self.get_subscribers(event):
1099 try:
1100 value = subscriber(event, *args, **kwargs)
1101 except Exception:
1102 logger.exception('Exception during event publication')
1103 value = None
1104 result.append(value)
1105 logger.debug('publish %s: args = %s, kwargs = %s, result = %s', event, args, kwargs, result)
1106 return result
1109#
1110# Simple sequencing
1111#
1112class Sequencer(object):
1114 def __init__(self):
1115 self._preds = {}
1116 self._succs = {}
1117 self._nodes = set() # nodes with no preds/succs
1119 def add_node(self, node):
1120 self._nodes.add(node)
1122 def remove_node(self, node, edges=False):
1123 if node in self._nodes:
1124 self._nodes.remove(node)
1125 if edges:
1126 for p in set(self._preds.get(node, ())):
1127 self.remove(p, node)
1128 for s in set(self._succs.get(node, ())):
1129 self.remove(node, s)
1130 # Remove empties
1131 for k, v in list(self._preds.items()):
1132 if not v:
1133 del self._preds[k]
1134 for k, v in list(self._succs.items()):
1135 if not v:
1136 del self._succs[k]
1138 def add(self, pred, succ):
1139 assert pred != succ
1140 self._preds.setdefault(succ, set()).add(pred)
1141 self._succs.setdefault(pred, set()).add(succ)
1143 def remove(self, pred, succ):
1144 assert pred != succ
1145 try:
1146 preds = self._preds[succ]
1147 succs = self._succs[pred]
1148 except KeyError: # pragma: no cover
1149 raise ValueError('%r not a successor of anything' % succ)
1150 try:
1151 preds.remove(pred)
1152 succs.remove(succ)
1153 except KeyError: # pragma: no cover
1154 raise ValueError('%r not a successor of %r' % (succ, pred))
1156 def is_step(self, step):
1157 return (step in self._preds or step in self._succs or step in self._nodes)
1159 def get_steps(self, final):
1160 if not self.is_step(final):
1161 raise ValueError('Unknown: %r' % final)
1162 result = []
1163 todo = []
1164 seen = set()
1165 todo.append(final)
1166 while todo:
1167 step = todo.pop(0)
1168 if step in seen:
1169 # if a step was already seen,
1170 # move it to the end (so it will appear earlier
1171 # when reversed on return) ... but not for the
1172 # final step, as that would be confusing for
1173 # users
1174 if step != final:
1175 result.remove(step)
1176 result.append(step)
1177 else:
1178 seen.add(step)
1179 result.append(step)
1180 preds = self._preds.get(step, ())
1181 todo.extend(preds)
1182 return reversed(result)
1184 @property
1185 def strong_connections(self):
1186 # http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
1187 index_counter = [0]
1188 stack = []
1189 lowlinks = {}
1190 index = {}
1191 result = []
1193 graph = self._succs
1195 def strongconnect(node):
1196 # set the depth index for this node to the smallest unused index
1197 index[node] = index_counter[0]
1198 lowlinks[node] = index_counter[0]
1199 index_counter[0] += 1
1200 stack.append(node)
1202 # Consider successors
1203 try:
1204 successors = graph[node]
1205 except Exception:
1206 successors = []
1207 for successor in successors:
1208 if successor not in lowlinks:
1209 # Successor has not yet been visited
1210 strongconnect(successor)
1211 lowlinks[node] = min(lowlinks[node], lowlinks[successor])
1212 elif successor in stack:
1213 # the successor is in the stack and hence in the current
1214 # strongly connected component (SCC)
1215 lowlinks[node] = min(lowlinks[node], index[successor])
1217 # If `node` is a root node, pop the stack and generate an SCC
1218 if lowlinks[node] == index[node]:
1219 connected_component = []
1221 while True:
1222 successor = stack.pop()
1223 connected_component.append(successor)
1224 if successor == node:
1225 break
1226 component = tuple(connected_component)
1227 # storing the result
1228 result.append(component)
1230 for node in graph:
1231 if node not in lowlinks:
1232 strongconnect(node)
1234 return result
1236 @property
1237 def dot(self):
1238 result = ['digraph G {']
1239 for succ in self._preds:
1240 preds = self._preds[succ]
1241 for pred in preds:
1242 result.append(' %s -> %s;' % (pred, succ))
1243 for node in self._nodes:
1244 result.append(' %s;' % node)
1245 result.append('}')
1246 return '\n'.join(result)
1249#
1250# Unarchiving functionality for zip, tar, tgz, tbz, whl
1251#
1253ARCHIVE_EXTENSIONS = ('.tar.gz', '.tar.bz2', '.tar', '.zip', '.tgz', '.tbz', '.whl')
1256def unarchive(archive_filename, dest_dir, format=None, check=True):
1258 # Symlinks seen so far in the archive, mapping the link's resolved path
1259 # to its resolved target. Earlier members don't exist on disk while we
1260 # check, so later member paths are resolved through this map lexically.
1261 links = {}
1263 def resolve(path):
1264 cur = dest_dir
1265 if not (path == dest_dir or path.startswith(dest_dir + os.sep)):
1266 return os.path.realpath(path)
1267 for part in path[len(dest_dir):].split(os.sep):
1268 if part in ('', '.'):
1269 continue
1270 if part == '..':
1271 cur = os.path.dirname(cur)
1272 else:
1273 cur = links.get(os.path.join(cur, part), os.path.join(cur, part))
1274 return os.path.realpath(cur)
1276 def check_path(path, base=None):
1277 if not isinstance(path, text_type):
1278 path = path.decode('utf-8')
1279 if base is None:
1280 base = dest_dir
1281 p = resolve(os.path.join(base, path))
1282 if not (p == dest_dir or p.startswith(dest_dir + os.sep)):
1283 raise ValueError('path outside destination: %r' % p)
1284 return p
1286 def check_link(member):
1287 # A symlink/hardlink member's name is validated like any other
1288 # member, but its target (linkname) is not covered by extractall's
1289 # name-based handling. An unchecked target lets a later member be
1290 # written through the link to a location outside dest_dir. Validate
1291 # the resolved target stays within dest_dir. Symlink targets are
1292 # relative to the member's own directory; hardlink targets are
1293 # relative to the archive root (i.e. dest_dir).
1294 if not (member.issym() or member.islnk()):
1295 return
1296 if member.islnk():
1297 check_path(member.linkname)
1298 return
1299 name = member.name.rstrip('/')
1300 parent = check_path(os.path.dirname(name) or '.')
1301 links[os.path.join(parent, os.path.basename(name))] = check_path(member.linkname, base=parent)
1303 dest_dir = os.path.realpath(dest_dir)
1304 archive = None
1305 if format is None:
1306 if archive_filename.endswith(('.zip', '.whl')):
1307 format = 'zip'
1308 elif archive_filename.endswith(('.tar.gz', '.tgz')):
1309 format = 'tgz'
1310 mode = 'r:gz'
1311 elif archive_filename.endswith(('.tar.bz2', '.tbz')):
1312 format = 'tbz'
1313 mode = 'r:bz2'
1314 elif archive_filename.endswith('.tar'):
1315 format = 'tar'
1316 mode = 'r'
1317 else: # pragma: no cover
1318 raise ValueError('Unknown format for %r' % archive_filename)
1319 try:
1320 if format == 'zip':
1321 archive = ZipFile(archive_filename, 'r')
1322 if check:
1323 names = archive.namelist()
1324 for name in names:
1325 check_path(name)
1326 else:
1327 archive = tarfile.open(archive_filename, mode)
1328 if check:
1329 for member in archive.getmembers():
1330 check_path(member.name)
1331 check_link(member)
1332 if format != 'zip' and sys.version_info[0] < 3:
1333 # See Python issue 17153. If the dest path contains Unicode,
1334 # tarfile extraction fails on Python 2.x if a member path name
1335 # contains non-ASCII characters - it leads to an implicit
1336 # bytes -> unicode conversion using ASCII to decode.
1337 for tarinfo in archive.getmembers():
1338 if not isinstance(tarinfo.name, text_type):
1339 tarinfo.name = tarinfo.name.decode('utf-8')
1341 # Limit extraction of dangerous items, if this Python
1342 # allows it easily. If not, just trust the input.
1343 # See: https://docs.python.org/3/library/tarfile.html#extraction-filters
1344 # data_filter is the stdlib default from Python 3.14 onwards; using the
1345 # laxer tar_filter here would downgrade that secure default.
1346 def extraction_filter(member, path):
1347 """Run tarfile.data_filter, but raise the expected ValueError"""
1348 # This is only called if the current Python has tarfile filters
1349 try:
1350 return tarfile.data_filter(member, path)
1351 except tarfile.FilterError as exc:
1352 raise ValueError(str(exc))
1354 archive.extraction_filter = extraction_filter
1356 archive.extractall(dest_dir)
1358 finally:
1359 if archive:
1360 archive.close()
1363def zip_dir(directory):
1364 """zip a directory tree into a BytesIO object"""
1365 result = io.BytesIO()
1366 dlen = len(directory)
1367 with ZipFile(result, "w") as zf:
1368 for root, dirs, files in os.walk(directory):
1369 for name in files:
1370 full = os.path.join(root, name)
1371 rel = root[dlen:]
1372 dest = os.path.join(rel, name)
1373 zf.write(full, dest)
1374 return result
1377#
1378# Simple progress bar
1379#
1381UNITS = ('', 'K', 'M', 'G', 'T', 'P')
1384class Progress(object):
1385 unknown = 'UNKNOWN'
1387 def __init__(self, minval=0, maxval=100):
1388 assert maxval is None or maxval >= minval
1389 self.min = self.cur = minval
1390 self.max = maxval
1391 self.started = None
1392 self.elapsed = 0
1393 self.done = False
1395 def update(self, curval):
1396 assert self.min <= curval
1397 assert self.max is None or curval <= self.max
1398 self.cur = curval
1399 now = time.time()
1400 if self.started is None:
1401 self.started = now
1402 else:
1403 self.elapsed = now - self.started
1405 def increment(self, incr):
1406 assert incr >= 0
1407 self.update(self.cur + incr)
1409 def start(self):
1410 self.update(self.min)
1411 return self
1413 def stop(self):
1414 if self.max is not None:
1415 self.update(self.max)
1416 self.done = True
1418 @property
1419 def maximum(self):
1420 return self.unknown if self.max is None else self.max
1422 @property
1423 def percentage(self):
1424 if self.done:
1425 result = '100 %'
1426 elif self.max is None:
1427 result = ' ?? %'
1428 else:
1429 v = 100.0 * (self.cur - self.min) / (self.max - self.min)
1430 result = '%3d %%' % v
1431 return result
1433 def format_duration(self, duration):
1434 if (duration <= 0) and self.max is None or self.cur == self.min:
1435 result = '??:??:??'
1436 # elif duration < 1:
1437 # result = '--:--:--'
1438 else:
1439 result = time.strftime('%H:%M:%S', time.gmtime(duration))
1440 return result
1442 @property
1443 def ETA(self):
1444 if self.done:
1445 prefix = 'Done'
1446 t = self.elapsed
1447 # import pdb; pdb.set_trace()
1448 else:
1449 prefix = 'ETA '
1450 if self.max is None:
1451 t = -1
1452 elif self.elapsed == 0 or (self.cur == self.min):
1453 t = 0
1454 else:
1455 # import pdb; pdb.set_trace()
1456 t = float(self.max - self.min)
1457 t /= self.cur - self.min
1458 t = (t - 1) * self.elapsed
1459 return '%s: %s' % (prefix, self.format_duration(t))
1461 @property
1462 def speed(self):
1463 if self.elapsed == 0:
1464 result = 0.0
1465 else:
1466 result = (self.cur - self.min) / self.elapsed
1467 for unit in UNITS:
1468 if result < 1000:
1469 break
1470 result /= 1000.0
1471 return '%d %sB/s' % (result, unit)
1474#
1475# Glob functionality
1476#
1478RICH_GLOB = re.compile(r'\{([^}]*)\}')
1479_CHECK_RECURSIVE_GLOB = re.compile(r'[^/\\,{]\*\*|\*\*[^/\\,}]')
1480_CHECK_MISMATCH_SET = re.compile(r'^[^{]*\}|\{[^}]*$')
1483def iglob(path_glob):
1484 """Extended globbing function that supports ** and {opt1,opt2,opt3}."""
1485 if _CHECK_RECURSIVE_GLOB.search(path_glob):
1486 msg = """invalid glob %r: recursive glob "**" must be used alone"""
1487 raise ValueError(msg % path_glob)
1488 if _CHECK_MISMATCH_SET.search(path_glob):
1489 msg = """invalid glob %r: mismatching set marker '{' or '}'"""
1490 raise ValueError(msg % path_glob)
1491 return _iglob(path_glob)
1494def _iglob(path_glob):
1495 rich_path_glob = RICH_GLOB.split(path_glob, 1)
1496 if len(rich_path_glob) > 1:
1497 assert len(rich_path_glob) == 3, rich_path_glob
1498 prefix, set, suffix = rich_path_glob
1499 for item in set.split(','):
1500 for path in _iglob(''.join((prefix, item, suffix))):
1501 yield path
1502 else:
1503 if '**' not in path_glob:
1504 for item in std_iglob(path_glob):
1505 yield item
1506 else:
1507 prefix, radical = path_glob.split('**', 1)
1508 if prefix == '':
1509 prefix = '.'
1510 if radical == '':
1511 radical = '*'
1512 else:
1513 # we support both
1514 radical = radical.lstrip('/')
1515 radical = radical.lstrip('\\')
1516 for path, dir, files in os.walk(prefix):
1517 path = os.path.normpath(path)
1518 for fn in _iglob(os.path.join(path, radical)):
1519 yield fn
1522if ssl:
1523 from .compat import (HTTPSHandler as BaseHTTPSHandler, match_hostname, CertificateError)
1525 #
1526 # HTTPSConnection which verifies certificates/matches domains
1527 #
1530 class HTTPSConnection(httplib.HTTPSConnection):
1531 ca_certs = None # set this to the path to the certs file (.pem)
1532 check_domain = True # only used if ca_certs is not None
1534 # noinspection PyPropertyAccess
1535 def connect(self):
1536 sock = socket.create_connection((self.host, self.port), self.timeout)
1537 if getattr(self, '_tunnel_host', False):
1538 self.sock = sock
1539 self._tunnel()
1541 context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
1542 if hasattr(ssl, 'OP_NO_SSLv2'):
1543 context.options |= ssl.OP_NO_SSLv2
1544 if getattr(self, 'cert_file', None):
1545 context.load_cert_chain(self.cert_file, self.key_file)
1546 kwargs = {}
1547 if self.ca_certs:
1548 context.verify_mode = ssl.CERT_REQUIRED
1549 context.load_verify_locations(cafile=self.ca_certs)
1550 if getattr(ssl, 'HAS_SNI', False):
1551 kwargs['server_hostname'] = self.host
1553 self.sock = context.wrap_socket(sock, **kwargs)
1554 if self.ca_certs and self.check_domain:
1555 try:
1556 match_hostname(self.sock.getpeercert(), self.host)
1557 logger.debug('Host verified: %s', self.host)
1558 except CertificateError: # pragma: no cover
1559 self.sock.shutdown(socket.SHUT_RDWR)
1560 self.sock.close()
1561 raise
1563 class HTTPSHandler(BaseHTTPSHandler):
1565 def __init__(self, ca_certs, check_domain=True):
1566 BaseHTTPSHandler.__init__(self)
1567 self.ca_certs = ca_certs
1568 self.check_domain = check_domain
1570 def _conn_maker(self, *args, **kwargs):
1571 """
1572 This is called to create a connection instance. Normally you'd
1573 pass a connection class to do_open, but it doesn't actually check for
1574 a class, and just expects a callable. As long as we behave just as a
1575 constructor would have, we should be OK. If it ever changes so that
1576 we *must* pass a class, we'll create an UnsafeHTTPSConnection class
1577 which just sets check_domain to False in the class definition, and
1578 choose which one to pass to do_open.
1579 """
1580 result = HTTPSConnection(*args, **kwargs)
1581 if self.ca_certs:
1582 result.ca_certs = self.ca_certs
1583 result.check_domain = self.check_domain
1584 return result
1586 def https_open(self, req):
1587 try:
1588 return self.do_open(self._conn_maker, req)
1589 except URLError as e:
1590 if 'certificate verify failed' in str(e.reason):
1591 raise CertificateError('Unable to verify server certificate '
1592 'for %s' % req.host)
1593 else:
1594 raise
1596 #
1597 # To prevent against mixing HTTP traffic with HTTPS (examples: A Man-In-The-
1598 # Middle proxy using HTTP listens on port 443, or an index mistakenly serves
1599 # HTML containing a http://xyz link when it should be https://xyz),
1600 # you can use the following handler class, which does not allow HTTP traffic.
1601 #
1602 # It works by inheriting from HTTPHandler - so build_opener won't add a
1603 # handler for HTTP itself.
1604 #
1605 class HTTPSOnlyHandler(HTTPSHandler, HTTPHandler):
1607 def http_open(self, req):
1608 raise URLError('Unexpected HTTP request on what should be a secure '
1609 'connection: %s' % req)
1612#
1613# XML-RPC with timeouts
1614#
1615class Transport(xmlrpclib.Transport):
1617 def __init__(self, timeout, use_datetime=0):
1618 self.timeout = timeout
1619 xmlrpclib.Transport.__init__(self, use_datetime)
1621 def make_connection(self, host):
1622 h, eh, x509 = self.get_host_info(host)
1623 if not self._connection or host != self._connection[0]:
1624 self._extra_headers = eh
1625 self._connection = host, httplib.HTTPConnection(h)
1626 return self._connection[1]
1629if ssl:
1631 class SafeTransport(xmlrpclib.SafeTransport):
1633 def __init__(self, timeout, use_datetime=0):
1634 self.timeout = timeout
1635 xmlrpclib.SafeTransport.__init__(self, use_datetime)
1637 def make_connection(self, host):
1638 h, eh, kwargs = self.get_host_info(host)
1639 if not kwargs:
1640 kwargs = {}
1641 kwargs['timeout'] = self.timeout
1642 if not self._connection or host != self._connection[0]:
1643 self._extra_headers = eh
1644 self._connection = host, httplib.HTTPSConnection(h, None, **kwargs)
1645 return self._connection[1]
1648class ServerProxy(xmlrpclib.ServerProxy):
1650 def __init__(self, uri, **kwargs):
1651 self.timeout = timeout = kwargs.pop('timeout', None)
1652 # The above classes only come into play if a timeout
1653 # is specified
1654 if timeout is not None:
1655 # scheme = splittype(uri) # deprecated as of Python 3.8
1656 scheme = urlparse(uri)[0]
1657 use_datetime = kwargs.get('use_datetime', 0)
1658 if scheme == 'https':
1659 tcls = SafeTransport
1660 else:
1661 tcls = Transport
1662 kwargs['transport'] = t = tcls(timeout, use_datetime=use_datetime)
1663 self.transport = t
1664 xmlrpclib.ServerProxy.__init__(self, uri, **kwargs)
1667#
1668# CSV functionality. This is provided because on 2.x, the csv module can't
1669# handle Unicode. However, we need to deal with Unicode in e.g. RECORD files.
1670#
1673def _csv_open(fn, mode, **kwargs):
1674 if sys.version_info[0] < 3:
1675 mode += 'b'
1676 else:
1677 kwargs['newline'] = ''
1678 # Python 3 determines encoding from locale. Force 'utf-8'
1679 # file encoding to match other forced utf-8 encoding
1680 kwargs['encoding'] = 'utf-8'
1681 return open(fn, mode, **kwargs)
1684class CSVBase(object):
1685 defaults = {
1686 'delimiter': str(','), # The strs are used because we need native
1687 'quotechar': str('"'), # str in the csv API (2.x won't take
1688 'lineterminator': str('\n') # Unicode)
1689 }
1691 def __enter__(self):
1692 return self
1694 def __exit__(self, *exc_info):
1695 self.stream.close()
1698class CSVReader(CSVBase):
1700 def __init__(self, **kwargs):
1701 if 'stream' in kwargs:
1702 stream = kwargs['stream']
1703 if sys.version_info[0] >= 3:
1704 # needs to be a text stream
1705 stream = codecs.getreader('utf-8')(stream)
1706 self.stream = stream
1707 else:
1708 self.stream = _csv_open(kwargs['path'], 'r')
1709 self.reader = csv.reader(self.stream, **self.defaults)
1711 def __iter__(self):
1712 return self
1714 def next(self):
1715 result = next(self.reader)
1716 if sys.version_info[0] < 3:
1717 for i, item in enumerate(result):
1718 if not isinstance(item, text_type):
1719 result[i] = item.decode('utf-8')
1720 return result
1722 __next__ = next
1725class CSVWriter(CSVBase):
1727 def __init__(self, fn, **kwargs):
1728 self.stream = _csv_open(fn, 'w')
1729 self.writer = csv.writer(self.stream, **self.defaults)
1731 def writerow(self, row):
1732 if sys.version_info[0] < 3:
1733 r = []
1734 for item in row:
1735 if isinstance(item, text_type):
1736 item = item.encode('utf-8')
1737 r.append(item)
1738 row = r
1739 self.writer.writerow(row)
1742#
1743# Configurator functionality
1744#
1747class Configurator(BaseConfigurator):
1749 value_converters = dict(BaseConfigurator.value_converters)
1750 value_converters['inc'] = 'inc_convert'
1752 def __init__(self, config, base=None):
1753 super(Configurator, self).__init__(config)
1754 self.base = base or os.getcwd()
1756 def configure_custom(self, config):
1758 def convert(o):
1759 if isinstance(o, (list, tuple)):
1760 result = type(o)([convert(i) for i in o])
1761 elif isinstance(o, dict):
1762 if '()' in o:
1763 result = self.configure_custom(o)
1764 else:
1765 result = {}
1766 for k in o:
1767 result[k] = convert(o[k])
1768 else:
1769 result = self.convert(o)
1770 return result
1772 c = config.pop('()')
1773 if not callable(c):
1774 c = self.resolve(c)
1775 props = config.pop('.', None)
1776 # Check for valid identifiers
1777 args = config.pop('[]', ())
1778 if args:
1779 args = tuple([convert(o) for o in args])
1780 items = [(k, convert(config[k])) for k in config if valid_ident(k)]
1781 kwargs = dict(items)
1782 result = c(*args, **kwargs)
1783 if props:
1784 for n, v in props.items():
1785 setattr(result, n, convert(v))
1786 return result
1788 def __getitem__(self, key):
1789 result = self.config[key]
1790 if isinstance(result, dict) and '()' in result:
1791 self.config[key] = result = self.configure_custom(result)
1792 return result
1794 def inc_convert(self, value):
1795 """Default converter for the inc:// protocol."""
1796 if not os.path.isabs(value):
1797 value = os.path.join(self.base, value)
1798 with text_open(value, 'r', encoding='utf-8') as f:
1799 result = json.load(f)
1800 return result
1803class SubprocessMixin(object):
1804 """
1805 Mixin for running subprocesses and capturing their output
1806 """
1808 def __init__(self, verbose=False, progress=None):
1809 self.verbose = verbose
1810 self.progress = progress
1812 def reader(self, stream, context):
1813 """
1814 Read lines from a subprocess' output stream and either pass to a progress
1815 callable (if specified) or write progress information to sys.stderr.
1816 """
1817 progress = self.progress
1818 verbose = self.verbose
1819 while True:
1820 s = stream.readline()
1821 if not s:
1822 break
1823 if progress is not None:
1824 progress(s, context)
1825 else:
1826 if not verbose:
1827 sys.stderr.write('.')
1828 else:
1829 sys.stderr.write(s.decode('utf-8'))
1830 sys.stderr.flush()
1831 stream.close()
1833 def run_command(self, cmd, **kwargs):
1834 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
1835 t1 = threading.Thread(target=self.reader, args=(p.stdout, 'stdout'))
1836 t1.start()
1837 t2 = threading.Thread(target=self.reader, args=(p.stderr, 'stderr'))
1838 t2.start()
1839 p.wait()
1840 t1.join()
1841 t2.join()
1842 if self.progress is not None:
1843 self.progress('done.', 'main')
1844 elif self.verbose:
1845 sys.stderr.write('done.\n')
1846 return p
1849def normalize_name(name):
1850 """Normalize a python package name a la PEP 503"""
1851 # https://peps.python.org/pep-0503/#normalized-names
1852 # Emulates re.sub(r"[-_.]+", "-", name).lower()
1853 # Much faster than re, and even faster than str.translate
1854 value = name.lower().replace("_", "-").replace(".", "-")
1855 while "--" in value:
1856 value = value.replace("--", "-")
1857 return value
1860# def _get_pypirc_command():
1861# """
1862# Get the distutils command for interacting with PyPI configurations.
1863# :return: the command.
1864# """
1865# from distutils.core import Distribution
1866# from distutils.config import PyPIRCCommand
1867# d = Distribution()
1868# return PyPIRCCommand(d)
1871class PyPIRCFile(object):
1873 DEFAULT_REPOSITORY = 'https://upload.pypi.org/legacy/'
1874 DEFAULT_REALM = 'pypi'
1876 def __init__(self, fn=None, url=None):
1877 if fn is None:
1878 fn = os.path.join(os.path.expanduser('~'), '.pypirc')
1879 self.filename = fn
1880 self.url = url
1882 def read(self):
1883 result = {}
1885 if os.path.exists(self.filename):
1886 repository = self.url or self.DEFAULT_REPOSITORY
1888 config = configparser.RawConfigParser()
1889 config.read(self.filename)
1890 sections = config.sections()
1891 if 'distutils' in sections:
1892 # let's get the list of servers
1893 index_servers = config.get('distutils', 'index-servers')
1894 _servers = [server.strip() for server in index_servers.split('\n') if server.strip() != '']
1895 if _servers == []:
1896 # nothing set, let's try to get the default pypi
1897 if 'pypi' in sections:
1898 _servers = ['pypi']
1899 else:
1900 for server in _servers:
1901 result = {'server': server}
1902 result['username'] = config.get(server, 'username')
1904 # optional params
1905 for key, default in (('repository', self.DEFAULT_REPOSITORY), ('realm', self.DEFAULT_REALM),
1906 ('password', None)):
1907 if config.has_option(server, key):
1908 result[key] = config.get(server, key)
1909 else:
1910 result[key] = default
1912 # work around people having "repository" for the "pypi"
1913 # section of their config set to the HTTP (rather than
1914 # HTTPS) URL
1915 if (server == 'pypi' and repository in (self.DEFAULT_REPOSITORY, 'pypi')):
1916 result['repository'] = self.DEFAULT_REPOSITORY
1917 elif (result['server'] != repository and result['repository'] != repository):
1918 result = {}
1919 elif 'server-login' in sections:
1920 # old format
1921 server = 'server-login'
1922 if config.has_option(server, 'repository'):
1923 repository = config.get(server, 'repository')
1924 else:
1925 repository = self.DEFAULT_REPOSITORY
1926 result = {
1927 'username': config.get(server, 'username'),
1928 'password': config.get(server, 'password'),
1929 'repository': repository,
1930 'server': server,
1931 'realm': self.DEFAULT_REALM
1932 }
1933 return result
1935 def update(self, username, password):
1936 # import pdb; pdb.set_trace()
1937 config = configparser.RawConfigParser()
1938 fn = self.filename
1939 config.read(fn)
1940 if not config.has_section('pypi'):
1941 config.add_section('pypi')
1942 config.set('pypi', 'username', username)
1943 config.set('pypi', 'password', password)
1944 with open(fn, 'w') as f:
1945 config.write(f)
1948def _load_pypirc(index):
1949 """
1950 Read the PyPI access configuration as supported by distutils.
1951 """
1952 return PyPIRCFile(url=index.url).read()
1955def _store_pypirc(index):
1956 PyPIRCFile().update(index.username, index.password)
1959#
1960# get_platform()/get_host_platform() copied from Python 3.10.a0 source, with some minor
1961# tweaks
1962#
1965def get_host_platform():
1966 """Return a string that identifies the current platform. This is used mainly to
1967 distinguish platform-specific build directories and platform-specific built
1968 distributions. Typically includes the OS name and version and the
1969 architecture (as supplied by 'os.uname()'), although the exact information
1970 included depends on the OS; eg. on Linux, the kernel version isn't
1971 particularly important.
1973 Examples of returned values:
1974 linux-i586
1975 linux-alpha (?)
1976 solaris-2.6-sun4u
1978 Windows will return one of:
1979 win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
1980 win32 (all others - specifically, sys.platform is returned)
1982 For other non-POSIX platforms, currently just returns 'sys.platform'.
1984 """
1985 if os.name == 'nt':
1986 if 'amd64' in sys.version.lower():
1987 return 'win-amd64'
1988 if '(arm)' in sys.version.lower():
1989 return 'win-arm32'
1990 if '(arm64)' in sys.version.lower():
1991 return 'win-arm64'
1992 return sys.platform
1994 # Set for cross builds explicitly
1995 if "_PYTHON_HOST_PLATFORM" in os.environ:
1996 return os.environ["_PYTHON_HOST_PLATFORM"]
1998 if os.name != 'posix' or not hasattr(os, 'uname'):
1999 # XXX what about the architecture? NT is Intel or Alpha,
2000 # Mac OS is M68k or PPC, etc.
2001 return sys.platform
2003 # Try to distinguish various flavours of Unix
2005 (osname, host, release, version, machine) = os.uname()
2007 # Convert the OS name to lowercase, remove '/' characters, and translate
2008 # spaces (for "Power Macintosh")
2009 osname = osname.lower().replace('/', '')
2010 machine = machine.replace(' ', '_').replace('/', '-')
2012 if osname[:5] == 'linux':
2013 # At least on Linux/Intel, 'machine' is the processor --
2014 # i386, etc.
2015 # XXX what about Alpha, SPARC, etc?
2016 return "%s-%s" % (osname, machine)
2018 elif osname[:5] == 'sunos':
2019 if release[0] >= '5': # SunOS 5 == Solaris 2
2020 osname = 'solaris'
2021 release = '%d.%s' % (int(release[0]) - 3, release[2:])
2022 # We can't use 'platform.architecture()[0]' because a
2023 # bootstrap problem. We use a dict to get an error
2024 # if some suspicious happens.
2025 bitness = {2147483647: '32bit', 9223372036854775807: '64bit'}
2026 machine += '.%s' % bitness[sys.maxsize]
2027 # fall through to standard osname-release-machine representation
2028 elif osname[:3] == 'aix':
2029 from _aix_support import aix_platform
2030 return aix_platform()
2031 elif osname[:6] == 'cygwin':
2032 osname = 'cygwin'
2033 rel_re = re.compile(r'[\d.]+', re.ASCII)
2034 m = rel_re.match(release)
2035 if m:
2036 release = m.group()
2037 elif osname[:6] == 'darwin':
2038 import _osx_support
2039 try:
2040 from distutils import sysconfig
2041 except ImportError:
2042 import sysconfig
2043 osname, release, machine = _osx_support.get_platform_osx(sysconfig.get_config_vars(), osname, release, machine)
2045 return '%s-%s-%s' % (osname, release, machine)
2048_TARGET_TO_PLAT = {
2049 'x86': 'win32',
2050 'x64': 'win-amd64',
2051 'arm': 'win-arm32',
2052}
2055def get_platform():
2056 if os.name != 'nt':
2057 return get_host_platform()
2058 cross_compilation_target = os.environ.get('VSCMD_ARG_TGT_ARCH')
2059 if cross_compilation_target not in _TARGET_TO_PLAT:
2060 return get_host_platform()
2061 return _TARGET_TO_PLAT[cross_compilation_target]
2064def is_in_directory(path, target):
2065 """
2066 Check if a path is inside a target directory. This doesn't check case (might be an issue on Windows)
2067 """
2068 path = os.path.abspath(path)
2069 target = os.path.abspath(target)
2070 return path == target or path.startswith(target + os.sep)