Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/distlib/util.py: 29%
1265 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:51 +0000
1#
2# Copyright (C) 2012-2021 The Python Software Foundation.
3# See LICENSE.txt and CONTRIBUTORS.txt.
4#
5import codecs
6from collections import deque
7import contextlib
8import csv
9from glob import iglob as std_iglob
10import io
11import json
12import logging
13import os
14import py_compile
15import re
16import socket
17try:
18 import ssl
19except ImportError: # pragma: no cover
20 ssl = None
21import subprocess
22import sys
23import tarfile
24import tempfile
25import textwrap
27try:
28 import threading
29except ImportError: # pragma: no cover
30 import dummy_threading as threading
31import time
33from . import DistlibException
34from .compat import (string_types, text_type, shutil, raw_input, StringIO,
35 cache_from_source, urlopen, urljoin, httplib, xmlrpclib,
36 splittype, HTTPHandler, BaseConfigurator, valid_ident,
37 Container, configparser, URLError, ZipFile, fsdecode,
38 unquote, urlparse)
40logger = logging.getLogger(__name__)
42#
43# Requirement parsing code as per PEP 508
44#
46IDENTIFIER = re.compile(r'^([\w\.-]+)\s*')
47VERSION_IDENTIFIER = re.compile(r'^([\w\.*+-]+)\s*')
48COMPARE_OP = re.compile(r'^(<=?|>=?|={2,3}|[~!]=)\s*')
49MARKER_OP = re.compile(r'^((<=?)|(>=?)|={2,3}|[~!]=|in|not\s+in)\s*')
50OR = re.compile(r'^or\b\s*')
51AND = re.compile(r'^and\b\s*')
52NON_SPACE = re.compile(r'(\S+)\s*')
53STRING_CHUNK = re.compile(r'([\s\w\.{}()*+#:;,/?!~`@$%^&=|<>\[\]-]+)')
56def parse_marker(marker_string):
57 """
58 Parse a marker string and return a dictionary containing a marker expression.
60 The dictionary will contain keys "op", "lhs" and "rhs" for non-terminals in
61 the expression grammar, or strings. A string contained in quotes is to be
62 interpreted as a literal string, and a string not contained in quotes is a
63 variable (such as os_name).
64 """
65 def marker_var(remaining):
66 # either identifier, or literal string
67 m = IDENTIFIER.match(remaining)
68 if m:
69 result = m.groups()[0]
70 remaining = remaining[m.end():]
71 elif not remaining:
72 raise SyntaxError('unexpected end of input')
73 else:
74 q = remaining[0]
75 if q not in '\'"':
76 raise SyntaxError('invalid expression: %s' % remaining)
77 oq = '\'"'.replace(q, '')
78 remaining = remaining[1:]
79 parts = [q]
80 while remaining:
81 # either a string chunk, or oq, or q to terminate
82 if remaining[0] == q:
83 break
84 elif remaining[0] == oq:
85 parts.append(oq)
86 remaining = remaining[1:]
87 else:
88 m = STRING_CHUNK.match(remaining)
89 if not m:
90 raise SyntaxError('error in string literal: %s' % remaining)
91 parts.append(m.groups()[0])
92 remaining = remaining[m.end():]
93 else:
94 s = ''.join(parts)
95 raise SyntaxError('unterminated string: %s' % s)
96 parts.append(q)
97 result = ''.join(parts)
98 remaining = remaining[1:].lstrip() # skip past closing quote
99 return result, remaining
101 def marker_expr(remaining):
102 if remaining and remaining[0] == '(':
103 result, remaining = marker(remaining[1:].lstrip())
104 if remaining[0] != ')':
105 raise SyntaxError('unterminated parenthesis: %s' % remaining)
106 remaining = remaining[1:].lstrip()
107 else:
108 lhs, remaining = marker_var(remaining)
109 while remaining:
110 m = MARKER_OP.match(remaining)
111 if not m:
112 break
113 op = m.groups()[0]
114 remaining = remaining[m.end():]
115 rhs, remaining = marker_var(remaining)
116 lhs = {'op': op, 'lhs': lhs, 'rhs': rhs}
117 result = lhs
118 return result, remaining
120 def marker_and(remaining):
121 lhs, remaining = marker_expr(remaining)
122 while remaining:
123 m = AND.match(remaining)
124 if not m:
125 break
126 remaining = remaining[m.end():]
127 rhs, remaining = marker_expr(remaining)
128 lhs = {'op': 'and', 'lhs': lhs, 'rhs': rhs}
129 return lhs, remaining
131 def marker(remaining):
132 lhs, remaining = marker_and(remaining)
133 while remaining:
134 m = OR.match(remaining)
135 if not m:
136 break
137 remaining = remaining[m.end():]
138 rhs, remaining = marker_and(remaining)
139 lhs = {'op': 'or', 'lhs': lhs, 'rhs': rhs}
140 return lhs, remaining
142 return marker(marker_string)
145def parse_requirement(req):
146 """
147 Parse a requirement passed in as a string. Return a Container
148 whose attributes contain the various parts of the requirement.
149 """
150 remaining = req.strip()
151 if not remaining or remaining.startswith('#'):
152 return None
153 m = IDENTIFIER.match(remaining)
154 if not m:
155 raise SyntaxError('name expected: %s' % remaining)
156 distname = m.groups()[0]
157 remaining = remaining[m.end():]
158 extras = mark_expr = versions = uri = None
159 if remaining and remaining[0] == '[':
160 i = remaining.find(']', 1)
161 if i < 0:
162 raise SyntaxError('unterminated extra: %s' % remaining)
163 s = remaining[1:i]
164 remaining = remaining[i + 1:].lstrip()
165 extras = []
166 while s:
167 m = IDENTIFIER.match(s)
168 if not m:
169 raise SyntaxError('malformed extra: %s' % s)
170 extras.append(m.groups()[0])
171 s = s[m.end():]
172 if not s:
173 break
174 if s[0] != ',':
175 raise SyntaxError('comma expected in extras: %s' % s)
176 s = s[1:].lstrip()
177 if not extras:
178 extras = None
179 if remaining:
180 if remaining[0] == '@':
181 # it's a URI
182 remaining = remaining[1:].lstrip()
183 m = NON_SPACE.match(remaining)
184 if not m:
185 raise SyntaxError('invalid URI: %s' % remaining)
186 uri = m.groups()[0]
187 t = urlparse(uri)
188 # there are issues with Python and URL parsing, so this test
189 # is a bit crude. See bpo-20271, bpo-23505. Python doesn't
190 # always parse invalid URLs correctly - it should raise
191 # exceptions for malformed URLs
192 if not (t.scheme and t.netloc):
193 raise SyntaxError('Invalid URL: %s' % uri)
194 remaining = remaining[m.end():].lstrip()
195 else:
197 def get_versions(ver_remaining):
198 """
199 Return a list of operator, version tuples if any are
200 specified, else None.
201 """
202 m = COMPARE_OP.match(ver_remaining)
203 versions = None
204 if m:
205 versions = []
206 while True:
207 op = m.groups()[0]
208 ver_remaining = ver_remaining[m.end():]
209 m = VERSION_IDENTIFIER.match(ver_remaining)
210 if not m:
211 raise SyntaxError('invalid version: %s' % ver_remaining)
212 v = m.groups()[0]
213 versions.append((op, v))
214 ver_remaining = ver_remaining[m.end():]
215 if not ver_remaining or ver_remaining[0] != ',':
216 break
217 ver_remaining = ver_remaining[1:].lstrip()
218 # Some packages have a trailing comma which would break things
219 # See issue #148
220 if not ver_remaining:
221 break
222 m = COMPARE_OP.match(ver_remaining)
223 if not m:
224 raise SyntaxError('invalid constraint: %s' % ver_remaining)
225 if not versions:
226 versions = None
227 return versions, ver_remaining
229 if remaining[0] != '(':
230 versions, remaining = get_versions(remaining)
231 else:
232 i = remaining.find(')', 1)
233 if i < 0:
234 raise SyntaxError('unterminated parenthesis: %s' % remaining)
235 s = remaining[1:i]
236 remaining = remaining[i + 1:].lstrip()
237 # As a special diversion from PEP 508, allow a version number
238 # a.b.c in parentheses as a synonym for ~= a.b.c (because this
239 # is allowed in earlier PEPs)
240 if COMPARE_OP.match(s):
241 versions, _ = get_versions(s)
242 else:
243 m = VERSION_IDENTIFIER.match(s)
244 if not m:
245 raise SyntaxError('invalid constraint: %s' % s)
246 v = m.groups()[0]
247 s = s[m.end():].lstrip()
248 if s:
249 raise SyntaxError('invalid constraint: %s' % s)
250 versions = [('~=', v)]
252 if remaining:
253 if remaining[0] != ';':
254 raise SyntaxError('invalid requirement: %s' % remaining)
255 remaining = remaining[1:].lstrip()
257 mark_expr, remaining = parse_marker(remaining)
259 if remaining and remaining[0] != '#':
260 raise SyntaxError('unexpected trailing data: %s' % remaining)
262 if not versions:
263 rs = distname
264 else:
265 rs = '%s %s' % (distname, ', '.join(['%s %s' % con for con in versions]))
266 return Container(name=distname, extras=extras, constraints=versions,
267 marker=mark_expr, url=uri, requirement=rs)
270def get_resources_dests(resources_root, rules):
271 """Find destinations for resources files"""
273 def get_rel_path(root, path):
274 # normalizes and returns a lstripped-/-separated path
275 root = root.replace(os.path.sep, '/')
276 path = path.replace(os.path.sep, '/')
277 assert path.startswith(root)
278 return path[len(root):].lstrip('/')
280 destinations = {}
281 for base, suffix, dest in rules:
282 prefix = os.path.join(resources_root, base)
283 for abs_base in iglob(prefix):
284 abs_glob = os.path.join(abs_base, suffix)
285 for abs_path in iglob(abs_glob):
286 resource_file = get_rel_path(resources_root, abs_path)
287 if dest is None: # remove the entry if it was here
288 destinations.pop(resource_file, None)
289 else:
290 rel_path = get_rel_path(abs_base, abs_path)
291 rel_dest = dest.replace(os.path.sep, '/').rstrip('/')
292 destinations[resource_file] = rel_dest + '/' + rel_path
293 return destinations
296def in_venv():
297 if hasattr(sys, 'real_prefix'):
298 # virtualenv venvs
299 result = True
300 else:
301 # PEP 405 venvs
302 result = sys.prefix != getattr(sys, 'base_prefix', sys.prefix)
303 return result
306def get_executable():
307# The __PYVENV_LAUNCHER__ dance is apparently no longer needed, as
308# changes to the stub launcher mean that sys.executable always points
309# to the stub on OS X
310# if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__'
311# in os.environ):
312# result = os.environ['__PYVENV_LAUNCHER__']
313# else:
314# result = sys.executable
315# return result
316 # Avoid normcasing: see issue #143
317 # result = os.path.normcase(sys.executable)
318 result = sys.executable
319 if not isinstance(result, text_type):
320 result = fsdecode(result)
321 return result
324def proceed(prompt, allowed_chars, error_prompt=None, default=None):
325 p = prompt
326 while True:
327 s = raw_input(p)
328 p = prompt
329 if not s and default:
330 s = default
331 if s:
332 c = s[0].lower()
333 if c in allowed_chars:
334 break
335 if error_prompt:
336 p = '%c: %s\n%s' % (c, error_prompt, prompt)
337 return c
340def extract_by_key(d, keys):
341 if isinstance(keys, string_types):
342 keys = keys.split()
343 result = {}
344 for key in keys:
345 if key in d:
346 result[key] = d[key]
347 return result
349def read_exports(stream):
350 if sys.version_info[0] >= 3:
351 # needs to be a text stream
352 stream = codecs.getreader('utf-8')(stream)
353 # Try to load as JSON, falling back on legacy format
354 data = stream.read()
355 stream = StringIO(data)
356 try:
357 jdata = json.load(stream)
358 result = jdata['extensions']['python.exports']['exports']
359 for group, entries in result.items():
360 for k, v in entries.items():
361 s = '%s = %s' % (k, v)
362 entry = get_export_entry(s)
363 assert entry is not None
364 entries[k] = entry
365 return result
366 except Exception:
367 stream.seek(0, 0)
369 def read_stream(cp, stream):
370 if hasattr(cp, 'read_file'):
371 cp.read_file(stream)
372 else:
373 cp.readfp(stream)
375 cp = configparser.ConfigParser()
376 try:
377 read_stream(cp, stream)
378 except configparser.MissingSectionHeaderError:
379 stream.close()
380 data = textwrap.dedent(data)
381 stream = StringIO(data)
382 read_stream(cp, stream)
384 result = {}
385 for key in cp.sections():
386 result[key] = entries = {}
387 for name, value in cp.items(key):
388 s = '%s = %s' % (name, value)
389 entry = get_export_entry(s)
390 assert entry is not None
391 #entry.dist = self
392 entries[name] = entry
393 return result
396def write_exports(exports, stream):
397 if sys.version_info[0] >= 3:
398 # needs to be a text stream
399 stream = codecs.getwriter('utf-8')(stream)
400 cp = configparser.ConfigParser()
401 for k, v in exports.items():
402 # TODO check k, v for valid values
403 cp.add_section(k)
404 for entry in v.values():
405 if entry.suffix is None:
406 s = entry.prefix
407 else:
408 s = '%s:%s' % (entry.prefix, entry.suffix)
409 if entry.flags:
410 s = '%s [%s]' % (s, ', '.join(entry.flags))
411 cp.set(k, entry.name, s)
412 cp.write(stream)
415@contextlib.contextmanager
416def tempdir():
417 td = tempfile.mkdtemp()
418 try:
419 yield td
420 finally:
421 shutil.rmtree(td)
423@contextlib.contextmanager
424def chdir(d):
425 cwd = os.getcwd()
426 try:
427 os.chdir(d)
428 yield
429 finally:
430 os.chdir(cwd)
433@contextlib.contextmanager
434def socket_timeout(seconds=15):
435 cto = socket.getdefaulttimeout()
436 try:
437 socket.setdefaulttimeout(seconds)
438 yield
439 finally:
440 socket.setdefaulttimeout(cto)
443class cached_property(object):
444 def __init__(self, func):
445 self.func = func
446 #for attr in ('__name__', '__module__', '__doc__'):
447 # setattr(self, attr, getattr(func, attr, None))
449 def __get__(self, obj, cls=None):
450 if obj is None:
451 return self
452 value = self.func(obj)
453 object.__setattr__(obj, self.func.__name__, value)
454 #obj.__dict__[self.func.__name__] = value = self.func(obj)
455 return value
457def convert_path(pathname):
458 """Return 'pathname' as a name that will work on the native filesystem.
460 The path is split on '/' and put back together again using the current
461 directory separator. Needed because filenames in the setup script are
462 always supplied in Unix style, and have to be converted to the local
463 convention before we can actually use them in the filesystem. Raises
464 ValueError on non-Unix-ish systems if 'pathname' either starts or
465 ends with a slash.
466 """
467 if os.sep == '/':
468 return pathname
469 if not pathname:
470 return pathname
471 if pathname[0] == '/':
472 raise ValueError("path '%s' cannot be absolute" % pathname)
473 if pathname[-1] == '/':
474 raise ValueError("path '%s' cannot end with '/'" % pathname)
476 paths = pathname.split('/')
477 while os.curdir in paths:
478 paths.remove(os.curdir)
479 if not paths:
480 return os.curdir
481 return os.path.join(*paths)
484class FileOperator(object):
485 def __init__(self, dry_run=False):
486 self.dry_run = dry_run
487 self.ensured = set()
488 self._init_record()
490 def _init_record(self):
491 self.record = False
492 self.files_written = set()
493 self.dirs_created = set()
495 def record_as_written(self, path):
496 if self.record:
497 self.files_written.add(path)
499 def newer(self, source, target):
500 """Tell if the target is newer than the source.
502 Returns true if 'source' exists and is more recently modified than
503 'target', or if 'source' exists and 'target' doesn't.
505 Returns false if both exist and 'target' is the same age or younger
506 than 'source'. Raise PackagingFileError if 'source' does not exist.
508 Note that this test is not very accurate: files created in the same
509 second will have the same "age".
510 """
511 if not os.path.exists(source):
512 raise DistlibException("file '%r' does not exist" %
513 os.path.abspath(source))
514 if not os.path.exists(target):
515 return True
517 return os.stat(source).st_mtime > os.stat(target).st_mtime
519 def copy_file(self, infile, outfile, check=True):
520 """Copy a file respecting dry-run and force flags.
521 """
522 self.ensure_dir(os.path.dirname(outfile))
523 logger.info('Copying %s to %s', infile, outfile)
524 if not self.dry_run:
525 msg = None
526 if check:
527 if os.path.islink(outfile):
528 msg = '%s is a symlink' % outfile
529 elif os.path.exists(outfile) and not os.path.isfile(outfile):
530 msg = '%s is a non-regular file' % outfile
531 if msg:
532 raise ValueError(msg + ' which would be overwritten')
533 shutil.copyfile(infile, outfile)
534 self.record_as_written(outfile)
536 def copy_stream(self, instream, outfile, encoding=None):
537 assert not os.path.isdir(outfile)
538 self.ensure_dir(os.path.dirname(outfile))
539 logger.info('Copying stream %s to %s', instream, outfile)
540 if not self.dry_run:
541 if encoding is None:
542 outstream = open(outfile, 'wb')
543 else:
544 outstream = codecs.open(outfile, 'w', encoding=encoding)
545 try:
546 shutil.copyfileobj(instream, outstream)
547 finally:
548 outstream.close()
549 self.record_as_written(outfile)
551 def write_binary_file(self, path, data):
552 self.ensure_dir(os.path.dirname(path))
553 if not self.dry_run:
554 if os.path.exists(path):
555 os.remove(path)
556 with open(path, 'wb') as f:
557 f.write(data)
558 self.record_as_written(path)
560 def write_text_file(self, path, data, encoding):
561 self.write_binary_file(path, data.encode(encoding))
563 def set_mode(self, bits, mask, files):
564 if os.name == 'posix' or (os.name == 'java' and os._name == 'posix'):
565 # Set the executable bits (owner, group, and world) on
566 # all the files specified.
567 for f in files:
568 if self.dry_run:
569 logger.info("changing mode of %s", f)
570 else:
571 mode = (os.stat(f).st_mode | bits) & mask
572 logger.info("changing mode of %s to %o", f, mode)
573 os.chmod(f, mode)
575 set_executable_mode = lambda s, f: s.set_mode(0o555, 0o7777, f)
577 def ensure_dir(self, path):
578 path = os.path.abspath(path)
579 if path not in self.ensured and not os.path.exists(path):
580 self.ensured.add(path)
581 d, f = os.path.split(path)
582 self.ensure_dir(d)
583 logger.info('Creating %s' % path)
584 if not self.dry_run:
585 os.mkdir(path)
586 if self.record:
587 self.dirs_created.add(path)
589 def byte_compile(self, path, optimize=False, force=False, prefix=None, hashed_invalidation=False):
590 dpath = cache_from_source(path, not optimize)
591 logger.info('Byte-compiling %s to %s', path, dpath)
592 if not self.dry_run:
593 if force or self.newer(path, dpath):
594 if not prefix:
595 diagpath = None
596 else:
597 assert path.startswith(prefix)
598 diagpath = path[len(prefix):]
599 compile_kwargs = {}
600 if hashed_invalidation and hasattr(py_compile, 'PycInvalidationMode'):
601 compile_kwargs['invalidation_mode'] = py_compile.PycInvalidationMode.CHECKED_HASH
602 py_compile.compile(path, dpath, diagpath, True, **compile_kwargs) # raise error
603 self.record_as_written(dpath)
604 return dpath
606 def ensure_removed(self, path):
607 if os.path.exists(path):
608 if os.path.isdir(path) and not os.path.islink(path):
609 logger.debug('Removing directory tree at %s', path)
610 if not self.dry_run:
611 shutil.rmtree(path)
612 if self.record:
613 if path in self.dirs_created:
614 self.dirs_created.remove(path)
615 else:
616 if os.path.islink(path):
617 s = 'link'
618 else:
619 s = 'file'
620 logger.debug('Removing %s %s', s, path)
621 if not self.dry_run:
622 os.remove(path)
623 if self.record:
624 if path in self.files_written:
625 self.files_written.remove(path)
627 def is_writable(self, path):
628 result = False
629 while not result:
630 if os.path.exists(path):
631 result = os.access(path, os.W_OK)
632 break
633 parent = os.path.dirname(path)
634 if parent == path:
635 break
636 path = parent
637 return result
639 def commit(self):
640 """
641 Commit recorded changes, turn off recording, return
642 changes.
643 """
644 assert self.record
645 result = self.files_written, self.dirs_created
646 self._init_record()
647 return result
649 def rollback(self):
650 if not self.dry_run:
651 for f in list(self.files_written):
652 if os.path.exists(f):
653 os.remove(f)
654 # dirs should all be empty now, except perhaps for
655 # __pycache__ subdirs
656 # reverse so that subdirs appear before their parents
657 dirs = sorted(self.dirs_created, reverse=True)
658 for d in dirs:
659 flist = os.listdir(d)
660 if flist:
661 assert flist == ['__pycache__']
662 sd = os.path.join(d, flist[0])
663 os.rmdir(sd)
664 os.rmdir(d) # should fail if non-empty
665 self._init_record()
667def resolve(module_name, dotted_path):
668 if module_name in sys.modules:
669 mod = sys.modules[module_name]
670 else:
671 mod = __import__(module_name)
672 if dotted_path is None:
673 result = mod
674 else:
675 parts = dotted_path.split('.')
676 result = getattr(mod, parts.pop(0))
677 for p in parts:
678 result = getattr(result, p)
679 return result
682class ExportEntry(object):
683 def __init__(self, name, prefix, suffix, flags):
684 self.name = name
685 self.prefix = prefix
686 self.suffix = suffix
687 self.flags = flags
689 @cached_property
690 def value(self):
691 return resolve(self.prefix, self.suffix)
693 def __repr__(self): # pragma: no cover
694 return '<ExportEntry %s = %s:%s %s>' % (self.name, self.prefix,
695 self.suffix, self.flags)
697 def __eq__(self, other):
698 if not isinstance(other, ExportEntry):
699 result = False
700 else:
701 result = (self.name == other.name and
702 self.prefix == other.prefix and
703 self.suffix == other.suffix and
704 self.flags == other.flags)
705 return result
707 __hash__ = object.__hash__
710ENTRY_RE = re.compile(r'''(?P<name>([^\[]\S*))
711 \s*=\s*(?P<callable>(\w+)([:\.]\w+)*)
712 \s*(\[\s*(?P<flags>[\w-]+(=\w+)?(,\s*\w+(=\w+)?)*)\s*\])?
713 ''', re.VERBOSE)
715def get_export_entry(specification):
716 m = ENTRY_RE.search(specification)
717 if not m:
718 result = None
719 if '[' in specification or ']' in specification:
720 raise DistlibException("Invalid specification "
721 "'%s'" % specification)
722 else:
723 d = m.groupdict()
724 name = d['name']
725 path = d['callable']
726 colons = path.count(':')
727 if colons == 0:
728 prefix, suffix = path, None
729 else:
730 if colons != 1:
731 raise DistlibException("Invalid specification "
732 "'%s'" % specification)
733 prefix, suffix = path.split(':')
734 flags = d['flags']
735 if flags is None:
736 if '[' in specification or ']' in specification:
737 raise DistlibException("Invalid specification "
738 "'%s'" % specification)
739 flags = []
740 else:
741 flags = [f.strip() for f in flags.split(',')]
742 result = ExportEntry(name, prefix, suffix, flags)
743 return result
746def get_cache_base(suffix=None):
747 """
748 Return the default base location for distlib caches. If the directory does
749 not exist, it is created. Use the suffix provided for the base directory,
750 and default to '.distlib' if it isn't provided.
752 On Windows, if LOCALAPPDATA is defined in the environment, then it is
753 assumed to be a directory, and will be the parent directory of the result.
754 On POSIX, and on Windows if LOCALAPPDATA is not defined, the user's home
755 directory - using os.expanduser('~') - will be the parent directory of
756 the result.
758 The result is just the directory '.distlib' in the parent directory as
759 determined above, or with the name specified with ``suffix``.
760 """
761 if suffix is None:
762 suffix = '.distlib'
763 if os.name == 'nt' and 'LOCALAPPDATA' in os.environ:
764 result = os.path.expandvars('$localappdata')
765 else:
766 # Assume posix, or old Windows
767 result = os.path.expanduser('~')
768 # we use 'isdir' instead of 'exists', because we want to
769 # fail if there's a file with that name
770 if os.path.isdir(result):
771 usable = os.access(result, os.W_OK)
772 if not usable:
773 logger.warning('Directory exists but is not writable: %s', result)
774 else:
775 try:
776 os.makedirs(result)
777 usable = True
778 except OSError:
779 logger.warning('Unable to create %s', result, exc_info=True)
780 usable = False
781 if not usable:
782 result = tempfile.mkdtemp()
783 logger.warning('Default location unusable, using %s', result)
784 return os.path.join(result, suffix)
787def path_to_cache_dir(path):
788 """
789 Convert an absolute path to a directory name for use in a cache.
791 The algorithm used is:
793 #. On Windows, any ``':'`` in the drive is replaced with ``'---'``.
794 #. Any occurrence of ``os.sep`` is replaced with ``'--'``.
795 #. ``'.cache'`` is appended.
796 """
797 d, p = os.path.splitdrive(os.path.abspath(path))
798 if d:
799 d = d.replace(':', '---')
800 p = p.replace(os.sep, '--')
801 return d + p + '.cache'
804def ensure_slash(s):
805 if not s.endswith('/'):
806 return s + '/'
807 return s
810def parse_credentials(netloc):
811 username = password = None
812 if '@' in netloc:
813 prefix, netloc = netloc.rsplit('@', 1)
814 if ':' not in prefix:
815 username = prefix
816 else:
817 username, password = prefix.split(':', 1)
818 if username:
819 username = unquote(username)
820 if password:
821 password = unquote(password)
822 return username, password, netloc
825def get_process_umask():
826 result = os.umask(0o22)
827 os.umask(result)
828 return result
830def is_string_sequence(seq):
831 result = True
832 i = None
833 for i, s in enumerate(seq):
834 if not isinstance(s, string_types):
835 result = False
836 break
837 assert i is not None
838 return result
840PROJECT_NAME_AND_VERSION = re.compile('([a-z0-9_]+([.-][a-z_][a-z0-9_]*)*)-'
841 '([a-z0-9_.+-]+)', re.I)
842PYTHON_VERSION = re.compile(r'-py(\d\.?\d?)')
845def split_filename(filename, project_name=None):
846 """
847 Extract name, version, python version from a filename (no extension)
849 Return name, version, pyver or None
850 """
851 result = None
852 pyver = None
853 filename = unquote(filename).replace(' ', '-')
854 m = PYTHON_VERSION.search(filename)
855 if m:
856 pyver = m.group(1)
857 filename = filename[:m.start()]
858 if project_name and len(filename) > len(project_name) + 1:
859 m = re.match(re.escape(project_name) + r'\b', filename)
860 if m:
861 n = m.end()
862 result = filename[:n], filename[n + 1:], pyver
863 if result is None:
864 m = PROJECT_NAME_AND_VERSION.match(filename)
865 if m:
866 result = m.group(1), m.group(3), pyver
867 return result
869# Allow spaces in name because of legacy dists like "Twisted Core"
870NAME_VERSION_RE = re.compile(r'(?P<name>[\w .-]+)\s*'
871 r'\(\s*(?P<ver>[^\s)]+)\)$')
873def parse_name_and_version(p):
874 """
875 A utility method used to get name and version from a string.
877 From e.g. a Provides-Dist value.
879 :param p: A value in a form 'foo (1.0)'
880 :return: The name and version as a tuple.
881 """
882 m = NAME_VERSION_RE.match(p)
883 if not m:
884 raise DistlibException('Ill-formed name/version string: \'%s\'' % p)
885 d = m.groupdict()
886 return d['name'].strip().lower(), d['ver']
888def get_extras(requested, available):
889 result = set()
890 requested = set(requested or [])
891 available = set(available or [])
892 if '*' in requested:
893 requested.remove('*')
894 result |= available
895 for r in requested:
896 if r == '-':
897 result.add(r)
898 elif r.startswith('-'):
899 unwanted = r[1:]
900 if unwanted not in available:
901 logger.warning('undeclared extra: %s' % unwanted)
902 if unwanted in result:
903 result.remove(unwanted)
904 else:
905 if r not in available:
906 logger.warning('undeclared extra: %s' % r)
907 result.add(r)
908 return result
909#
910# Extended metadata functionality
911#
913def _get_external_data(url):
914 result = {}
915 try:
916 # urlopen might fail if it runs into redirections,
917 # because of Python issue #13696. Fixed in locators
918 # using a custom redirect handler.
919 resp = urlopen(url)
920 headers = resp.info()
921 ct = headers.get('Content-Type')
922 if not ct.startswith('application/json'):
923 logger.debug('Unexpected response for JSON request: %s', ct)
924 else:
925 reader = codecs.getreader('utf-8')(resp)
926 #data = reader.read().decode('utf-8')
927 #result = json.loads(data)
928 result = json.load(reader)
929 except Exception as e:
930 logger.exception('Failed to get external data for %s: %s', url, e)
931 return result
933_external_data_base_url = 'https://www.red-dove.com/pypi/projects/'
935def get_project_data(name):
936 url = '%s/%s/project.json' % (name[0].upper(), name)
937 url = urljoin(_external_data_base_url, url)
938 result = _get_external_data(url)
939 return result
941def get_package_data(name, version):
942 url = '%s/%s/package-%s.json' % (name[0].upper(), name, version)
943 url = urljoin(_external_data_base_url, url)
944 return _get_external_data(url)
947class Cache(object):
948 """
949 A class implementing a cache for resources that need to live in the file system
950 e.g. shared libraries. This class was moved from resources to here because it
951 could be used by other modules, e.g. the wheel module.
952 """
954 def __init__(self, base):
955 """
956 Initialise an instance.
958 :param base: The base directory where the cache should be located.
959 """
960 # we use 'isdir' instead of 'exists', because we want to
961 # fail if there's a file with that name
962 if not os.path.isdir(base): # pragma: no cover
963 os.makedirs(base)
964 if (os.stat(base).st_mode & 0o77) != 0:
965 logger.warning('Directory \'%s\' is not private', base)
966 self.base = os.path.abspath(os.path.normpath(base))
968 def prefix_to_dir(self, prefix):
969 """
970 Converts a resource prefix to a directory name in the cache.
971 """
972 return path_to_cache_dir(prefix)
974 def clear(self):
975 """
976 Clear the cache.
977 """
978 not_removed = []
979 for fn in os.listdir(self.base):
980 fn = os.path.join(self.base, fn)
981 try:
982 if os.path.islink(fn) or os.path.isfile(fn):
983 os.remove(fn)
984 elif os.path.isdir(fn):
985 shutil.rmtree(fn)
986 except Exception:
987 not_removed.append(fn)
988 return not_removed
991class EventMixin(object):
992 """
993 A very simple publish/subscribe system.
994 """
995 def __init__(self):
996 self._subscribers = {}
998 def add(self, event, subscriber, append=True):
999 """
1000 Add a subscriber for an event.
1002 :param event: The name of an event.
1003 :param subscriber: The subscriber to be added (and called when the
1004 event is published).
1005 :param append: Whether to append or prepend the subscriber to an
1006 existing subscriber list for the event.
1007 """
1008 subs = self._subscribers
1009 if event not in subs:
1010 subs[event] = deque([subscriber])
1011 else:
1012 sq = subs[event]
1013 if append:
1014 sq.append(subscriber)
1015 else:
1016 sq.appendleft(subscriber)
1018 def remove(self, event, subscriber):
1019 """
1020 Remove a subscriber for an event.
1022 :param event: The name of an event.
1023 :param subscriber: The subscriber to be removed.
1024 """
1025 subs = self._subscribers
1026 if event not in subs:
1027 raise ValueError('No subscribers: %r' % event)
1028 subs[event].remove(subscriber)
1030 def get_subscribers(self, event):
1031 """
1032 Return an iterator for the subscribers for an event.
1033 :param event: The event to return subscribers for.
1034 """
1035 return iter(self._subscribers.get(event, ()))
1037 def publish(self, event, *args, **kwargs):
1038 """
1039 Publish a event and return a list of values returned by its
1040 subscribers.
1042 :param event: The event to publish.
1043 :param args: The positional arguments to pass to the event's
1044 subscribers.
1045 :param kwargs: The keyword arguments to pass to the event's
1046 subscribers.
1047 """
1048 result = []
1049 for subscriber in self.get_subscribers(event):
1050 try:
1051 value = subscriber(event, *args, **kwargs)
1052 except Exception:
1053 logger.exception('Exception during event publication')
1054 value = None
1055 result.append(value)
1056 logger.debug('publish %s: args = %s, kwargs = %s, result = %s',
1057 event, args, kwargs, result)
1058 return result
1060#
1061# Simple sequencing
1062#
1063class Sequencer(object):
1064 def __init__(self):
1065 self._preds = {}
1066 self._succs = {}
1067 self._nodes = set() # nodes with no preds/succs
1069 def add_node(self, node):
1070 self._nodes.add(node)
1072 def remove_node(self, node, edges=False):
1073 if node in self._nodes:
1074 self._nodes.remove(node)
1075 if edges:
1076 for p in set(self._preds.get(node, ())):
1077 self.remove(p, node)
1078 for s in set(self._succs.get(node, ())):
1079 self.remove(node, s)
1080 # Remove empties
1081 for k, v in list(self._preds.items()):
1082 if not v:
1083 del self._preds[k]
1084 for k, v in list(self._succs.items()):
1085 if not v:
1086 del self._succs[k]
1088 def add(self, pred, succ):
1089 assert pred != succ
1090 self._preds.setdefault(succ, set()).add(pred)
1091 self._succs.setdefault(pred, set()).add(succ)
1093 def remove(self, pred, succ):
1094 assert pred != succ
1095 try:
1096 preds = self._preds[succ]
1097 succs = self._succs[pred]
1098 except KeyError: # pragma: no cover
1099 raise ValueError('%r not a successor of anything' % succ)
1100 try:
1101 preds.remove(pred)
1102 succs.remove(succ)
1103 except KeyError: # pragma: no cover
1104 raise ValueError('%r not a successor of %r' % (succ, pred))
1106 def is_step(self, step):
1107 return (step in self._preds or step in self._succs or
1108 step in self._nodes)
1110 def get_steps(self, final):
1111 if not self.is_step(final):
1112 raise ValueError('Unknown: %r' % final)
1113 result = []
1114 todo = []
1115 seen = set()
1116 todo.append(final)
1117 while todo:
1118 step = todo.pop(0)
1119 if step in seen:
1120 # if a step was already seen,
1121 # move it to the end (so it will appear earlier
1122 # when reversed on return) ... but not for the
1123 # final step, as that would be confusing for
1124 # users
1125 if step != final:
1126 result.remove(step)
1127 result.append(step)
1128 else:
1129 seen.add(step)
1130 result.append(step)
1131 preds = self._preds.get(step, ())
1132 todo.extend(preds)
1133 return reversed(result)
1135 @property
1136 def strong_connections(self):
1137 #http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
1138 index_counter = [0]
1139 stack = []
1140 lowlinks = {}
1141 index = {}
1142 result = []
1144 graph = self._succs
1146 def strongconnect(node):
1147 # set the depth index for this node to the smallest unused index
1148 index[node] = index_counter[0]
1149 lowlinks[node] = index_counter[0]
1150 index_counter[0] += 1
1151 stack.append(node)
1153 # Consider successors
1154 try:
1155 successors = graph[node]
1156 except Exception:
1157 successors = []
1158 for successor in successors:
1159 if successor not in lowlinks:
1160 # Successor has not yet been visited
1161 strongconnect(successor)
1162 lowlinks[node] = min(lowlinks[node],lowlinks[successor])
1163 elif successor in stack:
1164 # the successor is in the stack and hence in the current
1165 # strongly connected component (SCC)
1166 lowlinks[node] = min(lowlinks[node],index[successor])
1168 # If `node` is a root node, pop the stack and generate an SCC
1169 if lowlinks[node] == index[node]:
1170 connected_component = []
1172 while True:
1173 successor = stack.pop()
1174 connected_component.append(successor)
1175 if successor == node: break
1176 component = tuple(connected_component)
1177 # storing the result
1178 result.append(component)
1180 for node in graph:
1181 if node not in lowlinks:
1182 strongconnect(node)
1184 return result
1186 @property
1187 def dot(self):
1188 result = ['digraph G {']
1189 for succ in self._preds:
1190 preds = self._preds[succ]
1191 for pred in preds:
1192 result.append(' %s -> %s;' % (pred, succ))
1193 for node in self._nodes:
1194 result.append(' %s;' % node)
1195 result.append('}')
1196 return '\n'.join(result)
1198#
1199# Unarchiving functionality for zip, tar, tgz, tbz, whl
1200#
1202ARCHIVE_EXTENSIONS = ('.tar.gz', '.tar.bz2', '.tar', '.zip',
1203 '.tgz', '.tbz', '.whl')
1205def unarchive(archive_filename, dest_dir, format=None, check=True):
1207 def check_path(path):
1208 if not isinstance(path, text_type):
1209 path = path.decode('utf-8')
1210 p = os.path.abspath(os.path.join(dest_dir, path))
1211 if not p.startswith(dest_dir) or p[plen] != os.sep:
1212 raise ValueError('path outside destination: %r' % p)
1214 dest_dir = os.path.abspath(dest_dir)
1215 plen = len(dest_dir)
1216 archive = None
1217 if format is None:
1218 if archive_filename.endswith(('.zip', '.whl')):
1219 format = 'zip'
1220 elif archive_filename.endswith(('.tar.gz', '.tgz')):
1221 format = 'tgz'
1222 mode = 'r:gz'
1223 elif archive_filename.endswith(('.tar.bz2', '.tbz')):
1224 format = 'tbz'
1225 mode = 'r:bz2'
1226 elif archive_filename.endswith('.tar'):
1227 format = 'tar'
1228 mode = 'r'
1229 else: # pragma: no cover
1230 raise ValueError('Unknown format for %r' % archive_filename)
1231 try:
1232 if format == 'zip':
1233 archive = ZipFile(archive_filename, 'r')
1234 if check:
1235 names = archive.namelist()
1236 for name in names:
1237 check_path(name)
1238 else:
1239 archive = tarfile.open(archive_filename, mode)
1240 if check:
1241 names = archive.getnames()
1242 for name in names:
1243 check_path(name)
1244 if format != 'zip' and sys.version_info[0] < 3:
1245 # See Python issue 17153. If the dest path contains Unicode,
1246 # tarfile extraction fails on Python 2.x if a member path name
1247 # contains non-ASCII characters - it leads to an implicit
1248 # bytes -> unicode conversion using ASCII to decode.
1249 for tarinfo in archive.getmembers():
1250 if not isinstance(tarinfo.name, text_type):
1251 tarinfo.name = tarinfo.name.decode('utf-8')
1253 # Limit extraction of dangerous items, if this Python
1254 # allows it easily. If not, just trust the input.
1255 # See: https://docs.python.org/3/library/tarfile.html#extraction-filters
1256 def extraction_filter(member, path):
1257 """Run tarfile.tar_filter, but raise the expected ValueError"""
1258 # This is only called if the current Python has tarfile filters
1259 try:
1260 return tarfile.tar_filter(member, path)
1261 except tarfile.FilterError as exc:
1262 raise ValueError(str(exc))
1263 archive.extraction_filter = extraction_filter
1265 archive.extractall(dest_dir)
1267 finally:
1268 if archive:
1269 archive.close()
1272def zip_dir(directory):
1273 """zip a directory tree into a BytesIO object"""
1274 result = io.BytesIO()
1275 dlen = len(directory)
1276 with ZipFile(result, "w") as zf:
1277 for root, dirs, files in os.walk(directory):
1278 for name in files:
1279 full = os.path.join(root, name)
1280 rel = root[dlen:]
1281 dest = os.path.join(rel, name)
1282 zf.write(full, dest)
1283 return result
1285#
1286# Simple progress bar
1287#
1289UNITS = ('', 'K', 'M', 'G','T','P')
1292class Progress(object):
1293 unknown = 'UNKNOWN'
1295 def __init__(self, minval=0, maxval=100):
1296 assert maxval is None or maxval >= minval
1297 self.min = self.cur = minval
1298 self.max = maxval
1299 self.started = None
1300 self.elapsed = 0
1301 self.done = False
1303 def update(self, curval):
1304 assert self.min <= curval
1305 assert self.max is None or curval <= self.max
1306 self.cur = curval
1307 now = time.time()
1308 if self.started is None:
1309 self.started = now
1310 else:
1311 self.elapsed = now - self.started
1313 def increment(self, incr):
1314 assert incr >= 0
1315 self.update(self.cur + incr)
1317 def start(self):
1318 self.update(self.min)
1319 return self
1321 def stop(self):
1322 if self.max is not None:
1323 self.update(self.max)
1324 self.done = True
1326 @property
1327 def maximum(self):
1328 return self.unknown if self.max is None else self.max
1330 @property
1331 def percentage(self):
1332 if self.done:
1333 result = '100 %'
1334 elif self.max is None:
1335 result = ' ?? %'
1336 else:
1337 v = 100.0 * (self.cur - self.min) / (self.max - self.min)
1338 result = '%3d %%' % v
1339 return result
1341 def format_duration(self, duration):
1342 if (duration <= 0) and self.max is None or self.cur == self.min:
1343 result = '??:??:??'
1344 #elif duration < 1:
1345 # result = '--:--:--'
1346 else:
1347 result = time.strftime('%H:%M:%S', time.gmtime(duration))
1348 return result
1350 @property
1351 def ETA(self):
1352 if self.done:
1353 prefix = 'Done'
1354 t = self.elapsed
1355 #import pdb; pdb.set_trace()
1356 else:
1357 prefix = 'ETA '
1358 if self.max is None:
1359 t = -1
1360 elif self.elapsed == 0 or (self.cur == self.min):
1361 t = 0
1362 else:
1363 #import pdb; pdb.set_trace()
1364 t = float(self.max - self.min)
1365 t /= self.cur - self.min
1366 t = (t - 1) * self.elapsed
1367 return '%s: %s' % (prefix, self.format_duration(t))
1369 @property
1370 def speed(self):
1371 if self.elapsed == 0:
1372 result = 0.0
1373 else:
1374 result = (self.cur - self.min) / self.elapsed
1375 for unit in UNITS:
1376 if result < 1000:
1377 break
1378 result /= 1000.0
1379 return '%d %sB/s' % (result, unit)
1381#
1382# Glob functionality
1383#
1385RICH_GLOB = re.compile(r'\{([^}]*)\}')
1386_CHECK_RECURSIVE_GLOB = re.compile(r'[^/\\,{]\*\*|\*\*[^/\\,}]')
1387_CHECK_MISMATCH_SET = re.compile(r'^[^{]*\}|\{[^}]*$')
1390def iglob(path_glob):
1391 """Extended globbing function that supports ** and {opt1,opt2,opt3}."""
1392 if _CHECK_RECURSIVE_GLOB.search(path_glob):
1393 msg = """invalid glob %r: recursive glob "**" must be used alone"""
1394 raise ValueError(msg % path_glob)
1395 if _CHECK_MISMATCH_SET.search(path_glob):
1396 msg = """invalid glob %r: mismatching set marker '{' or '}'"""
1397 raise ValueError(msg % path_glob)
1398 return _iglob(path_glob)
1401def _iglob(path_glob):
1402 rich_path_glob = RICH_GLOB.split(path_glob, 1)
1403 if len(rich_path_glob) > 1:
1404 assert len(rich_path_glob) == 3, rich_path_glob
1405 prefix, set, suffix = rich_path_glob
1406 for item in set.split(','):
1407 for path in _iglob(''.join((prefix, item, suffix))):
1408 yield path
1409 else:
1410 if '**' not in path_glob:
1411 for item in std_iglob(path_glob):
1412 yield item
1413 else:
1414 prefix, radical = path_glob.split('**', 1)
1415 if prefix == '':
1416 prefix = '.'
1417 if radical == '':
1418 radical = '*'
1419 else:
1420 # we support both
1421 radical = radical.lstrip('/')
1422 radical = radical.lstrip('\\')
1423 for path, dir, files in os.walk(prefix):
1424 path = os.path.normpath(path)
1425 for fn in _iglob(os.path.join(path, radical)):
1426 yield fn
1428if ssl:
1429 from .compat import (HTTPSHandler as BaseHTTPSHandler, match_hostname,
1430 CertificateError)
1433#
1434# HTTPSConnection which verifies certificates/matches domains
1435#
1437 class HTTPSConnection(httplib.HTTPSConnection):
1438 ca_certs = None # set this to the path to the certs file (.pem)
1439 check_domain = True # only used if ca_certs is not None
1441 # noinspection PyPropertyAccess
1442 def connect(self):
1443 sock = socket.create_connection((self.host, self.port), self.timeout)
1444 if getattr(self, '_tunnel_host', False):
1445 self.sock = sock
1446 self._tunnel()
1448 context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
1449 if hasattr(ssl, 'OP_NO_SSLv2'):
1450 context.options |= ssl.OP_NO_SSLv2
1451 if getattr(self, 'cert_file', None):
1452 context.load_cert_chain(self.cert_file, self.key_file)
1453 kwargs = {}
1454 if self.ca_certs:
1455 context.verify_mode = ssl.CERT_REQUIRED
1456 context.load_verify_locations(cafile=self.ca_certs)
1457 if getattr(ssl, 'HAS_SNI', False):
1458 kwargs['server_hostname'] = self.host
1460 self.sock = context.wrap_socket(sock, **kwargs)
1461 if self.ca_certs and self.check_domain:
1462 try:
1463 match_hostname(self.sock.getpeercert(), self.host)
1464 logger.debug('Host verified: %s', self.host)
1465 except CertificateError: # pragma: no cover
1466 self.sock.shutdown(socket.SHUT_RDWR)
1467 self.sock.close()
1468 raise
1470 class HTTPSHandler(BaseHTTPSHandler):
1471 def __init__(self, ca_certs, check_domain=True):
1472 BaseHTTPSHandler.__init__(self)
1473 self.ca_certs = ca_certs
1474 self.check_domain = check_domain
1476 def _conn_maker(self, *args, **kwargs):
1477 """
1478 This is called to create a connection instance. Normally you'd
1479 pass a connection class to do_open, but it doesn't actually check for
1480 a class, and just expects a callable. As long as we behave just as a
1481 constructor would have, we should be OK. If it ever changes so that
1482 we *must* pass a class, we'll create an UnsafeHTTPSConnection class
1483 which just sets check_domain to False in the class definition, and
1484 choose which one to pass to do_open.
1485 """
1486 result = HTTPSConnection(*args, **kwargs)
1487 if self.ca_certs:
1488 result.ca_certs = self.ca_certs
1489 result.check_domain = self.check_domain
1490 return result
1492 def https_open(self, req):
1493 try:
1494 return self.do_open(self._conn_maker, req)
1495 except URLError as e:
1496 if 'certificate verify failed' in str(e.reason):
1497 raise CertificateError('Unable to verify server certificate '
1498 'for %s' % req.host)
1499 else:
1500 raise
1502 #
1503 # To prevent against mixing HTTP traffic with HTTPS (examples: A Man-In-The-
1504 # Middle proxy using HTTP listens on port 443, or an index mistakenly serves
1505 # HTML containing a http://xyz link when it should be https://xyz),
1506 # you can use the following handler class, which does not allow HTTP traffic.
1507 #
1508 # It works by inheriting from HTTPHandler - so build_opener won't add a
1509 # handler for HTTP itself.
1510 #
1511 class HTTPSOnlyHandler(HTTPSHandler, HTTPHandler):
1512 def http_open(self, req):
1513 raise URLError('Unexpected HTTP request on what should be a secure '
1514 'connection: %s' % req)
1516#
1517# XML-RPC with timeouts
1518#
1519class Transport(xmlrpclib.Transport):
1520 def __init__(self, timeout, use_datetime=0):
1521 self.timeout = timeout
1522 xmlrpclib.Transport.__init__(self, use_datetime)
1524 def make_connection(self, host):
1525 h, eh, x509 = self.get_host_info(host)
1526 if not self._connection or host != self._connection[0]:
1527 self._extra_headers = eh
1528 self._connection = host, httplib.HTTPConnection(h)
1529 return self._connection[1]
1531if ssl:
1532 class SafeTransport(xmlrpclib.SafeTransport):
1533 def __init__(self, timeout, use_datetime=0):
1534 self.timeout = timeout
1535 xmlrpclib.SafeTransport.__init__(self, use_datetime)
1537 def make_connection(self, host):
1538 h, eh, kwargs = self.get_host_info(host)
1539 if not kwargs:
1540 kwargs = {}
1541 kwargs['timeout'] = self.timeout
1542 if not self._connection or host != self._connection[0]:
1543 self._extra_headers = eh
1544 self._connection = host, httplib.HTTPSConnection(h, None,
1545 **kwargs)
1546 return self._connection[1]
1549class ServerProxy(xmlrpclib.ServerProxy):
1550 def __init__(self, uri, **kwargs):
1551 self.timeout = timeout = kwargs.pop('timeout', None)
1552 # The above classes only come into play if a timeout
1553 # is specified
1554 if timeout is not None:
1555 # scheme = splittype(uri) # deprecated as of Python 3.8
1556 scheme = urlparse(uri)[0]
1557 use_datetime = kwargs.get('use_datetime', 0)
1558 if scheme == 'https':
1559 tcls = SafeTransport
1560 else:
1561 tcls = Transport
1562 kwargs['transport'] = t = tcls(timeout, use_datetime=use_datetime)
1563 self.transport = t
1564 xmlrpclib.ServerProxy.__init__(self, uri, **kwargs)
1566#
1567# CSV functionality. This is provided because on 2.x, the csv module can't
1568# handle Unicode. However, we need to deal with Unicode in e.g. RECORD files.
1569#
1571def _csv_open(fn, mode, **kwargs):
1572 if sys.version_info[0] < 3:
1573 mode += 'b'
1574 else:
1575 kwargs['newline'] = ''
1576 # Python 3 determines encoding from locale. Force 'utf-8'
1577 # file encoding to match other forced utf-8 encoding
1578 kwargs['encoding'] = 'utf-8'
1579 return open(fn, mode, **kwargs)
1582class CSVBase(object):
1583 defaults = {
1584 'delimiter': str(','), # The strs are used because we need native
1585 'quotechar': str('"'), # str in the csv API (2.x won't take
1586 'lineterminator': str('\n') # Unicode)
1587 }
1589 def __enter__(self):
1590 return self
1592 def __exit__(self, *exc_info):
1593 self.stream.close()
1596class CSVReader(CSVBase):
1597 def __init__(self, **kwargs):
1598 if 'stream' in kwargs:
1599 stream = kwargs['stream']
1600 if sys.version_info[0] >= 3:
1601 # needs to be a text stream
1602 stream = codecs.getreader('utf-8')(stream)
1603 self.stream = stream
1604 else:
1605 self.stream = _csv_open(kwargs['path'], 'r')
1606 self.reader = csv.reader(self.stream, **self.defaults)
1608 def __iter__(self):
1609 return self
1611 def next(self):
1612 result = next(self.reader)
1613 if sys.version_info[0] < 3:
1614 for i, item in enumerate(result):
1615 if not isinstance(item, text_type):
1616 result[i] = item.decode('utf-8')
1617 return result
1619 __next__ = next
1621class CSVWriter(CSVBase):
1622 def __init__(self, fn, **kwargs):
1623 self.stream = _csv_open(fn, 'w')
1624 self.writer = csv.writer(self.stream, **self.defaults)
1626 def writerow(self, row):
1627 if sys.version_info[0] < 3:
1628 r = []
1629 for item in row:
1630 if isinstance(item, text_type):
1631 item = item.encode('utf-8')
1632 r.append(item)
1633 row = r
1634 self.writer.writerow(row)
1636#
1637# Configurator functionality
1638#
1640class Configurator(BaseConfigurator):
1642 value_converters = dict(BaseConfigurator.value_converters)
1643 value_converters['inc'] = 'inc_convert'
1645 def __init__(self, config, base=None):
1646 super(Configurator, self).__init__(config)
1647 self.base = base or os.getcwd()
1649 def configure_custom(self, config):
1650 def convert(o):
1651 if isinstance(o, (list, tuple)):
1652 result = type(o)([convert(i) for i in o])
1653 elif isinstance(o, dict):
1654 if '()' in o:
1655 result = self.configure_custom(o)
1656 else:
1657 result = {}
1658 for k in o:
1659 result[k] = convert(o[k])
1660 else:
1661 result = self.convert(o)
1662 return result
1664 c = config.pop('()')
1665 if not callable(c):
1666 c = self.resolve(c)
1667 props = config.pop('.', None)
1668 # Check for valid identifiers
1669 args = config.pop('[]', ())
1670 if args:
1671 args = tuple([convert(o) for o in args])
1672 items = [(k, convert(config[k])) for k in config if valid_ident(k)]
1673 kwargs = dict(items)
1674 result = c(*args, **kwargs)
1675 if props:
1676 for n, v in props.items():
1677 setattr(result, n, convert(v))
1678 return result
1680 def __getitem__(self, key):
1681 result = self.config[key]
1682 if isinstance(result, dict) and '()' in result:
1683 self.config[key] = result = self.configure_custom(result)
1684 return result
1686 def inc_convert(self, value):
1687 """Default converter for the inc:// protocol."""
1688 if not os.path.isabs(value):
1689 value = os.path.join(self.base, value)
1690 with codecs.open(value, 'r', encoding='utf-8') as f:
1691 result = json.load(f)
1692 return result
1695class SubprocessMixin(object):
1696 """
1697 Mixin for running subprocesses and capturing their output
1698 """
1699 def __init__(self, verbose=False, progress=None):
1700 self.verbose = verbose
1701 self.progress = progress
1703 def reader(self, stream, context):
1704 """
1705 Read lines from a subprocess' output stream and either pass to a progress
1706 callable (if specified) or write progress information to sys.stderr.
1707 """
1708 progress = self.progress
1709 verbose = self.verbose
1710 while True:
1711 s = stream.readline()
1712 if not s:
1713 break
1714 if progress is not None:
1715 progress(s, context)
1716 else:
1717 if not verbose:
1718 sys.stderr.write('.')
1719 else:
1720 sys.stderr.write(s.decode('utf-8'))
1721 sys.stderr.flush()
1722 stream.close()
1724 def run_command(self, cmd, **kwargs):
1725 p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
1726 stderr=subprocess.PIPE, **kwargs)
1727 t1 = threading.Thread(target=self.reader, args=(p.stdout, 'stdout'))
1728 t1.start()
1729 t2 = threading.Thread(target=self.reader, args=(p.stderr, 'stderr'))
1730 t2.start()
1731 p.wait()
1732 t1.join()
1733 t2.join()
1734 if self.progress is not None:
1735 self.progress('done.', 'main')
1736 elif self.verbose:
1737 sys.stderr.write('done.\n')
1738 return p
1741def normalize_name(name):
1742 """Normalize a python package name a la PEP 503"""
1743 # https://www.python.org/dev/peps/pep-0503/#normalized-names
1744 return re.sub('[-_.]+', '-', name).lower()
1746# def _get_pypirc_command():
1747 # """
1748 # Get the distutils command for interacting with PyPI configurations.
1749 # :return: the command.
1750 # """
1751 # from distutils.core import Distribution
1752 # from distutils.config import PyPIRCCommand
1753 # d = Distribution()
1754 # return PyPIRCCommand(d)
1756class PyPIRCFile(object):
1758 DEFAULT_REPOSITORY = 'https://upload.pypi.org/legacy/'
1759 DEFAULT_REALM = 'pypi'
1761 def __init__(self, fn=None, url=None):
1762 if fn is None:
1763 fn = os.path.join(os.path.expanduser('~'), '.pypirc')
1764 self.filename = fn
1765 self.url = url
1767 def read(self):
1768 result = {}
1770 if os.path.exists(self.filename):
1771 repository = self.url or self.DEFAULT_REPOSITORY
1773 config = configparser.RawConfigParser()
1774 config.read(self.filename)
1775 sections = config.sections()
1776 if 'distutils' in sections:
1777 # let's get the list of servers
1778 index_servers = config.get('distutils', 'index-servers')
1779 _servers = [server.strip() for server in
1780 index_servers.split('\n')
1781 if server.strip() != '']
1782 if _servers == []:
1783 # nothing set, let's try to get the default pypi
1784 if 'pypi' in sections:
1785 _servers = ['pypi']
1786 else:
1787 for server in _servers:
1788 result = {'server': server}
1789 result['username'] = config.get(server, 'username')
1791 # optional params
1792 for key, default in (('repository', self.DEFAULT_REPOSITORY),
1793 ('realm', self.DEFAULT_REALM),
1794 ('password', None)):
1795 if config.has_option(server, key):
1796 result[key] = config.get(server, key)
1797 else:
1798 result[key] = default
1800 # work around people having "repository" for the "pypi"
1801 # section of their config set to the HTTP (rather than
1802 # HTTPS) URL
1803 if (server == 'pypi' and
1804 repository in (self.DEFAULT_REPOSITORY, 'pypi')):
1805 result['repository'] = self.DEFAULT_REPOSITORY
1806 elif (result['server'] != repository and
1807 result['repository'] != repository):
1808 result = {}
1809 elif 'server-login' in sections:
1810 # old format
1811 server = 'server-login'
1812 if config.has_option(server, 'repository'):
1813 repository = config.get(server, 'repository')
1814 else:
1815 repository = self.DEFAULT_REPOSITORY
1816 result = {
1817 'username': config.get(server, 'username'),
1818 'password': config.get(server, 'password'),
1819 'repository': repository,
1820 'server': server,
1821 'realm': self.DEFAULT_REALM
1822 }
1823 return result
1825 def update(self, username, password):
1826 # import pdb; pdb.set_trace()
1827 config = configparser.RawConfigParser()
1828 fn = self.filename
1829 config.read(fn)
1830 if not config.has_section('pypi'):
1831 config.add_section('pypi')
1832 config.set('pypi', 'username', username)
1833 config.set('pypi', 'password', password)
1834 with open(fn, 'w') as f:
1835 config.write(f)
1837def _load_pypirc(index):
1838 """
1839 Read the PyPI access configuration as supported by distutils.
1840 """
1841 return PyPIRCFile(url=index.url).read()
1843def _store_pypirc(index):
1844 PyPIRCFile().update(index.username, index.password)
1846#
1847# get_platform()/get_host_platform() copied from Python 3.10.a0 source, with some minor
1848# tweaks
1849#
1851def get_host_platform():
1852 """Return a string that identifies the current platform. This is used mainly to
1853 distinguish platform-specific build directories and platform-specific built
1854 distributions. Typically includes the OS name and version and the
1855 architecture (as supplied by 'os.uname()'), although the exact information
1856 included depends on the OS; eg. on Linux, the kernel version isn't
1857 particularly important.
1859 Examples of returned values:
1860 linux-i586
1861 linux-alpha (?)
1862 solaris-2.6-sun4u
1864 Windows will return one of:
1865 win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
1866 win32 (all others - specifically, sys.platform is returned)
1868 For other non-POSIX platforms, currently just returns 'sys.platform'.
1870 """
1871 if os.name == 'nt':
1872 if 'amd64' in sys.version.lower():
1873 return 'win-amd64'
1874 if '(arm)' in sys.version.lower():
1875 return 'win-arm32'
1876 if '(arm64)' in sys.version.lower():
1877 return 'win-arm64'
1878 return sys.platform
1880 # Set for cross builds explicitly
1881 if "_PYTHON_HOST_PLATFORM" in os.environ:
1882 return os.environ["_PYTHON_HOST_PLATFORM"]
1884 if os.name != 'posix' or not hasattr(os, 'uname'):
1885 # XXX what about the architecture? NT is Intel or Alpha,
1886 # Mac OS is M68k or PPC, etc.
1887 return sys.platform
1889 # Try to distinguish various flavours of Unix
1891 (osname, host, release, version, machine) = os.uname()
1893 # Convert the OS name to lowercase, remove '/' characters, and translate
1894 # spaces (for "Power Macintosh")
1895 osname = osname.lower().replace('/', '')
1896 machine = machine.replace(' ', '_').replace('/', '-')
1898 if osname[:5] == 'linux':
1899 # At least on Linux/Intel, 'machine' is the processor --
1900 # i386, etc.
1901 # XXX what about Alpha, SPARC, etc?
1902 return "%s-%s" % (osname, machine)
1904 elif osname[:5] == 'sunos':
1905 if release[0] >= '5': # SunOS 5 == Solaris 2
1906 osname = 'solaris'
1907 release = '%d.%s' % (int(release[0]) - 3, release[2:])
1908 # We can't use 'platform.architecture()[0]' because a
1909 # bootstrap problem. We use a dict to get an error
1910 # if some suspicious happens.
1911 bitness = {2147483647:'32bit', 9223372036854775807:'64bit'}
1912 machine += '.%s' % bitness[sys.maxsize]
1913 # fall through to standard osname-release-machine representation
1914 elif osname[:3] == 'aix':
1915 from _aix_support import aix_platform
1916 return aix_platform()
1917 elif osname[:6] == 'cygwin':
1918 osname = 'cygwin'
1919 rel_re = re.compile (r'[\d.]+', re.ASCII)
1920 m = rel_re.match(release)
1921 if m:
1922 release = m.group()
1923 elif osname[:6] == 'darwin':
1924 import _osx_support
1925 try:
1926 from distutils import sysconfig
1927 except ImportError:
1928 import sysconfig
1929 osname, release, machine = _osx_support.get_platform_osx(
1930 sysconfig.get_config_vars(),
1931 osname, release, machine)
1933 return '%s-%s-%s' % (osname, release, machine)
1936_TARGET_TO_PLAT = {
1937 'x86' : 'win32',
1938 'x64' : 'win-amd64',
1939 'arm' : 'win-arm32',
1940}
1943def get_platform():
1944 if os.name != 'nt':
1945 return get_host_platform()
1946 cross_compilation_target = os.environ.get('VSCMD_ARG_TGT_ARCH')
1947 if cross_compilation_target not in _TARGET_TO_PLAT:
1948 return get_host_platform()
1949 return _TARGET_TO_PLAT[cross_compilation_target]