Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# Copyright (C) 2012-2026 The Python Software Foundation.
3# See LICENSE.txt and CONTRIBUTORS.txt.
4#
5import codecs
6from collections import deque
7import contextlib
8import csv
9from glob import iglob as std_iglob
10import io
11import json
12import logging
13import os
14import py_compile
15import re
16import socket
17try:
18 import ssl
19except ImportError: # pragma: no cover
20 ssl = None
21import subprocess
22import sys
23import tarfile
24import tempfile
25import textwrap
27try:
28 import threading
29except ImportError: # pragma: no cover
30 import dummy_threading as threading
31import time
33from . import DistlibException
34from .compat import (string_types, text_type, shutil, raw_input, StringIO, cache_from_source, urlopen, urljoin, httplib,
35 xmlrpclib, HTTPHandler, BaseConfigurator, valid_ident, Container, configparser, URLError, ZipFile,
36 fsdecode, unquote, urlparse)
38logger = logging.getLogger(__name__)
40#
41# Requirement parsing code as per PEP 508
42#
44IDENTIFIER = re.compile(r'^([\w\.-]+)\s*')
45VERSION_IDENTIFIER = re.compile(r'^([\w\.*+-]+)\s*')
46COMPARE_OP = re.compile(r'^(<=?|>=?|={2,3}|[~!]=)\s*')
47MARKER_OP = re.compile(r'^((<=?)|(>=?)|={2,3}|[~!]=|in|not\s+in)\s*')
48OR = re.compile(r'^or\b\s*')
49AND = re.compile(r'^and\b\s*')
50NON_SPACE = re.compile(r'(\S+)\s*')
51STRING_CHUNK = re.compile(r'([\s\w\.{}()*+#:;,/?!~`@$%^&=|<>\[\]-]+)')
54def parse_marker(marker_string):
55 """
56 Parse a marker string and return a dictionary containing a marker expression.
58 The dictionary will contain keys "op", "lhs" and "rhs" for non-terminals in
59 the expression grammar, or strings. A string contained in quotes is to be
60 interpreted as a literal string, and a string not contained in quotes is a
61 variable (such as os_name).
62 """
64 def marker_var(remaining):
65 # either identifier, or literal string
66 m = IDENTIFIER.match(remaining)
67 if m:
68 result = m.groups()[0]
69 remaining = remaining[m.end():]
70 elif not remaining:
71 raise SyntaxError('unexpected end of input')
72 else:
73 q = remaining[0]
74 if q not in '\'"':
75 raise SyntaxError('invalid expression: %s' % remaining)
76 oq = '\'"'.replace(q, '')
77 remaining = remaining[1:]
78 parts = [q]
79 while remaining:
80 # either a string chunk, or oq, or q to terminate
81 if remaining[0] == q:
82 break
83 elif remaining[0] == oq:
84 parts.append(oq)
85 remaining = remaining[1:]
86 else:
87 m = STRING_CHUNK.match(remaining)
88 if not m:
89 raise SyntaxError('error in string literal: %s' % remaining)
90 parts.append(m.groups()[0])
91 remaining = remaining[m.end():]
92 else:
93 s = ''.join(parts)
94 raise SyntaxError('unterminated string: %s' % s)
95 parts.append(q)
96 result = ''.join(parts)
97 remaining = remaining[1:].lstrip() # skip past closing quote
98 return result, remaining
100 def marker_expr(remaining):
101 if remaining and remaining[0] == '(':
102 result, remaining = marker(remaining[1:].lstrip())
103 if remaining[0] != ')':
104 raise SyntaxError('unterminated parenthesis: %s' % remaining)
105 remaining = remaining[1:].lstrip()
106 else:
107 lhs, remaining = marker_var(remaining)
108 while remaining:
109 m = MARKER_OP.match(remaining)
110 if not m:
111 break
112 op = m.groups()[0]
113 remaining = remaining[m.end():]
114 rhs, remaining = marker_var(remaining)
115 lhs = {'op': op, 'lhs': lhs, 'rhs': rhs}
116 result = lhs
117 return result, remaining
119 def marker_and(remaining):
120 lhs, remaining = marker_expr(remaining)
121 while remaining:
122 m = AND.match(remaining)
123 if not m:
124 break
125 remaining = remaining[m.end():]
126 rhs, remaining = marker_expr(remaining)
127 lhs = {'op': 'and', 'lhs': lhs, 'rhs': rhs}
128 return lhs, remaining
130 def marker(remaining):
131 lhs, remaining = marker_and(remaining)
132 while remaining:
133 m = OR.match(remaining)
134 if not m:
135 break
136 remaining = remaining[m.end():]
137 rhs, remaining = marker_and(remaining)
138 lhs = {'op': 'or', 'lhs': lhs, 'rhs': rhs}
139 return lhs, remaining
141 return marker(marker_string)
144def parse_requirement(req):
145 """
146 Parse a requirement passed in as a string. Return a Container
147 whose attributes contain the various parts of the requirement.
148 """
149 remaining = req.strip()
150 if not remaining or remaining.startswith('#'):
151 return None
152 m = IDENTIFIER.match(remaining)
153 if not m:
154 raise SyntaxError('name expected: %s' % remaining)
155 distname = m.groups()[0]
156 remaining = remaining[m.end():]
157 extras = mark_expr = versions = uri = None
158 if remaining and remaining[0] == '[':
159 i = remaining.find(']', 1)
160 if i < 0:
161 raise SyntaxError('unterminated extra: %s' % remaining)
162 s = remaining[1:i]
163 remaining = remaining[i + 1:].lstrip()
164 extras = []
165 while s:
166 m = IDENTIFIER.match(s)
167 if not m:
168 raise SyntaxError('malformed extra: %s' % s)
169 extras.append(m.groups()[0])
170 s = s[m.end():]
171 if not s:
172 break
173 if s[0] != ',':
174 raise SyntaxError('comma expected in extras: %s' % s)
175 s = s[1:].lstrip()
176 if not extras:
177 extras = None
178 if remaining:
179 if remaining[0] == '@':
180 # it's a URI
181 remaining = remaining[1:].lstrip()
182 m = NON_SPACE.match(remaining)
183 if not m:
184 raise SyntaxError('invalid URI: %s' % remaining)
185 uri = m.groups()[0]
186 t = urlparse(uri)
187 # there are issues with Python and URL parsing, so this test
188 # is a bit crude. See bpo-20271, bpo-23505. Python doesn't
189 # always parse invalid URLs correctly - it should raise
190 # exceptions for malformed URLs
191 if not (t.scheme and t.netloc):
192 raise SyntaxError('Invalid URL: %s' % uri)
193 remaining = remaining[m.end():].lstrip()
194 else:
196 def get_versions(ver_remaining):
197 """
198 Return a list of operator, version tuples if any are
199 specified, else None.
200 """
201 m = COMPARE_OP.match(ver_remaining)
202 versions = None
203 if m:
204 versions = []
205 while True:
206 op = m.groups()[0]
207 ver_remaining = ver_remaining[m.end():]
208 m = VERSION_IDENTIFIER.match(ver_remaining)
209 if not m:
210 raise SyntaxError('invalid version: %s' % ver_remaining)
211 v = m.groups()[0]
212 versions.append((op, v))
213 ver_remaining = ver_remaining[m.end():]
214 if not ver_remaining or ver_remaining[0] != ',':
215 break
216 ver_remaining = ver_remaining[1:].lstrip()
217 # Some packages have a trailing comma which would break things
218 # See issue #148
219 if not ver_remaining:
220 break
221 m = COMPARE_OP.match(ver_remaining)
222 if not m:
223 raise SyntaxError('invalid constraint: %s' % ver_remaining)
224 if not versions:
225 versions = None
226 return versions, ver_remaining
228 if remaining[0] != '(':
229 versions, remaining = get_versions(remaining)
230 else:
231 i = remaining.find(')', 1)
232 if i < 0:
233 raise SyntaxError('unterminated parenthesis: %s' % remaining)
234 s = remaining[1:i]
235 remaining = remaining[i + 1:].lstrip()
236 # As a special diversion from PEP 508, allow a version number
237 # a.b.c in parentheses as a synonym for ~= a.b.c (because this
238 # is allowed in earlier PEPs)
239 if COMPARE_OP.match(s):
240 versions, _ = get_versions(s)
241 else:
242 m = VERSION_IDENTIFIER.match(s)
243 if not m:
244 raise SyntaxError('invalid constraint: %s' % s)
245 v = m.groups()[0]
246 s = s[m.end():].lstrip()
247 if s:
248 raise SyntaxError('invalid constraint: %s' % s)
249 versions = [('~=', v)]
251 if remaining:
252 if remaining[0] != ';':
253 raise SyntaxError('invalid requirement: %s' % remaining)
254 remaining = remaining[1:].lstrip()
256 mark_expr, remaining = parse_marker(remaining)
258 if remaining and remaining[0] != '#':
259 raise SyntaxError('unexpected trailing data: %s' % remaining)
261 if not versions:
262 rs = distname
263 else:
264 rs = '%s %s' % (distname, ', '.join(['%s %s' % con for con in versions]))
265 return Container(name=distname, extras=extras, constraints=versions, marker=mark_expr, url=uri, requirement=rs)
268def get_resources_dests(resources_root, rules):
269 """Find destinations for resources files"""
271 def get_rel_path(root, path):
272 # normalizes and returns a lstripped-/-separated path
273 root = root.replace(os.path.sep, '/')
274 path = path.replace(os.path.sep, '/')
275 assert path.startswith(root)
276 return path[len(root):].lstrip('/')
278 destinations = {}
279 for base, suffix, dest in rules:
280 prefix = os.path.join(resources_root, base)
281 for abs_base in iglob(prefix):
282 abs_glob = os.path.join(abs_base, suffix)
283 for abs_path in iglob(abs_glob):
284 resource_file = get_rel_path(resources_root, abs_path)
285 if dest is None: # remove the entry if it was here
286 destinations.pop(resource_file, None)
287 else:
288 rel_path = get_rel_path(abs_base, abs_path)
289 rel_dest = dest.replace(os.path.sep, '/').rstrip('/')
290 destinations[resource_file] = rel_dest + '/' + rel_path
291 return destinations
294def in_venv():
295 if hasattr(sys, 'real_prefix'):
296 # virtualenv venvs
297 result = True
298 else:
299 # PEP 405 venvs
300 result = sys.prefix != getattr(sys, 'base_prefix', sys.prefix)
301 return result
304def get_executable():
305 # The __PYVENV_LAUNCHER__ dance is apparently no longer needed, as
306 # changes to the stub launcher mean that sys.executable always points
307 # to the stub on OS X
308 # if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__'
309 # in os.environ):
310 # result = os.environ['__PYVENV_LAUNCHER__']
311 # else:
312 # result = sys.executable
313 # return result
314 # Avoid normcasing: see issue #143
315 # result = os.path.normcase(sys.executable)
316 result = sys.executable
317 if not isinstance(result, text_type):
318 result = fsdecode(result)
319 return result
322def proceed(prompt, allowed_chars, error_prompt=None, default=None):
323 p = prompt
324 while True:
325 s = raw_input(p)
326 p = prompt
327 if not s and default:
328 s = default
329 if s:
330 c = s[0].lower()
331 if c in allowed_chars:
332 break
333 if error_prompt:
334 p = '%c: %s\n%s' % (c, error_prompt, prompt)
335 return c
338def extract_by_key(d, keys):
339 if isinstance(keys, string_types):
340 keys = keys.split()
341 result = {}
342 for key in keys:
343 if key in d:
344 result[key] = d[key]
345 return result
348def read_exports(stream):
349 if sys.version_info[0] >= 3:
350 # needs to be a text stream
351 stream = codecs.getreader('utf-8')(stream)
352 # Try to load as JSON, falling back on legacy format
353 data = stream.read()
354 stream = StringIO(data)
355 try:
356 jdata = json.load(stream)
357 result = jdata['extensions']['python.exports']['exports']
358 for group, entries in result.items():
359 for k, v in entries.items():
360 s = '%s = %s' % (k, v)
361 entry = get_export_entry(s)
362 assert entry is not None
363 entries[k] = entry
364 return result
365 except Exception:
366 stream.seek(0, 0)
368 def read_stream(cp, stream):
369 if hasattr(cp, 'read_file'):
370 cp.read_file(stream)
371 else:
372 cp.readfp(stream)
374 cp = configparser.ConfigParser()
375 try:
376 read_stream(cp, stream)
377 except configparser.MissingSectionHeaderError:
378 stream.close()
379 data = textwrap.dedent(data)
380 stream = StringIO(data)
381 read_stream(cp, stream)
383 result = {}
384 for key in cp.sections():
385 result[key] = entries = {}
386 for name, value in cp.items(key):
387 s = '%s = %s' % (name, value)
388 entry = get_export_entry(s)
389 assert entry is not None
390 # entry.dist = self
391 entries[name] = entry
392 return result
395def write_exports(exports, stream):
396 if sys.version_info[0] >= 3:
397 # needs to be a text stream
398 stream = codecs.getwriter('utf-8')(stream)
399 cp = configparser.ConfigParser()
400 for k, v in exports.items():
401 # TODO check k, v for valid values
402 cp.add_section(k)
403 for entry in v.values():
404 if entry.suffix is None:
405 s = entry.prefix
406 else:
407 s = '%s:%s' % (entry.prefix, entry.suffix)
408 if entry.flags:
409 s = '%s [%s]' % (s, ', '.join(entry.flags))
410 cp.set(k, entry.name, s)
411 cp.write(stream)
414@contextlib.contextmanager
415def tempdir():
416 td = tempfile.mkdtemp()
417 try:
418 yield td
419 finally:
420 shutil.rmtree(td)
423@contextlib.contextmanager
424def chdir(d):
425 cwd = os.getcwd()
426 try:
427 os.chdir(d)
428 yield
429 finally:
430 os.chdir(cwd)
433@contextlib.contextmanager
434def socket_timeout(seconds=15):
435 cto = socket.getdefaulttimeout()
436 try:
437 socket.setdefaulttimeout(seconds)
438 yield
439 finally:
440 socket.setdefaulttimeout(cto)
443class cached_property(object):
445 def __init__(self, func):
446 self.func = func
447 # for attr in ('__name__', '__module__', '__doc__'):
448 # setattr(self, attr, getattr(func, attr, None))
450 def __get__(self, obj, cls=None):
451 if obj is None:
452 return self
453 value = self.func(obj)
454 object.__setattr__(obj, self.func.__name__, value)
455 # obj.__dict__[self.func.__name__] = value = self.func(obj)
456 return value
459def convert_path(pathname):
460 """Return 'pathname' as a name that will work on the native filesystem.
462 The path is split on '/' and put back together again using the current
463 directory separator. Needed because filenames in the setup script are
464 always supplied in Unix style, and have to be converted to the local
465 convention before we can actually use them in the filesystem. Raises
466 ValueError on non-Unix-ish systems if 'pathname' either starts or
467 ends with a slash.
468 """
469 if os.sep == '/':
470 return pathname
471 if not pathname:
472 return pathname
473 if pathname[0] == '/':
474 raise ValueError("path '%s' cannot be absolute" % pathname)
475 if pathname[-1] == '/':
476 raise ValueError("path '%s' cannot end with '/'" % pathname)
478 paths = pathname.split('/')
479 while os.curdir in paths:
480 paths.remove(os.curdir)
481 if not paths:
482 return os.curdir
483 return os.path.join(*paths)
486class FileOperator(object):
488 def __init__(self, dry_run=False):
489 self.dry_run = dry_run
490 self.ensured = set()
491 self._init_record()
493 def _init_record(self):
494 self.record = False
495 self.files_written = set()
496 self.dirs_created = set()
498 def record_as_written(self, path):
499 if self.record:
500 self.files_written.add(path)
502 def newer(self, source, target):
503 """Tell if the target is newer than the source.
505 Returns true if 'source' exists and is more recently modified than
506 'target', or if 'source' exists and 'target' doesn't.
508 Returns false if both exist and 'target' is the same age or younger
509 than 'source'. Raise PackagingFileError if 'source' does not exist.
511 Note that this test is not very accurate: files created in the same
512 second will have the same "age".
513 """
514 if not os.path.exists(source):
515 raise DistlibException("file '%r' does not exist" % os.path.abspath(source))
516 if not os.path.exists(target):
517 return True
519 return os.stat(source).st_mtime > os.stat(target).st_mtime
521 def copy_file(self, infile, outfile, check=True):
522 """Copy a file respecting dry-run and force flags.
523 """
524 self.ensure_dir(os.path.dirname(outfile))
525 logger.info('Copying %s to %s', infile, outfile)
526 if not self.dry_run:
527 msg = None
528 if check:
529 if os.path.islink(outfile):
530 msg = '%s is a symlink' % outfile
531 elif os.path.exists(outfile) and not os.path.isfile(outfile):
532 msg = '%s is a non-regular file' % outfile
533 if msg:
534 raise ValueError(msg + ' which would be overwritten')
535 shutil.copyfile(infile, outfile)
536 self.record_as_written(outfile)
538 def copy_stream(self, instream, outfile, encoding=None):
539 assert not os.path.isdir(outfile)
540 self.ensure_dir(os.path.dirname(outfile))
541 logger.info('Copying stream %s to %s', instream, outfile)
542 if not self.dry_run:
543 if encoding is None:
544 outstream = open(outfile, 'wb')
545 else:
546 outstream = codecs.open(outfile, 'w', encoding=encoding)
547 try:
548 shutil.copyfileobj(instream, outstream)
549 finally:
550 outstream.close()
551 self.record_as_written(outfile)
553 def write_binary_file(self, path, data):
554 self.ensure_dir(os.path.dirname(path))
555 if not self.dry_run:
556 if os.path.exists(path):
557 os.remove(path)
558 with open(path, 'wb') as f:
559 f.write(data)
560 self.record_as_written(path)
562 def write_text_file(self, path, data, encoding):
563 self.write_binary_file(path, data.encode(encoding))
565 def set_mode(self, bits, mask, files):
566 if os.name == 'posix' or (os.name == 'java' and os._name == 'posix'):
567 # Set the executable bits (owner, group, and world) on
568 # all the files specified.
569 for f in files:
570 if self.dry_run:
571 logger.info("changing mode of %s", f)
572 else:
573 mode = (os.stat(f).st_mode | bits) & mask
574 logger.info("changing mode of %s to %o", f, mode)
575 os.chmod(f, mode)
577 set_executable_mode = lambda s, f: s.set_mode(0o555, 0o7777, f)
579 def ensure_dir(self, path):
580 path = os.path.abspath(path)
581 if path not in self.ensured and not os.path.exists(path):
582 self.ensured.add(path)
583 d, f = os.path.split(path)
584 self.ensure_dir(d)
585 logger.info('Creating %s' % path)
586 if not self.dry_run:
587 os.mkdir(path)
588 if self.record:
589 self.dirs_created.add(path)
591 def byte_compile(self, path, optimize=False, force=False, prefix=None, hashed_invalidation=False):
592 if not optimize:
593 optimization = ''
594 else:
595 optimization = '1'
596 dpath = cache_from_source(path, optimization=optimization)
597 logger.info('Byte-compiling %s to %s', path, dpath)
598 if not self.dry_run:
599 if force or self.newer(path, dpath):
600 if not prefix:
601 diagpath = None
602 else:
603 assert path.startswith(prefix)
604 diagpath = path[len(prefix):]
605 compile_kwargs = {}
606 if hashed_invalidation and hasattr(py_compile, 'PycInvalidationMode'):
607 if not isinstance(hashed_invalidation, py_compile.PycInvalidationMode):
608 hashed_invalidation = py_compile.PycInvalidationMode.CHECKED_HASH
609 compile_kwargs['invalidation_mode'] = hashed_invalidation
610 py_compile.compile(path, dpath, diagpath, True, **compile_kwargs) # raise error
611 self.record_as_written(dpath)
612 return dpath
614 def ensure_removed(self, path):
615 if os.path.exists(path):
616 if os.path.isdir(path) and not os.path.islink(path):
617 logger.debug('Removing directory tree at %s', path)
618 if not self.dry_run:
619 shutil.rmtree(path)
620 if self.record:
621 if path in self.dirs_created:
622 self.dirs_created.remove(path)
623 else:
624 if os.path.islink(path):
625 s = 'link'
626 else:
627 s = 'file'
628 logger.debug('Removing %s %s', s, path)
629 if not self.dry_run:
630 os.remove(path)
631 if self.record:
632 if path in self.files_written:
633 self.files_written.remove(path)
635 def is_writable(self, path):
636 result = False
637 while not result:
638 if os.path.exists(path):
639 result = os.access(path, os.W_OK)
640 break
641 parent = os.path.dirname(path)
642 if parent == path:
643 break
644 path = parent
645 return result
647 def commit(self):
648 """
649 Commit recorded changes, turn off recording, return
650 changes.
651 """
652 assert self.record
653 result = self.files_written, self.dirs_created
654 self._init_record()
655 return result
657 def rollback(self):
658 if not self.dry_run:
659 for f in list(self.files_written):
660 if os.path.exists(f):
661 os.remove(f)
662 # dirs should all be empty now, except perhaps for
663 # __pycache__ subdirs
664 # reverse so that subdirs appear before their parents
665 dirs = sorted(self.dirs_created, reverse=True)
666 for d in dirs:
667 flist = os.listdir(d)
668 if flist:
669 assert flist == ['__pycache__']
670 sd = os.path.join(d, flist[0])
671 os.rmdir(sd)
672 os.rmdir(d) # should fail if non-empty
673 self._init_record()
676def resolve(module_name, dotted_path):
677 if module_name in sys.modules:
678 mod = sys.modules[module_name]
679 else:
680 mod = __import__(module_name)
681 if dotted_path is None:
682 result = mod
683 else:
684 parts = dotted_path.split('.')
685 result = getattr(mod, parts.pop(0))
686 for p in parts:
687 result = getattr(result, p)
688 return result
691class ExportEntry(object):
693 def __init__(self, name, prefix, suffix, flags):
694 self.name = name
695 self.prefix = prefix
696 self.suffix = suffix
697 self.flags = flags
699 @cached_property
700 def value(self):
701 return resolve(self.prefix, self.suffix)
703 def __repr__(self): # pragma: no cover
704 return '<ExportEntry %s = %s:%s %s>' % (self.name, self.prefix, self.suffix, self.flags)
706 def __eq__(self, other):
707 if not isinstance(other, ExportEntry):
708 result = False
709 else:
710 result = (self.name == other.name and self.prefix == other.prefix and self.suffix == other.suffix
711 and self.flags == other.flags)
712 return result
714 __hash__ = object.__hash__
717ENTRY_RE = re.compile(
718 r'''(?P<name>([^\[]\S*))
719 \s*=\s*(?P<callable>(\w+)([:\.]\w+)*)
720 \s*(\[\s*(?P<flags>[\w-]+(=\w+)?(,\s*\w+(=\w+)?)*)\s*\])?
721 ''', re.VERBOSE)
724def get_export_entry(specification):
725 m = ENTRY_RE.search(specification)
726 if not m:
727 result = None
728 if '[' in specification or ']' in specification:
729 raise DistlibException("Invalid specification "
730 "'%s'" % specification)
731 else:
732 d = m.groupdict()
733 name = d['name']
734 path = d['callable']
735 colons = path.count(':')
736 if colons == 0:
737 prefix, suffix = path, None
738 else:
739 if colons != 1:
740 raise DistlibException("Invalid specification "
741 "'%s'" % specification)
742 prefix, suffix = path.split(':')
743 flags = d['flags']
744 if flags is None:
745 if '[' in specification or ']' in specification:
746 raise DistlibException("Invalid specification "
747 "'%s'" % specification)
748 flags = []
749 else:
750 flags = [f.strip() for f in flags.split(',')]
751 result = ExportEntry(name, prefix, suffix, flags)
752 return result
755def get_cache_base(suffix=None):
756 """
757 Return the default base location for distlib caches. If the directory does
758 not exist, it is created. Use the suffix provided for the base directory,
759 and default to '.distlib' if it isn't provided.
761 On Windows, if LOCALAPPDATA is defined in the environment, then it is
762 assumed to be a directory, and will be the parent directory of the result.
763 On POSIX, and on Windows if LOCALAPPDATA is not defined, the user's home
764 directory - using os.expanduser('~') - will be the parent directory of
765 the result.
767 The result is just the directory '.distlib' in the parent directory as
768 determined above, or with the name specified with ``suffix``.
769 """
770 if suffix is None:
771 suffix = '.distlib'
772 if os.name == 'nt' and 'LOCALAPPDATA' in os.environ:
773 result = os.path.expandvars('$localappdata')
774 else:
775 # Assume posix, or old Windows
776 result = os.path.expanduser('~')
777 # we use 'isdir' instead of 'exists', because we want to
778 # fail if there's a file with that name
779 if os.path.isdir(result):
780 usable = os.access(result, os.W_OK)
781 if not usable:
782 logger.warning('Directory exists but is not writable: %s', result)
783 else:
784 try:
785 os.makedirs(result)
786 usable = True
787 except OSError:
788 logger.warning('Unable to create %s', result, exc_info=True)
789 usable = False
790 if not usable:
791 result = tempfile.mkdtemp()
792 logger.warning('Default location unusable, using %s', result)
793 return os.path.join(result, suffix)
796def path_to_cache_dir(path, use_abspath=True):
797 """
798 Convert an absolute path to a directory name for use in a cache.
800 The algorithm used is:
802 #. On Windows, any ``':'`` in the drive is replaced with ``'---'``.
803 #. Any occurrence of ``os.sep`` is replaced with ``'--'``.
804 #. ``'.cache'`` is appended.
805 """
806 d, p = os.path.splitdrive(os.path.abspath(path) if use_abspath else path)
807 if d:
808 d = d.replace(':', '---')
809 p = p.replace(os.sep, '--')
810 return d + p + '.cache'
813def ensure_slash(s):
814 if not s.endswith('/'):
815 return s + '/'
816 return s
819def parse_credentials(netloc):
820 username = password = None
821 if '@' in netloc:
822 prefix, netloc = netloc.rsplit('@', 1)
823 if ':' not in prefix:
824 username = prefix
825 else:
826 username, password = prefix.split(':', 1)
827 if username:
828 username = unquote(username)
829 if password:
830 password = unquote(password)
831 return username, password, netloc
834def get_process_umask():
835 result = os.umask(0o22)
836 os.umask(result)
837 return result
840def is_string_sequence(seq):
841 result = True
842 i = None
843 for i, s in enumerate(seq):
844 if not isinstance(s, string_types):
845 result = False
846 break
847 assert i is not None
848 return result
851PROJECT_NAME_AND_VERSION = re.compile('([a-z0-9_]+([.-][a-z_][a-z0-9_]*)*)-'
852 '([a-z0-9_.+-]+)', re.I)
853PYTHON_VERSION = re.compile(r'-py(\d\.?\d?)')
856def split_filename(filename, project_name=None):
857 """
858 Extract name, version, python version from a filename (no extension)
860 Return name, version, pyver or None
861 """
862 result = None
863 pyver = None
864 filename = unquote(filename).replace(' ', '-')
865 m = PYTHON_VERSION.search(filename)
866 if m:
867 pyver = m.group(1)
868 filename = filename[:m.start()]
869 if project_name and len(filename) > len(project_name) + 1:
870 m = re.match(re.escape(project_name) + r'\b', filename)
871 if m:
872 n = m.end()
873 result = filename[:n], filename[n + 1:], pyver
874 if result is None:
875 m = PROJECT_NAME_AND_VERSION.match(filename)
876 if m:
877 result = m.group(1), m.group(3), pyver
878 return result
881# Allow spaces in name because of legacy dists like "Twisted Core"
882NAME_VERSION_RE = re.compile(r'(?P<name>[\w .-]+)\s*'
883 r'\(\s*(?P<ver>[^\s)]+)\)$')
886def parse_name_and_version(p):
887 """
888 A utility method used to get name and version from a string.
890 From e.g. a Provides-Dist value.
892 :param p: A value in a form 'foo (1.0)'
893 :return: The name and version as a tuple.
894 """
895 m = NAME_VERSION_RE.match(p)
896 if not m:
897 raise DistlibException('Ill-formed name/version string: \'%s\'' % p)
898 d = m.groupdict()
899 return d['name'].strip().lower(), d['ver']
902def get_extras(requested, available):
903 result = set()
904 requested = set(requested or [])
905 available = set(available or [])
906 if '*' in requested:
907 requested.remove('*')
908 result |= available
909 for r in requested:
910 if r == '-':
911 result.add(r)
912 elif r.startswith('-'):
913 unwanted = r[1:]
914 if unwanted not in available:
915 logger.warning('undeclared extra: %s' % unwanted)
916 if unwanted in result:
917 result.remove(unwanted)
918 else:
919 if r not in available:
920 logger.warning('undeclared extra: %s' % r)
921 result.add(r)
922 return result
925#
926# Extended metadata functionality
927#
930def _get_external_data(url):
931 result = {}
932 try:
933 # urlopen might fail if it runs into redirections,
934 # because of Python issue #13696. Fixed in locators
935 # using a custom redirect handler.
936 resp = urlopen(url)
937 headers = resp.info()
938 ct = headers.get('Content-Type')
939 if not ct.startswith('application/json'):
940 logger.debug('Unexpected response for JSON request: %s', ct)
941 else:
942 reader = codecs.getreader('utf-8')(resp)
943 # data = reader.read().decode('utf-8')
944 # result = json.loads(data)
945 result = json.load(reader)
946 except Exception as e:
947 logger.exception('Failed to get external data for %s: %s', url, e)
948 return result
951_external_data_base_url = 'https://www.red-dove.com/pypi/projects/'
954def get_project_data(name):
955 url = '%s/%s/project.json' % (name[0].upper(), name)
956 url = urljoin(_external_data_base_url, url)
957 result = _get_external_data(url)
958 return result
961def get_package_data(name, version):
962 url = '%s/%s/package-%s.json' % (name[0].upper(), name, version)
963 url = urljoin(_external_data_base_url, url)
964 return _get_external_data(url)
967class Cache(object):
968 """
969 A class implementing a cache for resources that need to live in the file system
970 e.g. shared libraries. This class was moved from resources to here because it
971 could be used by other modules, e.g. the wheel module.
972 """
974 def __init__(self, base):
975 """
976 Initialise an instance.
978 :param base: The base directory where the cache should be located.
979 """
980 # we use 'isdir' instead of 'exists', because we want to
981 # fail if there's a file with that name
982 if not os.path.isdir(base): # pragma: no cover
983 os.makedirs(base)
984 if (os.stat(base).st_mode & 0o77) != 0:
985 logger.warning('Directory \'%s\' is not private', base)
986 self.base = os.path.abspath(os.path.normpath(base))
988 def prefix_to_dir(self, prefix, use_abspath=True):
989 """
990 Converts a resource prefix to a directory name in the cache.
991 """
992 return path_to_cache_dir(prefix, use_abspath=use_abspath)
994 def clear(self):
995 """
996 Clear the cache.
997 """
998 not_removed = []
999 for fn in os.listdir(self.base):
1000 fn = os.path.join(self.base, fn)
1001 try:
1002 if os.path.islink(fn) or os.path.isfile(fn):
1003 os.remove(fn)
1004 elif os.path.isdir(fn):
1005 shutil.rmtree(fn)
1006 except Exception:
1007 not_removed.append(fn)
1008 return not_removed
1011class EventMixin(object):
1012 """
1013 A very simple publish/subscribe system.
1014 """
1016 def __init__(self):
1017 self._subscribers = {}
1019 def add(self, event, subscriber, append=True):
1020 """
1021 Add a subscriber for an event.
1023 :param event: The name of an event.
1024 :param subscriber: The subscriber to be added (and called when the
1025 event is published).
1026 :param append: Whether to append or prepend the subscriber to an
1027 existing subscriber list for the event.
1028 """
1029 subs = self._subscribers
1030 if event not in subs:
1031 subs[event] = deque([subscriber])
1032 else:
1033 sq = subs[event]
1034 if append:
1035 sq.append(subscriber)
1036 else:
1037 sq.appendleft(subscriber)
1039 def remove(self, event, subscriber):
1040 """
1041 Remove a subscriber for an event.
1043 :param event: The name of an event.
1044 :param subscriber: The subscriber to be removed.
1045 """
1046 subs = self._subscribers
1047 if event not in subs:
1048 raise ValueError('No subscribers: %r' % event)
1049 subs[event].remove(subscriber)
1051 def get_subscribers(self, event):
1052 """
1053 Return an iterator for the subscribers for an event.
1054 :param event: The event to return subscribers for.
1055 """
1056 return iter(self._subscribers.get(event, ()))
1058 def publish(self, event, *args, **kwargs):
1059 """
1060 Publish a event and return a list of values returned by its
1061 subscribers.
1063 :param event: The event to publish.
1064 :param args: The positional arguments to pass to the event's
1065 subscribers.
1066 :param kwargs: The keyword arguments to pass to the event's
1067 subscribers.
1068 """
1069 result = []
1070 for subscriber in self.get_subscribers(event):
1071 try:
1072 value = subscriber(event, *args, **kwargs)
1073 except Exception:
1074 logger.exception('Exception during event publication')
1075 value = None
1076 result.append(value)
1077 logger.debug('publish %s: args = %s, kwargs = %s, result = %s', event, args, kwargs, result)
1078 return result
1081#
1082# Simple sequencing
1083#
1084class Sequencer(object):
1086 def __init__(self):
1087 self._preds = {}
1088 self._succs = {}
1089 self._nodes = set() # nodes with no preds/succs
1091 def add_node(self, node):
1092 self._nodes.add(node)
1094 def remove_node(self, node, edges=False):
1095 if node in self._nodes:
1096 self._nodes.remove(node)
1097 if edges:
1098 for p in set(self._preds.get(node, ())):
1099 self.remove(p, node)
1100 for s in set(self._succs.get(node, ())):
1101 self.remove(node, s)
1102 # Remove empties
1103 for k, v in list(self._preds.items()):
1104 if not v:
1105 del self._preds[k]
1106 for k, v in list(self._succs.items()):
1107 if not v:
1108 del self._succs[k]
1110 def add(self, pred, succ):
1111 assert pred != succ
1112 self._preds.setdefault(succ, set()).add(pred)
1113 self._succs.setdefault(pred, set()).add(succ)
1115 def remove(self, pred, succ):
1116 assert pred != succ
1117 try:
1118 preds = self._preds[succ]
1119 succs = self._succs[pred]
1120 except KeyError: # pragma: no cover
1121 raise ValueError('%r not a successor of anything' % succ)
1122 try:
1123 preds.remove(pred)
1124 succs.remove(succ)
1125 except KeyError: # pragma: no cover
1126 raise ValueError('%r not a successor of %r' % (succ, pred))
1128 def is_step(self, step):
1129 return (step in self._preds or step in self._succs or step in self._nodes)
1131 def get_steps(self, final):
1132 if not self.is_step(final):
1133 raise ValueError('Unknown: %r' % final)
1134 result = []
1135 todo = []
1136 seen = set()
1137 todo.append(final)
1138 while todo:
1139 step = todo.pop(0)
1140 if step in seen:
1141 # if a step was already seen,
1142 # move it to the end (so it will appear earlier
1143 # when reversed on return) ... but not for the
1144 # final step, as that would be confusing for
1145 # users
1146 if step != final:
1147 result.remove(step)
1148 result.append(step)
1149 else:
1150 seen.add(step)
1151 result.append(step)
1152 preds = self._preds.get(step, ())
1153 todo.extend(preds)
1154 return reversed(result)
1156 @property
1157 def strong_connections(self):
1158 # http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
1159 index_counter = [0]
1160 stack = []
1161 lowlinks = {}
1162 index = {}
1163 result = []
1165 graph = self._succs
1167 def strongconnect(node):
1168 # set the depth index for this node to the smallest unused index
1169 index[node] = index_counter[0]
1170 lowlinks[node] = index_counter[0]
1171 index_counter[0] += 1
1172 stack.append(node)
1174 # Consider successors
1175 try:
1176 successors = graph[node]
1177 except Exception:
1178 successors = []
1179 for successor in successors:
1180 if successor not in lowlinks:
1181 # Successor has not yet been visited
1182 strongconnect(successor)
1183 lowlinks[node] = min(lowlinks[node], lowlinks[successor])
1184 elif successor in stack:
1185 # the successor is in the stack and hence in the current
1186 # strongly connected component (SCC)
1187 lowlinks[node] = min(lowlinks[node], index[successor])
1189 # If `node` is a root node, pop the stack and generate an SCC
1190 if lowlinks[node] == index[node]:
1191 connected_component = []
1193 while True:
1194 successor = stack.pop()
1195 connected_component.append(successor)
1196 if successor == node:
1197 break
1198 component = tuple(connected_component)
1199 # storing the result
1200 result.append(component)
1202 for node in graph:
1203 if node not in lowlinks:
1204 strongconnect(node)
1206 return result
1208 @property
1209 def dot(self):
1210 result = ['digraph G {']
1211 for succ in self._preds:
1212 preds = self._preds[succ]
1213 for pred in preds:
1214 result.append(' %s -> %s;' % (pred, succ))
1215 for node in self._nodes:
1216 result.append(' %s;' % node)
1217 result.append('}')
1218 return '\n'.join(result)
1221#
1222# Unarchiving functionality for zip, tar, tgz, tbz, whl
1223#
1225ARCHIVE_EXTENSIONS = ('.tar.gz', '.tar.bz2', '.tar', '.zip', '.tgz', '.tbz', '.whl')
1228def unarchive(archive_filename, dest_dir, format=None, check=True):
1230 def check_path(path, base=None):
1231 if not isinstance(path, text_type):
1232 path = path.decode('utf-8')
1233 if base is None:
1234 base = dest_dir
1235 p = os.path.abspath(os.path.join(base, path))
1236 if not p.startswith(dest_dir) or p[plen] != os.sep:
1237 raise ValueError('path outside destination: %r' % p)
1239 def check_link(member):
1240 # A symlink/hardlink member's name is validated like any other
1241 # member, but its target (linkname) is not covered by extractall's
1242 # name-based handling. An unchecked target lets a later member be
1243 # written through the link to a location outside dest_dir. Validate
1244 # the resolved target stays within dest_dir. Symlink targets are
1245 # relative to the member's own directory; hardlink targets are
1246 # relative to the archive root (i.e. dest_dir).
1247 if not (member.issym() or member.islnk()):
1248 return
1249 if member.issym():
1250 link_base = os.path.dirname(os.path.join(dest_dir, member.name))
1251 else:
1252 link_base = dest_dir
1253 check_path(member.linkname, base=link_base)
1255 dest_dir = os.path.abspath(dest_dir)
1256 plen = len(dest_dir)
1257 archive = None
1258 if format is None:
1259 if archive_filename.endswith(('.zip', '.whl')):
1260 format = 'zip'
1261 elif archive_filename.endswith(('.tar.gz', '.tgz')):
1262 format = 'tgz'
1263 mode = 'r:gz'
1264 elif archive_filename.endswith(('.tar.bz2', '.tbz')):
1265 format = 'tbz'
1266 mode = 'r:bz2'
1267 elif archive_filename.endswith('.tar'):
1268 format = 'tar'
1269 mode = 'r'
1270 else: # pragma: no cover
1271 raise ValueError('Unknown format for %r' % archive_filename)
1272 try:
1273 if format == 'zip':
1274 archive = ZipFile(archive_filename, 'r')
1275 if check:
1276 names = archive.namelist()
1277 for name in names:
1278 check_path(name)
1279 else:
1280 archive = tarfile.open(archive_filename, mode)
1281 if check:
1282 for member in archive.getmembers():
1283 check_path(member.name)
1284 check_link(member)
1285 if format != 'zip' and sys.version_info[0] < 3:
1286 # See Python issue 17153. If the dest path contains Unicode,
1287 # tarfile extraction fails on Python 2.x if a member path name
1288 # contains non-ASCII characters - it leads to an implicit
1289 # bytes -> unicode conversion using ASCII to decode.
1290 for tarinfo in archive.getmembers():
1291 if not isinstance(tarinfo.name, text_type):
1292 tarinfo.name = tarinfo.name.decode('utf-8')
1294 # Limit extraction of dangerous items, if this Python
1295 # allows it easily. If not, just trust the input.
1296 # See: https://docs.python.org/3/library/tarfile.html#extraction-filters
1297 def extraction_filter(member, path):
1298 """Run tarfile.tar_filter, but raise the expected ValueError"""
1299 # This is only called if the current Python has tarfile filters
1300 try:
1301 return tarfile.tar_filter(member, path)
1302 except tarfile.FilterError as exc:
1303 raise ValueError(str(exc))
1305 archive.extraction_filter = extraction_filter
1307 archive.extractall(dest_dir)
1309 finally:
1310 if archive:
1311 archive.close()
1314def zip_dir(directory):
1315 """zip a directory tree into a BytesIO object"""
1316 result = io.BytesIO()
1317 dlen = len(directory)
1318 with ZipFile(result, "w") as zf:
1319 for root, dirs, files in os.walk(directory):
1320 for name in files:
1321 full = os.path.join(root, name)
1322 rel = root[dlen:]
1323 dest = os.path.join(rel, name)
1324 zf.write(full, dest)
1325 return result
1328#
1329# Simple progress bar
1330#
1332UNITS = ('', 'K', 'M', 'G', 'T', 'P')
1335class Progress(object):
1336 unknown = 'UNKNOWN'
1338 def __init__(self, minval=0, maxval=100):
1339 assert maxval is None or maxval >= minval
1340 self.min = self.cur = minval
1341 self.max = maxval
1342 self.started = None
1343 self.elapsed = 0
1344 self.done = False
1346 def update(self, curval):
1347 assert self.min <= curval
1348 assert self.max is None or curval <= self.max
1349 self.cur = curval
1350 now = time.time()
1351 if self.started is None:
1352 self.started = now
1353 else:
1354 self.elapsed = now - self.started
1356 def increment(self, incr):
1357 assert incr >= 0
1358 self.update(self.cur + incr)
1360 def start(self):
1361 self.update(self.min)
1362 return self
1364 def stop(self):
1365 if self.max is not None:
1366 self.update(self.max)
1367 self.done = True
1369 @property
1370 def maximum(self):
1371 return self.unknown if self.max is None else self.max
1373 @property
1374 def percentage(self):
1375 if self.done:
1376 result = '100 %'
1377 elif self.max is None:
1378 result = ' ?? %'
1379 else:
1380 v = 100.0 * (self.cur - self.min) / (self.max - self.min)
1381 result = '%3d %%' % v
1382 return result
1384 def format_duration(self, duration):
1385 if (duration <= 0) and self.max is None or self.cur == self.min:
1386 result = '??:??:??'
1387 # elif duration < 1:
1388 # result = '--:--:--'
1389 else:
1390 result = time.strftime('%H:%M:%S', time.gmtime(duration))
1391 return result
1393 @property
1394 def ETA(self):
1395 if self.done:
1396 prefix = 'Done'
1397 t = self.elapsed
1398 # import pdb; pdb.set_trace()
1399 else:
1400 prefix = 'ETA '
1401 if self.max is None:
1402 t = -1
1403 elif self.elapsed == 0 or (self.cur == self.min):
1404 t = 0
1405 else:
1406 # import pdb; pdb.set_trace()
1407 t = float(self.max - self.min)
1408 t /= self.cur - self.min
1409 t = (t - 1) * self.elapsed
1410 return '%s: %s' % (prefix, self.format_duration(t))
1412 @property
1413 def speed(self):
1414 if self.elapsed == 0:
1415 result = 0.0
1416 else:
1417 result = (self.cur - self.min) / self.elapsed
1418 for unit in UNITS:
1419 if result < 1000:
1420 break
1421 result /= 1000.0
1422 return '%d %sB/s' % (result, unit)
1425#
1426# Glob functionality
1427#
1429RICH_GLOB = re.compile(r'\{([^}]*)\}')
1430_CHECK_RECURSIVE_GLOB = re.compile(r'[^/\\,{]\*\*|\*\*[^/\\,}]')
1431_CHECK_MISMATCH_SET = re.compile(r'^[^{]*\}|\{[^}]*$')
1434def iglob(path_glob):
1435 """Extended globbing function that supports ** and {opt1,opt2,opt3}."""
1436 if _CHECK_RECURSIVE_GLOB.search(path_glob):
1437 msg = """invalid glob %r: recursive glob "**" must be used alone"""
1438 raise ValueError(msg % path_glob)
1439 if _CHECK_MISMATCH_SET.search(path_glob):
1440 msg = """invalid glob %r: mismatching set marker '{' or '}'"""
1441 raise ValueError(msg % path_glob)
1442 return _iglob(path_glob)
1445def _iglob(path_glob):
1446 rich_path_glob = RICH_GLOB.split(path_glob, 1)
1447 if len(rich_path_glob) > 1:
1448 assert len(rich_path_glob) == 3, rich_path_glob
1449 prefix, set, suffix = rich_path_glob
1450 for item in set.split(','):
1451 for path in _iglob(''.join((prefix, item, suffix))):
1452 yield path
1453 else:
1454 if '**' not in path_glob:
1455 for item in std_iglob(path_glob):
1456 yield item
1457 else:
1458 prefix, radical = path_glob.split('**', 1)
1459 if prefix == '':
1460 prefix = '.'
1461 if radical == '':
1462 radical = '*'
1463 else:
1464 # we support both
1465 radical = radical.lstrip('/')
1466 radical = radical.lstrip('\\')
1467 for path, dir, files in os.walk(prefix):
1468 path = os.path.normpath(path)
1469 for fn in _iglob(os.path.join(path, radical)):
1470 yield fn
1473if ssl:
1474 from .compat import (HTTPSHandler as BaseHTTPSHandler, match_hostname, CertificateError)
1476 #
1477 # HTTPSConnection which verifies certificates/matches domains
1478 #
1481 class HTTPSConnection(httplib.HTTPSConnection):
1482 ca_certs = None # set this to the path to the certs file (.pem)
1483 check_domain = True # only used if ca_certs is not None
1485 # noinspection PyPropertyAccess
1486 def connect(self):
1487 sock = socket.create_connection((self.host, self.port), self.timeout)
1488 if getattr(self, '_tunnel_host', False):
1489 self.sock = sock
1490 self._tunnel()
1492 context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
1493 if hasattr(ssl, 'OP_NO_SSLv2'):
1494 context.options |= ssl.OP_NO_SSLv2
1495 if getattr(self, 'cert_file', None):
1496 context.load_cert_chain(self.cert_file, self.key_file)
1497 kwargs = {}
1498 if self.ca_certs:
1499 context.verify_mode = ssl.CERT_REQUIRED
1500 context.load_verify_locations(cafile=self.ca_certs)
1501 if getattr(ssl, 'HAS_SNI', False):
1502 kwargs['server_hostname'] = self.host
1504 self.sock = context.wrap_socket(sock, **kwargs)
1505 if self.ca_certs and self.check_domain:
1506 try:
1507 match_hostname(self.sock.getpeercert(), self.host)
1508 logger.debug('Host verified: %s', self.host)
1509 except CertificateError: # pragma: no cover
1510 self.sock.shutdown(socket.SHUT_RDWR)
1511 self.sock.close()
1512 raise
1514 class HTTPSHandler(BaseHTTPSHandler):
1516 def __init__(self, ca_certs, check_domain=True):
1517 BaseHTTPSHandler.__init__(self)
1518 self.ca_certs = ca_certs
1519 self.check_domain = check_domain
1521 def _conn_maker(self, *args, **kwargs):
1522 """
1523 This is called to create a connection instance. Normally you'd
1524 pass a connection class to do_open, but it doesn't actually check for
1525 a class, and just expects a callable. As long as we behave just as a
1526 constructor would have, we should be OK. If it ever changes so that
1527 we *must* pass a class, we'll create an UnsafeHTTPSConnection class
1528 which just sets check_domain to False in the class definition, and
1529 choose which one to pass to do_open.
1530 """
1531 result = HTTPSConnection(*args, **kwargs)
1532 if self.ca_certs:
1533 result.ca_certs = self.ca_certs
1534 result.check_domain = self.check_domain
1535 return result
1537 def https_open(self, req):
1538 try:
1539 return self.do_open(self._conn_maker, req)
1540 except URLError as e:
1541 if 'certificate verify failed' in str(e.reason):
1542 raise CertificateError('Unable to verify server certificate '
1543 'for %s' % req.host)
1544 else:
1545 raise
1547 #
1548 # To prevent against mixing HTTP traffic with HTTPS (examples: A Man-In-The-
1549 # Middle proxy using HTTP listens on port 443, or an index mistakenly serves
1550 # HTML containing a http://xyz link when it should be https://xyz),
1551 # you can use the following handler class, which does not allow HTTP traffic.
1552 #
1553 # It works by inheriting from HTTPHandler - so build_opener won't add a
1554 # handler for HTTP itself.
1555 #
1556 class HTTPSOnlyHandler(HTTPSHandler, HTTPHandler):
1558 def http_open(self, req):
1559 raise URLError('Unexpected HTTP request on what should be a secure '
1560 'connection: %s' % req)
1563#
1564# XML-RPC with timeouts
1565#
1566class Transport(xmlrpclib.Transport):
1568 def __init__(self, timeout, use_datetime=0):
1569 self.timeout = timeout
1570 xmlrpclib.Transport.__init__(self, use_datetime)
1572 def make_connection(self, host):
1573 h, eh, x509 = self.get_host_info(host)
1574 if not self._connection or host != self._connection[0]:
1575 self._extra_headers = eh
1576 self._connection = host, httplib.HTTPConnection(h)
1577 return self._connection[1]
1580if ssl:
1582 class SafeTransport(xmlrpclib.SafeTransport):
1584 def __init__(self, timeout, use_datetime=0):
1585 self.timeout = timeout
1586 xmlrpclib.SafeTransport.__init__(self, use_datetime)
1588 def make_connection(self, host):
1589 h, eh, kwargs = self.get_host_info(host)
1590 if not kwargs:
1591 kwargs = {}
1592 kwargs['timeout'] = self.timeout
1593 if not self._connection or host != self._connection[0]:
1594 self._extra_headers = eh
1595 self._connection = host, httplib.HTTPSConnection(h, None, **kwargs)
1596 return self._connection[1]
1599class ServerProxy(xmlrpclib.ServerProxy):
1601 def __init__(self, uri, **kwargs):
1602 self.timeout = timeout = kwargs.pop('timeout', None)
1603 # The above classes only come into play if a timeout
1604 # is specified
1605 if timeout is not None:
1606 # scheme = splittype(uri) # deprecated as of Python 3.8
1607 scheme = urlparse(uri)[0]
1608 use_datetime = kwargs.get('use_datetime', 0)
1609 if scheme == 'https':
1610 tcls = SafeTransport
1611 else:
1612 tcls = Transport
1613 kwargs['transport'] = t = tcls(timeout, use_datetime=use_datetime)
1614 self.transport = t
1615 xmlrpclib.ServerProxy.__init__(self, uri, **kwargs)
1618#
1619# CSV functionality. This is provided because on 2.x, the csv module can't
1620# handle Unicode. However, we need to deal with Unicode in e.g. RECORD files.
1621#
1624def _csv_open(fn, mode, **kwargs):
1625 if sys.version_info[0] < 3:
1626 mode += 'b'
1627 else:
1628 kwargs['newline'] = ''
1629 # Python 3 determines encoding from locale. Force 'utf-8'
1630 # file encoding to match other forced utf-8 encoding
1631 kwargs['encoding'] = 'utf-8'
1632 return open(fn, mode, **kwargs)
1635class CSVBase(object):
1636 defaults = {
1637 'delimiter': str(','), # The strs are used because we need native
1638 'quotechar': str('"'), # str in the csv API (2.x won't take
1639 'lineterminator': str('\n') # Unicode)
1640 }
1642 def __enter__(self):
1643 return self
1645 def __exit__(self, *exc_info):
1646 self.stream.close()
1649class CSVReader(CSVBase):
1651 def __init__(self, **kwargs):
1652 if 'stream' in kwargs:
1653 stream = kwargs['stream']
1654 if sys.version_info[0] >= 3:
1655 # needs to be a text stream
1656 stream = codecs.getreader('utf-8')(stream)
1657 self.stream = stream
1658 else:
1659 self.stream = _csv_open(kwargs['path'], 'r')
1660 self.reader = csv.reader(self.stream, **self.defaults)
1662 def __iter__(self):
1663 return self
1665 def next(self):
1666 result = next(self.reader)
1667 if sys.version_info[0] < 3:
1668 for i, item in enumerate(result):
1669 if not isinstance(item, text_type):
1670 result[i] = item.decode('utf-8')
1671 return result
1673 __next__ = next
1676class CSVWriter(CSVBase):
1678 def __init__(self, fn, **kwargs):
1679 self.stream = _csv_open(fn, 'w')
1680 self.writer = csv.writer(self.stream, **self.defaults)
1682 def writerow(self, row):
1683 if sys.version_info[0] < 3:
1684 r = []
1685 for item in row:
1686 if isinstance(item, text_type):
1687 item = item.encode('utf-8')
1688 r.append(item)
1689 row = r
1690 self.writer.writerow(row)
1693#
1694# Configurator functionality
1695#
1698class Configurator(BaseConfigurator):
1700 value_converters = dict(BaseConfigurator.value_converters)
1701 value_converters['inc'] = 'inc_convert'
1703 def __init__(self, config, base=None):
1704 super(Configurator, self).__init__(config)
1705 self.base = base or os.getcwd()
1707 def configure_custom(self, config):
1709 def convert(o):
1710 if isinstance(o, (list, tuple)):
1711 result = type(o)([convert(i) for i in o])
1712 elif isinstance(o, dict):
1713 if '()' in o:
1714 result = self.configure_custom(o)
1715 else:
1716 result = {}
1717 for k in o:
1718 result[k] = convert(o[k])
1719 else:
1720 result = self.convert(o)
1721 return result
1723 c = config.pop('()')
1724 if not callable(c):
1725 c = self.resolve(c)
1726 props = config.pop('.', None)
1727 # Check for valid identifiers
1728 args = config.pop('[]', ())
1729 if args:
1730 args = tuple([convert(o) for o in args])
1731 items = [(k, convert(config[k])) for k in config if valid_ident(k)]
1732 kwargs = dict(items)
1733 result = c(*args, **kwargs)
1734 if props:
1735 for n, v in props.items():
1736 setattr(result, n, convert(v))
1737 return result
1739 def __getitem__(self, key):
1740 result = self.config[key]
1741 if isinstance(result, dict) and '()' in result:
1742 self.config[key] = result = self.configure_custom(result)
1743 return result
1745 def inc_convert(self, value):
1746 """Default converter for the inc:// protocol."""
1747 if not os.path.isabs(value):
1748 value = os.path.join(self.base, value)
1749 with codecs.open(value, 'r', encoding='utf-8') as f:
1750 result = json.load(f)
1751 return result
1754class SubprocessMixin(object):
1755 """
1756 Mixin for running subprocesses and capturing their output
1757 """
1759 def __init__(self, verbose=False, progress=None):
1760 self.verbose = verbose
1761 self.progress = progress
1763 def reader(self, stream, context):
1764 """
1765 Read lines from a subprocess' output stream and either pass to a progress
1766 callable (if specified) or write progress information to sys.stderr.
1767 """
1768 progress = self.progress
1769 verbose = self.verbose
1770 while True:
1771 s = stream.readline()
1772 if not s:
1773 break
1774 if progress is not None:
1775 progress(s, context)
1776 else:
1777 if not verbose:
1778 sys.stderr.write('.')
1779 else:
1780 sys.stderr.write(s.decode('utf-8'))
1781 sys.stderr.flush()
1782 stream.close()
1784 def run_command(self, cmd, **kwargs):
1785 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
1786 t1 = threading.Thread(target=self.reader, args=(p.stdout, 'stdout'))
1787 t1.start()
1788 t2 = threading.Thread(target=self.reader, args=(p.stderr, 'stderr'))
1789 t2.start()
1790 p.wait()
1791 t1.join()
1792 t2.join()
1793 if self.progress is not None:
1794 self.progress('done.', 'main')
1795 elif self.verbose:
1796 sys.stderr.write('done.\n')
1797 return p
1800def normalize_name(name):
1801 """Normalize a python package name a la PEP 503"""
1802 # https://peps.python.org/pep-0503/#normalized-names
1803 # Emulates re.sub(r"[-_.]+", "-", name).lower()
1804 # Much faster than re, and even faster than str.translate
1805 value = name.lower().replace("_", "-").replace(".", "-")
1806 while "--" in value:
1807 value = value.replace("--", "-")
1808 return value
1811# def _get_pypirc_command():
1812# """
1813# Get the distutils command for interacting with PyPI configurations.
1814# :return: the command.
1815# """
1816# from distutils.core import Distribution
1817# from distutils.config import PyPIRCCommand
1818# d = Distribution()
1819# return PyPIRCCommand(d)
1822class PyPIRCFile(object):
1824 DEFAULT_REPOSITORY = 'https://upload.pypi.org/legacy/'
1825 DEFAULT_REALM = 'pypi'
1827 def __init__(self, fn=None, url=None):
1828 if fn is None:
1829 fn = os.path.join(os.path.expanduser('~'), '.pypirc')
1830 self.filename = fn
1831 self.url = url
1833 def read(self):
1834 result = {}
1836 if os.path.exists(self.filename):
1837 repository = self.url or self.DEFAULT_REPOSITORY
1839 config = configparser.RawConfigParser()
1840 config.read(self.filename)
1841 sections = config.sections()
1842 if 'distutils' in sections:
1843 # let's get the list of servers
1844 index_servers = config.get('distutils', 'index-servers')
1845 _servers = [server.strip() for server in index_servers.split('\n') if server.strip() != '']
1846 if _servers == []:
1847 # nothing set, let's try to get the default pypi
1848 if 'pypi' in sections:
1849 _servers = ['pypi']
1850 else:
1851 for server in _servers:
1852 result = {'server': server}
1853 result['username'] = config.get(server, 'username')
1855 # optional params
1856 for key, default in (('repository', self.DEFAULT_REPOSITORY), ('realm', self.DEFAULT_REALM),
1857 ('password', None)):
1858 if config.has_option(server, key):
1859 result[key] = config.get(server, key)
1860 else:
1861 result[key] = default
1863 # work around people having "repository" for the "pypi"
1864 # section of their config set to the HTTP (rather than
1865 # HTTPS) URL
1866 if (server == 'pypi' and repository in (self.DEFAULT_REPOSITORY, 'pypi')):
1867 result['repository'] = self.DEFAULT_REPOSITORY
1868 elif (result['server'] != repository and result['repository'] != repository):
1869 result = {}
1870 elif 'server-login' in sections:
1871 # old format
1872 server = 'server-login'
1873 if config.has_option(server, 'repository'):
1874 repository = config.get(server, 'repository')
1875 else:
1876 repository = self.DEFAULT_REPOSITORY
1877 result = {
1878 'username': config.get(server, 'username'),
1879 'password': config.get(server, 'password'),
1880 'repository': repository,
1881 'server': server,
1882 'realm': self.DEFAULT_REALM
1883 }
1884 return result
1886 def update(self, username, password):
1887 # import pdb; pdb.set_trace()
1888 config = configparser.RawConfigParser()
1889 fn = self.filename
1890 config.read(fn)
1891 if not config.has_section('pypi'):
1892 config.add_section('pypi')
1893 config.set('pypi', 'username', username)
1894 config.set('pypi', 'password', password)
1895 with open(fn, 'w') as f:
1896 config.write(f)
1899def _load_pypirc(index):
1900 """
1901 Read the PyPI access configuration as supported by distutils.
1902 """
1903 return PyPIRCFile(url=index.url).read()
1906def _store_pypirc(index):
1907 PyPIRCFile().update(index.username, index.password)
1910#
1911# get_platform()/get_host_platform() copied from Python 3.10.a0 source, with some minor
1912# tweaks
1913#
1916def get_host_platform():
1917 """Return a string that identifies the current platform. This is used mainly to
1918 distinguish platform-specific build directories and platform-specific built
1919 distributions. Typically includes the OS name and version and the
1920 architecture (as supplied by 'os.uname()'), although the exact information
1921 included depends on the OS; eg. on Linux, the kernel version isn't
1922 particularly important.
1924 Examples of returned values:
1925 linux-i586
1926 linux-alpha (?)
1927 solaris-2.6-sun4u
1929 Windows will return one of:
1930 win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
1931 win32 (all others - specifically, sys.platform is returned)
1933 For other non-POSIX platforms, currently just returns 'sys.platform'.
1935 """
1936 if os.name == 'nt':
1937 if 'amd64' in sys.version.lower():
1938 return 'win-amd64'
1939 if '(arm)' in sys.version.lower():
1940 return 'win-arm32'
1941 if '(arm64)' in sys.version.lower():
1942 return 'win-arm64'
1943 return sys.platform
1945 # Set for cross builds explicitly
1946 if "_PYTHON_HOST_PLATFORM" in os.environ:
1947 return os.environ["_PYTHON_HOST_PLATFORM"]
1949 if os.name != 'posix' or not hasattr(os, 'uname'):
1950 # XXX what about the architecture? NT is Intel or Alpha,
1951 # Mac OS is M68k or PPC, etc.
1952 return sys.platform
1954 # Try to distinguish various flavours of Unix
1956 (osname, host, release, version, machine) = os.uname()
1958 # Convert the OS name to lowercase, remove '/' characters, and translate
1959 # spaces (for "Power Macintosh")
1960 osname = osname.lower().replace('/', '')
1961 machine = machine.replace(' ', '_').replace('/', '-')
1963 if osname[:5] == 'linux':
1964 # At least on Linux/Intel, 'machine' is the processor --
1965 # i386, etc.
1966 # XXX what about Alpha, SPARC, etc?
1967 return "%s-%s" % (osname, machine)
1969 elif osname[:5] == 'sunos':
1970 if release[0] >= '5': # SunOS 5 == Solaris 2
1971 osname = 'solaris'
1972 release = '%d.%s' % (int(release[0]) - 3, release[2:])
1973 # We can't use 'platform.architecture()[0]' because a
1974 # bootstrap problem. We use a dict to get an error
1975 # if some suspicious happens.
1976 bitness = {2147483647: '32bit', 9223372036854775807: '64bit'}
1977 machine += '.%s' % bitness[sys.maxsize]
1978 # fall through to standard osname-release-machine representation
1979 elif osname[:3] == 'aix':
1980 from _aix_support import aix_platform
1981 return aix_platform()
1982 elif osname[:6] == 'cygwin':
1983 osname = 'cygwin'
1984 rel_re = re.compile(r'[\d.]+', re.ASCII)
1985 m = rel_re.match(release)
1986 if m:
1987 release = m.group()
1988 elif osname[:6] == 'darwin':
1989 import _osx_support
1990 try:
1991 from distutils import sysconfig
1992 except ImportError:
1993 import sysconfig
1994 osname, release, machine = _osx_support.get_platform_osx(sysconfig.get_config_vars(), osname, release, machine)
1996 return '%s-%s-%s' % (osname, release, machine)
1999_TARGET_TO_PLAT = {
2000 'x86': 'win32',
2001 'x64': 'win-amd64',
2002 'arm': 'win-arm32',
2003}
2006def get_platform():
2007 if os.name != 'nt':
2008 return get_host_platform()
2009 cross_compilation_target = os.environ.get('VSCMD_ARG_TGT_ARCH')
2010 if cross_compilation_target not in _TARGET_TO_PLAT:
2011 return get_host_platform()
2012 return _TARGET_TO_PLAT[cross_compilation_target]
2015def is_in_directory(path, target):
2016 """
2017 Check if a path is inside a target directory. This doesn't check case (might be an issue on Windows)
2018 """
2019 path = os.path.abspath(path)
2020 target = os.path.abspath(target)
2021 return path == target or path.startswith(target + os.sep)