You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

940 lines
29 KiB

4 years ago
  1. from __future__ import absolute_import
  2. import contextlib
  3. import errno
  4. import io
  5. import locale
  6. # we have a submodule named 'logging' which would shadow this if we used the
  7. # regular name:
  8. import logging as std_logging
  9. import os
  10. import posixpath
  11. import re
  12. import shutil
  13. import stat
  14. import subprocess
  15. import sys
  16. import tarfile
  17. import zipfile
  18. from collections import deque
  19. from pip._vendor import pkg_resources
  20. # NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is
  21. # why we ignore the type on this import.
  22. from pip._vendor.retrying import retry # type: ignore
  23. from pip._vendor.six import PY2
  24. from pip._vendor.six.moves import input
  25. from pip._vendor.six.moves.urllib import parse as urllib_parse
  26. from pip._internal.exceptions import CommandError, InstallationError
  27. from pip._internal.locations import (
  28. running_under_virtualenv, site_packages, user_site, virtualenv_no_global,
  29. write_delete_marker_file,
  30. )
  31. from pip._internal.utils.compat import (
  32. WINDOWS, console_to_str, expanduser, stdlib_pkgs,
  33. )
  34. if PY2:
  35. from io import BytesIO as StringIO
  36. else:
  37. from io import StringIO
  38. __all__ = ['rmtree', 'display_path', 'backup_dir',
  39. 'ask', 'splitext',
  40. 'format_size', 'is_installable_dir',
  41. 'is_svn_page', 'file_contents',
  42. 'split_leading_dir', 'has_leading_dir',
  43. 'normalize_path',
  44. 'renames', 'get_prog',
  45. 'unzip_file', 'untar_file', 'unpack_file', 'call_subprocess',
  46. 'captured_stdout', 'ensure_dir',
  47. 'ARCHIVE_EXTENSIONS', 'SUPPORTED_EXTENSIONS',
  48. 'get_installed_version', 'remove_auth_from_url']
  49. logger = std_logging.getLogger(__name__)
  50. BZ2_EXTENSIONS = ('.tar.bz2', '.tbz')
  51. XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', '.tar.lz', '.tar.lzma')
  52. ZIP_EXTENSIONS = ('.zip', '.whl')
  53. TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar')
  54. ARCHIVE_EXTENSIONS = (
  55. ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS)
  56. SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS
  57. try:
  58. import bz2 # noqa
  59. SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS
  60. except ImportError:
  61. logger.debug('bz2 module is not available')
  62. try:
  63. # Only for Python 3.3+
  64. import lzma # noqa
  65. SUPPORTED_EXTENSIONS += XZ_EXTENSIONS
  66. except ImportError:
  67. logger.debug('lzma module is not available')
  68. def import_or_raise(pkg_or_module_string, ExceptionType, *args, **kwargs):
  69. try:
  70. return __import__(pkg_or_module_string)
  71. except ImportError:
  72. raise ExceptionType(*args, **kwargs)
  73. def ensure_dir(path):
  74. """os.path.makedirs without EEXIST."""
  75. try:
  76. os.makedirs(path)
  77. except OSError as e:
  78. if e.errno != errno.EEXIST:
  79. raise
  80. def get_prog():
  81. try:
  82. prog = os.path.basename(sys.argv[0])
  83. if prog in ('__main__.py', '-c'):
  84. return "%s -m pip" % sys.executable
  85. else:
  86. return prog
  87. except (AttributeError, TypeError, IndexError):
  88. pass
  89. return 'pip'
  90. # Retry every half second for up to 3 seconds
  91. @retry(stop_max_delay=3000, wait_fixed=500)
  92. def rmtree(dir, ignore_errors=False):
  93. shutil.rmtree(dir, ignore_errors=ignore_errors,
  94. onerror=rmtree_errorhandler)
  95. def rmtree_errorhandler(func, path, exc_info):
  96. """On Windows, the files in .svn are read-only, so when rmtree() tries to
  97. remove them, an exception is thrown. We catch that here, remove the
  98. read-only attribute, and hopefully continue without problems."""
  99. # if file type currently read only
  100. if os.stat(path).st_mode & stat.S_IREAD:
  101. # convert to read/write
  102. os.chmod(path, stat.S_IWRITE)
  103. # use the original function to repeat the operation
  104. func(path)
  105. return
  106. else:
  107. raise
  108. def display_path(path):
  109. """Gives the display value for a given path, making it relative to cwd
  110. if possible."""
  111. path = os.path.normcase(os.path.abspath(path))
  112. if sys.version_info[0] == 2:
  113. path = path.decode(sys.getfilesystemencoding(), 'replace')
  114. path = path.encode(sys.getdefaultencoding(), 'replace')
  115. if path.startswith(os.getcwd() + os.path.sep):
  116. path = '.' + path[len(os.getcwd()):]
  117. return path
  118. def backup_dir(dir, ext='.bak'):
  119. """Figure out the name of a directory to back up the given dir to
  120. (adding .bak, .bak2, etc)"""
  121. n = 1
  122. extension = ext
  123. while os.path.exists(dir + extension):
  124. n += 1
  125. extension = ext + str(n)
  126. return dir + extension
  127. def ask_path_exists(message, options):
  128. for action in os.environ.get('PIP_EXISTS_ACTION', '').split():
  129. if action in options:
  130. return action
  131. return ask(message, options)
  132. def ask(message, options):
  133. """Ask the message interactively, with the given possible responses"""
  134. while 1:
  135. if os.environ.get('PIP_NO_INPUT'):
  136. raise Exception(
  137. 'No input was expected ($PIP_NO_INPUT set); question: %s' %
  138. message
  139. )
  140. response = input(message)
  141. response = response.strip().lower()
  142. if response not in options:
  143. print(
  144. 'Your response (%r) was not one of the expected responses: '
  145. '%s' % (response, ', '.join(options))
  146. )
  147. else:
  148. return response
  149. def format_size(bytes):
  150. if bytes > 1000 * 1000:
  151. return '%.1fMB' % (bytes / 1000.0 / 1000)
  152. elif bytes > 10 * 1000:
  153. return '%ikB' % (bytes / 1000)
  154. elif bytes > 1000:
  155. return '%.1fkB' % (bytes / 1000.0)
  156. else:
  157. return '%ibytes' % bytes
  158. def is_installable_dir(path):
  159. """Is path is a directory containing setup.py or pyproject.toml?
  160. """
  161. if not os.path.isdir(path):
  162. return False
  163. setup_py = os.path.join(path, 'setup.py')
  164. if os.path.isfile(setup_py):
  165. return True
  166. pyproject_toml = os.path.join(path, 'pyproject.toml')
  167. if os.path.isfile(pyproject_toml):
  168. return True
  169. return False
  170. def is_svn_page(html):
  171. """
  172. Returns true if the page appears to be the index page of an svn repository
  173. """
  174. return (re.search(r'<title>[^<]*Revision \d+:', html) and
  175. re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I))
  176. def file_contents(filename):
  177. with open(filename, 'rb') as fp:
  178. return fp.read().decode('utf-8')
  179. def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE):
  180. """Yield pieces of data from a file-like object until EOF."""
  181. while True:
  182. chunk = file.read(size)
  183. if not chunk:
  184. break
  185. yield chunk
  186. def split_leading_dir(path):
  187. path = path.lstrip('/').lstrip('\\')
  188. if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) or
  189. '\\' not in path):
  190. return path.split('/', 1)
  191. elif '\\' in path:
  192. return path.split('\\', 1)
  193. else:
  194. return path, ''
  195. def has_leading_dir(paths):
  196. """Returns true if all the paths have the same leading path name
  197. (i.e., everything is in one subdirectory in an archive)"""
  198. common_prefix = None
  199. for path in paths:
  200. prefix, rest = split_leading_dir(path)
  201. if not prefix:
  202. return False
  203. elif common_prefix is None:
  204. common_prefix = prefix
  205. elif prefix != common_prefix:
  206. return False
  207. return True
  208. def normalize_path(path, resolve_symlinks=True):
  209. """
  210. Convert a path to its canonical, case-normalized, absolute version.
  211. """
  212. path = expanduser(path)
  213. if resolve_symlinks:
  214. path = os.path.realpath(path)
  215. else:
  216. path = os.path.abspath(path)
  217. return os.path.normcase(path)
  218. def splitext(path):
  219. """Like os.path.splitext, but take off .tar too"""
  220. base, ext = posixpath.splitext(path)
  221. if base.lower().endswith('.tar'):
  222. ext = base[-4:] + ext
  223. base = base[:-4]
  224. return base, ext
  225. def renames(old, new):
  226. """Like os.renames(), but handles renaming across devices."""
  227. # Implementation borrowed from os.renames().
  228. head, tail = os.path.split(new)
  229. if head and tail and not os.path.exists(head):
  230. os.makedirs(head)
  231. shutil.move(old, new)
  232. head, tail = os.path.split(old)
  233. if head and tail:
  234. try:
  235. os.removedirs(head)
  236. except OSError:
  237. pass
  238. def is_local(path):
  239. """
  240. Return True if path is within sys.prefix, if we're running in a virtualenv.
  241. If we're not in a virtualenv, all paths are considered "local."
  242. """
  243. if not running_under_virtualenv():
  244. return True
  245. return normalize_path(path).startswith(normalize_path(sys.prefix))
  246. def dist_is_local(dist):
  247. """
  248. Return True if given Distribution object is installed locally
  249. (i.e. within current virtualenv).
  250. Always True if we're not in a virtualenv.
  251. """
  252. return is_local(dist_location(dist))
  253. def dist_in_usersite(dist):
  254. """
  255. Return True if given Distribution is installed in user site.
  256. """
  257. norm_path = normalize_path(dist_location(dist))
  258. return norm_path.startswith(normalize_path(user_site))
  259. def dist_in_site_packages(dist):
  260. """
  261. Return True if given Distribution is installed in
  262. sysconfig.get_python_lib().
  263. """
  264. return normalize_path(
  265. dist_location(dist)
  266. ).startswith(normalize_path(site_packages))
  267. def dist_is_editable(dist):
  268. """Is distribution an editable install?"""
  269. for path_item in sys.path:
  270. egg_link = os.path.join(path_item, dist.project_name + '.egg-link')
  271. if os.path.isfile(egg_link):
  272. return True
  273. return False
  274. def get_installed_distributions(local_only=True,
  275. skip=stdlib_pkgs,
  276. include_editables=True,
  277. editables_only=False,
  278. user_only=False):
  279. """
  280. Return a list of installed Distribution objects.
  281. If ``local_only`` is True (default), only return installations
  282. local to the current virtualenv, if in a virtualenv.
  283. ``skip`` argument is an iterable of lower-case project names to
  284. ignore; defaults to stdlib_pkgs
  285. If ``include_editables`` is False, don't report editables.
  286. If ``editables_only`` is True , only report editables.
  287. If ``user_only`` is True , only report installations in the user
  288. site directory.
  289. """
  290. if local_only:
  291. local_test = dist_is_local
  292. else:
  293. def local_test(d):
  294. return True
  295. if include_editables:
  296. def editable_test(d):
  297. return True
  298. else:
  299. def editable_test(d):
  300. return not dist_is_editable(d)
  301. if editables_only:
  302. def editables_only_test(d):
  303. return dist_is_editable(d)
  304. else:
  305. def editables_only_test(d):
  306. return True
  307. if user_only:
  308. user_test = dist_in_usersite
  309. else:
  310. def user_test(d):
  311. return True
  312. return [d for d in pkg_resources.working_set
  313. if local_test(d) and
  314. d.key not in skip and
  315. editable_test(d) and
  316. editables_only_test(d) and
  317. user_test(d)
  318. ]
  319. def egg_link_path(dist):
  320. """
  321. Return the path for the .egg-link file if it exists, otherwise, None.
  322. There's 3 scenarios:
  323. 1) not in a virtualenv
  324. try to find in site.USER_SITE, then site_packages
  325. 2) in a no-global virtualenv
  326. try to find in site_packages
  327. 3) in a yes-global virtualenv
  328. try to find in site_packages, then site.USER_SITE
  329. (don't look in global location)
  330. For #1 and #3, there could be odd cases, where there's an egg-link in 2
  331. locations.
  332. This method will just return the first one found.
  333. """
  334. sites = []
  335. if running_under_virtualenv():
  336. if virtualenv_no_global():
  337. sites.append(site_packages)
  338. else:
  339. sites.append(site_packages)
  340. if user_site:
  341. sites.append(user_site)
  342. else:
  343. if user_site:
  344. sites.append(user_site)
  345. sites.append(site_packages)
  346. for site in sites:
  347. egglink = os.path.join(site, dist.project_name) + '.egg-link'
  348. if os.path.isfile(egglink):
  349. return egglink
  350. def dist_location(dist):
  351. """
  352. Get the site-packages location of this distribution. Generally
  353. this is dist.location, except in the case of develop-installed
  354. packages, where dist.location is the source code location, and we
  355. want to know where the egg-link file is.
  356. """
  357. egg_link = egg_link_path(dist)
  358. if egg_link:
  359. return egg_link
  360. return dist.location
  361. def current_umask():
  362. """Get the current umask which involves having to set it temporarily."""
  363. mask = os.umask(0)
  364. os.umask(mask)
  365. return mask
  366. def unzip_file(filename, location, flatten=True):
  367. """
  368. Unzip the file (with path `filename`) to the destination `location`. All
  369. files are written based on system defaults and umask (i.e. permissions are
  370. not preserved), except that regular file members with any execute
  371. permissions (user, group, or world) have "chmod +x" applied after being
  372. written. Note that for windows, any execute changes using os.chmod are
  373. no-ops per the python docs.
  374. """
  375. ensure_dir(location)
  376. zipfp = open(filename, 'rb')
  377. try:
  378. zip = zipfile.ZipFile(zipfp, allowZip64=True)
  379. leading = has_leading_dir(zip.namelist()) and flatten
  380. for info in zip.infolist():
  381. name = info.filename
  382. data = zip.read(name)
  383. fn = name
  384. if leading:
  385. fn = split_leading_dir(name)[1]
  386. fn = os.path.join(location, fn)
  387. dir = os.path.dirname(fn)
  388. if fn.endswith('/') or fn.endswith('\\'):
  389. # A directory
  390. ensure_dir(fn)
  391. else:
  392. ensure_dir(dir)
  393. fp = open(fn, 'wb')
  394. try:
  395. fp.write(data)
  396. finally:
  397. fp.close()
  398. mode = info.external_attr >> 16
  399. # if mode and regular file and any execute permissions for
  400. # user/group/world?
  401. if mode and stat.S_ISREG(mode) and mode & 0o111:
  402. # make dest file have execute for user/group/world
  403. # (chmod +x) no-op on windows per python docs
  404. os.chmod(fn, (0o777 - current_umask() | 0o111))
  405. finally:
  406. zipfp.close()
  407. def untar_file(filename, location):
  408. """
  409. Untar the file (with path `filename`) to the destination `location`.
  410. All files are written based on system defaults and umask (i.e. permissions
  411. are not preserved), except that regular file members with any execute
  412. permissions (user, group, or world) have "chmod +x" applied after being
  413. written. Note that for windows, any execute changes using os.chmod are
  414. no-ops per the python docs.
  415. """
  416. ensure_dir(location)
  417. if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'):
  418. mode = 'r:gz'
  419. elif filename.lower().endswith(BZ2_EXTENSIONS):
  420. mode = 'r:bz2'
  421. elif filename.lower().endswith(XZ_EXTENSIONS):
  422. mode = 'r:xz'
  423. elif filename.lower().endswith('.tar'):
  424. mode = 'r'
  425. else:
  426. logger.warning(
  427. 'Cannot determine compression type for file %s', filename,
  428. )
  429. mode = 'r:*'
  430. tar = tarfile.open(filename, mode)
  431. try:
  432. # note: python<=2.5 doesn't seem to know about pax headers, filter them
  433. leading = has_leading_dir([
  434. member.name for member in tar.getmembers()
  435. if member.name != 'pax_global_header'
  436. ])
  437. for member in tar.getmembers():
  438. fn = member.name
  439. if fn == 'pax_global_header':
  440. continue
  441. if leading:
  442. fn = split_leading_dir(fn)[1]
  443. path = os.path.join(location, fn)
  444. if member.isdir():
  445. ensure_dir(path)
  446. elif member.issym():
  447. try:
  448. tar._extract_member(member, path)
  449. except Exception as exc:
  450. # Some corrupt tar files seem to produce this
  451. # (specifically bad symlinks)
  452. logger.warning(
  453. 'In the tar file %s the member %s is invalid: %s',
  454. filename, member.name, exc,
  455. )
  456. continue
  457. else:
  458. try:
  459. fp = tar.extractfile(member)
  460. except (KeyError, AttributeError) as exc:
  461. # Some corrupt tar files seem to produce this
  462. # (specifically bad symlinks)
  463. logger.warning(
  464. 'In the tar file %s the member %s is invalid: %s',
  465. filename, member.name, exc,
  466. )
  467. continue
  468. ensure_dir(os.path.dirname(path))
  469. with open(path, 'wb') as destfp:
  470. shutil.copyfileobj(fp, destfp)
  471. fp.close()
  472. # Update the timestamp (useful for cython compiled files)
  473. tar.utime(member, path)
  474. # member have any execute permissions for user/group/world?
  475. if member.mode & 0o111:
  476. # make dest file have execute for user/group/world
  477. # no-op on windows per python docs
  478. os.chmod(path, (0o777 - current_umask() | 0o111))
  479. finally:
  480. tar.close()
  481. def unpack_file(filename, location, content_type, link):
  482. filename = os.path.realpath(filename)
  483. if (content_type == 'application/zip' or
  484. filename.lower().endswith(ZIP_EXTENSIONS) or
  485. zipfile.is_zipfile(filename)):
  486. unzip_file(
  487. filename,
  488. location,
  489. flatten=not filename.endswith('.whl')
  490. )
  491. elif (content_type == 'application/x-gzip' or
  492. tarfile.is_tarfile(filename) or
  493. filename.lower().endswith(
  494. TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS)):
  495. untar_file(filename, location)
  496. elif (content_type and content_type.startswith('text/html') and
  497. is_svn_page(file_contents(filename))):
  498. # We don't really care about this
  499. from pip._internal.vcs.subversion import Subversion
  500. Subversion('svn+' + link.url).unpack(location)
  501. else:
  502. # FIXME: handle?
  503. # FIXME: magic signatures?
  504. logger.critical(
  505. 'Cannot unpack file %s (downloaded from %s, content-type: %s); '
  506. 'cannot detect archive format',
  507. filename, location, content_type,
  508. )
  509. raise InstallationError(
  510. 'Cannot determine archive format of %s' % location
  511. )
  512. def call_subprocess(cmd, show_stdout=True, cwd=None,
  513. on_returncode='raise',
  514. command_desc=None,
  515. extra_environ=None, unset_environ=None, spinner=None):
  516. """
  517. Args:
  518. unset_environ: an iterable of environment variable names to unset
  519. prior to calling subprocess.Popen().
  520. """
  521. if unset_environ is None:
  522. unset_environ = []
  523. # This function's handling of subprocess output is confusing and I
  524. # previously broke it terribly, so as penance I will write a long comment
  525. # explaining things.
  526. #
  527. # The obvious thing that affects output is the show_stdout=
  528. # kwarg. show_stdout=True means, let the subprocess write directly to our
  529. # stdout. Even though it is nominally the default, it is almost never used
  530. # inside pip (and should not be used in new code without a very good
  531. # reason); as of 2016-02-22 it is only used in a few places inside the VCS
  532. # wrapper code. Ideally we should get rid of it entirely, because it
  533. # creates a lot of complexity here for a rarely used feature.
  534. #
  535. # Most places in pip set show_stdout=False. What this means is:
  536. # - We connect the child stdout to a pipe, which we read.
  537. # - By default, we hide the output but show a spinner -- unless the
  538. # subprocess exits with an error, in which case we show the output.
  539. # - If the --verbose option was passed (= loglevel is DEBUG), then we show
  540. # the output unconditionally. (But in this case we don't want to show
  541. # the output a second time if it turns out that there was an error.)
  542. #
  543. # stderr is always merged with stdout (even if show_stdout=True).
  544. if show_stdout:
  545. stdout = None
  546. else:
  547. stdout = subprocess.PIPE
  548. if command_desc is None:
  549. cmd_parts = []
  550. for part in cmd:
  551. if ' ' in part or '\n' in part or '"' in part or "'" in part:
  552. part = '"%s"' % part.replace('"', '\\"')
  553. cmd_parts.append(part)
  554. command_desc = ' '.join(cmd_parts)
  555. logger.debug("Running command %s", command_desc)
  556. env = os.environ.copy()
  557. if extra_environ:
  558. env.update(extra_environ)
  559. for name in unset_environ:
  560. env.pop(name, None)
  561. try:
  562. proc = subprocess.Popen(
  563. cmd, stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
  564. stdout=stdout, cwd=cwd, env=env,
  565. )
  566. proc.stdin.close()
  567. except Exception as exc:
  568. logger.critical(
  569. "Error %s while executing command %s", exc, command_desc,
  570. )
  571. raise
  572. all_output = []
  573. if stdout is not None:
  574. while True:
  575. line = console_to_str(proc.stdout.readline())
  576. if not line:
  577. break
  578. line = line.rstrip()
  579. all_output.append(line + '\n')
  580. if logger.getEffectiveLevel() <= std_logging.DEBUG:
  581. # Show the line immediately
  582. logger.debug(line)
  583. else:
  584. # Update the spinner
  585. if spinner is not None:
  586. spinner.spin()
  587. try:
  588. proc.wait()
  589. finally:
  590. if proc.stdout:
  591. proc.stdout.close()
  592. if spinner is not None:
  593. if proc.returncode:
  594. spinner.finish("error")
  595. else:
  596. spinner.finish("done")
  597. if proc.returncode:
  598. if on_returncode == 'raise':
  599. if (logger.getEffectiveLevel() > std_logging.DEBUG and
  600. not show_stdout):
  601. logger.info(
  602. 'Complete output from command %s:', command_desc,
  603. )
  604. logger.info(
  605. ''.join(all_output) +
  606. '\n----------------------------------------'
  607. )
  608. raise InstallationError(
  609. 'Command "%s" failed with error code %s in %s'
  610. % (command_desc, proc.returncode, cwd))
  611. elif on_returncode == 'warn':
  612. logger.warning(
  613. 'Command "%s" had error code %s in %s',
  614. command_desc, proc.returncode, cwd,
  615. )
  616. elif on_returncode == 'ignore':
  617. pass
  618. else:
  619. raise ValueError('Invalid value: on_returncode=%s' %
  620. repr(on_returncode))
  621. if not show_stdout:
  622. return ''.join(all_output)
  623. def read_text_file(filename):
  624. """Return the contents of *filename*.
  625. Try to decode the file contents with utf-8, the preferred system encoding
  626. (e.g., cp1252 on some Windows machines), and latin1, in that order.
  627. Decoding a byte string with latin1 will never raise an error. In the worst
  628. case, the returned string will contain some garbage characters.
  629. """
  630. with open(filename, 'rb') as fp:
  631. data = fp.read()
  632. encodings = ['utf-8', locale.getpreferredencoding(False), 'latin1']
  633. for enc in encodings:
  634. try:
  635. data = data.decode(enc)
  636. except UnicodeDecodeError:
  637. continue
  638. break
  639. assert type(data) != bytes # Latin1 should have worked.
  640. return data
  641. def _make_build_dir(build_dir):
  642. os.makedirs(build_dir)
  643. write_delete_marker_file(build_dir)
  644. class FakeFile(object):
  645. """Wrap a list of lines in an object with readline() to make
  646. ConfigParser happy."""
  647. def __init__(self, lines):
  648. self._gen = (l for l in lines)
  649. def readline(self):
  650. try:
  651. try:
  652. return next(self._gen)
  653. except NameError:
  654. return self._gen.next()
  655. except StopIteration:
  656. return ''
  657. def __iter__(self):
  658. return self._gen
  659. class StreamWrapper(StringIO):
  660. @classmethod
  661. def from_stream(cls, orig_stream):
  662. cls.orig_stream = orig_stream
  663. return cls()
  664. # compileall.compile_dir() needs stdout.encoding to print to stdout
  665. @property
  666. def encoding(self):
  667. return self.orig_stream.encoding
  668. @contextlib.contextmanager
  669. def captured_output(stream_name):
  670. """Return a context manager used by captured_stdout/stdin/stderr
  671. that temporarily replaces the sys stream *stream_name* with a StringIO.
  672. Taken from Lib/support/__init__.py in the CPython repo.
  673. """
  674. orig_stdout = getattr(sys, stream_name)
  675. setattr(sys, stream_name, StreamWrapper.from_stream(orig_stdout))
  676. try:
  677. yield getattr(sys, stream_name)
  678. finally:
  679. setattr(sys, stream_name, orig_stdout)
  680. def captured_stdout():
  681. """Capture the output of sys.stdout:
  682. with captured_stdout() as stdout:
  683. print('hello')
  684. self.assertEqual(stdout.getvalue(), 'hello\n')
  685. Taken from Lib/support/__init__.py in the CPython repo.
  686. """
  687. return captured_output('stdout')
  688. class cached_property(object):
  689. """A property that is only computed once per instance and then replaces
  690. itself with an ordinary attribute. Deleting the attribute resets the
  691. property.
  692. Source: https://github.com/bottlepy/bottle/blob/0.11.5/bottle.py#L175
  693. """
  694. def __init__(self, func):
  695. self.__doc__ = getattr(func, '__doc__')
  696. self.func = func
  697. def __get__(self, obj, cls):
  698. if obj is None:
  699. # We're being accessed from the class itself, not from an object
  700. return self
  701. value = obj.__dict__[self.func.__name__] = self.func(obj)
  702. return value
  703. def get_installed_version(dist_name, working_set=None):
  704. """Get the installed version of dist_name avoiding pkg_resources cache"""
  705. # Create a requirement that we'll look for inside of setuptools.
  706. req = pkg_resources.Requirement.parse(dist_name)
  707. if working_set is None:
  708. # We want to avoid having this cached, so we need to construct a new
  709. # working set each time.
  710. working_set = pkg_resources.WorkingSet()
  711. # Get the installed distribution from our working set
  712. dist = working_set.find(req)
  713. # Check to see if we got an installed distribution or not, if we did
  714. # we want to return it's version.
  715. return dist.version if dist else None
  716. def consume(iterator):
  717. """Consume an iterable at C speed."""
  718. deque(iterator, maxlen=0)
  719. # Simulates an enum
  720. def enum(*sequential, **named):
  721. enums = dict(zip(sequential, range(len(sequential))), **named)
  722. reverse = {value: key for key, value in enums.items()}
  723. enums['reverse_mapping'] = reverse
  724. return type('Enum', (), enums)
  725. def make_vcs_requirement_url(repo_url, rev, egg_project_name, subdir=None):
  726. """
  727. Return the URL for a VCS requirement.
  728. Args:
  729. repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+").
  730. """
  731. req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name)
  732. if subdir:
  733. req += '&subdirectory={}'.format(subdir)
  734. return req
  735. def split_auth_from_netloc(netloc):
  736. """
  737. Parse out and remove the auth information from a netloc.
  738. Returns: (netloc, (username, password)).
  739. """
  740. if '@' not in netloc:
  741. return netloc, (None, None)
  742. # Split from the right because that's how urllib.parse.urlsplit()
  743. # behaves if more than one @ is present (which can be checked using
  744. # the password attribute of urlsplit()'s return value).
  745. auth, netloc = netloc.rsplit('@', 1)
  746. if ':' in auth:
  747. # Split from the left because that's how urllib.parse.urlsplit()
  748. # behaves if more than one : is present (which again can be checked
  749. # using the password attribute of the return value)
  750. user_pass = tuple(auth.split(':', 1))
  751. else:
  752. user_pass = auth, None
  753. return netloc, user_pass
  754. def remove_auth_from_url(url):
  755. # Return a copy of url with 'username:password@' removed.
  756. # username/pass params are passed to subversion through flags
  757. # and are not recognized in the url.
  758. # parsed url
  759. purl = urllib_parse.urlsplit(url)
  760. netloc, user_pass = split_auth_from_netloc(purl.netloc)
  761. # stripped url
  762. url_pieces = (
  763. purl.scheme, netloc, purl.path, purl.query, purl.fragment
  764. )
  765. surl = urllib_parse.urlunsplit(url_pieces)
  766. return surl
  767. def protect_pip_from_modification_on_windows(modifying_pip):
  768. """Protection of pip.exe from modification on Windows
  769. On Windows, any operation modifying pip should be run as:
  770. python -m pip ...
  771. """
  772. pip_names = [
  773. "pip.exe",
  774. "pip{}.exe".format(sys.version_info[0]),
  775. "pip{}.{}.exe".format(*sys.version_info[:2])
  776. ]
  777. # See https://github.com/pypa/pip/issues/1299 for more discussion
  778. should_show_use_python_msg = (
  779. modifying_pip and
  780. WINDOWS and
  781. os.path.basename(sys.argv[0]) in pip_names
  782. )
  783. if should_show_use_python_msg:
  784. new_command = [
  785. sys.executable, "-m", "pip"
  786. ] + sys.argv[1:]
  787. raise CommandError(
  788. 'To modify pip, please run the following command:\n{}'
  789. .format(" ".join(new_command))
  790. )