You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

921 lines
32 KiB

4 years ago
  1. from __future__ import absolute_import
  2. import cgi
  3. import email.utils
  4. import getpass
  5. import json
  6. import logging
  7. import mimetypes
  8. import os
  9. import platform
  10. import re
  11. import shutil
  12. import sys
  13. from pip._vendor import requests, six, urllib3
  14. from pip._vendor.cachecontrol import CacheControlAdapter
  15. from pip._vendor.cachecontrol.caches import FileCache
  16. from pip._vendor.lockfile import LockError
  17. from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
  18. from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
  19. from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
  20. from pip._vendor.requests.structures import CaseInsensitiveDict
  21. from pip._vendor.requests.utils import get_netrc_auth
  22. # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is
  23. # why we ignore the type on this import
  24. from pip._vendor.six.moves import xmlrpc_client # type: ignore
  25. from pip._vendor.six.moves.urllib import parse as urllib_parse
  26. from pip._vendor.six.moves.urllib import request as urllib_request
  27. from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote
  28. from pip._vendor.urllib3.util import IS_PYOPENSSL
  29. import pip
  30. from pip._internal.exceptions import HashMismatch, InstallationError
  31. from pip._internal.locations import write_delete_marker_file
  32. from pip._internal.models.index import PyPI
  33. from pip._internal.utils.encoding import auto_decode
  34. from pip._internal.utils.filesystem import check_path_owner
  35. from pip._internal.utils.glibc import libc_ver
  36. from pip._internal.utils.logging import indent_log
  37. from pip._internal.utils.misc import (
  38. ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, call_subprocess, consume,
  39. display_path, format_size, get_installed_version, rmtree, splitext,
  40. unpack_file,
  41. )
  42. from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM
  43. from pip._internal.utils.temp_dir import TempDirectory
  44. from pip._internal.utils.ui import DownloadProgressProvider
  45. from pip._internal.vcs import vcs
  46. try:
  47. import ssl # noqa
  48. except ImportError:
  49. ssl = None
  50. HAS_TLS = (ssl is not None) or IS_PYOPENSSL
  51. __all__ = ['get_file_content',
  52. 'is_url', 'url_to_path', 'path_to_url',
  53. 'is_archive_file', 'unpack_vcs_link',
  54. 'unpack_file_url', 'is_vcs_url', 'is_file_url',
  55. 'unpack_http_url', 'unpack_url']
  56. logger = logging.getLogger(__name__)
  57. def user_agent():
  58. """
  59. Return a string representing the user agent.
  60. """
  61. data = {
  62. "installer": {"name": "pip", "version": pip.__version__},
  63. "python": platform.python_version(),
  64. "implementation": {
  65. "name": platform.python_implementation(),
  66. },
  67. }
  68. if data["implementation"]["name"] == 'CPython':
  69. data["implementation"]["version"] = platform.python_version()
  70. elif data["implementation"]["name"] == 'PyPy':
  71. if sys.pypy_version_info.releaselevel == 'final':
  72. pypy_version_info = sys.pypy_version_info[:3]
  73. else:
  74. pypy_version_info = sys.pypy_version_info
  75. data["implementation"]["version"] = ".".join(
  76. [str(x) for x in pypy_version_info]
  77. )
  78. elif data["implementation"]["name"] == 'Jython':
  79. # Complete Guess
  80. data["implementation"]["version"] = platform.python_version()
  81. elif data["implementation"]["name"] == 'IronPython':
  82. # Complete Guess
  83. data["implementation"]["version"] = platform.python_version()
  84. if sys.platform.startswith("linux"):
  85. from pip._vendor import distro
  86. distro_infos = dict(filter(
  87. lambda x: x[1],
  88. zip(["name", "version", "id"], distro.linux_distribution()),
  89. ))
  90. libc = dict(filter(
  91. lambda x: x[1],
  92. zip(["lib", "version"], libc_ver()),
  93. ))
  94. if libc:
  95. distro_infos["libc"] = libc
  96. if distro_infos:
  97. data["distro"] = distro_infos
  98. if sys.platform.startswith("darwin") and platform.mac_ver()[0]:
  99. data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]}
  100. if platform.system():
  101. data.setdefault("system", {})["name"] = platform.system()
  102. if platform.release():
  103. data.setdefault("system", {})["release"] = platform.release()
  104. if platform.machine():
  105. data["cpu"] = platform.machine()
  106. if HAS_TLS:
  107. data["openssl_version"] = ssl.OPENSSL_VERSION
  108. setuptools_version = get_installed_version("setuptools")
  109. if setuptools_version is not None:
  110. data["setuptools_version"] = setuptools_version
  111. return "{data[installer][name]}/{data[installer][version]} {json}".format(
  112. data=data,
  113. json=json.dumps(data, separators=(",", ":"), sort_keys=True),
  114. )
  115. class MultiDomainBasicAuth(AuthBase):
  116. def __init__(self, prompting=True):
  117. self.prompting = prompting
  118. self.passwords = {}
  119. def __call__(self, req):
  120. parsed = urllib_parse.urlparse(req.url)
  121. # Get the netloc without any embedded credentials
  122. netloc = parsed.netloc.rsplit("@", 1)[-1]
  123. # Set the url of the request to the url without any credentials
  124. req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:])
  125. # Use any stored credentials that we have for this netloc
  126. username, password = self.passwords.get(netloc, (None, None))
  127. # Extract credentials embedded in the url if we have none stored
  128. if username is None:
  129. username, password = self.parse_credentials(parsed.netloc)
  130. # Get creds from netrc if we still don't have them
  131. if username is None and password is None:
  132. netrc_auth = get_netrc_auth(req.url)
  133. username, password = netrc_auth if netrc_auth else (None, None)
  134. if username or password:
  135. # Store the username and password
  136. self.passwords[netloc] = (username, password)
  137. # Send the basic auth with this request
  138. req = HTTPBasicAuth(username or "", password or "")(req)
  139. # Attach a hook to handle 401 responses
  140. req.register_hook("response", self.handle_401)
  141. return req
  142. def handle_401(self, resp, **kwargs):
  143. # We only care about 401 responses, anything else we want to just
  144. # pass through the actual response
  145. if resp.status_code != 401:
  146. return resp
  147. # We are not able to prompt the user so simply return the response
  148. if not self.prompting:
  149. return resp
  150. parsed = urllib_parse.urlparse(resp.url)
  151. # Prompt the user for a new username and password
  152. username = six.moves.input("User for %s: " % parsed.netloc)
  153. password = getpass.getpass("Password: ")
  154. # Store the new username and password to use for future requests
  155. if username or password:
  156. self.passwords[parsed.netloc] = (username, password)
  157. # Consume content and release the original connection to allow our new
  158. # request to reuse the same one.
  159. resp.content
  160. resp.raw.release_conn()
  161. # Add our new username and password to the request
  162. req = HTTPBasicAuth(username or "", password or "")(resp.request)
  163. # Send our new request
  164. new_resp = resp.connection.send(req, **kwargs)
  165. new_resp.history.append(resp)
  166. return new_resp
  167. def parse_credentials(self, netloc):
  168. if "@" in netloc:
  169. userinfo = netloc.rsplit("@", 1)[0]
  170. if ":" in userinfo:
  171. user, pwd = userinfo.split(":", 1)
  172. return (urllib_unquote(user), urllib_unquote(pwd))
  173. return urllib_unquote(userinfo), None
  174. return None, None
  175. class LocalFSAdapter(BaseAdapter):
  176. def send(self, request, stream=None, timeout=None, verify=None, cert=None,
  177. proxies=None):
  178. pathname = url_to_path(request.url)
  179. resp = Response()
  180. resp.status_code = 200
  181. resp.url = request.url
  182. try:
  183. stats = os.stat(pathname)
  184. except OSError as exc:
  185. resp.status_code = 404
  186. resp.raw = exc
  187. else:
  188. modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
  189. content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
  190. resp.headers = CaseInsensitiveDict({
  191. "Content-Type": content_type,
  192. "Content-Length": stats.st_size,
  193. "Last-Modified": modified,
  194. })
  195. resp.raw = open(pathname, "rb")
  196. resp.close = resp.raw.close
  197. return resp
  198. def close(self):
  199. pass
  200. class SafeFileCache(FileCache):
  201. """
  202. A file based cache which is safe to use even when the target directory may
  203. not be accessible or writable.
  204. """
  205. def __init__(self, *args, **kwargs):
  206. super(SafeFileCache, self).__init__(*args, **kwargs)
  207. # Check to ensure that the directory containing our cache directory
  208. # is owned by the user current executing pip. If it does not exist
  209. # we will check the parent directory until we find one that does exist.
  210. # If it is not owned by the user executing pip then we will disable
  211. # the cache and log a warning.
  212. if not check_path_owner(self.directory):
  213. logger.warning(
  214. "The directory '%s' or its parent directory is not owned by "
  215. "the current user and the cache has been disabled. Please "
  216. "check the permissions and owner of that directory. If "
  217. "executing pip with sudo, you may want sudo's -H flag.",
  218. self.directory,
  219. )
  220. # Set our directory to None to disable the Cache
  221. self.directory = None
  222. def get(self, *args, **kwargs):
  223. # If we don't have a directory, then the cache should be a no-op.
  224. if self.directory is None:
  225. return
  226. try:
  227. return super(SafeFileCache, self).get(*args, **kwargs)
  228. except (LockError, OSError, IOError):
  229. # We intentionally silence this error, if we can't access the cache
  230. # then we can just skip caching and process the request as if
  231. # caching wasn't enabled.
  232. pass
  233. def set(self, *args, **kwargs):
  234. # If we don't have a directory, then the cache should be a no-op.
  235. if self.directory is None:
  236. return
  237. try:
  238. return super(SafeFileCache, self).set(*args, **kwargs)
  239. except (LockError, OSError, IOError):
  240. # We intentionally silence this error, if we can't access the cache
  241. # then we can just skip caching and process the request as if
  242. # caching wasn't enabled.
  243. pass
  244. def delete(self, *args, **kwargs):
  245. # If we don't have a directory, then the cache should be a no-op.
  246. if self.directory is None:
  247. return
  248. try:
  249. return super(SafeFileCache, self).delete(*args, **kwargs)
  250. except (LockError, OSError, IOError):
  251. # We intentionally silence this error, if we can't access the cache
  252. # then we can just skip caching and process the request as if
  253. # caching wasn't enabled.
  254. pass
  255. class InsecureHTTPAdapter(HTTPAdapter):
  256. def cert_verify(self, conn, url, verify, cert):
  257. conn.cert_reqs = 'CERT_NONE'
  258. conn.ca_certs = None
  259. class PipSession(requests.Session):
  260. timeout = None
  261. def __init__(self, *args, **kwargs):
  262. retries = kwargs.pop("retries", 0)
  263. cache = kwargs.pop("cache", None)
  264. insecure_hosts = kwargs.pop("insecure_hosts", [])
  265. super(PipSession, self).__init__(*args, **kwargs)
  266. # Attach our User Agent to the request
  267. self.headers["User-Agent"] = user_agent()
  268. # Attach our Authentication handler to the session
  269. self.auth = MultiDomainBasicAuth()
  270. # Create our urllib3.Retry instance which will allow us to customize
  271. # how we handle retries.
  272. retries = urllib3.Retry(
  273. # Set the total number of retries that a particular request can
  274. # have.
  275. total=retries,
  276. # A 503 error from PyPI typically means that the Fastly -> Origin
  277. # connection got interrupted in some way. A 503 error in general
  278. # is typically considered a transient error so we'll go ahead and
  279. # retry it.
  280. # A 500 may indicate transient error in Amazon S3
  281. # A 520 or 527 - may indicate transient error in CloudFlare
  282. status_forcelist=[500, 503, 520, 527],
  283. # Add a small amount of back off between failed requests in
  284. # order to prevent hammering the service.
  285. backoff_factor=0.25,
  286. )
  287. # We want to _only_ cache responses on securely fetched origins. We do
  288. # this because we can't validate the response of an insecurely fetched
  289. # origin, and we don't want someone to be able to poison the cache and
  290. # require manual eviction from the cache to fix it.
  291. if cache:
  292. secure_adapter = CacheControlAdapter(
  293. cache=SafeFileCache(cache, use_dir_lock=True),
  294. max_retries=retries,
  295. )
  296. else:
  297. secure_adapter = HTTPAdapter(max_retries=retries)
  298. # Our Insecure HTTPAdapter disables HTTPS validation. It does not
  299. # support caching (see above) so we'll use it for all http:// URLs as
  300. # well as any https:// host that we've marked as ignoring TLS errors
  301. # for.
  302. insecure_adapter = InsecureHTTPAdapter(max_retries=retries)
  303. self.mount("https://", secure_adapter)
  304. self.mount("http://", insecure_adapter)
  305. # Enable file:// urls
  306. self.mount("file://", LocalFSAdapter())
  307. # We want to use a non-validating adapter for any requests which are
  308. # deemed insecure.
  309. for host in insecure_hosts:
  310. self.mount("https://{}/".format(host), insecure_adapter)
  311. def request(self, method, url, *args, **kwargs):
  312. # Allow setting a default timeout on a session
  313. kwargs.setdefault("timeout", self.timeout)
  314. # Dispatch the actual request
  315. return super(PipSession, self).request(method, url, *args, **kwargs)
  316. def get_file_content(url, comes_from=None, session=None):
  317. """Gets the content of a file; it may be a filename, file: URL, or
  318. http: URL. Returns (location, content). Content is unicode.
  319. :param url: File path or url.
  320. :param comes_from: Origin description of requirements.
  321. :param session: Instance of pip.download.PipSession.
  322. """
  323. if session is None:
  324. raise TypeError(
  325. "get_file_content() missing 1 required keyword argument: 'session'"
  326. )
  327. match = _scheme_re.search(url)
  328. if match:
  329. scheme = match.group(1).lower()
  330. if (scheme == 'file' and comes_from and
  331. comes_from.startswith('http')):
  332. raise InstallationError(
  333. 'Requirements file %s references URL %s, which is local'
  334. % (comes_from, url))
  335. if scheme == 'file':
  336. path = url.split(':', 1)[1]
  337. path = path.replace('\\', '/')
  338. match = _url_slash_drive_re.match(path)
  339. if match:
  340. path = match.group(1) + ':' + path.split('|', 1)[1]
  341. path = urllib_parse.unquote(path)
  342. if path.startswith('/'):
  343. path = '/' + path.lstrip('/')
  344. url = path
  345. else:
  346. # FIXME: catch some errors
  347. resp = session.get(url)
  348. resp.raise_for_status()
  349. return resp.url, resp.text
  350. try:
  351. with open(url, 'rb') as f:
  352. content = auto_decode(f.read())
  353. except IOError as exc:
  354. raise InstallationError(
  355. 'Could not open requirements file: %s' % str(exc)
  356. )
  357. return url, content
  358. _scheme_re = re.compile(r'^(http|https|file):', re.I)
  359. _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)
  360. def is_url(name):
  361. """Returns true if the name looks like a URL"""
  362. if ':' not in name:
  363. return False
  364. scheme = name.split(':', 1)[0].lower()
  365. return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes
  366. def url_to_path(url):
  367. """
  368. Convert a file: URL to a path.
  369. """
  370. assert url.startswith('file:'), (
  371. "You can only turn file: urls into filenames (not %r)" % url)
  372. _, netloc, path, _, _ = urllib_parse.urlsplit(url)
  373. # if we have a UNC path, prepend UNC share notation
  374. if netloc:
  375. netloc = '\\\\' + netloc
  376. path = urllib_request.url2pathname(netloc + path)
  377. return path
  378. def path_to_url(path):
  379. """
  380. Convert a path to a file: URL. The path will be made absolute and have
  381. quoted path parts.
  382. """
  383. path = os.path.normpath(os.path.abspath(path))
  384. url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path))
  385. return url
  386. def is_archive_file(name):
  387. """Return True if `name` is a considered as an archive file."""
  388. ext = splitext(name)[1].lower()
  389. if ext in ARCHIVE_EXTENSIONS:
  390. return True
  391. return False
  392. def unpack_vcs_link(link, location):
  393. vcs_backend = _get_used_vcs_backend(link)
  394. vcs_backend.unpack(location)
  395. def _get_used_vcs_backend(link):
  396. for backend in vcs.backends:
  397. if link.scheme in backend.schemes:
  398. vcs_backend = backend(link.url)
  399. return vcs_backend
  400. def is_vcs_url(link):
  401. return bool(_get_used_vcs_backend(link))
  402. def is_file_url(link):
  403. return link.url.lower().startswith('file:')
  404. def is_dir_url(link):
  405. """Return whether a file:// Link points to a directory.
  406. ``link`` must not have any other scheme but file://. Call is_file_url()
  407. first.
  408. """
  409. link_path = url_to_path(link.url_without_fragment)
  410. return os.path.isdir(link_path)
  411. def _progress_indicator(iterable, *args, **kwargs):
  412. return iterable
  413. def _download_url(resp, link, content_file, hashes, progress_bar):
  414. try:
  415. total_length = int(resp.headers['content-length'])
  416. except (ValueError, KeyError, TypeError):
  417. total_length = 0
  418. cached_resp = getattr(resp, "from_cache", False)
  419. if logger.getEffectiveLevel() > logging.INFO:
  420. show_progress = False
  421. elif cached_resp:
  422. show_progress = False
  423. elif total_length > (40 * 1000):
  424. show_progress = True
  425. elif not total_length:
  426. show_progress = True
  427. else:
  428. show_progress = False
  429. show_url = link.show_url
  430. def resp_read(chunk_size):
  431. try:
  432. # Special case for urllib3.
  433. for chunk in resp.raw.stream(
  434. chunk_size,
  435. # We use decode_content=False here because we don't
  436. # want urllib3 to mess with the raw bytes we get
  437. # from the server. If we decompress inside of
  438. # urllib3 then we cannot verify the checksum
  439. # because the checksum will be of the compressed
  440. # file. This breakage will only occur if the
  441. # server adds a Content-Encoding header, which
  442. # depends on how the server was configured:
  443. # - Some servers will notice that the file isn't a
  444. # compressible file and will leave the file alone
  445. # and with an empty Content-Encoding
  446. # - Some servers will notice that the file is
  447. # already compressed and will leave the file
  448. # alone and will add a Content-Encoding: gzip
  449. # header
  450. # - Some servers won't notice anything at all and
  451. # will take a file that's already been compressed
  452. # and compress it again and set the
  453. # Content-Encoding: gzip header
  454. #
  455. # By setting this not to decode automatically we
  456. # hope to eliminate problems with the second case.
  457. decode_content=False):
  458. yield chunk
  459. except AttributeError:
  460. # Standard file-like object.
  461. while True:
  462. chunk = resp.raw.read(chunk_size)
  463. if not chunk:
  464. break
  465. yield chunk
  466. def written_chunks(chunks):
  467. for chunk in chunks:
  468. content_file.write(chunk)
  469. yield chunk
  470. progress_indicator = _progress_indicator
  471. if link.netloc == PyPI.netloc:
  472. url = show_url
  473. else:
  474. url = link.url_without_fragment
  475. if show_progress: # We don't show progress on cached responses
  476. progress_indicator = DownloadProgressProvider(progress_bar,
  477. max=total_length)
  478. if total_length:
  479. logger.info("Downloading %s (%s)", url, format_size(total_length))
  480. else:
  481. logger.info("Downloading %s", url)
  482. elif cached_resp:
  483. logger.info("Using cached %s", url)
  484. else:
  485. logger.info("Downloading %s", url)
  486. logger.debug('Downloading from URL %s', link)
  487. downloaded_chunks = written_chunks(
  488. progress_indicator(
  489. resp_read(CONTENT_CHUNK_SIZE),
  490. CONTENT_CHUNK_SIZE
  491. )
  492. )
  493. if hashes:
  494. hashes.check_against_chunks(downloaded_chunks)
  495. else:
  496. consume(downloaded_chunks)
  497. def _copy_file(filename, location, link):
  498. copy = True
  499. download_location = os.path.join(location, link.filename)
  500. if os.path.exists(download_location):
  501. response = ask_path_exists(
  502. 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' %
  503. display_path(download_location), ('i', 'w', 'b', 'a'))
  504. if response == 'i':
  505. copy = False
  506. elif response == 'w':
  507. logger.warning('Deleting %s', display_path(download_location))
  508. os.remove(download_location)
  509. elif response == 'b':
  510. dest_file = backup_dir(download_location)
  511. logger.warning(
  512. 'Backing up %s to %s',
  513. display_path(download_location),
  514. display_path(dest_file),
  515. )
  516. shutil.move(download_location, dest_file)
  517. elif response == 'a':
  518. sys.exit(-1)
  519. if copy:
  520. shutil.copy(filename, download_location)
  521. logger.info('Saved %s', display_path(download_location))
  522. def unpack_http_url(link, location, download_dir=None,
  523. session=None, hashes=None, progress_bar="on"):
  524. if session is None:
  525. raise TypeError(
  526. "unpack_http_url() missing 1 required keyword argument: 'session'"
  527. )
  528. with TempDirectory(kind="unpack") as temp_dir:
  529. # If a download dir is specified, is the file already downloaded there?
  530. already_downloaded_path = None
  531. if download_dir:
  532. already_downloaded_path = _check_download_dir(link,
  533. download_dir,
  534. hashes)
  535. if already_downloaded_path:
  536. from_path = already_downloaded_path
  537. content_type = mimetypes.guess_type(from_path)[0]
  538. else:
  539. # let's download to a tmp dir
  540. from_path, content_type = _download_http_url(link,
  541. session,
  542. temp_dir.path,
  543. hashes,
  544. progress_bar)
  545. # unpack the archive to the build dir location. even when only
  546. # downloading archives, they have to be unpacked to parse dependencies
  547. unpack_file(from_path, location, content_type, link)
  548. # a download dir is specified; let's copy the archive there
  549. if download_dir and not already_downloaded_path:
  550. _copy_file(from_path, download_dir, link)
  551. if not already_downloaded_path:
  552. os.unlink(from_path)
  553. def unpack_file_url(link, location, download_dir=None, hashes=None):
  554. """Unpack link into location.
  555. If download_dir is provided and link points to a file, make a copy
  556. of the link file inside download_dir.
  557. """
  558. link_path = url_to_path(link.url_without_fragment)
  559. # If it's a url to a local directory
  560. if is_dir_url(link):
  561. if os.path.isdir(location):
  562. rmtree(location)
  563. shutil.copytree(link_path, location, symlinks=True)
  564. if download_dir:
  565. logger.info('Link is a directory, ignoring download_dir')
  566. return
  567. # If --require-hashes is off, `hashes` is either empty, the
  568. # link's embedded hash, or MissingHashes; it is required to
  569. # match. If --require-hashes is on, we are satisfied by any
  570. # hash in `hashes` matching: a URL-based or an option-based
  571. # one; no internet-sourced hash will be in `hashes`.
  572. if hashes:
  573. hashes.check_against_path(link_path)
  574. # If a download dir is specified, is the file already there and valid?
  575. already_downloaded_path = None
  576. if download_dir:
  577. already_downloaded_path = _check_download_dir(link,
  578. download_dir,
  579. hashes)
  580. if already_downloaded_path:
  581. from_path = already_downloaded_path
  582. else:
  583. from_path = link_path
  584. content_type = mimetypes.guess_type(from_path)[0]
  585. # unpack the archive to the build dir location. even when only downloading
  586. # archives, they have to be unpacked to parse dependencies
  587. unpack_file(from_path, location, content_type, link)
  588. # a download dir is specified and not already downloaded
  589. if download_dir and not already_downloaded_path:
  590. _copy_file(from_path, download_dir, link)
  591. def _copy_dist_from_dir(link_path, location):
  592. """Copy distribution files in `link_path` to `location`.
  593. Invoked when user requests to install a local directory. E.g.:
  594. pip install .
  595. pip install ~/dev/git-repos/python-prompt-toolkit
  596. """
  597. # Note: This is currently VERY SLOW if you have a lot of data in the
  598. # directory, because it copies everything with `shutil.copytree`.
  599. # What it should really do is build an sdist and install that.
  600. # See https://github.com/pypa/pip/issues/2195
  601. if os.path.isdir(location):
  602. rmtree(location)
  603. # build an sdist
  604. setup_py = 'setup.py'
  605. sdist_args = [sys.executable]
  606. sdist_args.append('-c')
  607. sdist_args.append(SETUPTOOLS_SHIM % setup_py)
  608. sdist_args.append('sdist')
  609. sdist_args += ['--dist-dir', location]
  610. logger.info('Running setup.py sdist for %s', link_path)
  611. with indent_log():
  612. call_subprocess(sdist_args, cwd=link_path, show_stdout=False)
  613. # unpack sdist into `location`
  614. sdist = os.path.join(location, os.listdir(location)[0])
  615. logger.info('Unpacking sdist %s into %s', sdist, location)
  616. unpack_file(sdist, location, content_type=None, link=None)
  617. class PipXmlrpcTransport(xmlrpc_client.Transport):
  618. """Provide a `xmlrpclib.Transport` implementation via a `PipSession`
  619. object.
  620. """
  621. def __init__(self, index_url, session, use_datetime=False):
  622. xmlrpc_client.Transport.__init__(self, use_datetime)
  623. index_parts = urllib_parse.urlparse(index_url)
  624. self._scheme = index_parts.scheme
  625. self._session = session
  626. def request(self, host, handler, request_body, verbose=False):
  627. parts = (self._scheme, host, handler, None, None, None)
  628. url = urllib_parse.urlunparse(parts)
  629. try:
  630. headers = {'Content-Type': 'text/xml'}
  631. response = self._session.post(url, data=request_body,
  632. headers=headers, stream=True)
  633. response.raise_for_status()
  634. self.verbose = verbose
  635. return self.parse_response(response.raw)
  636. except requests.HTTPError as exc:
  637. logger.critical(
  638. "HTTP error %s while getting %s",
  639. exc.response.status_code, url,
  640. )
  641. raise
  642. def unpack_url(link, location, download_dir=None,
  643. only_download=False, session=None, hashes=None,
  644. progress_bar="on"):
  645. """Unpack link.
  646. If link is a VCS link:
  647. if only_download, export into download_dir and ignore location
  648. else unpack into location
  649. for other types of link:
  650. - unpack into location
  651. - if download_dir, copy the file into download_dir
  652. - if only_download, mark location for deletion
  653. :param hashes: A Hashes object, one of whose embedded hashes must match,
  654. or HashMismatch will be raised. If the Hashes is empty, no matches are
  655. required, and unhashable types of requirements (like VCS ones, which
  656. would ordinarily raise HashUnsupported) are allowed.
  657. """
  658. # non-editable vcs urls
  659. if is_vcs_url(link):
  660. unpack_vcs_link(link, location)
  661. # file urls
  662. elif is_file_url(link):
  663. unpack_file_url(link, location, download_dir, hashes=hashes)
  664. # http urls
  665. else:
  666. if session is None:
  667. session = PipSession()
  668. unpack_http_url(
  669. link,
  670. location,
  671. download_dir,
  672. session,
  673. hashes=hashes,
  674. progress_bar=progress_bar
  675. )
  676. if only_download:
  677. write_delete_marker_file(location)
  678. def _download_http_url(link, session, temp_dir, hashes, progress_bar):
  679. """Download link url into temp_dir using provided session"""
  680. target_url = link.url.split('#', 1)[0]
  681. try:
  682. resp = session.get(
  683. target_url,
  684. # We use Accept-Encoding: identity here because requests
  685. # defaults to accepting compressed responses. This breaks in
  686. # a variety of ways depending on how the server is configured.
  687. # - Some servers will notice that the file isn't a compressible
  688. # file and will leave the file alone and with an empty
  689. # Content-Encoding
  690. # - Some servers will notice that the file is already
  691. # compressed and will leave the file alone and will add a
  692. # Content-Encoding: gzip header
  693. # - Some servers won't notice anything at all and will take
  694. # a file that's already been compressed and compress it again
  695. # and set the Content-Encoding: gzip header
  696. # By setting this to request only the identity encoding We're
  697. # hoping to eliminate the third case. Hopefully there does not
  698. # exist a server which when given a file will notice it is
  699. # already compressed and that you're not asking for a
  700. # compressed file and will then decompress it before sending
  701. # because if that's the case I don't think it'll ever be
  702. # possible to make this work.
  703. headers={"Accept-Encoding": "identity"},
  704. stream=True,
  705. )
  706. resp.raise_for_status()
  707. except requests.HTTPError as exc:
  708. logger.critical(
  709. "HTTP error %s while getting %s", exc.response.status_code, link,
  710. )
  711. raise
  712. content_type = resp.headers.get('content-type', '')
  713. filename = link.filename # fallback
  714. # Have a look at the Content-Disposition header for a better guess
  715. content_disposition = resp.headers.get('content-disposition')
  716. if content_disposition:
  717. type, params = cgi.parse_header(content_disposition)
  718. # We use ``or`` here because we don't want to use an "empty" value
  719. # from the filename param.
  720. filename = params.get('filename') or filename
  721. ext = splitext(filename)[1]
  722. if not ext:
  723. ext = mimetypes.guess_extension(content_type)
  724. if ext:
  725. filename += ext
  726. if not ext and link.url != resp.url:
  727. ext = os.path.splitext(resp.url)[1]
  728. if ext:
  729. filename += ext
  730. file_path = os.path.join(temp_dir, filename)
  731. with open(file_path, 'wb') as content_file:
  732. _download_url(resp, link, content_file, hashes, progress_bar)
  733. return file_path, content_type
  734. def _check_download_dir(link, download_dir, hashes):
  735. """ Check download_dir for previously downloaded file with correct hash
  736. If a correct file is found return its path else None
  737. """
  738. download_path = os.path.join(download_dir, link.filename)
  739. if os.path.exists(download_path):
  740. # If already downloaded, does its hash match?
  741. logger.info('File was already downloaded %s', download_path)
  742. if hashes:
  743. try:
  744. hashes.check_against_path(download_path)
  745. except HashMismatch:
  746. logger.warning(
  747. 'Previously-downloaded file %s has bad hash. '
  748. 'Re-downloading.',
  749. download_path
  750. )
  751. os.unlink(download_path)
  752. return None
  753. return download_path
  754. return None