You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

355 lines
14 KiB

4 years ago
  1. """Prepares a distribution for installation
  2. """
  3. import logging
  4. import os
  5. from pip._vendor import pkg_resources, requests
  6. from pip._internal.build_env import BuildEnvironment
  7. from pip._internal.download import (
  8. is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path,
  9. )
  10. from pip._internal.exceptions import (
  11. DirectoryUrlHashUnsupported, HashUnpinned, InstallationError,
  12. PreviousBuildDirError, VcsHashUnsupported,
  13. )
  14. from pip._internal.utils.compat import expanduser
  15. from pip._internal.utils.hashes import MissingHashes
  16. from pip._internal.utils.logging import indent_log
  17. from pip._internal.utils.misc import display_path, normalize_path
  18. from pip._internal.vcs import vcs
  19. logger = logging.getLogger(__name__)
  20. def make_abstract_dist(req):
  21. """Factory to make an abstract dist object.
  22. Preconditions: Either an editable req with a source_dir, or satisfied_by or
  23. a wheel link, or a non-editable req with a source_dir.
  24. :return: A concrete DistAbstraction.
  25. """
  26. if req.editable:
  27. return IsSDist(req)
  28. elif req.link and req.link.is_wheel:
  29. return IsWheel(req)
  30. else:
  31. return IsSDist(req)
  32. class DistAbstraction(object):
  33. """Abstracts out the wheel vs non-wheel Resolver.resolve() logic.
  34. The requirements for anything installable are as follows:
  35. - we must be able to determine the requirement name
  36. (or we can't correctly handle the non-upgrade case).
  37. - we must be able to generate a list of run-time dependencies
  38. without installing any additional packages (or we would
  39. have to either burn time by doing temporary isolated installs
  40. or alternatively violate pips 'don't start installing unless
  41. all requirements are available' rule - neither of which are
  42. desirable).
  43. - for packages with setup requirements, we must also be able
  44. to determine their requirements without installing additional
  45. packages (for the same reason as run-time dependencies)
  46. - we must be able to create a Distribution object exposing the
  47. above metadata.
  48. """
  49. def __init__(self, req):
  50. self.req = req
  51. def dist(self, finder):
  52. """Return a setuptools Dist object."""
  53. raise NotImplementedError(self.dist)
  54. def prep_for_dist(self, finder, build_isolation):
  55. """Ensure that we can get a Dist for this requirement."""
  56. raise NotImplementedError(self.dist)
  57. class IsWheel(DistAbstraction):
  58. def dist(self, finder):
  59. return list(pkg_resources.find_distributions(
  60. self.req.source_dir))[0]
  61. def prep_for_dist(self, finder, build_isolation):
  62. # FIXME:https://github.com/pypa/pip/issues/1112
  63. pass
  64. class IsSDist(DistAbstraction):
  65. def dist(self, finder):
  66. dist = self.req.get_dist()
  67. # FIXME: shouldn't be globally added.
  68. if finder and dist.has_metadata('dependency_links.txt'):
  69. finder.add_dependency_links(
  70. dist.get_metadata_lines('dependency_links.txt')
  71. )
  72. return dist
  73. def prep_for_dist(self, finder, build_isolation):
  74. # Prepare for building. We need to:
  75. # 1. Load pyproject.toml (if it exists)
  76. # 2. Set up the build environment
  77. self.req.load_pyproject_toml()
  78. should_isolate = self.req.use_pep517 and build_isolation
  79. if should_isolate:
  80. # Isolate in a BuildEnvironment and install the build-time
  81. # requirements.
  82. self.req.build_env = BuildEnvironment()
  83. self.req.build_env.install_requirements(
  84. finder, self.req.pyproject_requires,
  85. "Installing build dependencies"
  86. )
  87. missing = []
  88. if self.req.requirements_to_check:
  89. check = self.req.requirements_to_check
  90. missing = self.req.build_env.missing_requirements(check)
  91. if missing:
  92. logger.warning(
  93. "Missing build requirements in pyproject.toml for %s.",
  94. self.req,
  95. )
  96. logger.warning(
  97. "The project does not specify a build backend, and pip "
  98. "cannot fall back to setuptools without %s.",
  99. " and ".join(map(repr, sorted(missing)))
  100. )
  101. self.req.run_egg_info()
  102. self.req.assert_source_matches_version()
  103. class Installed(DistAbstraction):
  104. def dist(self, finder):
  105. return self.req.satisfied_by
  106. def prep_for_dist(self, finder, build_isolation):
  107. pass
  108. class RequirementPreparer(object):
  109. """Prepares a Requirement
  110. """
  111. def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir,
  112. progress_bar, build_isolation, req_tracker):
  113. super(RequirementPreparer, self).__init__()
  114. self.src_dir = src_dir
  115. self.build_dir = build_dir
  116. self.req_tracker = req_tracker
  117. # Where still packed archives should be written to. If None, they are
  118. # not saved, and are deleted immediately after unpacking.
  119. self.download_dir = download_dir
  120. # Where still-packed .whl files should be written to. If None, they are
  121. # written to the download_dir parameter. Separate to download_dir to
  122. # permit only keeping wheel archives for pip wheel.
  123. if wheel_download_dir:
  124. wheel_download_dir = normalize_path(wheel_download_dir)
  125. self.wheel_download_dir = wheel_download_dir
  126. # NOTE
  127. # download_dir and wheel_download_dir overlap semantically and may
  128. # be combined if we're willing to have non-wheel archives present in
  129. # the wheelhouse output by 'pip wheel'.
  130. self.progress_bar = progress_bar
  131. # Is build isolation allowed?
  132. self.build_isolation = build_isolation
  133. @property
  134. def _download_should_save(self):
  135. # TODO: Modify to reduce indentation needed
  136. if self.download_dir:
  137. self.download_dir = expanduser(self.download_dir)
  138. if os.path.exists(self.download_dir):
  139. return True
  140. else:
  141. logger.critical('Could not find download directory')
  142. raise InstallationError(
  143. "Could not find or access download directory '%s'"
  144. % display_path(self.download_dir))
  145. return False
  146. def prepare_linked_requirement(self, req, session, finder,
  147. upgrade_allowed, require_hashes):
  148. """Prepare a requirement that would be obtained from req.link
  149. """
  150. # TODO: Breakup into smaller functions
  151. if req.link and req.link.scheme == 'file':
  152. path = url_to_path(req.link.url)
  153. logger.info('Processing %s', display_path(path))
  154. else:
  155. logger.info('Collecting %s', req)
  156. with indent_log():
  157. # @@ if filesystem packages are not marked
  158. # editable in a req, a non deterministic error
  159. # occurs when the script attempts to unpack the
  160. # build directory
  161. req.ensure_has_source_dir(self.build_dir)
  162. # If a checkout exists, it's unwise to keep going. version
  163. # inconsistencies are logged later, but do not fail the
  164. # installation.
  165. # FIXME: this won't upgrade when there's an existing
  166. # package unpacked in `req.source_dir`
  167. # package unpacked in `req.source_dir`
  168. if os.path.exists(os.path.join(req.source_dir, 'setup.py')):
  169. raise PreviousBuildDirError(
  170. "pip can't proceed with requirements '%s' due to a"
  171. " pre-existing build directory (%s). This is "
  172. "likely due to a previous installation that failed"
  173. ". pip is being responsible and not assuming it "
  174. "can delete this. Please delete it and try again."
  175. % (req, req.source_dir)
  176. )
  177. req.populate_link(finder, upgrade_allowed, require_hashes)
  178. # We can't hit this spot and have populate_link return None.
  179. # req.satisfied_by is None here (because we're
  180. # guarded) and upgrade has no impact except when satisfied_by
  181. # is not None.
  182. # Then inside find_requirement existing_applicable -> False
  183. # If no new versions are found, DistributionNotFound is raised,
  184. # otherwise a result is guaranteed.
  185. assert req.link
  186. link = req.link
  187. # Now that we have the real link, we can tell what kind of
  188. # requirements we have and raise some more informative errors
  189. # than otherwise. (For example, we can raise VcsHashUnsupported
  190. # for a VCS URL rather than HashMissing.)
  191. if require_hashes:
  192. # We could check these first 2 conditions inside
  193. # unpack_url and save repetition of conditions, but then
  194. # we would report less-useful error messages for
  195. # unhashable requirements, complaining that there's no
  196. # hash provided.
  197. if is_vcs_url(link):
  198. raise VcsHashUnsupported()
  199. elif is_file_url(link) and is_dir_url(link):
  200. raise DirectoryUrlHashUnsupported()
  201. if not req.original_link and not req.is_pinned:
  202. # Unpinned packages are asking for trouble when a new
  203. # version is uploaded. This isn't a security check, but
  204. # it saves users a surprising hash mismatch in the
  205. # future.
  206. #
  207. # file:/// URLs aren't pinnable, so don't complain
  208. # about them not being pinned.
  209. raise HashUnpinned()
  210. hashes = req.hashes(trust_internet=not require_hashes)
  211. if require_hashes and not hashes:
  212. # Known-good hashes are missing for this requirement, so
  213. # shim it with a facade object that will provoke hash
  214. # computation and then raise a HashMissing exception
  215. # showing the user what the hash should be.
  216. hashes = MissingHashes()
  217. try:
  218. download_dir = self.download_dir
  219. # We always delete unpacked sdists after pip ran.
  220. autodelete_unpacked = True
  221. if req.link.is_wheel and self.wheel_download_dir:
  222. # when doing 'pip wheel` we download wheels to a
  223. # dedicated dir.
  224. download_dir = self.wheel_download_dir
  225. if req.link.is_wheel:
  226. if download_dir:
  227. # When downloading, we only unpack wheels to get
  228. # metadata.
  229. autodelete_unpacked = True
  230. else:
  231. # When installing a wheel, we use the unpacked
  232. # wheel.
  233. autodelete_unpacked = False
  234. unpack_url(
  235. req.link, req.source_dir,
  236. download_dir, autodelete_unpacked,
  237. session=session, hashes=hashes,
  238. progress_bar=self.progress_bar
  239. )
  240. except requests.HTTPError as exc:
  241. logger.critical(
  242. 'Could not install requirement %s because of error %s',
  243. req,
  244. exc,
  245. )
  246. raise InstallationError(
  247. 'Could not install requirement %s because of HTTP '
  248. 'error %s for URL %s' %
  249. (req, exc, req.link)
  250. )
  251. abstract_dist = make_abstract_dist(req)
  252. with self.req_tracker.track(req):
  253. abstract_dist.prep_for_dist(finder, self.build_isolation)
  254. if self._download_should_save:
  255. # Make a .zip of the source_dir we already created.
  256. if req.link.scheme in vcs.all_schemes:
  257. req.archive(self.download_dir)
  258. return abstract_dist
  259. def prepare_editable_requirement(self, req, require_hashes, use_user_site,
  260. finder):
  261. """Prepare an editable requirement
  262. """
  263. assert req.editable, "cannot prepare a non-editable req as editable"
  264. logger.info('Obtaining %s', req)
  265. with indent_log():
  266. if require_hashes:
  267. raise InstallationError(
  268. 'The editable requirement %s cannot be installed when '
  269. 'requiring hashes, because there is no single file to '
  270. 'hash.' % req
  271. )
  272. req.ensure_has_source_dir(self.src_dir)
  273. req.update_editable(not self._download_should_save)
  274. abstract_dist = make_abstract_dist(req)
  275. with self.req_tracker.track(req):
  276. abstract_dist.prep_for_dist(finder, self.build_isolation)
  277. if self._download_should_save:
  278. req.archive(self.download_dir)
  279. req.check_if_exists(use_user_site)
  280. return abstract_dist
  281. def prepare_installed_requirement(self, req, require_hashes, skip_reason):
  282. """Prepare an already-installed requirement
  283. """
  284. assert req.satisfied_by, "req should have been satisfied but isn't"
  285. assert skip_reason is not None, (
  286. "did not get skip reason skipped but req.satisfied_by "
  287. "is set to %r" % (req.satisfied_by,)
  288. )
  289. logger.info(
  290. 'Requirement %s: %s (%s)',
  291. skip_reason, req, req.satisfied_by.version
  292. )
  293. with indent_log():
  294. if require_hashes:
  295. logger.debug(
  296. 'Since it is already installed, we are trusting this '
  297. 'package without checking its hash. To ensure a '
  298. 'completely repeatable environment, install into an '
  299. 'empty virtualenv.'
  300. )
  301. abstract_dist = Installed(req)
  302. return abstract_dist