You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1094 lines
39 KiB

4 years ago
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2012 The Python Software Foundation.
  4. # See LICENSE.txt and CONTRIBUTORS.txt.
  5. #
  6. """Implementation of the Metadata for Python packages PEPs.
  7. Supports all metadata formats (1.0, 1.1, 1.2, and 2.0 experimental).
  8. """
  9. from __future__ import unicode_literals
  10. import codecs
  11. from email import message_from_file
  12. import json
  13. import logging
  14. import re
  15. from . import DistlibException, __version__
  16. from .compat import StringIO, string_types, text_type
  17. from .markers import interpret
  18. from .util import extract_by_key, get_extras
  19. from .version import get_scheme, PEP440_VERSION_RE
  20. logger = logging.getLogger(__name__)
  21. class MetadataMissingError(DistlibException):
  22. """A required metadata is missing"""
  23. class MetadataConflictError(DistlibException):
  24. """Attempt to read or write metadata fields that are conflictual."""
  25. class MetadataUnrecognizedVersionError(DistlibException):
  26. """Unknown metadata version number."""
  27. class MetadataInvalidError(DistlibException):
  28. """A metadata value is invalid"""
  29. # public API of this module
  30. __all__ = ['Metadata', 'PKG_INFO_ENCODING', 'PKG_INFO_PREFERRED_VERSION']
  31. # Encoding used for the PKG-INFO files
  32. PKG_INFO_ENCODING = 'utf-8'
  33. # preferred version. Hopefully will be changed
  34. # to 1.2 once PEP 345 is supported everywhere
  35. PKG_INFO_PREFERRED_VERSION = '1.1'
  36. _LINE_PREFIX_1_2 = re.compile('\n \\|')
  37. _LINE_PREFIX_PRE_1_2 = re.compile('\n ')
  38. _241_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform',
  39. 'Summary', 'Description',
  40. 'Keywords', 'Home-page', 'Author', 'Author-email',
  41. 'License')
  42. _314_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform',
  43. 'Supported-Platform', 'Summary', 'Description',
  44. 'Keywords', 'Home-page', 'Author', 'Author-email',
  45. 'License', 'Classifier', 'Download-URL', 'Obsoletes',
  46. 'Provides', 'Requires')
  47. _314_MARKERS = ('Obsoletes', 'Provides', 'Requires', 'Classifier',
  48. 'Download-URL')
  49. _345_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform',
  50. 'Supported-Platform', 'Summary', 'Description',
  51. 'Keywords', 'Home-page', 'Author', 'Author-email',
  52. 'Maintainer', 'Maintainer-email', 'License',
  53. 'Classifier', 'Download-URL', 'Obsoletes-Dist',
  54. 'Project-URL', 'Provides-Dist', 'Requires-Dist',
  55. 'Requires-Python', 'Requires-External')
  56. _345_MARKERS = ('Provides-Dist', 'Requires-Dist', 'Requires-Python',
  57. 'Obsoletes-Dist', 'Requires-External', 'Maintainer',
  58. 'Maintainer-email', 'Project-URL')
  59. _426_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform',
  60. 'Supported-Platform', 'Summary', 'Description',
  61. 'Keywords', 'Home-page', 'Author', 'Author-email',
  62. 'Maintainer', 'Maintainer-email', 'License',
  63. 'Classifier', 'Download-URL', 'Obsoletes-Dist',
  64. 'Project-URL', 'Provides-Dist', 'Requires-Dist',
  65. 'Requires-Python', 'Requires-External', 'Private-Version',
  66. 'Obsoleted-By', 'Setup-Requires-Dist', 'Extension',
  67. 'Provides-Extra')
  68. _426_MARKERS = ('Private-Version', 'Provides-Extra', 'Obsoleted-By',
  69. 'Setup-Requires-Dist', 'Extension')
  70. # See issue #106: Sometimes 'Requires' occurs wrongly in the metadata. Include
  71. # it in the tuple literal below to allow it (for now)
  72. _566_FIELDS = _426_FIELDS + ('Description-Content-Type', 'Requires')
  73. _566_MARKERS = ('Description-Content-Type',)
  74. _ALL_FIELDS = set()
  75. _ALL_FIELDS.update(_241_FIELDS)
  76. _ALL_FIELDS.update(_314_FIELDS)
  77. _ALL_FIELDS.update(_345_FIELDS)
  78. _ALL_FIELDS.update(_426_FIELDS)
  79. _ALL_FIELDS.update(_566_FIELDS)
  80. EXTRA_RE = re.compile(r'''extra\s*==\s*("([^"]+)"|'([^']+)')''')
  81. def _version2fieldlist(version):
  82. if version == '1.0':
  83. return _241_FIELDS
  84. elif version == '1.1':
  85. return _314_FIELDS
  86. elif version == '1.2':
  87. return _345_FIELDS
  88. elif version in ('1.3', '2.1'):
  89. return _345_FIELDS + _566_FIELDS
  90. elif version == '2.0':
  91. return _426_FIELDS
  92. raise MetadataUnrecognizedVersionError(version)
  93. def _best_version(fields):
  94. """Detect the best version depending on the fields used."""
  95. def _has_marker(keys, markers):
  96. for marker in markers:
  97. if marker in keys:
  98. return True
  99. return False
  100. keys = []
  101. for key, value in fields.items():
  102. if value in ([], 'UNKNOWN', None):
  103. continue
  104. keys.append(key)
  105. possible_versions = ['1.0', '1.1', '1.2', '1.3', '2.0', '2.1']
  106. # first let's try to see if a field is not part of one of the version
  107. for key in keys:
  108. if key not in _241_FIELDS and '1.0' in possible_versions:
  109. possible_versions.remove('1.0')
  110. logger.debug('Removed 1.0 due to %s', key)
  111. if key not in _314_FIELDS and '1.1' in possible_versions:
  112. possible_versions.remove('1.1')
  113. logger.debug('Removed 1.1 due to %s', key)
  114. if key not in _345_FIELDS and '1.2' in possible_versions:
  115. possible_versions.remove('1.2')
  116. logger.debug('Removed 1.2 due to %s', key)
  117. if key not in _566_FIELDS and '1.3' in possible_versions:
  118. possible_versions.remove('1.3')
  119. logger.debug('Removed 1.3 due to %s', key)
  120. if key not in _566_FIELDS and '2.1' in possible_versions:
  121. if key != 'Description': # In 2.1, description allowed after headers
  122. possible_versions.remove('2.1')
  123. logger.debug('Removed 2.1 due to %s', key)
  124. if key not in _426_FIELDS and '2.0' in possible_versions:
  125. possible_versions.remove('2.0')
  126. logger.debug('Removed 2.0 due to %s', key)
  127. # possible_version contains qualified versions
  128. if len(possible_versions) == 1:
  129. return possible_versions[0] # found !
  130. elif len(possible_versions) == 0:
  131. logger.debug('Out of options - unknown metadata set: %s', fields)
  132. raise MetadataConflictError('Unknown metadata set')
  133. # let's see if one unique marker is found
  134. is_1_1 = '1.1' in possible_versions and _has_marker(keys, _314_MARKERS)
  135. is_1_2 = '1.2' in possible_versions and _has_marker(keys, _345_MARKERS)
  136. is_2_1 = '2.1' in possible_versions and _has_marker(keys, _566_MARKERS)
  137. is_2_0 = '2.0' in possible_versions and _has_marker(keys, _426_MARKERS)
  138. if int(is_1_1) + int(is_1_2) + int(is_2_1) + int(is_2_0) > 1:
  139. raise MetadataConflictError('You used incompatible 1.1/1.2/2.0/2.1 fields')
  140. # we have the choice, 1.0, or 1.2, or 2.0
  141. # - 1.0 has a broken Summary field but works with all tools
  142. # - 1.1 is to avoid
  143. # - 1.2 fixes Summary but has little adoption
  144. # - 2.0 adds more features and is very new
  145. if not is_1_1 and not is_1_2 and not is_2_1 and not is_2_0:
  146. # we couldn't find any specific marker
  147. if PKG_INFO_PREFERRED_VERSION in possible_versions:
  148. return PKG_INFO_PREFERRED_VERSION
  149. if is_1_1:
  150. return '1.1'
  151. if is_1_2:
  152. return '1.2'
  153. if is_2_1:
  154. return '2.1'
  155. return '2.0'
  156. _ATTR2FIELD = {
  157. 'metadata_version': 'Metadata-Version',
  158. 'name': 'Name',
  159. 'version': 'Version',
  160. 'platform': 'Platform',
  161. 'supported_platform': 'Supported-Platform',
  162. 'summary': 'Summary',
  163. 'description': 'Description',
  164. 'keywords': 'Keywords',
  165. 'home_page': 'Home-page',
  166. 'author': 'Author',
  167. 'author_email': 'Author-email',
  168. 'maintainer': 'Maintainer',
  169. 'maintainer_email': 'Maintainer-email',
  170. 'license': 'License',
  171. 'classifier': 'Classifier',
  172. 'download_url': 'Download-URL',
  173. 'obsoletes_dist': 'Obsoletes-Dist',
  174. 'provides_dist': 'Provides-Dist',
  175. 'requires_dist': 'Requires-Dist',
  176. 'setup_requires_dist': 'Setup-Requires-Dist',
  177. 'requires_python': 'Requires-Python',
  178. 'requires_external': 'Requires-External',
  179. 'requires': 'Requires',
  180. 'provides': 'Provides',
  181. 'obsoletes': 'Obsoletes',
  182. 'project_url': 'Project-URL',
  183. 'private_version': 'Private-Version',
  184. 'obsoleted_by': 'Obsoleted-By',
  185. 'extension': 'Extension',
  186. 'provides_extra': 'Provides-Extra',
  187. }
  188. _PREDICATE_FIELDS = ('Requires-Dist', 'Obsoletes-Dist', 'Provides-Dist')
  189. _VERSIONS_FIELDS = ('Requires-Python',)
  190. _VERSION_FIELDS = ('Version',)
  191. _LISTFIELDS = ('Platform', 'Classifier', 'Obsoletes',
  192. 'Requires', 'Provides', 'Obsoletes-Dist',
  193. 'Provides-Dist', 'Requires-Dist', 'Requires-External',
  194. 'Project-URL', 'Supported-Platform', 'Setup-Requires-Dist',
  195. 'Provides-Extra', 'Extension')
  196. _LISTTUPLEFIELDS = ('Project-URL',)
  197. _ELEMENTSFIELD = ('Keywords',)
  198. _UNICODEFIELDS = ('Author', 'Maintainer', 'Summary', 'Description')
  199. _MISSING = object()
  200. _FILESAFE = re.compile('[^A-Za-z0-9.]+')
  201. def _get_name_and_version(name, version, for_filename=False):
  202. """Return the distribution name with version.
  203. If for_filename is true, return a filename-escaped form."""
  204. if for_filename:
  205. # For both name and version any runs of non-alphanumeric or '.'
  206. # characters are replaced with a single '-'. Additionally any
  207. # spaces in the version string become '.'
  208. name = _FILESAFE.sub('-', name)
  209. version = _FILESAFE.sub('-', version.replace(' ', '.'))
  210. return '%s-%s' % (name, version)
  211. class LegacyMetadata(object):
  212. """The legacy metadata of a release.
  213. Supports versions 1.0, 1.1 and 1.2 (auto-detected). You can
  214. instantiate the class with one of these arguments (or none):
  215. - *path*, the path to a metadata file
  216. - *fileobj* give a file-like object with metadata as content
  217. - *mapping* is a dict-like object
  218. - *scheme* is a version scheme name
  219. """
  220. # TODO document the mapping API and UNKNOWN default key
  221. def __init__(self, path=None, fileobj=None, mapping=None,
  222. scheme='default'):
  223. if [path, fileobj, mapping].count(None) < 2:
  224. raise TypeError('path, fileobj and mapping are exclusive')
  225. self._fields = {}
  226. self.requires_files = []
  227. self._dependencies = None
  228. self.scheme = scheme
  229. if path is not None:
  230. self.read(path)
  231. elif fileobj is not None:
  232. self.read_file(fileobj)
  233. elif mapping is not None:
  234. self.update(mapping)
  235. self.set_metadata_version()
  236. def set_metadata_version(self):
  237. self._fields['Metadata-Version'] = _best_version(self._fields)
  238. def _write_field(self, fileobj, name, value):
  239. fileobj.write('%s: %s\n' % (name, value))
  240. def __getitem__(self, name):
  241. return self.get(name)
  242. def __setitem__(self, name, value):
  243. return self.set(name, value)
  244. def __delitem__(self, name):
  245. field_name = self._convert_name(name)
  246. try:
  247. del self._fields[field_name]
  248. except KeyError:
  249. raise KeyError(name)
  250. def __contains__(self, name):
  251. return (name in self._fields or
  252. self._convert_name(name) in self._fields)
  253. def _convert_name(self, name):
  254. if name in _ALL_FIELDS:
  255. return name
  256. name = name.replace('-', '_').lower()
  257. return _ATTR2FIELD.get(name, name)
  258. def _default_value(self, name):
  259. if name in _LISTFIELDS or name in _ELEMENTSFIELD:
  260. return []
  261. return 'UNKNOWN'
  262. def _remove_line_prefix(self, value):
  263. if self.metadata_version in ('1.0', '1.1'):
  264. return _LINE_PREFIX_PRE_1_2.sub('\n', value)
  265. else:
  266. return _LINE_PREFIX_1_2.sub('\n', value)
  267. def __getattr__(self, name):
  268. if name in _ATTR2FIELD:
  269. return self[name]
  270. raise AttributeError(name)
  271. #
  272. # Public API
  273. #
  274. # dependencies = property(_get_dependencies, _set_dependencies)
  275. def get_fullname(self, filesafe=False):
  276. """Return the distribution name with version.
  277. If filesafe is true, return a filename-escaped form."""
  278. return _get_name_and_version(self['Name'], self['Version'], filesafe)
  279. def is_field(self, name):
  280. """return True if name is a valid metadata key"""
  281. name = self._convert_name(name)
  282. return name in _ALL_FIELDS
  283. def is_multi_field(self, name):
  284. name = self._convert_name(name)
  285. return name in _LISTFIELDS
  286. def read(self, filepath):
  287. """Read the metadata values from a file path."""
  288. fp = codecs.open(filepath, 'r', encoding='utf-8')
  289. try:
  290. self.read_file(fp)
  291. finally:
  292. fp.close()
  293. def read_file(self, fileob):
  294. """Read the metadata values from a file object."""
  295. msg = message_from_file(fileob)
  296. self._fields['Metadata-Version'] = msg['metadata-version']
  297. # When reading, get all the fields we can
  298. for field in _ALL_FIELDS:
  299. if field not in msg:
  300. continue
  301. if field in _LISTFIELDS:
  302. # we can have multiple lines
  303. values = msg.get_all(field)
  304. if field in _LISTTUPLEFIELDS and values is not None:
  305. values = [tuple(value.split(',')) for value in values]
  306. self.set(field, values)
  307. else:
  308. # single line
  309. value = msg[field]
  310. if value is not None and value != 'UNKNOWN':
  311. self.set(field, value)
  312. # logger.debug('Attempting to set metadata for %s', self)
  313. # self.set_metadata_version()
  314. def write(self, filepath, skip_unknown=False):
  315. """Write the metadata fields to filepath."""
  316. fp = codecs.open(filepath, 'w', encoding='utf-8')
  317. try:
  318. self.write_file(fp, skip_unknown)
  319. finally:
  320. fp.close()
  321. def write_file(self, fileobject, skip_unknown=False):
  322. """Write the PKG-INFO format data to a file object."""
  323. self.set_metadata_version()
  324. for field in _version2fieldlist(self['Metadata-Version']):
  325. values = self.get(field)
  326. if skip_unknown and values in ('UNKNOWN', [], ['UNKNOWN']):
  327. continue
  328. if field in _ELEMENTSFIELD:
  329. self._write_field(fileobject, field, ','.join(values))
  330. continue
  331. if field not in _LISTFIELDS:
  332. if field == 'Description':
  333. if self.metadata_version in ('1.0', '1.1'):
  334. values = values.replace('\n', '\n ')
  335. else:
  336. values = values.replace('\n', '\n |')
  337. values = [values]
  338. if field in _LISTTUPLEFIELDS:
  339. values = [','.join(value) for value in values]
  340. for value in values:
  341. self._write_field(fileobject, field, value)
  342. def update(self, other=None, **kwargs):
  343. """Set metadata values from the given iterable `other` and kwargs.
  344. Behavior is like `dict.update`: If `other` has a ``keys`` method,
  345. they are looped over and ``self[key]`` is assigned ``other[key]``.
  346. Else, ``other`` is an iterable of ``(key, value)`` iterables.
  347. Keys that don't match a metadata field or that have an empty value are
  348. dropped.
  349. """
  350. def _set(key, value):
  351. if key in _ATTR2FIELD and value:
  352. self.set(self._convert_name(key), value)
  353. if not other:
  354. # other is None or empty container
  355. pass
  356. elif hasattr(other, 'keys'):
  357. for k in other.keys():
  358. _set(k, other[k])
  359. else:
  360. for k, v in other:
  361. _set(k, v)
  362. if kwargs:
  363. for k, v in kwargs.items():
  364. _set(k, v)
  365. def set(self, name, value):
  366. """Control then set a metadata field."""
  367. name = self._convert_name(name)
  368. if ((name in _ELEMENTSFIELD or name == 'Platform') and
  369. not isinstance(value, (list, tuple))):
  370. if isinstance(value, string_types):
  371. value = [v.strip() for v in value.split(',')]
  372. else:
  373. value = []
  374. elif (name in _LISTFIELDS and
  375. not isinstance(value, (list, tuple))):
  376. if isinstance(value, string_types):
  377. value = [value]
  378. else:
  379. value = []
  380. if logger.isEnabledFor(logging.WARNING):
  381. project_name = self['Name']
  382. scheme = get_scheme(self.scheme)
  383. if name in _PREDICATE_FIELDS and value is not None:
  384. for v in value:
  385. # check that the values are valid
  386. if not scheme.is_valid_matcher(v.split(';')[0]):
  387. logger.warning(
  388. "'%s': '%s' is not valid (field '%s')",
  389. project_name, v, name)
  390. # FIXME this rejects UNKNOWN, is that right?
  391. elif name in _VERSIONS_FIELDS and value is not None:
  392. if not scheme.is_valid_constraint_list(value):
  393. logger.warning("'%s': '%s' is not a valid version (field '%s')",
  394. project_name, value, name)
  395. elif name in _VERSION_FIELDS and value is not None:
  396. if not scheme.is_valid_version(value):
  397. logger.warning("'%s': '%s' is not a valid version (field '%s')",
  398. project_name, value, name)
  399. if name in _UNICODEFIELDS:
  400. if name == 'Description':
  401. value = self._remove_line_prefix(value)
  402. self._fields[name] = value
  403. def get(self, name, default=_MISSING):
  404. """Get a metadata field."""
  405. name = self._convert_name(name)
  406. if name not in self._fields:
  407. if default is _MISSING:
  408. default = self._default_value(name)
  409. return default
  410. if name in _UNICODEFIELDS:
  411. value = self._fields[name]
  412. return value
  413. elif name in _LISTFIELDS:
  414. value = self._fields[name]
  415. if value is None:
  416. return []
  417. res = []
  418. for val in value:
  419. if name not in _LISTTUPLEFIELDS:
  420. res.append(val)
  421. else:
  422. # That's for Project-URL
  423. res.append((val[0], val[1]))
  424. return res
  425. elif name in _ELEMENTSFIELD:
  426. value = self._fields[name]
  427. if isinstance(value, string_types):
  428. return value.split(',')
  429. return self._fields[name]
  430. def check(self, strict=False):
  431. """Check if the metadata is compliant. If strict is True then raise if
  432. no Name or Version are provided"""
  433. self.set_metadata_version()
  434. # XXX should check the versions (if the file was loaded)
  435. missing, warnings = [], []
  436. for attr in ('Name', 'Version'): # required by PEP 345
  437. if attr not in self:
  438. missing.append(attr)
  439. if strict and missing != []:
  440. msg = 'missing required metadata: %s' % ', '.join(missing)
  441. raise MetadataMissingError(msg)
  442. for attr in ('Home-page', 'Author'):
  443. if attr not in self:
  444. missing.append(attr)
  445. # checking metadata 1.2 (XXX needs to check 1.1, 1.0)
  446. if self['Metadata-Version'] != '1.2':
  447. return missing, warnings
  448. scheme = get_scheme(self.scheme)
  449. def are_valid_constraints(value):
  450. for v in value:
  451. if not scheme.is_valid_matcher(v.split(';')[0]):
  452. return False
  453. return True
  454. for fields, controller in ((_PREDICATE_FIELDS, are_valid_constraints),
  455. (_VERSIONS_FIELDS,
  456. scheme.is_valid_constraint_list),
  457. (_VERSION_FIELDS,
  458. scheme.is_valid_version)):
  459. for field in fields:
  460. value = self.get(field, None)
  461. if value is not None and not controller(value):
  462. warnings.append("Wrong value for '%s': %s" % (field, value))
  463. return missing, warnings
  464. def todict(self, skip_missing=False):
  465. """Return fields as a dict.
  466. Field names will be converted to use the underscore-lowercase style
  467. instead of hyphen-mixed case (i.e. home_page instead of Home-page).
  468. """
  469. self.set_metadata_version()
  470. mapping_1_0 = (
  471. ('metadata_version', 'Metadata-Version'),
  472. ('name', 'Name'),
  473. ('version', 'Version'),
  474. ('summary', 'Summary'),
  475. ('home_page', 'Home-page'),
  476. ('author', 'Author'),
  477. ('author_email', 'Author-email'),
  478. ('license', 'License'),
  479. ('description', 'Description'),
  480. ('keywords', 'Keywords'),
  481. ('platform', 'Platform'),
  482. ('classifiers', 'Classifier'),
  483. ('download_url', 'Download-URL'),
  484. )
  485. data = {}
  486. for key, field_name in mapping_1_0:
  487. if not skip_missing or field_name in self._fields:
  488. data[key] = self[field_name]
  489. if self['Metadata-Version'] == '1.2':
  490. mapping_1_2 = (
  491. ('requires_dist', 'Requires-Dist'),
  492. ('requires_python', 'Requires-Python'),
  493. ('requires_external', 'Requires-External'),
  494. ('provides_dist', 'Provides-Dist'),
  495. ('obsoletes_dist', 'Obsoletes-Dist'),
  496. ('project_url', 'Project-URL'),
  497. ('maintainer', 'Maintainer'),
  498. ('maintainer_email', 'Maintainer-email'),
  499. )
  500. for key, field_name in mapping_1_2:
  501. if not skip_missing or field_name in self._fields:
  502. if key != 'project_url':
  503. data[key] = self[field_name]
  504. else:
  505. data[key] = [','.join(u) for u in self[field_name]]
  506. elif self['Metadata-Version'] == '1.1':
  507. mapping_1_1 = (
  508. ('provides', 'Provides'),
  509. ('requires', 'Requires'),
  510. ('obsoletes', 'Obsoletes'),
  511. )
  512. for key, field_name in mapping_1_1:
  513. if not skip_missing or field_name in self._fields:
  514. data[key] = self[field_name]
  515. return data
  516. def add_requirements(self, requirements):
  517. if self['Metadata-Version'] == '1.1':
  518. # we can't have 1.1 metadata *and* Setuptools requires
  519. for field in ('Obsoletes', 'Requires', 'Provides'):
  520. if field in self:
  521. del self[field]
  522. self['Requires-Dist'] += requirements
  523. # Mapping API
  524. # TODO could add iter* variants
  525. def keys(self):
  526. return list(_version2fieldlist(self['Metadata-Version']))
  527. def __iter__(self):
  528. for key in self.keys():
  529. yield key
  530. def values(self):
  531. return [self[key] for key in self.keys()]
  532. def items(self):
  533. return [(key, self[key]) for key in self.keys()]
  534. def __repr__(self):
  535. return '<%s %s %s>' % (self.__class__.__name__, self.name,
  536. self.version)
  537. METADATA_FILENAME = 'pydist.json'
  538. WHEEL_METADATA_FILENAME = 'metadata.json'
  539. LEGACY_METADATA_FILENAME = 'METADATA'
  540. class Metadata(object):
  541. """
  542. The metadata of a release. This implementation uses 2.0 (JSON)
  543. metadata where possible. If not possible, it wraps a LegacyMetadata
  544. instance which handles the key-value metadata format.
  545. """
  546. METADATA_VERSION_MATCHER = re.compile(r'^\d+(\.\d+)*$')
  547. NAME_MATCHER = re.compile('^[0-9A-Z]([0-9A-Z_.-]*[0-9A-Z])?$', re.I)
  548. VERSION_MATCHER = PEP440_VERSION_RE
  549. SUMMARY_MATCHER = re.compile('.{1,2047}')
  550. METADATA_VERSION = '2.0'
  551. GENERATOR = 'distlib (%s)' % __version__
  552. MANDATORY_KEYS = {
  553. 'name': (),
  554. 'version': (),
  555. 'summary': ('legacy',),
  556. }
  557. INDEX_KEYS = ('name version license summary description author '
  558. 'author_email keywords platform home_page classifiers '
  559. 'download_url')
  560. DEPENDENCY_KEYS = ('extras run_requires test_requires build_requires '
  561. 'dev_requires provides meta_requires obsoleted_by '
  562. 'supports_environments')
  563. SYNTAX_VALIDATORS = {
  564. 'metadata_version': (METADATA_VERSION_MATCHER, ()),
  565. 'name': (NAME_MATCHER, ('legacy',)),
  566. 'version': (VERSION_MATCHER, ('legacy',)),
  567. 'summary': (SUMMARY_MATCHER, ('legacy',)),
  568. }
  569. __slots__ = ('_legacy', '_data', 'scheme')
  570. def __init__(self, path=None, fileobj=None, mapping=None,
  571. scheme='default'):
  572. if [path, fileobj, mapping].count(None) < 2:
  573. raise TypeError('path, fileobj and mapping are exclusive')
  574. self._legacy = None
  575. self._data = None
  576. self.scheme = scheme
  577. #import pdb; pdb.set_trace()
  578. if mapping is not None:
  579. try:
  580. self._validate_mapping(mapping, scheme)
  581. self._data = mapping
  582. except MetadataUnrecognizedVersionError:
  583. self._legacy = LegacyMetadata(mapping=mapping, scheme=scheme)
  584. self.validate()
  585. else:
  586. data = None
  587. if path:
  588. with open(path, 'rb') as f:
  589. data = f.read()
  590. elif fileobj:
  591. data = fileobj.read()
  592. if data is None:
  593. # Initialised with no args - to be added
  594. self._data = {
  595. 'metadata_version': self.METADATA_VERSION,
  596. 'generator': self.GENERATOR,
  597. }
  598. else:
  599. if not isinstance(data, text_type):
  600. data = data.decode('utf-8')
  601. try:
  602. self._data = json.loads(data)
  603. self._validate_mapping(self._data, scheme)
  604. except ValueError:
  605. # Note: MetadataUnrecognizedVersionError does not
  606. # inherit from ValueError (it's a DistlibException,
  607. # which should not inherit from ValueError).
  608. # The ValueError comes from the json.load - if that
  609. # succeeds and we get a validation error, we want
  610. # that to propagate
  611. self._legacy = LegacyMetadata(fileobj=StringIO(data),
  612. scheme=scheme)
  613. self.validate()
  614. common_keys = set(('name', 'version', 'license', 'keywords', 'summary'))
  615. none_list = (None, list)
  616. none_dict = (None, dict)
  617. mapped_keys = {
  618. 'run_requires': ('Requires-Dist', list),
  619. 'build_requires': ('Setup-Requires-Dist', list),
  620. 'dev_requires': none_list,
  621. 'test_requires': none_list,
  622. 'meta_requires': none_list,
  623. 'extras': ('Provides-Extra', list),
  624. 'modules': none_list,
  625. 'namespaces': none_list,
  626. 'exports': none_dict,
  627. 'commands': none_dict,
  628. 'classifiers': ('Classifier', list),
  629. 'source_url': ('Download-URL', None),
  630. 'metadata_version': ('Metadata-Version', None),
  631. }
  632. del none_list, none_dict
  633. def __getattribute__(self, key):
  634. common = object.__getattribute__(self, 'common_keys')
  635. mapped = object.__getattribute__(self, 'mapped_keys')
  636. if key in mapped:
  637. lk, maker = mapped[key]
  638. if self._legacy:
  639. if lk is None:
  640. result = None if maker is None else maker()
  641. else:
  642. result = self._legacy.get(lk)
  643. else:
  644. value = None if maker is None else maker()
  645. if key not in ('commands', 'exports', 'modules', 'namespaces',
  646. 'classifiers'):
  647. result = self._data.get(key, value)
  648. else:
  649. # special cases for PEP 459
  650. sentinel = object()
  651. result = sentinel
  652. d = self._data.get('extensions')
  653. if d:
  654. if key == 'commands':
  655. result = d.get('python.commands', value)
  656. elif key == 'classifiers':
  657. d = d.get('python.details')
  658. if d:
  659. result = d.get(key, value)
  660. else:
  661. d = d.get('python.exports')
  662. if not d:
  663. d = self._data.get('python.exports')
  664. if d:
  665. result = d.get(key, value)
  666. if result is sentinel:
  667. result = value
  668. elif key not in common:
  669. result = object.__getattribute__(self, key)
  670. elif self._legacy:
  671. result = self._legacy.get(key)
  672. else:
  673. result = self._data.get(key)
  674. return result
  675. def _validate_value(self, key, value, scheme=None):
  676. if key in self.SYNTAX_VALIDATORS:
  677. pattern, exclusions = self.SYNTAX_VALIDATORS[key]
  678. if (scheme or self.scheme) not in exclusions:
  679. m = pattern.match(value)
  680. if not m:
  681. raise MetadataInvalidError("'%s' is an invalid value for "
  682. "the '%s' property" % (value,
  683. key))
  684. def __setattr__(self, key, value):
  685. self._validate_value(key, value)
  686. common = object.__getattribute__(self, 'common_keys')
  687. mapped = object.__getattribute__(self, 'mapped_keys')
  688. if key in mapped:
  689. lk, _ = mapped[key]
  690. if self._legacy:
  691. if lk is None:
  692. raise NotImplementedError
  693. self._legacy[lk] = value
  694. elif key not in ('commands', 'exports', 'modules', 'namespaces',
  695. 'classifiers'):
  696. self._data[key] = value
  697. else:
  698. # special cases for PEP 459
  699. d = self._data.setdefault('extensions', {})
  700. if key == 'commands':
  701. d['python.commands'] = value
  702. elif key == 'classifiers':
  703. d = d.setdefault('python.details', {})
  704. d[key] = value
  705. else:
  706. d = d.setdefault('python.exports', {})
  707. d[key] = value
  708. elif key not in common:
  709. object.__setattr__(self, key, value)
  710. else:
  711. if key == 'keywords':
  712. if isinstance(value, string_types):
  713. value = value.strip()
  714. if value:
  715. value = value.split()
  716. else:
  717. value = []
  718. if self._legacy:
  719. self._legacy[key] = value
  720. else:
  721. self._data[key] = value
  722. @property
  723. def name_and_version(self):
  724. return _get_name_and_version(self.name, self.version, True)
  725. @property
  726. def provides(self):
  727. if self._legacy:
  728. result = self._legacy['Provides-Dist']
  729. else:
  730. result = self._data.setdefault('provides', [])
  731. s = '%s (%s)' % (self.name, self.version)
  732. if s not in result:
  733. result.append(s)
  734. return result
  735. @provides.setter
  736. def provides(self, value):
  737. if self._legacy:
  738. self._legacy['Provides-Dist'] = value
  739. else:
  740. self._data['provides'] = value
  741. def get_requirements(self, reqts, extras=None, env=None):
  742. """
  743. Base method to get dependencies, given a set of extras
  744. to satisfy and an optional environment context.
  745. :param reqts: A list of sometimes-wanted dependencies,
  746. perhaps dependent on extras and environment.
  747. :param extras: A list of optional components being requested.
  748. :param env: An optional environment for marker evaluation.
  749. """
  750. if self._legacy:
  751. result = reqts
  752. else:
  753. result = []
  754. extras = get_extras(extras or [], self.extras)
  755. for d in reqts:
  756. if 'extra' not in d and 'environment' not in d:
  757. # unconditional
  758. include = True
  759. else:
  760. if 'extra' not in d:
  761. # Not extra-dependent - only environment-dependent
  762. include = True
  763. else:
  764. include = d.get('extra') in extras
  765. if include:
  766. # Not excluded because of extras, check environment
  767. marker = d.get('environment')
  768. if marker:
  769. include = interpret(marker, env)
  770. if include:
  771. result.extend(d['requires'])
  772. for key in ('build', 'dev', 'test'):
  773. e = ':%s:' % key
  774. if e in extras:
  775. extras.remove(e)
  776. # A recursive call, but it should terminate since 'test'
  777. # has been removed from the extras
  778. reqts = self._data.get('%s_requires' % key, [])
  779. result.extend(self.get_requirements(reqts, extras=extras,
  780. env=env))
  781. return result
  782. @property
  783. def dictionary(self):
  784. if self._legacy:
  785. return self._from_legacy()
  786. return self._data
  787. @property
  788. def dependencies(self):
  789. if self._legacy:
  790. raise NotImplementedError
  791. else:
  792. return extract_by_key(self._data, self.DEPENDENCY_KEYS)
  793. @dependencies.setter
  794. def dependencies(self, value):
  795. if self._legacy:
  796. raise NotImplementedError
  797. else:
  798. self._data.update(value)
  799. def _validate_mapping(self, mapping, scheme):
  800. if mapping.get('metadata_version') != self.METADATA_VERSION:
  801. raise MetadataUnrecognizedVersionError()
  802. missing = []
  803. for key, exclusions in self.MANDATORY_KEYS.items():
  804. if key not in mapping:
  805. if scheme not in exclusions:
  806. missing.append(key)
  807. if missing:
  808. msg = 'Missing metadata items: %s' % ', '.join(missing)
  809. raise MetadataMissingError(msg)
  810. for k, v in mapping.items():
  811. self._validate_value(k, v, scheme)
  812. def validate(self):
  813. if self._legacy:
  814. missing, warnings = self._legacy.check(True)
  815. if missing or warnings:
  816. logger.warning('Metadata: missing: %s, warnings: %s',
  817. missing, warnings)
  818. else:
  819. self._validate_mapping(self._data, self.scheme)
  820. def todict(self):
  821. if self._legacy:
  822. return self._legacy.todict(True)
  823. else:
  824. result = extract_by_key(self._data, self.INDEX_KEYS)
  825. return result
  826. def _from_legacy(self):
  827. assert self._legacy and not self._data
  828. result = {
  829. 'metadata_version': self.METADATA_VERSION,
  830. 'generator': self.GENERATOR,
  831. }
  832. lmd = self._legacy.todict(True) # skip missing ones
  833. for k in ('name', 'version', 'license', 'summary', 'description',
  834. 'classifier'):
  835. if k in lmd:
  836. if k == 'classifier':
  837. nk = 'classifiers'
  838. else:
  839. nk = k
  840. result[nk] = lmd[k]
  841. kw = lmd.get('Keywords', [])
  842. if kw == ['']:
  843. kw = []
  844. result['keywords'] = kw
  845. keys = (('requires_dist', 'run_requires'),
  846. ('setup_requires_dist', 'build_requires'))
  847. for ok, nk in keys:
  848. if ok in lmd and lmd[ok]:
  849. result[nk] = [{'requires': lmd[ok]}]
  850. result['provides'] = self.provides
  851. author = {}
  852. maintainer = {}
  853. return result
  854. LEGACY_MAPPING = {
  855. 'name': 'Name',
  856. 'version': 'Version',
  857. 'license': 'License',
  858. 'summary': 'Summary',
  859. 'description': 'Description',
  860. 'classifiers': 'Classifier',
  861. }
  862. def _to_legacy(self):
  863. def process_entries(entries):
  864. reqts = set()
  865. for e in entries:
  866. extra = e.get('extra')
  867. env = e.get('environment')
  868. rlist = e['requires']
  869. for r in rlist:
  870. if not env and not extra:
  871. reqts.add(r)
  872. else:
  873. marker = ''
  874. if extra:
  875. marker = 'extra == "%s"' % extra
  876. if env:
  877. if marker:
  878. marker = '(%s) and %s' % (env, marker)
  879. else:
  880. marker = env
  881. reqts.add(';'.join((r, marker)))
  882. return reqts
  883. assert self._data and not self._legacy
  884. result = LegacyMetadata()
  885. nmd = self._data
  886. for nk, ok in self.LEGACY_MAPPING.items():
  887. if nk in nmd:
  888. result[ok] = nmd[nk]
  889. r1 = process_entries(self.run_requires + self.meta_requires)
  890. r2 = process_entries(self.build_requires + self.dev_requires)
  891. if self.extras:
  892. result['Provides-Extra'] = sorted(self.extras)
  893. result['Requires-Dist'] = sorted(r1)
  894. result['Setup-Requires-Dist'] = sorted(r2)
  895. # TODO: other fields such as contacts
  896. return result
  897. def write(self, path=None, fileobj=None, legacy=False, skip_unknown=True):
  898. if [path, fileobj].count(None) != 1:
  899. raise ValueError('Exactly one of path and fileobj is needed')
  900. self.validate()
  901. if legacy:
  902. if self._legacy:
  903. legacy_md = self._legacy
  904. else:
  905. legacy_md = self._to_legacy()
  906. if path:
  907. legacy_md.write(path, skip_unknown=skip_unknown)
  908. else:
  909. legacy_md.write_file(fileobj, skip_unknown=skip_unknown)
  910. else:
  911. if self._legacy:
  912. d = self._from_legacy()
  913. else:
  914. d = self._data
  915. if fileobj:
  916. json.dump(d, fileobj, ensure_ascii=True, indent=2,
  917. sort_keys=True)
  918. else:
  919. with codecs.open(path, 'w', 'utf-8') as f:
  920. json.dump(d, f, ensure_ascii=True, indent=2,
  921. sort_keys=True)
  922. def add_requirements(self, requirements):
  923. if self._legacy:
  924. self._legacy.add_requirements(requirements)
  925. else:
  926. run_requires = self._data.setdefault('run_requires', [])
  927. always = None
  928. for entry in run_requires:
  929. if 'environment' not in entry and 'extra' not in entry:
  930. always = entry
  931. break
  932. if always is None:
  933. always = { 'requires': requirements }
  934. run_requires.insert(0, always)
  935. else:
  936. rset = set(always['requires']) | set(requirements)
  937. always['requires'] = sorted(rset)
  938. def __repr__(self):
  939. name = self.name or '(no name)'
  940. version = self.version or 'no version'
  941. return '<%s %s %s (%s)>' % (self.__class__.__name__,
  942. self.metadata_version, name, version)