install.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512
  1. """
  2. Operations on existing wheel files, including basic installation.
  3. """
  4. # XXX see patched pip to install
  5. from __future__ import print_function
  6. import csv
  7. import hashlib
  8. import os.path
  9. import re
  10. import shutil
  11. import sys
  12. import warnings
  13. import zipfile
  14. from . import signatures
  15. from .paths import get_install_paths
  16. from .pep425tags import get_supported
  17. from .pkginfo import read_pkg_info_bytes
  18. from .util import (
  19. urlsafe_b64encode, from_json, urlsafe_b64decode, native, binary, HashingFile, open_for_csv)
  20. try:
  21. _big_number = sys.maxsize
  22. except NameError:
  23. _big_number = sys.maxint
  24. # The next major version after this version of the 'wheel' tool:
  25. VERSION_TOO_HIGH = (1, 0)
  26. # Non-greedy matching of an optional build number may be too clever (more
  27. # invalid wheel filenames will match). Separate regex for .dist-info?
  28. WHEEL_INFO_RE = re.compile(
  29. r"""^(?P<namever>(?P<name>.+?)-(?P<ver>\d.*?))(-(?P<build>\d.*?))?
  30. -(?P<pyver>[a-z].+?)-(?P<abi>.+?)-(?P<plat>.+?)(\.whl|\.dist-info)$""",
  31. re.VERBOSE).match
  32. def parse_version(version):
  33. """Use parse_version from pkg_resources or distutils as available."""
  34. global parse_version
  35. try:
  36. from pkg_resources import parse_version
  37. except ImportError:
  38. from distutils.version import LooseVersion as parse_version
  39. return parse_version(version)
  40. class reify(object):
  41. """Put the result of a method which uses this (non-data)
  42. descriptor decorator in the instance dict after the first call,
  43. effectively replacing the decorator with an instance variable.
  44. """
  45. def __init__(self, wrapped):
  46. self.wrapped = wrapped
  47. self.__doc__ = wrapped.__doc__
  48. def __get__(self, inst, objtype=None):
  49. if inst is None:
  50. return self
  51. val = self.wrapped(inst)
  52. setattr(inst, self.wrapped.__name__, val)
  53. return val
  54. class BadWheelFile(ValueError):
  55. pass
  56. class WheelFile(object):
  57. """Parse wheel-specific attributes from a wheel (.whl) file and offer
  58. basic installation and verification support.
  59. WheelFile can be used to simply parse a wheel filename by avoiding the
  60. methods that require the actual file contents."""
  61. WHEEL_INFO = "WHEEL"
  62. RECORD = "RECORD"
  63. def __init__(self,
  64. filename,
  65. fp=None,
  66. append=False,
  67. context=get_supported):
  68. """
  69. :param fp: A seekable file-like object or None to open(filename).
  70. :param append: Open archive in append mode.
  71. :param context: Function returning list of supported tags. Wheels
  72. must have the same context to be sortable.
  73. """
  74. self.filename = filename
  75. self.fp = fp
  76. self.append = append
  77. self.context = context
  78. basename = os.path.basename(filename)
  79. self.parsed_filename = WHEEL_INFO_RE(basename)
  80. if not basename.endswith('.whl') or self.parsed_filename is None:
  81. raise BadWheelFile("Bad filename '%s'" % filename)
  82. def __repr__(self):
  83. return self.filename
  84. @property
  85. def distinfo_name(self):
  86. return "%s.dist-info" % self.parsed_filename.group('namever')
  87. @property
  88. def datadir_name(self):
  89. return "%s.data" % self.parsed_filename.group('namever')
  90. @property
  91. def record_name(self):
  92. return "%s/%s" % (self.distinfo_name, self.RECORD)
  93. @property
  94. def wheelinfo_name(self):
  95. return "%s/%s" % (self.distinfo_name, self.WHEEL_INFO)
  96. @property
  97. def tags(self):
  98. """A wheel file is compatible with the Cartesian product of the
  99. period-delimited tags in its filename.
  100. To choose a wheel file among several candidates having the same
  101. distribution version 'ver', an installer ranks each triple of
  102. (pyver, abi, plat) that its Python installation can run, sorting
  103. the wheels by the best-ranked tag it supports and then by their
  104. arity which is just len(list(compatibility_tags)).
  105. """
  106. tags = self.parsed_filename.groupdict()
  107. for pyver in tags['pyver'].split('.'):
  108. for abi in tags['abi'].split('.'):
  109. for plat in tags['plat'].split('.'):
  110. yield (pyver, abi, plat)
  111. compatibility_tags = tags
  112. @property
  113. def arity(self):
  114. """The number of compatibility tags the wheel declares."""
  115. return len(list(self.compatibility_tags))
  116. @property
  117. def rank(self):
  118. """
  119. Lowest index of any of this wheel's tags in self.context(), and the
  120. arity e.g. (0, 1)
  121. """
  122. return self.compatibility_rank(self.context())
  123. @property
  124. def compatible(self):
  125. return self.rank[0] != _big_number # bad API!
  126. # deprecated:
  127. def compatibility_rank(self, supported):
  128. """Rank the wheel against the supported tags. Smaller ranks are more
  129. compatible!
  130. :param supported: A list of compatibility tags that the current
  131. Python implemenation can run.
  132. """
  133. preferences = []
  134. for tag in self.compatibility_tags:
  135. try:
  136. preferences.append(supported.index(tag))
  137. # Tag not present
  138. except ValueError:
  139. pass
  140. if len(preferences):
  141. return (min(preferences), self.arity)
  142. return (_big_number, 0)
  143. # deprecated
  144. def supports_current_python(self, x):
  145. assert self.context == x, 'context mismatch'
  146. return self.compatible
  147. # Comparability.
  148. # Wheels are equal if they refer to the same file.
  149. # If two wheels are not equal, compare based on (in this order):
  150. # 1. Name
  151. # 2. Version
  152. # 3. Compatibility rank
  153. # 4. Filename (as a tiebreaker)
  154. @property
  155. def _sort_key(self):
  156. return (self.parsed_filename.group('name'),
  157. parse_version(self.parsed_filename.group('ver')),
  158. tuple(-x for x in self.rank),
  159. self.filename)
  160. def __eq__(self, other):
  161. return self.filename == other.filename
  162. def __ne__(self, other):
  163. return self.filename != other.filename
  164. def __lt__(self, other):
  165. if self.context != other.context:
  166. raise TypeError("{0}.context != {1}.context".format(self, other))
  167. return self._sort_key < other._sort_key
  168. # XXX prune
  169. sn = self.parsed_filename.group('name')
  170. on = other.parsed_filename.group('name')
  171. if sn != on:
  172. return sn < on
  173. sv = parse_version(self.parsed_filename.group('ver'))
  174. ov = parse_version(other.parsed_filename.group('ver'))
  175. if sv != ov:
  176. return sv < ov
  177. # Compatibility
  178. if self.context != other.context:
  179. raise TypeError("{0}.context != {1}.context".format(self, other))
  180. sc = self.rank
  181. oc = other.rank
  182. if sc is not None and oc is not None and sc != oc:
  183. # Smaller compatibility ranks are "better" than larger ones,
  184. # so we have to reverse the sense of the comparison here!
  185. return sc > oc
  186. elif sc is None and oc is not None:
  187. return False
  188. return self.filename < other.filename
  189. def __gt__(self, other):
  190. return other < self
  191. def __le__(self, other):
  192. return self == other or self < other
  193. def __ge__(self, other):
  194. return self == other or other < self
  195. #
  196. # Methods using the file's contents:
  197. #
  198. @reify
  199. def zipfile(self):
  200. mode = "r"
  201. if self.append:
  202. mode = "a"
  203. vzf = VerifyingZipFile(self.fp if self.fp else self.filename, mode)
  204. if not self.append:
  205. self.verify(vzf)
  206. return vzf
  207. @reify
  208. def parsed_wheel_info(self):
  209. """Parse wheel metadata (the .data/WHEEL file)"""
  210. return read_pkg_info_bytes(self.zipfile.read(self.wheelinfo_name))
  211. def check_version(self):
  212. version = self.parsed_wheel_info['Wheel-Version']
  213. if tuple(map(int, version.split('.'))) >= VERSION_TOO_HIGH:
  214. raise ValueError("Wheel version is too high")
  215. @reify
  216. def install_paths(self):
  217. """
  218. Consult distutils to get the install paths for our dist. A dict with
  219. ('purelib', 'platlib', 'headers', 'scripts', 'data').
  220. We use the name from our filename as the dist name, which means headers
  221. could be installed in the wrong place if the filesystem-escaped name
  222. is different than the Name. Who cares?
  223. """
  224. name = self.parsed_filename.group('name')
  225. return get_install_paths(name)
  226. def install(self, force=False, overrides={}):
  227. """
  228. Install the wheel into site-packages.
  229. """
  230. # Utility to get the target directory for a particular key
  231. def get_path(key):
  232. return overrides.get(key) or self.install_paths[key]
  233. # The base target location is either purelib or platlib
  234. if self.parsed_wheel_info['Root-Is-Purelib'] == 'true':
  235. root = get_path('purelib')
  236. else:
  237. root = get_path('platlib')
  238. # Parse all the names in the archive
  239. name_trans = {}
  240. for info in self.zipfile.infolist():
  241. name = info.filename
  242. # Zip files can contain entries representing directories.
  243. # These end in a '/'.
  244. # We ignore these, as we create directories on demand.
  245. if name.endswith('/'):
  246. continue
  247. # Pathnames in a zipfile namelist are always /-separated.
  248. # In theory, paths could start with ./ or have other oddities
  249. # but this won't happen in practical cases of well-formed wheels.
  250. # We'll cover the simple case of an initial './' as it's both easy
  251. # to do and more common than most other oddities.
  252. if name.startswith('./'):
  253. name = name[2:]
  254. # Split off the base directory to identify files that are to be
  255. # installed in non-root locations
  256. basedir, sep, filename = name.partition('/')
  257. if sep and basedir == self.datadir_name:
  258. # Data file. Target destination is elsewhere
  259. key, sep, filename = filename.partition('/')
  260. if not sep:
  261. raise ValueError("Invalid filename in wheel: {0}".format(name))
  262. target = get_path(key)
  263. else:
  264. # Normal file. Target destination is root
  265. key = ''
  266. target = root
  267. filename = name
  268. # Map the actual filename from the zipfile to its intended target
  269. # directory and the pathname relative to that directory.
  270. dest = os.path.normpath(os.path.join(target, filename))
  271. name_trans[info] = (key, target, filename, dest)
  272. # We're now ready to start processing the actual install. The process
  273. # is as follows:
  274. # 1. Prechecks - is the wheel valid, is its declared architecture
  275. # OK, etc. [[Responsibility of the caller]]
  276. # 2. Overwrite check - do any of the files to be installed already
  277. # exist?
  278. # 3. Actual install - put the files in their target locations.
  279. # 4. Update RECORD - write a suitably modified RECORD file to
  280. # reflect the actual installed paths.
  281. if not force:
  282. for info, v in name_trans.items():
  283. k = info.filename
  284. key, target, filename, dest = v
  285. if os.path.exists(dest):
  286. raise ValueError(
  287. "Wheel file {0} would overwrite {1}. Use force if this is intended".format(
  288. k, dest))
  289. # Get the name of our executable, for use when replacing script
  290. # wrapper hashbang lines.
  291. # We encode it using getfilesystemencoding, as that is "the name of
  292. # the encoding used to convert Unicode filenames into system file
  293. # names".
  294. exename = sys.executable.encode(sys.getfilesystemencoding())
  295. record_data = []
  296. record_name = self.distinfo_name + '/RECORD'
  297. for info, (key, target, filename, dest) in name_trans.items():
  298. name = info.filename
  299. source = self.zipfile.open(info)
  300. # Skip the RECORD file
  301. if name == record_name:
  302. continue
  303. ddir = os.path.dirname(dest)
  304. if not os.path.isdir(ddir):
  305. os.makedirs(ddir)
  306. temp_filename = dest + '.part'
  307. try:
  308. with HashingFile(temp_filename, 'wb') as destination:
  309. if key == 'scripts':
  310. hashbang = source.readline()
  311. if hashbang.startswith(b'#!python'):
  312. hashbang = b'#!' + exename + binary(os.linesep)
  313. destination.write(hashbang)
  314. shutil.copyfileobj(source, destination)
  315. except BaseException:
  316. if os.path.exists(temp_filename):
  317. os.unlink(temp_filename)
  318. raise
  319. os.rename(temp_filename, dest)
  320. reldest = os.path.relpath(dest, root)
  321. reldest.replace(os.sep, '/')
  322. record_data.append((reldest, destination.digest(), destination.length))
  323. destination.close()
  324. source.close()
  325. # preserve attributes (especially +x bit for scripts)
  326. attrs = info.external_attr >> 16
  327. if attrs: # tends to be 0 if Windows.
  328. os.chmod(dest, info.external_attr >> 16)
  329. record_name = os.path.join(root, self.record_name)
  330. with open_for_csv(record_name, 'w+') as record_file:
  331. writer = csv.writer(record_file)
  332. for reldest, digest, length in sorted(record_data):
  333. writer.writerow((reldest, digest, length))
  334. writer.writerow((self.record_name, '', ''))
  335. def verify(self, zipfile=None):
  336. """Configure the VerifyingZipFile `zipfile` by verifying its signature
  337. and setting expected hashes for every hash in RECORD.
  338. Caller must complete the verification process by completely reading
  339. every file in the archive (e.g. with extractall)."""
  340. sig = None
  341. if zipfile is None:
  342. zipfile = self.zipfile
  343. zipfile.strict = True
  344. record_name = '/'.join((self.distinfo_name, 'RECORD'))
  345. sig_name = '/'.join((self.distinfo_name, 'RECORD.jws'))
  346. # tolerate s/mime signatures:
  347. smime_sig_name = '/'.join((self.distinfo_name, 'RECORD.p7s'))
  348. zipfile.set_expected_hash(record_name, None)
  349. zipfile.set_expected_hash(sig_name, None)
  350. zipfile.set_expected_hash(smime_sig_name, None)
  351. record = zipfile.read(record_name)
  352. record_digest = urlsafe_b64encode(hashlib.sha256(record).digest())
  353. try:
  354. sig = from_json(native(zipfile.read(sig_name)))
  355. except KeyError: # no signature
  356. pass
  357. if sig:
  358. headers, payload = signatures.verify(sig)
  359. if payload['hash'] != "sha256=" + native(record_digest):
  360. msg = "RECORD.jws claimed RECORD hash {} != computed hash {}."
  361. raise BadWheelFile(msg.format(payload['hash'],
  362. native(record_digest)))
  363. reader = csv.reader((native(r, 'utf-8') for r in record.splitlines()))
  364. for row in reader:
  365. filename = row[0]
  366. hash = row[1]
  367. if not hash:
  368. if filename not in (record_name, sig_name):
  369. print("%s has no hash!" % filename, file=sys.stderr)
  370. continue
  371. algo, data = row[1].split('=', 1)
  372. assert algo == "sha256", "Unsupported hash algorithm"
  373. zipfile.set_expected_hash(filename, urlsafe_b64decode(binary(data)))
  374. class VerifyingZipFile(zipfile.ZipFile):
  375. """ZipFile that can assert that each of its extracted contents matches
  376. an expected sha256 hash. Note that each file must be completely read in
  377. order for its hash to be checked."""
  378. def __init__(self, file, mode="r",
  379. compression=zipfile.ZIP_STORED,
  380. allowZip64=True):
  381. super(VerifyingZipFile, self).__init__(file, mode, compression, allowZip64)
  382. self.strict = False
  383. self._expected_hashes = {}
  384. self._hash_algorithm = hashlib.sha256
  385. def set_expected_hash(self, name, hash):
  386. """
  387. :param name: name of zip entry
  388. :param hash: bytes of hash (or None for "don't care")
  389. """
  390. self._expected_hashes[name] = hash
  391. def open(self, name_or_info, mode="r", pwd=None):
  392. """Return file-like object for 'name'."""
  393. # A non-monkey-patched version would contain most of zipfile.py
  394. ef = super(VerifyingZipFile, self).open(name_or_info, mode, pwd)
  395. if isinstance(name_or_info, zipfile.ZipInfo):
  396. name = name_or_info.filename
  397. else:
  398. name = name_or_info
  399. if name in self._expected_hashes and self._expected_hashes[name] is not None:
  400. expected_hash = self._expected_hashes[name]
  401. try:
  402. _update_crc_orig = ef._update_crc
  403. except AttributeError:
  404. warnings.warn('Need ZipExtFile._update_crc to implement '
  405. 'file hash verification (in Python >= 2.7)')
  406. return ef
  407. running_hash = self._hash_algorithm()
  408. if hasattr(ef, '_eof'): # py33
  409. def _update_crc(data):
  410. _update_crc_orig(data)
  411. running_hash.update(data)
  412. if ef._eof and running_hash.digest() != expected_hash:
  413. raise BadWheelFile("Bad hash for file %r" % ef.name)
  414. else:
  415. def _update_crc(data, eof=None):
  416. _update_crc_orig(data, eof=eof)
  417. running_hash.update(data)
  418. if eof and running_hash.digest() != expected_hash:
  419. raise BadWheelFile("Bad hash for file %r" % ef.name)
  420. ef._update_crc = _update_crc
  421. elif self.strict and name not in self._expected_hashes:
  422. raise BadWheelFile("No expected hash for file %r" % ef.name)
  423. return ef
  424. def pop(self):
  425. """Truncate the last file off this zipfile.
  426. Assumes infolist() is in the same order as the files (true for
  427. ordinary zip files created by Python)"""
  428. if not self.fp:
  429. raise RuntimeError(
  430. "Attempt to pop from ZIP archive that was already closed")
  431. last = self.infolist().pop()
  432. del self.NameToInfo[last.filename]
  433. self.fp.seek(last.header_offset, os.SEEK_SET)
  434. self.fp.truncate()
  435. self._didModify = True