From 02255fa024cb101e72e3f5ff7119f20c60abd27b Mon Sep 17 00:00:00 2001 From: Erik Rose Date: Wed, 2 Dec 2015 22:38:48 -0500 Subject: [PATCH] Upgrade peep to 2.5, for compatibility with pip 7.x. --- letsencrypt_auto/pieces/peep.py | 155 +++++++++++++++++++++----------- 1 file changed, 105 insertions(+), 50 deletions(-) diff --git a/letsencrypt_auto/pieces/peep.py b/letsencrypt_auto/pieces/peep.py index 23e917ac6..6b9393a5e 100755 --- a/letsencrypt_auto/pieces/peep.py +++ b/letsencrypt_auto/pieces/peep.py @@ -26,13 +26,13 @@ try: xrange = xrange except NameError: xrange = range -from base64 import urlsafe_b64encode +from base64 import urlsafe_b64encode, urlsafe_b64decode +from binascii import hexlify import cgi from collections import defaultdict from functools import wraps from hashlib import sha256 -from itertools import chain -from linecache import getline +from itertools import chain, islice import mimetypes from optparse import OptionParser from os.path import join, basename, splitext, isdir @@ -104,8 +104,18 @@ except ImportError: DownloadProgressBar = DownloadProgressSpinner = NullProgressBar +__version__ = 2, 5, 0 -__version__ = 2, 4, 1 +try: + from pip.index import FormatControl # noqa + FORMAT_CONTROL_ARG = 'format_control' + + # The line-numbering bug will be fixed in pip 8. All 7.x releases had it. + PIP_MAJOR_VERSION = int(pip.__version__.split('.')[0]) + PIP_COUNTS_COMMENTS = PIP_MAJOR_VERSION >= 8 +except ImportError: + FORMAT_CONTROL_ARG = 'use_wheel' # pre-7 + PIP_COUNTS_COMMENTS = True ITS_FINE_ITS_FINE = 0 @@ -149,6 +159,45 @@ def encoded_hash(sha): return urlsafe_b64encode(sha.digest()).decode('ascii').rstrip('=') +def path_and_line(req): + """Return the path and line number of the file from which an + InstallRequirement came. + + """ + path, line = (re.match(r'-r (.*) \(line (\d+)\)$', + req.comes_from).groups()) + return path, int(line) + + +def hashes_above(path, line_number): + """Yield hashes from contiguous comment lines before line ``line_number``. + + """ + def hash_lists(path): + """Yield lists of hashes appearing between non-comment lines. + + The lists will be in order of appearance and, for each non-empty + list, their place in the results will coincide with that of the + line number of the corresponding result from `parse_requirements` + (which changed in pip 7.0 to not count comments). + + """ + hashes = [] + with open(path) as file: + for lineno, line in enumerate(file, 1): + match = HASH_COMMENT_RE.match(line) + if match: # Accumulate this hash. + hashes.append(match.groupdict()['hash']) + if not IGNORED_LINE_RE.match(line): + yield hashes # Report hashes seen so far. + hashes = [] + elif PIP_COUNTS_COMMENTS: + # Comment: count as normal req but have no hashes. + yield [] + + return next(islice(hash_lists(path), line_number - 1, None)) + + def run_pip(initial_args): """Delegate to pip the given args (starting with the subcommand), and raise ``PipException`` if something goes wrong.""" @@ -217,6 +266,8 @@ def requirement_args(argv, want_paths=False, want_other=False): if want_other: yield arg +# any line that is a comment or just whitespace +IGNORED_LINE_RE = re.compile(r'^(\s*#.*)?\s*$') HASH_COMMENT_RE = re.compile( r""" @@ -311,7 +362,7 @@ def package_finder(argv): # Carry over PackageFinder kwargs that have [about] the same names as # options attr names: possible_options = [ - 'find_links', 'use_wheel', 'allow_external', 'allow_unverified', + 'find_links', FORMAT_CONTROL_ARG, 'allow_external', 'allow_unverified', 'allow_all_external', ('allow_all_prereleases', 'pre'), 'process_dependency_links'] kwargs = {} @@ -434,36 +485,10 @@ class DownloadedReq(object): return True return False - def _path_and_line(self): - """Return the path and line number of the file from which our - InstallRequirement came. - - """ - path, line = (re.match(r'-r (.*) \(line (\d+)\)$', - self._req.comes_from).groups()) - return path, int(line) - @memoize # Avoid hitting the file[cache] over and over. def _expected_hashes(self): """Return a list of known-good hashes for this package.""" - - def hashes_above(path, line_number): - """Yield hashes from contiguous comment lines before line - ``line_number``. - - """ - for line_number in xrange(line_number - 1, 0, -1): - line = getline(path, line_number) - match = HASH_COMMENT_RE.match(line) - if match: - yield match.groupdict()['hash'] - elif not line.lstrip().startswith('#'): - # If we hit a non-comment line, abort - break - - hashes = list(hashes_above(*self._path_and_line())) - hashes.reverse() # because we read them backwards - return hashes + return hashes_above(*path_and_line(self._req)) def _download(self, link): """Download a file, and return its name within my temp dir. @@ -783,6 +808,22 @@ def first_every_last(iterable, first, every, last): last(item) +def _parse_requirements(path, finder): + try: + # list() so the generator that is parse_requirements() actually runs + # far enough to report a TypeError + return list(parse_requirements( + path, options=EmptyOptions(), finder=finder)) + except TypeError: + # session is a required kwarg as of pip 6.0 and will raise + # a TypeError if missing. It needs to be a PipSession instance, + # but in older versions we can't import it from pip.download + # (nor do we need it at all) so we only import it in this except block + from pip.download import PipSession + return list(parse_requirements( + path, options=EmptyOptions(), session=PipSession(), finder=finder)) + + def downloaded_reqs_from_path(path, argv): """Return a list of DownloadedReqs representing the requirements parsed out of a given requirements file. @@ -792,22 +833,8 @@ def downloaded_reqs_from_path(path, argv): """ finder = package_finder(argv) - - def downloaded_reqs(parsed_reqs): - """Just avoid repeating this list comp.""" - return [DownloadedReq(req, argv, finder) for req in parsed_reqs] - - try: - return downloaded_reqs(parse_requirements( - path, options=EmptyOptions(), finder=finder)) - except TypeError: - # session is a required kwarg as of pip 6.0 and will raise - # a TypeError if missing. It needs to be a PipSession instance, - # but in older versions we can't import it from pip.download - # (nor do we need it at all) so we only import it in this except block - from pip.download import PipSession - return downloaded_reqs(parse_requirements( - path, options=EmptyOptions(), session=PipSession(), finder=finder)) + return [DownloadedReq(req, argv, finder) for req in + _parse_requirements(path, finder)] def peep_install(argv): @@ -823,7 +850,7 @@ def peep_install(argv): try: req_paths = list(requirement_args(argv, want_paths=True)) if not req_paths: - out("You have to ``this`` specify one or more requirements files with the -r option, because\n" + out("You have to specify one or more requirements files with the -r option, because\n" "otherwise there's nowhere for peep to look up the hashes.\n") return COMMAND_LINE_ERROR @@ -864,10 +891,38 @@ def peep_install(argv): print(''.join(output)) +def peep_port(paths): + """Convert a peep requirements file to one compatble with pip-8 hashing. + + Loses comments and tromps on URLs, so the result will need a little manual + massaging, but the hard part--the hash conversion--is done for you. + + """ + if not paths: + print('Please specify one or more requirements files so I have ' + 'something to port.\n') + return COMMAND_LINE_ERROR + for req in chain.from_iterable( + _parse_requirements(path, package_finder(argv)) for path in paths): + hashes = [hexlify(urlsafe_b64decode((hash + '=').encode('ascii'))).decode('ascii') + for hash in hashes_above(*path_and_line(req))] + if not hashes: + print(req.req) + elif len(hashes) == 1: + print('%s --hash=sha256:%s' % (req.req, hashes[0])) + else: + print('%s' % req.req, end='') + for hash in hashes: + print(' \\') + print(' --hash=sha256:%s' % hash, end='') + print() + + def main(): """Be the top-level entrypoint. Return a shell status code.""" commands = {'hash': peep_hash, - 'install': peep_install} + 'install': peep_install, + 'port': peep_port} try: if len(argv) >= 2 and argv[1] in commands: return commands[argv[1]](argv[2:])