Upgrade peep to 2.5, for compatibility with pip 7.x.

letsencrypt_auto/pieces/peep.py

@@ -26,13 +26,13 @@ try:
     xrange = xrange
 except NameError:
     xrange = range
-from base64 import urlsafe_b64encode
+from base64 import urlsafe_b64encode, urlsafe_b64decode
+from binascii import hexlify
 import cgi
 from collections import defaultdict
 from functools import wraps
 from hashlib import sha256
-from itertools import chain
-from linecache import getline
+from itertools import chain, islice
 import mimetypes
 from optparse import OptionParser
 from os.path import join, basename, splitext, isdir
@@ -104,8 +104,18 @@ except ImportError:
 
     DownloadProgressBar = DownloadProgressSpinner = NullProgressBar
 
+__version__ = 2, 5, 0
 
-__version__ = 2, 4, 1
+try:
+    from pip.index import FormatControl  # noqa
+    FORMAT_CONTROL_ARG = 'format_control'
+
+    # The line-numbering bug will be fixed in pip 8. All 7.x releases had it.
+    PIP_MAJOR_VERSION = int(pip.__version__.split('.')[0])
+    PIP_COUNTS_COMMENTS = PIP_MAJOR_VERSION >= 8
+except ImportError:
+    FORMAT_CONTROL_ARG = 'use_wheel'  # pre-7
+    PIP_COUNTS_COMMENTS = True
 
 
 ITS_FINE_ITS_FINE = 0
@@ -149,6 +159,45 @@ def encoded_hash(sha):
     return urlsafe_b64encode(sha.digest()).decode('ascii').rstrip('=')
 
 
+def path_and_line(req):
+    """Return the path and line number of the file from which an
+    InstallRequirement came.
+
+    """
+    path, line = (re.match(r'-r (.*) \(line (\d+)\)$',
+                           req.comes_from).groups())
+    return path, int(line)
+
+
+def hashes_above(path, line_number):
+    """Yield hashes from contiguous comment lines before line ``line_number``.
+
+    """
+    def hash_lists(path):
+        """Yield lists of hashes appearing between non-comment lines.
+
+        The lists will be in order of appearance and, for each non-empty
+        list, their place in the results will coincide with that of the
+        line number of the corresponding result from `parse_requirements`
+        (which changed in pip 7.0 to not count comments).
+
+        """
+        hashes = []
+        with open(path) as file:
+            for lineno, line in enumerate(file, 1):
+                match = HASH_COMMENT_RE.match(line)
+                if match:  # Accumulate this hash.
+                    hashes.append(match.groupdict()['hash'])
+                if not IGNORED_LINE_RE.match(line):
+                    yield hashes  # Report hashes seen so far.
+                    hashes = []
+                elif PIP_COUNTS_COMMENTS:
+                    # Comment: count as normal req but have no hashes.
+                    yield []
+
+    return next(islice(hash_lists(path), line_number - 1, None))
+
+
 def run_pip(initial_args):
     """Delegate to pip the given args (starting with the subcommand), and raise
     ``PipException`` if something goes wrong."""
@@ -217,6 +266,8 @@ def requirement_args(argv, want_paths=False, want_other=False):
             if want_other:
                 yield arg
 
+# any line that is a comment or just whitespace
+IGNORED_LINE_RE = re.compile(r'^(\s*#.*)?\s*$')
 
 HASH_COMMENT_RE = re.compile(
     r"""
@@ -311,7 +362,7 @@ def package_finder(argv):
     # Carry over PackageFinder kwargs that have [about] the same names as
     # options attr names:
     possible_options = [
-        'find_links', 'use_wheel', 'allow_external', 'allow_unverified',
+        'find_links', FORMAT_CONTROL_ARG, 'allow_external', 'allow_unverified',
         'allow_all_external', ('allow_all_prereleases', 'pre'),
         'process_dependency_links']
     kwargs = {}
@@ -434,36 +485,10 @@ class DownloadedReq(object):
                     return True
         return False
 
-    def _path_and_line(self):
-        """Return the path and line number of the file from which our
-        InstallRequirement came.
-
-        """
-        path, line = (re.match(r'-r (.*) \(line (\d+)\)$',
-                               self._req.comes_from).groups())
-        return path, int(line)
-
     @memoize  # Avoid hitting the file[cache] over and over.
     def _expected_hashes(self):
         """Return a list of known-good hashes for this package."""
-
-        def hashes_above(path, line_number):
-            """Yield hashes from contiguous comment lines before line
-            ``line_number``.
-
-            """
-            for line_number in xrange(line_number - 1, 0, -1):
-                line = getline(path, line_number)
-                match = HASH_COMMENT_RE.match(line)
-                if match:
-                    yield match.groupdict()['hash']
-                elif not line.lstrip().startswith('#'):
-                    # If we hit a non-comment line, abort
-                    break
-
-        hashes = list(hashes_above(*self._path_and_line()))
-        hashes.reverse()  # because we read them backwards
-        return hashes
+        return hashes_above(*path_and_line(self._req))
 
     def _download(self, link):
         """Download a file, and return its name within my temp dir.
@@ -783,6 +808,22 @@ def first_every_last(iterable, first, every, last):
         last(item)
 
 
+def _parse_requirements(path, finder):
+    try:
+        # list() so the generator that is parse_requirements() actually runs
+        # far enough to report a TypeError
+        return list(parse_requirements(
+            path, options=EmptyOptions(), finder=finder))
+    except TypeError:
+        # session is a required kwarg as of pip 6.0 and will raise
+        # a TypeError if missing. It needs to be a PipSession instance,
+        # but in older versions we can't import it from pip.download
+        # (nor do we need it at all) so we only import it in this except block
+        from pip.download import PipSession
+        return list(parse_requirements(
+            path, options=EmptyOptions(), session=PipSession(), finder=finder))
+
+
 def downloaded_reqs_from_path(path, argv):
     """Return a list of DownloadedReqs representing the requirements parsed
     out of a given requirements file.
@@ -792,22 +833,8 @@ def downloaded_reqs_from_path(path, argv):
 
     """
     finder = package_finder(argv)
-
-    def downloaded_reqs(parsed_reqs):
-        """Just avoid repeating this list comp."""
-        return [DownloadedReq(req, argv, finder) for req in parsed_reqs]
-
-    try:
-        return downloaded_reqs(parse_requirements(
-            path, options=EmptyOptions(), finder=finder))
-    except TypeError:
-        # session is a required kwarg as of pip 6.0 and will raise
-        # a TypeError if missing. It needs to be a PipSession instance,
-        # but in older versions we can't import it from pip.download
-        # (nor do we need it at all) so we only import it in this except block
-        from pip.download import PipSession
-        return downloaded_reqs(parse_requirements(
-            path, options=EmptyOptions(), session=PipSession(), finder=finder))
+    return [DownloadedReq(req, argv, finder) for req in
+            _parse_requirements(path, finder)]
 
 
 def peep_install(argv):
@@ -823,7 +850,7 @@ def peep_install(argv):
     try:
         req_paths = list(requirement_args(argv, want_paths=True))
         if not req_paths:
-            out("You have to ``this`` specify one or more requirements files with the -r option, because\n"
+            out("You have to specify one or more requirements files with the -r option, because\n"
                 "otherwise there's nowhere for peep to look up the hashes.\n")
             return COMMAND_LINE_ERROR
 
@@ -864,10 +891,38 @@ def peep_install(argv):
         print(''.join(output))
 
 
+def peep_port(paths):
+    """Convert a peep requirements file to one compatble with pip-8 hashing.
+
+    Loses comments and tromps on URLs, so the result will need a little manual
+    massaging, but the hard part--the hash conversion--is done for you.
+
+    """
+    if not paths:
+        print('Please specify one or more requirements files so I have '
+              'something to port.\n')
+        return COMMAND_LINE_ERROR
+    for req in chain.from_iterable(
+            _parse_requirements(path, package_finder(argv)) for path in paths):
+        hashes = [hexlify(urlsafe_b64decode((hash + '=').encode('ascii'))).decode('ascii')
+                  for hash in hashes_above(*path_and_line(req))]
+        if not hashes:
+            print(req.req)
+        elif len(hashes) == 1:
+            print('%s --hash=sha256:%s' % (req.req, hashes[0]))
+        else:
+            print('%s' % req.req, end='')
+            for hash in hashes:
+                print(' \\')
+                print('    --hash=sha256:%s' % hash, end='')
+            print()
+
+
 def main():
     """Be the top-level entrypoint. Return a shell status code."""
     commands = {'hash': peep_hash,
-                'install': peep_install}
+                'install': peep_install,
+                'port': peep_port}
     try:
         if len(argv) >= 2 and argv[1] in commands:
             return commands[argv[1]](argv[2:])