Merge pull request #6786 from certbot/pin-deps_continued

This continues from the work of @sydneyli in PR #6671
I didn't do much here. Basically added support for reading data from sys.stdin to both tools/merge_requirements.py and tools/strip_hashes.py as well as support for reading files from paths passed as cli parameters to strip_hashes.py.

Reading the filepaths was not strictly required, but I thought would be a good thing to do in order to keep the tooling usage options consistent.

Fixes #6581

* Generate constraints file to pin deps in Docker images

Dockerfiles pin versions using constraints file

Pulling out strip_hashes and add --no-deps flag

* Add stdin option for merge_requirements

Add stdin and file path support to strip_hashes

* Address review comments

Dockerfile

@@ -6,7 +6,13 @@ VOLUME /etc/letsencrypt /var/lib/letsencrypt
 WORKDIR /opt/certbot
 
 COPY CHANGELOG.md README.rst setup.py src/
+
+# Generate constraints file to pin dependency versions
 COPY letsencrypt-auto-source/pieces/dependency-requirements.txt .
+COPY tools /opt/certbot/tools
+RUN sh -c 'cat dependency-requirements.txt | /opt/certbot/tools/strip_hashes.py > unhashed_requirements.txt'
+RUN sh -c 'cat tools/dev_constraints.txt unhashed_requirements.txt | /opt/certbot/tools/merge_requirements.py > docker_constraints.txt'
+
 COPY acme src/acme
 COPY certbot src/certbot
 
@@ -23,7 +29,7 @@ RUN apk add --no-cache --virtual .build-deps \
         musl-dev \
         libffi-dev \
     && pip install -r /opt/certbot/dependency-requirements.txt \
-    && pip install --no-cache-dir \
+    && pip install --no-cache-dir --no-deps \
         --editable /opt/certbot/src/acme \
         --editable /opt/certbot/src \
     && apk del .build-deps

certbot-dns-cloudflare/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-cloudflare
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudflare

certbot-dns-cloudxns/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-cloudxns
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudxns

certbot-dns-digitalocean/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-digitalocean
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-digitalocean

certbot-dns-dnsimple/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-dnsimple
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsimple

certbot-dns-dnsmadeeasy/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-dnsmadeeasy
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsmadeeasy

certbot-dns-gehirn/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-gehirn
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-gehirn
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-gehirn

certbot-dns-google/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-google
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-google
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-google

certbot-dns-linode/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-linode
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-linode
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-linode

certbot-dns-luadns/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-luadns
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-luadns
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-luadns

certbot-dns-nsone/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-nsone
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-nsone
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-nsone

certbot-dns-ovh/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-ovh
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-ovh
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-ovh

certbot-dns-rfc2136/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-rfc2136
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-rfc2136

certbot-dns-route53/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-route53
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-route53
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-route53

certbot-dns-sakuracloud/Dockerfile

@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-sakuracloud
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-sakuracloud
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-sakuracloud

tools/dev_constraints.txt

@@ -1,4 +1,4 @@
-# Specifies Python package versions for development.
+# Specifies Python package versions for development and building Docker images.
 # It includes in particular packages not specified in letsencrypt-auto's requirements file.
 # Some dev package versions specified here may be overridden by higher level constraints
 # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt).

tools/merge_requirements.py

@@ -10,27 +10,36 @@ from __future__ import print_function
 import sys
 
 
-def read_file(file_path):
-    """Reads in a Python requirements file.
+def process_entries(entries):
+    """
     Ignore empty lines, comments and editable requirements
 
-    :param str file_path: path to requirements file
+    :param list entries: List of entries
 
     :returns: mapping from a project to its pinned version
     :rtype: dict
-
     """
     data = {}
-    with open(file_path) as file_h:
-        for line in file_h:
-            line = line.strip()
-            if line and not line.startswith('#') and not line.startswith('-e'):
-                project, version = line.split('==')
-                if not version:
-                    raise ValueError("Unexpected syntax '{0}'".format(line))
-                data[project] = version
+    for e in entries:
+        e = e.strip()
+        if e and not e.startswith('#') and not e.startswith('-e'):
+            project, version = e.split('==')
+            if not version:
+                raise ValueError("Unexpected syntax '{0}'".format(e))
+            data[project] = version
     return data
 
+def read_file(file_path):
+    """Reads in a Python requirements file.
+
+    :param str file_path: path to requirements file
+
+    :returns: list of entries in the file
+    :rtype: list
+
+    """
+    with open(file_path) as file_h:
+        return file_h.readlines()
 
 def output_requirements(requirements):
     """Prepare print requirements to stdout.
@@ -46,14 +55,25 @@ def main(*paths):
     """Merges multiple requirements files together and prints the result.
 
     Requirement files specified later in the list take precedence over earlier
-    files.
+    files. Files are read from file paths passed from the command line arguments.
 
-    :param tuple paths: paths to requirements files
+    If no command line arguments are defined, data is read from stdin instead.
+
+    :param tuple paths: paths to requirements files provided on command line
 
     """
     data = {}
-    for path in paths:
-        data.update(read_file(path))
+    if paths:
+        for path in paths:
+            data.update(process_entries(read_file(path)))
+    else:
+        # Need to check if interactive to avoid blocking if nothing is piped
+        if not sys.stdin.isatty():
+            stdin_data = []
+            for line in sys.stdin:
+                stdin_data.append(line)
+            data.update(process_entries(stdin_data))
+
     return output_requirements(data)
 
 

tools/pip_install.py

@@ -19,6 +19,7 @@ import tempfile
 
 import merge_requirements as merge_module
 import readlink
+import strip_hashes
 
 
 def find_tools_path():
@@ -47,10 +48,8 @@ def certbot_normal_processing(tools_path, test_constraints):
     with open(certbot_requirements, 'r') as fd:
         data = fd.readlines()
     with open(test_constraints, 'w') as fd:
-        for line in data:
-            search = re.search(r'^(\S*==\S*).*$', line)
-            if search:
-                fd.write('{0}{1}'.format(search.group(1), os.linesep))
+        data = "\n".join(strip_hashes.process_entries(data))
+        fd.write(data)
 
 
 def merge_requirements(tools_path, requirements, test_constraints, all_constraints):

tools/strip_hashes.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""Removes hash information from requirement files passed to it as file path
+arguments or simply piped to stdin."""
+
+import re
+import sys
+
+
+def process_entries(entries):
+    """Strips off hash strings from dependencies.
+
+    :param list entries: List of entries
+
+    :returns: list of dependencies without hashes
+    :rtype: list
+    """
+    out_lines = []
+    for e in entries:
+        e = e.strip()
+        search = re.search(r'^(\S*==\S*).*$', e)
+        if search:
+            out_lines.append(search.group(1))
+    return out_lines
+
+def main(*paths):
+    """
+    Reads dependency definitions from a (list of) file(s) provided on the
+    command line. If no command line arguments are present, data is read from
+    stdin instead.
+
+    Hashes are removed from returned entries.
+    """
+
+    deps = []
+    if paths:
+        for path in paths:
+            with open(path) as file_h:
+                deps += process_entries(file_h.readlines())
+    else:
+        # Need to check if interactive to avoid blocking if nothing is piped
+        if not sys.stdin.isatty():
+            stdin_data = []
+            for line in sys.stdin:
+                stdin_data.append(line)
+            deps += process_entries(stdin_data)
+
+    return "\n".join(deps)
+
+if __name__ == '__main__':
+    print(main(*sys.argv[1:]))  # pylint: disable=star-args