From 7c731599a0140868acdb609bc20345f0bd0bd3aa Mon Sep 17 00:00:00 2001
From: sydneyli <sydney@eff.org>
Date: Fri, 18 Jan 2019 17:09:19 -0800
Subject: [PATCH 1/3] Generate constraints file to pin deps in Docker images

Dockerfiles pin versions using constraints file

Pulling out strip_hashes and add --no-deps flag
---
 Dockerfile                          |  8 +++++++-
 certbot-dns-cloudflare/Dockerfile   |  2 +-
 certbot-dns-cloudxns/Dockerfile     |  2 +-
 certbot-dns-digitalocean/Dockerfile |  2 +-
 certbot-dns-dnsimple/Dockerfile     |  2 +-
 certbot-dns-dnsmadeeasy/Dockerfile  |  2 +-
 certbot-dns-gehirn/Dockerfile       |  2 +-
 certbot-dns-google/Dockerfile       |  2 +-
 certbot-dns-linode/Dockerfile       |  2 +-
 certbot-dns-luadns/Dockerfile       |  2 +-
 certbot-dns-nsone/Dockerfile        |  2 +-
 certbot-dns-ovh/Dockerfile          |  2 +-
 certbot-dns-rfc2136/Dockerfile      |  2 +-
 certbot-dns-route53/Dockerfile      |  2 +-
 certbot-dns-sakuracloud/Dockerfile  |  2 +-
 tools/dev_constraints.txt           |  2 +-
 tools/pip_install.py                |  6 ++----
 tools/strip_hashes.py               | 16 ++++++++++++++++
 18 files changed, 40 insertions(+), 20 deletions(-)
 create mode 100755 tools/strip_hashes.py

diff --git a/Dockerfile b/Dockerfile
index 8f434e89c..828f5ec94 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,7 +6,13 @@ VOLUME /etc/letsencrypt /var/lib/letsencrypt
 WORKDIR /opt/certbot
 
 COPY CHANGELOG.md README.rst setup.py src/
+
+# Generate constraints file to pin dependency versions
 COPY letsencrypt-auto-source/pieces/dependency-requirements.txt .
+COPY tools /opt/certbot/tools
+RUN sh -c 'cat dependency-requirements.txt | /opt/certbot/tools/strip_hashes.py > unhashed_requirements.txt'
+RUN sh -c 'cat tools/dev_constraints.txt unhashed_requirements.txt | /opt/certbot/tools/merge_requirements.py > docker_constraints.txt'
+
 COPY acme src/acme
 COPY certbot src/certbot
 
@@ -23,7 +29,7 @@ RUN apk add --no-cache --virtual .build-deps \
         musl-dev \
         libffi-dev \
     && pip install -r /opt/certbot/dependency-requirements.txt \
-    && pip install --no-cache-dir \
+    && pip install --no-cache-dir --no-deps \
         --editable /opt/certbot/src/acme \
         --editable /opt/certbot/src \
     && apk del .build-deps
diff --git a/certbot-dns-cloudflare/Dockerfile b/certbot-dns-cloudflare/Dockerfile
index 27dcc8751..adbf715fa 100644
--- a/certbot-dns-cloudflare/Dockerfile
+++ b/certbot-dns-cloudflare/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-cloudflare
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-cloudflare
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudflare
diff --git a/certbot-dns-cloudxns/Dockerfile b/certbot-dns-cloudxns/Dockerfile
index cc84ea65b..48c88c35c 100644
--- a/certbot-dns-cloudxns/Dockerfile
+++ b/certbot-dns-cloudxns/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-cloudxns
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-cloudxns
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-cloudxns
diff --git a/certbot-dns-digitalocean/Dockerfile b/certbot-dns-digitalocean/Dockerfile
index 8bdd0619f..342e0e876 100644
--- a/certbot-dns-digitalocean/Dockerfile
+++ b/certbot-dns-digitalocean/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-digitalocean
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-digitalocean
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-digitalocean
diff --git a/certbot-dns-dnsimple/Dockerfile b/certbot-dns-dnsimple/Dockerfile
index 38d2be80e..724675339 100644
--- a/certbot-dns-dnsimple/Dockerfile
+++ b/certbot-dns-dnsimple/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-dnsimple
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-dnsimple
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsimple
diff --git a/certbot-dns-dnsmadeeasy/Dockerfile b/certbot-dns-dnsmadeeasy/Dockerfile
index ff7936925..1480baf4f 100644
--- a/certbot-dns-dnsmadeeasy/Dockerfile
+++ b/certbot-dns-dnsmadeeasy/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-dnsmadeeasy
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-dnsmadeeasy
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-dnsmadeeasy
diff --git a/certbot-dns-gehirn/Dockerfile b/certbot-dns-gehirn/Dockerfile
index 48ad902b5..7dce0e521 100644
--- a/certbot-dns-gehirn/Dockerfile
+++ b/certbot-dns-gehirn/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-gehirn
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-gehirn
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-gehirn
diff --git a/certbot-dns-google/Dockerfile b/certbot-dns-google/Dockerfile
index 4a258d0ee..5750b31d9 100644
--- a/certbot-dns-google/Dockerfile
+++ b/certbot-dns-google/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-google
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-google
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-google
diff --git a/certbot-dns-linode/Dockerfile b/certbot-dns-linode/Dockerfile
index 2e237b521..6db8b59fb 100644
--- a/certbot-dns-linode/Dockerfile
+++ b/certbot-dns-linode/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-linode
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-linode
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-linode
diff --git a/certbot-dns-luadns/Dockerfile b/certbot-dns-luadns/Dockerfile
index 6efb4d777..efc9f36d6 100644
--- a/certbot-dns-luadns/Dockerfile
+++ b/certbot-dns-luadns/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-luadns
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-luadns
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-luadns
diff --git a/certbot-dns-nsone/Dockerfile b/certbot-dns-nsone/Dockerfile
index 88fc13c57..de541e850 100644
--- a/certbot-dns-nsone/Dockerfile
+++ b/certbot-dns-nsone/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-nsone
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-nsone
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-nsone
diff --git a/certbot-dns-ovh/Dockerfile b/certbot-dns-ovh/Dockerfile
index e8da96d95..37e488dc4 100644
--- a/certbot-dns-ovh/Dockerfile
+++ b/certbot-dns-ovh/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-ovh
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-ovh
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-ovh
diff --git a/certbot-dns-rfc2136/Dockerfile b/certbot-dns-rfc2136/Dockerfile
index 1b8feb2f8..3ebb6a72e 100644
--- a/certbot-dns-rfc2136/Dockerfile
+++ b/certbot-dns-rfc2136/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-rfc2136
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-rfc2136
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-rfc2136
diff --git a/certbot-dns-route53/Dockerfile b/certbot-dns-route53/Dockerfile
index a1b8d6caf..e1825c11d 100644
--- a/certbot-dns-route53/Dockerfile
+++ b/certbot-dns-route53/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-route53
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-route53
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-route53
diff --git a/certbot-dns-sakuracloud/Dockerfile b/certbot-dns-sakuracloud/Dockerfile
index 694773f61..9fa9b3c22 100644
--- a/certbot-dns-sakuracloud/Dockerfile
+++ b/certbot-dns-sakuracloud/Dockerfile
@@ -2,4 +2,4 @@ FROM certbot/certbot
 
 COPY . src/certbot-dns-sakuracloud
 
-RUN pip install --no-cache-dir --editable src/certbot-dns-sakuracloud
+RUN pip install --constraint docker_constraints.txt --no-cache-dir --editable src/certbot-dns-sakuracloud
diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt
index 88340cb00..f9ad7329b 100644
--- a/tools/dev_constraints.txt
+++ b/tools/dev_constraints.txt
@@ -1,4 +1,4 @@
-# Specifies Python package versions for development.
+# Specifies Python package versions for development and building Docker images.
 # It includes in particular packages not specified in letsencrypt-auto's requirements file.
 # Some dev package versions specified here may be overridden by higher level constraints
 # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt).
diff --git a/tools/pip_install.py b/tools/pip_install.py
index dd6302b48..475799eb1 100755
--- a/tools/pip_install.py
+++ b/tools/pip_install.py
@@ -19,6 +19,7 @@ import tempfile
 
 import merge_requirements as merge_module
 import readlink
+import strip_hashes
 
 
 def find_tools_path():
@@ -47,10 +48,7 @@ def certbot_normal_processing(tools_path, test_constraints):
     with open(certbot_requirements, 'r') as fd:
         data = fd.readlines()
     with open(test_constraints, 'w') as fd:
-        for line in data:
-            search = re.search(r'^(\S*==\S*).*$', line)
-            if search:
-                fd.write('{0}{1}'.format(search.group(1), os.linesep))
+        fd.write(strip_hashes.main(data))
 
 
 def merge_requirements(tools_path, requirements, test_constraints, all_constraints):
diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py
new file mode 100755
index 000000000..c5591e2f4
--- /dev/null
+++ b/tools/strip_hashes.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+
+import os
+import re
+import sys
+
+def main(args):
+    out_lines = []
+    for line in args:
+        search = re.search(r'^(\S*==\S*).*$', line)
+        if search:
+            out_lines.append(search.group(1))
+    return os.linesep.join(out_lines)
+
+if __name__ == '__main__':
+    print(main(sys.argv[1:]))

From 8bda10541a67fc66c0b4da0ccc5bc8bacf741f25 Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joona@kuori.org>
Date: Wed, 20 Feb 2019 17:00:59 +0200
Subject: [PATCH 2/3] Add stdin option for merge_requirements

Add stdin and file path support to strip_hashes
---
 tools/merge_requirements.py | 43 ++++++++++++++++++++++++++-----------
 tools/pip_install.py        |  3 ++-
 tools/strip_hashes.py       | 39 ++++++++++++++++++++++++++++-----
 3 files changed, 66 insertions(+), 19 deletions(-)

diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py
index 4205e6bcf..521d28b8f 100755
--- a/tools/merge_requirements.py
+++ b/tools/merge_requirements.py
@@ -10,27 +10,36 @@ from __future__ import print_function
 import sys
 
 
-def read_file(file_path):
-    """Reads in a Python requirements file.
+def process_entries(entries):
+    """
     Ignore empty lines, comments and editable requirements
 
-    :param str file_path: path to requirements file
+    :param list entries: List of entries
 
     :returns: mapping from a project to its pinned version
     :rtype: dict
-
     """
     data = {}
-    with open(file_path) as file_h:
-        for line in file_h:
-            line = line.strip()
-            if line and not line.startswith('#') and not line.startswith('-e'):
-                project, version = line.split('==')
-                if not version:
-                    raise ValueError("Unexpected syntax '{0}'".format(line))
-                data[project] = version
+    for e in entries:
+        e = e.strip()
+        if e and not e.startswith('#') and not e.startswith('-e'):
+            project, version = e.split('==')
+            if not version:
+                raise ValueError("Unexpected syntax '{0}'".format(e))
+            data[project] = version
     return data
 
+def read_file(file_path):
+    """Reads in a Python requirements file.
+
+    :param str file_path: path to requirements file
+
+    :returns: list of entries in the file
+    :rtype: list
+
+    """
+    with open(file_path) as file_h:
+        return file_h.readlines()
 
 def output_requirements(requirements):
     """Prepare print requirements to stdout.
@@ -53,7 +62,15 @@ def main(*paths):
     """
     data = {}
     for path in paths:
-        data.update(read_file(path))
+        data.update(process_entries(read_file(path)))
+
+    # Need to check if interactive to avoid blocking if nothing is piped
+    if not sys.stdin.isatty():
+        stdin_data = []
+        for line in sys.stdin:
+            stdin_data.append(line)
+        data.update(process_entries(stdin_data))
+
     return output_requirements(data)
 
 
diff --git a/tools/pip_install.py b/tools/pip_install.py
index 475799eb1..8f0437d9c 100755
--- a/tools/pip_install.py
+++ b/tools/pip_install.py
@@ -48,7 +48,8 @@ def certbot_normal_processing(tools_path, test_constraints):
     with open(certbot_requirements, 'r') as fd:
         data = fd.readlines()
     with open(test_constraints, 'w') as fd:
-        fd.write(strip_hashes.main(data))
+        data = os.linesep.join(strip_hashes.process_entries(data))
+        fd.write(data)
 
 
 def merge_requirements(tools_path, requirements, test_constraints, all_constraints):
diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py
index c5591e2f4..7e7809458 100755
--- a/tools/strip_hashes.py
+++ b/tools/strip_hashes.py
@@ -1,16 +1,45 @@
 #!/usr/bin/env python
+"""Removes hash information from requirement files passed to it as file path
+arguments or simply piped to stdin."""
 
 import os
 import re
 import sys
 
-def main(args):
+
+def process_entries(entries):
+    """Strips off hash strings from dependencies.
+
+    :param list entries: List of entries
+
+    :returns: list of dependencies without hashes
+    :rtype: list
+    """
     out_lines = []
-    for line in args:
-        search = re.search(r'^(\S*==\S*).*$', line)
+    for e in entries:
+        e = e.strip()
+        search = re.search(r'^(\S*==\S*).*$', e)
         if search:
             out_lines.append(search.group(1))
-    return os.linesep.join(out_lines)
+    return out_lines
+
+def main(*paths):
+    """Reads dependency definitions from a (list of) file(s) or stdin and
+    removes hashes from returned entries"""
+
+    deps = []
+    for path in paths:
+        with open(path) as file_h:
+            deps += process_entries(file_h.readlines())
+
+    # Need to check if interactive to avoid blocking if nothing is piped
+    if not sys.stdin.isatty():
+        stdin_data = []
+        for line in sys.stdin:
+            stdin_data.append(line)
+        deps += process_entries(stdin_data)
+
+    return os.linesep.join(deps)
 
 if __name__ == '__main__':
-    print(main(sys.argv[1:]))
+    print(main(*sys.argv[1:]))  # pylint: disable=star-args

From d8a3fa3904ecddd00be871847c773c9370780449 Mon Sep 17 00:00:00 2001
From: Joona Hoikkala <joona@kuori.org>
Date: Mon, 4 Mar 2019 15:52:38 +0200
Subject: [PATCH 3/3] Address review comments

---
 tools/merge_requirements.py | 25 ++++++++++++++-----------
 tools/pip_install.py        |  2 +-
 tools/strip_hashes.py       | 33 +++++++++++++++++++--------------
 3 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py
index 521d28b8f..0d41d12c4 100755
--- a/tools/merge_requirements.py
+++ b/tools/merge_requirements.py
@@ -55,21 +55,24 @@ def main(*paths):
     """Merges multiple requirements files together and prints the result.
 
     Requirement files specified later in the list take precedence over earlier
-    files.
+    files. Files are read from file paths passed from the command line arguments.
 
-    :param tuple paths: paths to requirements files
+    If no command line arguments are defined, data is read from stdin instead.
+
+    :param tuple paths: paths to requirements files provided on command line
 
     """
     data = {}
-    for path in paths:
-        data.update(process_entries(read_file(path)))
-
-    # Need to check if interactive to avoid blocking if nothing is piped
-    if not sys.stdin.isatty():
-        stdin_data = []
-        for line in sys.stdin:
-            stdin_data.append(line)
-        data.update(process_entries(stdin_data))
+    if paths:
+        for path in paths:
+            data.update(process_entries(read_file(path)))
+    else:
+        # Need to check if interactive to avoid blocking if nothing is piped
+        if not sys.stdin.isatty():
+            stdin_data = []
+            for line in sys.stdin:
+                stdin_data.append(line)
+            data.update(process_entries(stdin_data))
 
     return output_requirements(data)
 
diff --git a/tools/pip_install.py b/tools/pip_install.py
index 8f0437d9c..15dc2f0c0 100755
--- a/tools/pip_install.py
+++ b/tools/pip_install.py
@@ -48,7 +48,7 @@ def certbot_normal_processing(tools_path, test_constraints):
     with open(certbot_requirements, 'r') as fd:
         data = fd.readlines()
     with open(test_constraints, 'w') as fd:
-        data = os.linesep.join(strip_hashes.process_entries(data))
+        data = "\n".join(strip_hashes.process_entries(data))
         fd.write(data)
 
 
diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py
index 7e7809458..988e72eb8 100755
--- a/tools/strip_hashes.py
+++ b/tools/strip_hashes.py
@@ -2,7 +2,6 @@
 """Removes hash information from requirement files passed to it as file path
 arguments or simply piped to stdin."""
 
-import os
 import re
 import sys
 
@@ -24,22 +23,28 @@ def process_entries(entries):
     return out_lines
 
 def main(*paths):
-    """Reads dependency definitions from a (list of) file(s) or stdin and
-    removes hashes from returned entries"""
+    """
+    Reads dependency definitions from a (list of) file(s) provided on the
+    command line. If no command line arguments are present, data is read from
+    stdin instead.
+
+    Hashes are removed from returned entries.
+    """
 
     deps = []
-    for path in paths:
-        with open(path) as file_h:
-            deps += process_entries(file_h.readlines())
+    if paths:
+        for path in paths:
+            with open(path) as file_h:
+                deps += process_entries(file_h.readlines())
+    else:
+        # Need to check if interactive to avoid blocking if nothing is piped
+        if not sys.stdin.isatty():
+            stdin_data = []
+            for line in sys.stdin:
+                stdin_data.append(line)
+            deps += process_entries(stdin_data)
 
-    # Need to check if interactive to avoid blocking if nothing is piped
-    if not sys.stdin.isatty():
-        stdin_data = []
-        for line in sys.stdin:
-            stdin_data.append(line)
-        deps += process_entries(stdin_data)
-
-    return os.linesep.join(deps)
+    return "\n".join(deps)
 
 if __name__ == '__main__':
     print(main(*sys.argv[1:]))  # pylint: disable=star-args