Merge branch 'master' into pin-deps-dns-plugins

.travis.yml

@@ -63,6 +63,10 @@ matrix:
       env: TOXENV=nginxroundtrip
 
     # These environments are executed on cron events only
+    - python: "3.7"
+      dist: xenial
+      env: TOXENV=py37 CERTBOT_NO_PIN=1
+      if: type = cron
     - python: "2.7"
       env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest
       sudo: required

Dockerfile

@@ -6,6 +6,7 @@ VOLUME /etc/letsencrypt /var/lib/letsencrypt
 WORKDIR /opt/certbot
 
 COPY CHANGELOG.md README.rst setup.py src/
+COPY letsencrypt-auto-source/pieces/dependency-requirements.txt .
 COPY acme src/acme
 COPY certbot src/certbot
 
@@ -21,6 +22,7 @@ RUN apk add --no-cache --virtual .build-deps \
         openssl-dev \
         musl-dev \
         libffi-dev \
+    && pip install -r /opt/certbot/dependency-requirements.txt \
     && pip install --no-cache-dir \
         --editable /opt/certbot/src/acme \
         --editable /opt/certbot/src \

acme/acme/__init__.py

@@ -1,14 +1,9 @@
 """ACME protocol implementation.
 
-This module is an implementation of the `ACME protocol`_. Latest
-supported version: `draft-ietf-acme-01`_.
-
+This module is an implementation of the `ACME protocol`_.
 
 .. _`ACME protocol`: https://ietf-wg-acme.github.io/acme
 
-.. _`draft-ietf-acme-01`:
-  https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01
-
 """
 import sys
 

acme/acme/messages.py

@@ -1,7 +1,10 @@
 """ACME protocol messages."""
-import collections
 import six
 import json
+try:
+    from collections.abc import Hashable  # pylint: disable=no-name-in-module
+except ImportError:
+    from collections import Hashable
 
 import josepy as jose
 
@@ -107,7 +110,7 @@ class Error(jose.JSONObjectWithFields, errors.Error):
             if part is not None).decode()
 
 
-class _Constant(jose.JSONDeSerializable, collections.Hashable):  # type: ignore
+class _Constant(jose.JSONDeSerializable, Hashable):  # type: ignore
     """ACME constant."""
     __slots__ = ('name',)
     POSSIBLE_NAMES = NotImplemented

certbot-dns-dnsimple/local-oldest-requirements.txt

@@ -1,3 +1,2 @@
 -e acme[dev]
 -e .[dev]
-dns-lexicon==2.2.1

certbot-dns-ovh/local-oldest-requirements.txt

@@ -1,2 +1,3 @@
 -e acme[dev]
 -e .[dev]
+dns-lexicon==2.7.14

letsencrypt-auto-source/letsencrypt-auto

@@ -333,63 +333,11 @@ BootstrapDebCommon() {
   fi
 
   augeas_pkg="libaugeas0 augeas-lenses"
-  AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2`
 
   if [ "$ASSUME_YES" = 1 ]; then
     YES_FLAG="-y"
   fi
 
-  AddBackportRepo() {
-    # ARGS:
-    BACKPORT_NAME="$1"
-    BACKPORT_SOURCELINE="$2"
-    say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME."
-    if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then
-      # This can theoretically error if sources.list.d is empty, but in that case we don't care.
-      if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then
-        if [ "$ASSUME_YES" = 1 ]; then
-          /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..."
-          sleep 1s
-          /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..."
-          sleep 1s
-          /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..."
-          sleep 1s
-          add_backports=1
-        else
-          read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response
-          case $response in
-            [yY][eE][sS]|[yY]|"")
-              add_backports=1;;
-            *)
-              add_backports=0;;
-          esac
-        fi
-        if [ "$add_backports" = 1 ]; then
-          sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list"
-          apt-get $QUIET_FLAG update
-        fi
-      fi
-    fi
-    if [ "$add_backports" != 0 ]; then
-      apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg
-      augeas_pkg=
-    fi
-  }
-
-
-  if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then
-    if lsb_release -a | grep -q wheezy ; then
-      AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main"
-    elif lsb_release -a | grep -q precise ; then
-      # XXX add ARM case
-      AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse"
-    else
-      echo "No libaugeas0 version is available that's new enough to run the"
-      echo "Certbot apache plugin..."
-    fi
-    # XXX add a case for ubuntu PPAs
-  fi
-
   apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \
     python \
     python-dev \

letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh

@@ -43,63 +43,11 @@ BootstrapDebCommon() {
   fi
 
   augeas_pkg="libaugeas0 augeas-lenses"
-  AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2`
 
   if [ "$ASSUME_YES" = 1 ]; then
     YES_FLAG="-y"
   fi
 
-  AddBackportRepo() {
-    # ARGS:
-    BACKPORT_NAME="$1"
-    BACKPORT_SOURCELINE="$2"
-    say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME."
-    if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then
-      # This can theoretically error if sources.list.d is empty, but in that case we don't care.
-      if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then
-        if [ "$ASSUME_YES" = 1 ]; then
-          /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..."
-          sleep 1s
-          /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..."
-          sleep 1s
-          /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..."
-          sleep 1s
-          add_backports=1
-        else
-          read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response
-          case $response in
-            [yY][eE][sS]|[yY]|"")
-              add_backports=1;;
-            *)
-              add_backports=0;;
-          esac
-        fi
-        if [ "$add_backports" = 1 ]; then
-          sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list"
-          apt-get $QUIET_FLAG update
-        fi
-      fi
-    fi
-    if [ "$add_backports" != 0 ]; then
-      apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg
-      augeas_pkg=
-    fi
-  }
-
-
-  if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then
-    if lsb_release -a | grep -q wheezy ; then
-      AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main"
-    elif lsb_release -a | grep -q precise ; then
-      # XXX add ARM case
-      AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse"
-    else
-      echo "No libaugeas0 version is available that's new enough to run the"
-      echo "Certbot apache plugin..."
-    fi
-    # XXX add a case for ubuntu PPAs
-  fi
-
   apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \
     python \
     python-dev \

pytest.ini

@@ -3,10 +3,14 @@
 # directly.
 [pytest]
 addopts = --numprocesses auto --pyargs
-# ResourceWarnings are ignored as errors, since they're raised at close
-# decodestring: https://github.com/rthalley/dnspython/issues/338
-# ignore our own TLS-SNI-01 warning
+# In general, all warnings are treated as errors. Here are the exceptions:
+#   1- decodestring: https://github.com/rthalley/dnspython/issues/338
+#   2- ignore our own TLS-SNI-01 warning
+#   3- ignore warn for importing abstract classes from collections instead of collections.abc,
+#      too much third party dependencies are still relying on this behavior,
+#      but it should be corrected to allow Certbot compatiblity with Python >= 3.8
 filterwarnings =
     error
     ignore:decodestring:DeprecationWarning
     ignore:TLS-SNI-01:DeprecationWarning
+    ignore:.*collections\.abc:DeprecationWarning

tools/dev_constraints.txt

@@ -1,5 +1,7 @@
-# Specifies Python package versions for packages not specified in
-# letsencrypt-auto's requirements file.
+# Specifies Python package versions for development.
+# It includes in particular packages not specified in letsencrypt-auto's requirements file.
+# Some dev package versions specified here may be overridden by higher level constraints
+# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt).
 alabaster==0.7.10
 apipkg==1.4
 asn1crypto==0.22.0

tools/install_and_test.py

@@ -17,16 +17,15 @@ import re
 SKIP_PROJECTS_ON_WINDOWS = [
     'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot']
 
+
 def call_with_print(command, cwd=None):
     print(command)
     subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd())
 
+
 def main(args):
-    if os.environ.get('CERTBOT_NO_PIN') == '1':
-        command = [sys.executable, '-m', 'pip', '-e']
-    else:
-        script_dir = os.path.dirname(os.path.abspath(__file__))
-        command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')]
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')]
 
     new_args = []
     for arg in args:

tools/merge_requirements.py

@@ -5,13 +5,14 @@ Requirements files specified later take precedence over earlier ones. Only
 simple SomeProject==1.2.3 format is currently supported.
 
 """
-
 from __future__ import print_function
 
 import sys
 
+
 def read_file(file_path):
     """Reads in a Python requirements file.
+    Ignore empty lines, comments and editable requirements
 
     :param str file_path: path to requirements file
 
@@ -19,16 +20,16 @@ def read_file(file_path):
     :rtype: dict
 
     """
-    d = {}
-    with open(file_path) as f:
-        for line in f:
+    data = {}
+    with open(file_path) as file_h:
+        for line in file_h:
             line = line.strip()
-            if line and not line.startswith('#'):
+            if line and not line.startswith('#') and not line.startswith('-e'):
                 project, version = line.split('==')
                 if not version:
                     raise ValueError("Unexpected syntax '{0}'".format(line))
-                d[project] = version
-    return d
+                data[project] = version
+    return data
 
 
 def output_requirements(requirements):
@@ -37,25 +38,24 @@ def output_requirements(requirements):
     :param dict requirements: mapping from a project to its pinned version
 
     """
-    return '\n'.join('{0}=={1}'.format(k, v)
-          for k, v in sorted(requirements.items()))
+    return '\n'.join('{0}=={1}'.format(key, value)
+                     for key, value in sorted(requirements.items()))
 
 
-def main(*files):
+def main(*paths):
     """Merges multiple requirements files together and prints the result.
 
     Requirement files specified later in the list take precedence over earlier
     files.
 
-    :param tuple files: paths to requirements files
+    :param tuple paths: paths to requirements files
 
     """
-    d = {}
-    for f in files:
-        d.update(read_file(f))
-    return output_requirements(d)
+    data = {}
+    for path in paths:
+        data.update(read_file(path))
+    return output_requirements(data)
 
 
 if __name__ == '__main__':
-    merged_requirements = main(*sys.argv[1:])
-    print(merged_requirements)
+    print(main(*sys.argv[1:]))  # pylint: disable=star-args

tools/oldest_constraints.txt

@@ -50,11 +50,10 @@ ConfigArgParse==0.10.0
 funcsigs==0.4
 zope.hookable==4.0.4
 
-# Ubuntu Bionic constraints
-# Formerly only lexicon==2.2.1 is available on Bionic. But we cannot put that here because some
-# DNS plugins require higher versions. We put to the least minimal Lexicon version to ensure
-# that Lexicon 2.x works with Certbot.
-dns-lexicon==2.7.14
+# Ubuntu Bionic constraints.
+# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins
+# using their local-oldest-requirements.txt
+dns-lexicon==2.2.1
 
 # Plugin constraints
 # These aren't necessarily the oldest versions we need to support

tools/pip_install.py

@@ -32,10 +32,10 @@ def certbot_oldest_processing(tools_path, args, test_constraints):
     # remove any extras such as [dev]
     pkg_dir = re.sub(r'\[\w+\]', '', args[1])
     requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt')
+    shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints)
     # packages like acme don't have any local oldest requirements
     if not os.path.isfile(requirements):
-        requirements = None
-    shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints)
+        return None
 
     return requirements
 
@@ -53,11 +53,19 @@ def certbot_normal_processing(tools_path, test_constraints):
                 fd.write('{0}{1}'.format(search.group(1), os.linesep))
 
 
-def merge_requirements(tools_path, test_constraints, all_constraints):
-    merged_requirements = merge_module.main(
-        os.path.join(tools_path, 'dev_constraints.txt'),
-        test_constraints
-    )
+def merge_requirements(tools_path, requirements, test_constraints, all_constraints):
+    # Order of the files in the merge function matters.
+    # Indeed version retained for a given package will be the last version
+    # found when following all requirements in the given order.
+    # Here is the order by increasing priority:
+    # 1) The general development constraints (tools/dev_constraints.txt)
+    # 2) The general tests constraints (oldest_requirements.txt or
+    #    certbot-auto's dependency-requirements.txt for the normal processing)
+    # 3) The local requirement file, typically local-oldest-requirement in oldest tests
+    files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints]
+    if requirements:
+        files.append(requirements)
+    merged_requirements = merge_module.main(*files)
     with open(all_constraints, 'w') as fd:
         fd.write(merged_requirements)
 
@@ -80,19 +88,25 @@ def main(args):
         test_constraints = os.path.join(working_dir, 'test_constraints.txt')
         all_constraints = os.path.join(working_dir, 'all_constraints.txt')
 
-        requirements = None
-        if os.environ.get('CERTBOT_OLDEST') == '1':
-            requirements = certbot_oldest_processing(tools_path, args, test_constraints)
+        if os.environ.get('CERTBOT_NO_PIN') == '1':
+            # With unpinned dependencies, there is no constraint
+            call_with_print('"{0}" -m pip install {1}'
+                            .format(sys.executable, ' '.join(args)))
         else:
-            certbot_normal_processing(tools_path, test_constraints)
+            # Otherwise, we merge requirements to build the constraints and pin dependencies
+            requirements = None
+            if os.environ.get('CERTBOT_OLDEST') == '1':
+                requirements = certbot_oldest_processing(tools_path, args, test_constraints)
+            else:
+                certbot_normal_processing(tools_path, test_constraints)
 
-        merge_requirements(tools_path, test_constraints, all_constraints)
-        if requirements:
-            call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"'
-                            .format(sys.executable, all_constraints, requirements))
+            merge_requirements(tools_path, requirements, test_constraints, all_constraints)
+            if requirements:
+                call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"'
+                                .format(sys.executable, all_constraints, requirements))
 
-        call_with_print('"{0}" -m pip install --constraint "{1}" {2}'
-                        .format(sys.executable, all_constraints, ' '.join(args)))
+            call_with_print('"{0}" -m pip install --constraint "{1}" {2}'
+                            .format(sys.executable, all_constraints, ' '.join(args)))
     finally:
         if os.environ.get('TRAVIS'):
             print('travis_fold:end:install_certbot_deps')

tox.ini

@@ -64,6 +64,8 @@ source_paths =
     tests/lock_test.py
 
 [testenv]
+passenv =
+    CERTBOT_NO_PIN
 commands =
     {[base]install_and_test} {[base]all_packages}
     python tests/lock_test.py