From 6f388945cde7e6ec68a83378dea9c6122b86d2c9 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 17 Jan 2019 09:41:46 -0800 Subject: [PATCH 1/6] Remove reference to latest draft implemented. (#6669) --- acme/acme/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index bab5b40ce..d91072a3b 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -1,14 +1,9 @@ """ACME protocol implementation. -This module is an implementation of the `ACME protocol`_. Latest -supported version: `draft-ietf-acme-01`_. - +This module is an implementation of the `ACME protocol`_. .. _`ACME protocol`: https://ietf-wg-acme.github.io/acme -.. _`draft-ietf-acme-01`: - https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 - """ import sys From 835f4b9271b7b5ee3cb3f54254a015e28f7bffdc Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Thu, 17 Jan 2019 22:02:35 +0100 Subject: [PATCH 2/6] Allow to execute a tox target without pinned dependencies (#6590) This PR passes the CERTBOT_NO_PIN environment variable to the unit tests tox envs. By setting CERTBOT_NO_PIN to 1 before executing a given tox env, certbot dependencies will be installed at their latest version instead of the usual pinned version. I also moved the unpin logic one layer below to allow it to be used potentially more widely, and avoid unnecessary merging constraints operation in this case. As warnings are errors now, latest versions of Python will break now the tests, because collections launch a warning when some classes are imported from collections instead of collections.abc. Certbot code is patched, and warning is ignored for now, because a lot of third party libraries still depend on this behavior. * Allow to execute a tox target without pinned dependencies * Correct lint * Retrigger build. * Remove debug code * Added test against unpinned dependencies from test-everything-unpinned-dependencies branch * Remove duplicated assertion to pass TRAVIS and APPVEYOR in default tox environment. --- .travis.yml | 4 ++++ acme/acme/messages.py | 7 +++++-- pytest.ini | 10 +++++++--- tools/install_and_test.py | 9 ++++----- tools/pip_install.py | 27 +++++++++++++++++---------- tox.ini | 2 ++ 6 files changed, 39 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38543845a..16cb6f23f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,6 +63,10 @@ matrix: env: TOXENV=nginxroundtrip # These environments are executed on cron events only + - python: "3.7" + dist: xenial + env: TOXENV=py37 CERTBOT_NO_PIN=1 + if: type = cron - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest sudo: required diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 4400a6c31..7c82c8507 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,7 +1,10 @@ """ACME protocol messages.""" -import collections import six import json +try: + from collections.abc import Hashable # pylint: disable=no-name-in-module +except ImportError: + from collections import Hashable import josepy as jose @@ -107,7 +110,7 @@ class Error(jose.JSONObjectWithFields, errors.Error): if part is not None).decode() -class _Constant(jose.JSONDeSerializable, collections.Hashable): # type: ignore +class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore """ACME constant.""" __slots__ = ('name',) POSSIBLE_NAMES = NotImplemented diff --git a/pytest.ini b/pytest.ini index 9a5807f34..8f009045c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,10 +3,14 @@ # directly. [pytest] addopts = --numprocesses auto --pyargs -# ResourceWarnings are ignored as errors, since they're raised at close -# decodestring: https://github.com/rthalley/dnspython/issues/338 -# ignore our own TLS-SNI-01 warning +# In general, all warnings are treated as errors. Here are the exceptions: +# 1- decodestring: https://github.com/rthalley/dnspython/issues/338 +# 2- ignore our own TLS-SNI-01 warning +# 3- ignore warn for importing abstract classes from collections instead of collections.abc, +# too much third party dependencies are still relying on this behavior, +# but it should be corrected to allow Certbot compatiblity with Python >= 3.8 filterwarnings = error ignore:decodestring:DeprecationWarning ignore:TLS-SNI-01:DeprecationWarning + ignore:.*collections\.abc:DeprecationWarning diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 79a7c2264..0142eeea4 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -17,16 +17,15 @@ import re SKIP_PROJECTS_ON_WINDOWS = [ 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + def call_with_print(command, cwd=None): print(command) subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + def main(args): - if os.environ.get('CERTBOT_NO_PIN') == '1': - command = [sys.executable, '-m', 'pip', '-e'] - else: - script_dir = os.path.dirname(os.path.abspath(__file__)) - command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] + script_dir = os.path.dirname(os.path.abspath(__file__)) + command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] new_args = [] for arg in args: diff --git a/tools/pip_install.py b/tools/pip_install.py index 4466729e0..332422019 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -80,19 +80,26 @@ def main(args): test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') - requirements = None - if os.environ.get('CERTBOT_OLDEST') == '1': - requirements = certbot_oldest_processing(tools_path, args, test_constraints) + if os.environ.get('CERTBOT_NO_PIN') == '1': + # With unpinned dependencies, there is no constraint + call_with_print('"{0}" -m pip install --ignore-installed {1}' + .format(sys.executable, ' '.join(args))) else: - certbot_normal_processing(tools_path, test_constraints) + # Otherwise, we merge requirements to build the constraints and pin dependencies + requirements = None + if os.environ.get('CERTBOT_OLDEST') == '1': + requirements = certbot_oldest_processing(tools_path, args, test_constraints) + else: + certbot_normal_processing(tools_path, test_constraints) - merge_requirements(tools_path, test_constraints, all_constraints) - if requirements: - call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' - .format(sys.executable, all_constraints, requirements)) + merge_requirements(tools_path, test_constraints, all_constraints) - call_with_print('"{0}" -m pip install --constraint "{1}" {2}' - .format(sys.executable, all_constraints, ' '.join(args))) + if requirements: + call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' + .format(sys.executable, all_constraints, requirements)) + + call_with_print('"{0}" -m pip install --constraint "{1}" {2}' + .format(sys.executable, all_constraints, ' '.join(args))) finally: if os.environ.get('TRAVIS'): print('travis_fold:end:install_certbot_deps') diff --git a/tox.ini b/tox.ini index 36057b5df..363f06bf2 100644 --- a/tox.ini +++ b/tox.ini @@ -64,6 +64,8 @@ source_paths = tests/lock_test.py [testenv] +passenv = + CERTBOT_NO_PIN commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py From bb1242c04767c8686de39af78f4f562ae33ded1d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 18 Jan 2019 00:21:13 +0100 Subject: [PATCH 3/6] Process isolated local oldest requirements for DNS plugins (#6653) * Include local-oldest-requirements in the constraints when oldest is invoked * Add oldest constraints for lexicon * Skip editable requirements during merge * Pin to lexicon 2.2.1 for oldest requirements. Override locally oldest if needed for specific providers. --- .../local-oldest-requirements.txt | 1 - certbot-dns-ovh/local-oldest-requirements.txt | 1 + tools/dev_constraints.txt | 6 ++-- tools/merge_requirements.py | 34 +++++++++---------- tools/oldest_constraints.txt | 9 +++-- tools/pip_install.py | 25 +++++++++----- 6 files changed, 42 insertions(+), 34 deletions(-) diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt index 5ecd3cbc7..65f5a758e 100644 --- a/certbot-dns-dnsimple/local-oldest-requirements.txt +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -1,3 +1,2 @@ -e acme[dev] -e .[dev] -dns-lexicon==2.2.1 \ No newline at end of file diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt index 65f5a758e..01cbcb317 100644 --- a/certbot-dns-ovh/local-oldest-requirements.txt +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -1,2 +1,3 @@ -e acme[dev] -e .[dev] +dns-lexicon==2.7.14 diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 111dc5495..88340cb00 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,5 +1,7 @@ -# Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. +# Specifies Python package versions for development. +# It includes in particular packages not specified in letsencrypt-auto's requirements file. +# Some dev package versions specified here may be overridden by higher level constraints +# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). alabaster==0.7.10 apipkg==1.4 asn1crypto==0.22.0 diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index ad44a55d0..4205e6bcf 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -5,13 +5,14 @@ Requirements files specified later take precedence over earlier ones. Only simple SomeProject==1.2.3 format is currently supported. """ - from __future__ import print_function import sys + def read_file(file_path): """Reads in a Python requirements file. + Ignore empty lines, comments and editable requirements :param str file_path: path to requirements file @@ -19,16 +20,16 @@ def read_file(file_path): :rtype: dict """ - d = {} - with open(file_path) as f: - for line in f: + data = {} + with open(file_path) as file_h: + for line in file_h: line = line.strip() - if line and not line.startswith('#'): + if line and not line.startswith('#') and not line.startswith('-e'): project, version = line.split('==') if not version: raise ValueError("Unexpected syntax '{0}'".format(line)) - d[project] = version - return d + data[project] = version + return data def output_requirements(requirements): @@ -37,25 +38,24 @@ def output_requirements(requirements): :param dict requirements: mapping from a project to its pinned version """ - return '\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items())) + return '\n'.join('{0}=={1}'.format(key, value) + for key, value in sorted(requirements.items())) -def main(*files): +def main(*paths): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier files. - :param tuple files: paths to requirements files + :param tuple paths: paths to requirements files """ - d = {} - for f in files: - d.update(read_file(f)) - return output_requirements(d) + data = {} + for path in paths: + data.update(read_file(path)) + return output_requirements(data) if __name__ == '__main__': - merged_requirements = main(*sys.argv[1:]) - print(merged_requirements) + print(main(*sys.argv[1:])) # pylint: disable=star-args diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 642af660e..e48d6b13c 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -50,11 +50,10 @@ ConfigArgParse==0.10.0 funcsigs==0.4 zope.hookable==4.0.4 -# Ubuntu Bionic constraints -# Formerly only lexicon==2.2.1 is available on Bionic. But we cannot put that here because some -# DNS plugins require higher versions. We put to the least minimal Lexicon version to ensure -# that Lexicon 2.x works with Certbot. -dns-lexicon==2.7.14 +# Ubuntu Bionic constraints. +# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins +# using their local-oldest-requirements.txt +dns-lexicon==2.2.1 # Plugin constraints # These aren't necessarily the oldest versions we need to support diff --git a/tools/pip_install.py b/tools/pip_install.py index 332422019..a03ed70d7 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -32,10 +32,10 @@ def certbot_oldest_processing(tools_path, args, test_constraints): # remove any extras such as [dev] pkg_dir = re.sub(r'\[\w+\]', '', args[1]) requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') + shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) # packages like acme don't have any local oldest requirements if not os.path.isfile(requirements): - requirements = None - shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) + return None return requirements @@ -53,11 +53,19 @@ def certbot_normal_processing(tools_path, test_constraints): fd.write('{0}{1}'.format(search.group(1), os.linesep)) -def merge_requirements(tools_path, test_constraints, all_constraints): - merged_requirements = merge_module.main( - os.path.join(tools_path, 'dev_constraints.txt'), - test_constraints - ) +def merge_requirements(tools_path, requirements, test_constraints, all_constraints): + # Order of the files in the merge function matters. + # Indeed version retained for a given package will be the last version + # found when following all requirements in the given order. + # Here is the order by increasing priority: + # 1) The general development constraints (tools/dev_constraints.txt) + # 2) The general tests constraints (oldest_requirements.txt or + # certbot-auto's dependency-requirements.txt for the normal processing) + # 3) The local requirement file, typically local-oldest-requirement in oldest tests + files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] + if requirements: + files.append(requirements) + merged_requirements = merge_module.main(*files) with open(all_constraints, 'w') as fd: fd.write(merged_requirements) @@ -92,8 +100,7 @@ def main(args): else: certbot_normal_processing(tools_path, test_constraints) - merge_requirements(tools_path, test_constraints, all_constraints) - + merge_requirements(tools_path, requirements, test_constraints, all_constraints) if requirements: call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' .format(sys.executable, all_constraints, requirements)) From 39eb66a8ad6e326118c36319818ad144fab6e85d Mon Sep 17 00:00:00 2001 From: sydneyli Date: Thu, 17 Jan 2019 16:00:39 -0800 Subject: [PATCH 4/6] pin dependency-requirements.txt in Dockerfile (#6670) Fixes #6580 --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d1296b30f..f3626dc8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot COPY CHANGELOG.md README.rst setup.py src/ +COPY letsencrypt-auto-source/pieces/dependency-requirements.txt . COPY acme src/acme COPY certbot src/certbot @@ -21,6 +22,7 @@ RUN apk add --no-cache --virtual .build-deps \ openssl-dev \ musl-dev \ libffi-dev \ + && pip install -r /opt/certbot/dependency-requirements.txt \ && pip install --no-cache-dir \ --editable /opt/certbot/src/acme \ --editable /opt/certbot/src \ From b58d0e722d4d3b54ede48a67aeb25d4fa6d6fc45 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 18 Jan 2019 05:27:31 -0800 Subject: [PATCH 5/6] Remove certbot-auto code for Wheezy and Precise (#6672) * Remove lsb_release dependency. * Remove more code. --- letsencrypt-auto-source/letsencrypt-auto | 52 ------------------- .../pieces/bootstrappers/deb_common.sh | 52 ------------------- 2 files changed, 104 deletions(-) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index c08a08160..72d32af8d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -333,63 +333,11 @@ BootstrapDebCommon() { fi augeas_pkg="libaugeas0 augeas-lenses" - AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` if [ "$ASSUME_YES" = 1 ]; then YES_FLAG="-y" fi - AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - add_backports=1 - else - read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response - case $response in - [yY][eE][sS]|[yY]|"") - add_backports=1;; - *) - add_backports=0;; - esac - fi - if [ "$add_backports" = 1 ]; then - sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - apt-get $QUIET_FLAG update - fi - fi - fi - if [ "$add_backports" != 0 ]; then - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - } - - - if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Certbot apache plugin..." - fi - # XXX add a case for ubuntu PPAs - fi - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ python \ python-dev \ diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh index eb22225e4..93bdc63b4 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh @@ -43,63 +43,11 @@ BootstrapDebCommon() { fi augeas_pkg="libaugeas0 augeas-lenses" - AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` if [ "$ASSUME_YES" = 1 ]; then YES_FLAG="-y" fi - AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - add_backports=1 - else - read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response - case $response in - [yY][eE][sS]|[yY]|"") - add_backports=1;; - *) - add_backports=0;; - esac - fi - if [ "$add_backports" = 1 ]; then - sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - apt-get $QUIET_FLAG update - fi - fi - fi - if [ "$add_backports" != 0 ]; then - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - } - - - if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Certbot apache plugin..." - fi - # XXX add a case for ubuntu PPAs - fi - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ python \ python-dev \ From 0f1d78e8974319dde66bf8f8b12acf8419f3e6b7 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 18 Jan 2019 18:47:43 +0100 Subject: [PATCH 6/6] Do not ignore editable distributions already installed. (#6675) This PR fix tests when run with unpinned dependencies. Option --ignore-installed is removed from pip command, allowing to use local dev modules (like acme) that are not still released. Be warned that these tests will mostly work correctly if the relevant virtualenv in .tox does not exist yet. Otherwise, already installed distribution, potentially not up-to-date, will be reused without update from pip if they match the minimal requirements in relevant projects. This is unlikely to be an issue on CI systems, that will start from a fresh system. We may build a dedicated tox environment for unpinned dependencies tests purpose, and ensure consistency of these tests on all situations. Fixes #6674 --- tools/pip_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pip_install.py b/tools/pip_install.py index a03ed70d7..dd6302b48 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -90,7 +90,7 @@ def main(args): if os.environ.get('CERTBOT_NO_PIN') == '1': # With unpinned dependencies, there is no constraint - call_with_print('"{0}" -m pip install --ignore-installed {1}' + call_with_print('"{0}" -m pip install {1}' .format(sys.executable, ' '.join(args))) else: # Otherwise, we merge requirements to build the constraints and pin dependencies