diff --git a/.travis.yml b/.travis.yml index 2b8eafc13..16cb6f23f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,14 +4,13 @@ cache: directories: - $HOME/.cache/pip -before_install: - - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3 && brew upgrade python && brew link python)' - before_script: - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi' + - export TOX_TESTENV_PASSENV=TRAVIS matrix: include: + # These environments are always executed - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install sudo: required @@ -57,11 +56,119 @@ matrix: before_install: addons: - python: "2.7" - env: TOXENV=apacheconftest + env: TOXENV=apacheconftest-with-pebble sudo: required + services: docker - python: "2.7" env: TOXENV=nginxroundtrip + # These environments are executed on cron events only + - python: "3.7" + dist: xenial + env: TOXENV=py37 CERTBOT_NO_PIN=1 + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest + sudo: required + services: docker + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest + sudo: required + services: docker + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest + sudo: required + services: docker + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest + sudo: required + services: docker + if: type = cron + - python: "3.4" + env: TOXENV=py34 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.4" + env: TOXENV=py34 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - python: "3.5" + env: TOXENV=py35 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.5" + env: TOXENV=py35 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - python: "3.6" + env: TOXENV=py36 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.6" + env: TOXENV=py36 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - python: "3.7" + dist: xenial + env: TOXENV=py37 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.7" + dist: xenial + env: TOXENV=py37 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - sudo: required + env: TOXENV=le_auto_xenial + services: docker + if: type = cron + - sudo: required + env: TOXENV=le_auto_jessie + services: docker + if: type = cron + - sudo: required + env: TOXENV=le_auto_centos6 + services: docker + if: type = cron + - sudo: required + env: TOXENV=docker_dev + services: docker + addons: + apt: + packages: # don't install nginx and apache + - libaugeas0 + if: type = cron + - language: generic + env: TOXENV=py27 + os: osx + addons: + homebrew: + packages: + - augeas + - python2 + if: type = cron + - language: generic + env: TOXENV=py3 + os: osx + addons: + homebrew: + packages: + - augeas + - python3 + if: type = cron + + # Only build pushes to the master branch, PRs, and branches beginning with # `test-` or of the form `digit(s).digit(s).x`. This reduces the number of diff --git a/CHANGELOG.md b/CHANGELOG.md index 6356bb714..01263b751 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,44 @@ # Certbot change log -Certbot adheres to [Semantic Versioning](http://semver.org/). +Certbot adheres to [Semantic Versioning](https://semver.org/). -## 0.30.0 - master +## 0.31.0 - master + +### Added + +* Avoid to process again challenges that are already validated + when a certificate is issued. + +### Changed + +* Lexicon-based DNS plugins are now fully compatible with Lexicon 3.x (support + on 2.x branch is maintained). + +### Fixed + +* Fixed accessing josepy contents through acme.jose when the full acme.jose + path is used. +* Clarify behavior for deleting certs as part of revocation. + +Despite us having broken lockstep, we are continuing to release new versions of +all Certbot components during releases for the time being, however, the only +package with changes other than its version number was: + +* acme +* certbot +* certbot-dns-cloudxns +* certbot-dns-dnsimple +* certbot-dns-dnsmadeeasy +* certbot-dns-gehirn +* certbot-dns-linode +* certbot-dns-luadns +* certbot-dns-nsone +* certbot-dns-ovh +* certbot-dns-sakuracloud + +More details about these changes can be found on our GitHub repo. + +## 0.30.0 - 2019-01-02 ### Added @@ -12,20 +48,24 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Copied account management functionality from the `register` subcommand to the `update_account` subcommand. -* Marked usage `register --update-registration` for deprecation and +* Marked usage `register --update-registration` for deprecation and removal in a future release. * Apache plugin now respects CERTBOT_DOCS environment variable when adding command line defaults. ### Fixed -* +* Older modules in the josepy library can now be accessed through acme.jose + like it could in previous versions of acme. This is only done to preserve + backwards compatibility and support for doing this with new modules in josepy + will not be added. Users of the acme library should switch to using josepy + directly if they haven't done so already. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: -* +* acme More details about these changes can be found on our GitHub repo. diff --git a/Dockerfile b/Dockerfile index d1296b30f..f3626dc8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot COPY CHANGELOG.md README.rst setup.py src/ +COPY letsencrypt-auto-source/pieces/dependency-requirements.txt . COPY acme src/acme COPY certbot src/certbot @@ -21,6 +22,7 @@ RUN apk add --no-cache --virtual .build-deps \ openssl-dev \ musl-dev \ libffi-dev \ + && pip install -r /opt/certbot/dependency-requirements.txt \ && pip install --no-cache-dir \ --editable /opt/certbot/src/acme \ --editable /opt/certbot/src \ diff --git a/README.rst b/README.rst index 62681c7eb..f55581268 100644 --- a/README.rst +++ b/README.rst @@ -99,8 +99,8 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme |build-status| |coverage| |docs| |container| -.. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master - :target: https://travis-ci.org/certbot/certbot +.. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master + :target: https://travis-ci.com/certbot/certbot :alt: Travis CI status .. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index e8a0b16a8..d91072a3b 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -1,12 +1,22 @@ """ACME protocol implementation. -This module is an implementation of the `ACME protocol`_. Latest -supported version: `draft-ietf-acme-01`_. - +This module is an implementation of the `ACME protocol`_. .. _`ACME protocol`: https://ietf-wg-acme.github.io/acme -.. _`draft-ietf-acme-01`: - https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 - """ +import sys + +# This code exists to keep backwards compatibility with people using acme.jose +# before it became the standalone josepy package. +# +# It is based on +# https://github.com/requests/requests/blob/1278ecdf71a312dc2268f3bfc0aabfab3c006dcf/requests/packages.py + +import josepy as jose + +for mod in list(sys.modules): + # This traversal is apparently necessary such that the identities are + # preserved (acme.jose.* is josepy.*) + if mod == 'josepy' or mod.startswith('josepy.'): + sys.modules['acme.' + mod.replace('josepy', 'jose', 1)] = sys.modules[mod] diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 33ae3886b..b3d0f1921 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -707,6 +707,7 @@ class ClientTest(ClientTestBase): self.certr, self.rsn) + class ClientV2Test(ClientTestBase): """Tests for acme.client.ClientV2.""" @@ -950,7 +951,6 @@ class ClientNetworkTest(unittest.TestCase): self.assertEqual(jws.signature.combined.kid, u'acct-uri') self.assertEqual(jws.signature.combined.url, u'url') - def test_check_response_not_ok_jobj_no_error(self): self.response.ok = False self.response.json.return_value = {} @@ -1113,8 +1113,8 @@ class ClientNetworkTest(unittest.TestCase): # Requests Library Exceptions except requests.exceptions.ConnectionError as z: #pragma: no cover - self.assertTrue("('Connection aborted.', error(111, 'Connection refused'))" - == str(z) or "[WinError 10061]" in str(z)) + self.assertTrue("'Connection aborted.'" in str(z) or "[WinError 10061]" in str(z)) + class ClientNetworkWithMockedResponseTest(unittest.TestCase): """Tests for acme.client.ClientNetwork which mock out response.""" diff --git a/acme/acme/jose_test.py b/acme/acme/jose_test.py new file mode 100644 index 000000000..340624a4f --- /dev/null +++ b/acme/acme/jose_test.py @@ -0,0 +1,53 @@ +"""Tests for acme.jose shim.""" +import importlib +import unittest + +class JoseTest(unittest.TestCase): + """Tests for acme.jose shim.""" + + def _test_it(self, submodule, attribute): + if submodule: + acme_jose_path = 'acme.jose.' + submodule + josepy_path = 'josepy.' + submodule + else: + acme_jose_path = 'acme.jose' + josepy_path = 'josepy' + acme_jose_mod = importlib.import_module(acme_jose_path) + josepy_mod = importlib.import_module(josepy_path) + + self.assertIs(acme_jose_mod, josepy_mod) + self.assertIs(getattr(acme_jose_mod, attribute), getattr(josepy_mod, attribute)) + + # We use the imports below with eval, but pylint doesn't + # understand that. + # pylint: disable=eval-used,unused-variable + import acme + import josepy + acme_jose_mod = eval(acme_jose_path) + josepy_mod = eval(josepy_path) + self.assertIs(acme_jose_mod, josepy_mod) + self.assertIs(getattr(acme_jose_mod, attribute), getattr(josepy_mod, attribute)) + + def test_top_level(self): + self._test_it('', 'RS512') + + def test_submodules(self): + # This test ensures that the modules in josepy that were + # available at the time it was moved into its own package are + # available under acme.jose. Backwards compatibility with new + # modules or testing code is not maintained. + mods_and_attrs = [('b64', 'b64decode',), + ('errors', 'Error',), + ('interfaces', 'JSONDeSerializable',), + ('json_util', 'Field',), + ('jwa', 'HS256',), + ('jwk', 'JWK',), + ('jws', 'JWS',), + ('util', 'ImmutableMap',),] + + for mod, attr in mods_and_attrs: + self._test_it(mod, attr) + + +if __name__ == '__main__': + unittest.main() # pragma: no cover diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 4400a6c31..7c82c8507 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,7 +1,10 @@ """ACME protocol messages.""" -import collections import six import json +try: + from collections.abc import Hashable # pylint: disable=no-name-in-module +except ImportError: + from collections import Hashable import josepy as jose @@ -107,7 +110,7 @@ class Error(jose.JSONObjectWithFields, errors.Error): if part is not None).decode() -class _Constant(jose.JSONDeSerializable, collections.Hashable): # type: ignore +class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore """ACME constant.""" __slots__ = ('name',) POSSIBLE_NAMES = NotImplemented diff --git a/acme/setup.py b/acme/setup.py index 53a4eda27..eac3974fa 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -3,21 +3,21 @@ from setuptools import find_packages from setuptools.command.test import test as TestCommand import sys -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) - 'cryptography>=0.8', + 'cryptography>=1.2.3', # formerly known as acme.jose: 'josepy>=1.0.0', # Connection.set_tlsext_host_name (>=0.13) 'mock', - 'PyOpenSSL>=0.13', + 'PyOpenSSL>=0.13.1', 'pyrfc3339', 'pytz', - 'requests[security]>=2.4.1', # security extras added in 2.4.1 + 'requests[security]>=2.6.0', # security extras added in 2.4.1 'requests-toolbelt>=0.3.0', 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible diff --git a/appveyor.yml b/appveyor.yml index ce2b5998c..2b6b82747 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,6 +24,7 @@ install: build: off test_script: + - set TOX_TESTENV_PASSENV=APPVEYOR # Test env is set by TOXENV env variable - tox diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test index dcbba9d3e..4838a6eee 100755 --- a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test @@ -3,6 +3,11 @@ # A hackish script to see if the client is behaving as expected # with each of the "passing" conf files. +if [ -z "$SERVER" ]; then + echo "Please set SERVER to the ACME server's directory URL." + exit 1 +fi + export EA=/etc/apache2/ TESTDIR="`dirname $0`" cd $TESTDIR/passing @@ -56,13 +61,16 @@ if [ "$1" = --debian-modules ] ; then done fi +CERTBOT_CMD="sudo $(command -v certbot) --server $SERVER -vvvv" +CERTBOT_CMD="$CERTBOT_CMD --debug --apache --register-unsafely-without-email" +CERTBOT_CMD="$CERTBOT_CMD --agree-tos certonly -t --no-verify-ssl" FAILS=0 trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... Setup - RESULT=`echo c | sudo $(command -v certbot) -vvvv --debug --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + RESULT=`echo c | $CERTBOT_CMD 2>&1` if echo $RESULT | grep -Eq \("Which names would you like"\|"mod_macro is not yet"\) ; then echo passed else diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 2562e6f3f..14d6cacb6 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-auto b/certbot-auto index 6003afcf4..be2c3679b 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.29.1" +LE_AUTO_VERSION="0.30.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1212,9 +1212,9 @@ requests-toolbelt==0.8.0 \ chardet==3.0.2 \ --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 -urllib3==1.21.1 \ - --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ - --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 certifi==2017.4.17 \ --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a @@ -1230,18 +1230,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.29.1 \ - --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ - --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 -acme==0.29.1 \ - --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ - --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c -certbot-apache==0.29.1 \ - --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ - --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 -certbot-nginx==0.29.1 \ - --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ - --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d +certbot==0.30.0 \ + --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ + --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 +acme==0.30.0 \ + --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ + --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 +certbot-apache==0.30.0 \ + --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ + --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf +certbot-nginx==0.30.0 \ + --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ + --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 405f8ff73..f519ed422 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 4f12e78f7..ff33293fe 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py index 658db6072..5132137f8 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py @@ -69,13 +69,15 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key, secret_key, ttl): super(_CloudXNSLexiconClient, self).__init__() - self.provider = cloudxns.Provider({ - 'provider_name': 'cloudxns', + config = dns_common_lexicon.build_lexicon_config('cloudxns', { + 'ttl': ttl, + }, { 'auth_username': api_key, 'auth_token': secret_key, - 'ttl': ttl, }) + self.provider = cloudxns.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('400 Client Error:'): diff --git a/certbot-dns-cloudxns/local-oldest-requirements.txt b/certbot-dns-cloudxns/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-cloudxns/local-oldest-requirements.txt +++ b/certbot-dns-cloudxns/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 92ed37f35..1a6f900d8 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -2,14 +2,14 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 0fd52c4b0..2f7fa37d6 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py index 3eb56e37c..ad2a3fa30 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py @@ -65,12 +65,14 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, token, ttl): super(_DNSimpleLexiconClient, self).__init__() - self.provider = dnsimple.Provider({ - 'provider_name': 'dnssimple', - 'auth_token': token, + config = dns_common_lexicon.build_lexicon_config('dnssimple', { 'ttl': ttl, + }, { + 'auth_token': token, }) + self.provider = dnsimple.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('401 Client Error: Unauthorized for url:'): diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-dnsimple/local-oldest-requirements.txt +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index eec35b472..1a2ce5d92 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -2,14 +2,14 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py index 4236ce37a..4b63cb4b5 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py @@ -71,13 +71,15 @@ class _DNSMadeEasyLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key, secret_key, ttl): super(_DNSMadeEasyLexiconClient, self).__init__() - self.provider = dnsmadeeasy.Provider({ - 'provider_name': 'dnsmadeeasy', + config = dns_common_lexicon.build_lexicon_config('dnsmadeeasy', { + 'ttl': ttl, + }, { 'auth_username': api_key, 'auth_token': secret_key, - 'ttl': ttl, }) + self.provider = dnsmadeeasy.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and str(e).startswith('404 Client Error: Not Found for url:'): return diff --git a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt +++ b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 21899f3d8..0a99f452d 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -2,14 +2,14 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py index 9c35e72ab..edf530072 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py @@ -72,13 +72,15 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_token, api_secret, ttl): super(_GehirnLexiconClient, self).__init__() - self.provider = gehirn.Provider({ - 'provider_name': 'gehirn', + config = dns_common_lexicon.build_lexicon_config('gehirn', { + 'ttl': ttl, + }, { 'auth_token': api_token, 'auth_secret': api_secret, - 'ttl': ttl, }) + self.provider = gehirn.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return # Expected errors when zone name guess is wrong diff --git a/certbot-dns-gehirn/local-oldest-requirements.txt b/certbot-dns-gehirn/local-oldest-requirements.txt new file mode 100644 index 000000000..65f5a758e --- /dev/null +++ b/certbot-dns-gehirn/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 2bca63ccf..f4a75379c 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -2,12 +2,12 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', 'dns-lexicon>=2.1.22', 'mock', 'setuptools', diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index ee484e402..c99ad38aa 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/dns_linode.py index 01da2cf60..4e0500fa0 100644 --- a/certbot-dns-linode/certbot_dns_linode/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode.py @@ -54,6 +54,7 @@ class Authenticator(dns_common.DNSAuthenticator): def _get_linode_client(self): return _LinodeLexiconClient(self.credentials.conf('key')) + class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): """ Encapsulates all communication with the Linode API. @@ -61,11 +62,13 @@ class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key): super(_LinodeLexiconClient, self).__init__() - self.provider = linode.Provider({ - 'provider_name': 'linode', - 'auth_token': api_key + + config = dns_common_lexicon.build_lexicon_config('linode', {}, { + 'auth_token': api_key, }) + self.provider = linode.Provider(config) + def _handle_general_error(self, e, domain_name): if not str(e).startswith('Domain not found'): return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}' diff --git a/certbot-dns-linode/local-oldest-requirements.txt b/certbot-dns-linode/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-linode/local-oldest-requirements.txt +++ b/certbot-dns-linode/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index c91aabd4f..31c2c20bc 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,12 +1,12 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', 'dns-lexicon>=2.2.1', 'mock', 'setuptools', diff --git a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py b/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py index bd6a16f69..7cdd4c8e1 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py +++ b/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py @@ -68,13 +68,15 @@ class _LuaDNSLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, email, token, ttl): super(_LuaDNSLexiconClient, self).__init__() - self.provider = luadns.Provider({ - 'provider_name': 'luadns', + config = dns_common_lexicon.build_lexicon_config('luadns', { + 'ttl': ttl, + }, { 'auth_username': email, 'auth_token': token, - 'ttl': ttl, }) + self.provider = luadns.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('401 Client Error: Unauthorized for url:'): diff --git a/certbot-dns-luadns/local-oldest-requirements.txt b/certbot-dns-luadns/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-luadns/local-oldest-requirements.txt +++ b/certbot-dns-luadns/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index 9f0fe149d..31472e8cf 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -2,14 +2,14 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py b/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py index 5f33efbba..3e23df11c 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py +++ b/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py @@ -65,12 +65,14 @@ class _NS1LexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key, ttl): super(_NS1LexiconClient, self).__init__() - self.provider = nsone.Provider({ - 'provider_name': 'nsone', - 'auth_token': api_key, + config = dns_common_lexicon.build_lexicon_config('nsone', { 'ttl': ttl, + }, { + 'auth_token': api_key, }) + self.provider = nsone.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:') or \ str(e).startswith("400 Client Error: Bad Request for url:")): diff --git a/certbot-dns-nsone/local-oldest-requirements.txt b/certbot-dns-nsone/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-nsone/local-oldest-requirements.txt +++ b/certbot-dns-nsone/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index 4ebfeb3e1..41b99cc73 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -2,14 +2,14 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py index 578ee8e89..84771b0a8 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py +++ b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py @@ -77,15 +77,17 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, endpoint, application_key, application_secret, consumer_key, ttl): super(_OVHLexiconClient, self).__init__() - self.provider = ovh.Provider({ - 'provider_name': 'ovh', + config = dns_common_lexicon.build_lexicon_config('ovh', { + 'ttl': ttl, + }, { 'auth_entrypoint': endpoint, 'auth_application_key': application_key, 'auth_application_secret': application_secret, 'auth_consumer_key': consumer_key, - 'ttl': ttl, }) + self.provider = ovh.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('400 Client Error:'): diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt index 8368d266e..01cbcb317 100644 --- a/certbot-dns-ovh/local-oldest-requirements.txt +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -1,2 +1,3 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] +dns-lexicon==2.7.14 diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index cbd77b901..5b3329568 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -2,14 +2,14 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index 2ea6699df..edf7b6ba6 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index 84ca63f00..69c2c7ed3 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,7 +1,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py index b892330f5..7fd6d3ef5 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py @@ -75,13 +75,15 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_token, api_secret, ttl): super(_SakuraCloudLexiconClient, self).__init__() - self.provider = sakuracloud.Provider({ - 'provider_name': 'sakuracloud', + config = dns_common_lexicon.build_lexicon_config('sakuracloud', { + 'ttl': ttl, + }, { 'auth_token': api_token, 'auth_secret': api_secret, - 'ttl': ttl, }) + self.provider = sakuracloud.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return # Expected errors when zone name guess is wrong diff --git a/certbot-dns-sakuracloud/local-oldest-requirements.txt b/certbot-dns-sakuracloud/local-oldest-requirements.txt new file mode 100644 index 000000000..65f5a758e --- /dev/null +++ b/certbot-dns-sakuracloud/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index cebd48e41..4ebfc6e1d 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -2,12 +2,12 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', 'dns-lexicon>=2.1.23', 'mock', 'setuptools', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 970089460..70e11f62b 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -2,7 +2,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.30.0.dev0' +version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. diff --git a/certbot-nginx/tests/boulder-integration.conf.sh b/certbot-nginx/tests/boulder-integration.conf.sh index 4374f9094..470eab28e 100755 --- a/certbot-nginx/tests/boulder-integration.conf.sh +++ b/certbot-nginx/tests/boulder-integration.conf.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Based on # https://www.exratione.com/2014/03/running-nginx-as-a-non-root-user/ # https://github.com/exratione/non-root-nginx/blob/9a77f62e5d5cb9c9026fd62eece76b9514011019/nginx.conf @@ -52,7 +53,7 @@ http { listen 5002 $default_server; # IPv6. listen [::]:5002 $default_server; - server_name nginx.wtf nginx2.wtf; + server_name nginx.wtf nginx-tls.wtf nginx2.wtf; root $root/webroot; diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 2a24e645f..547502ccf 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -39,8 +39,6 @@ nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf -certbot_test_nginx --domains nginx.wtf run --preferred-challenges tls-sni -test_deployment_and_rollback nginx.wtf certbot_test_nginx --domains nginx2.wtf --preferred-challenges http test_deployment_and_rollback nginx2.wtf # Overlapping location block and server-block-level return 301 @@ -66,4 +64,4 @@ test_deployment_and_rollback nginx6.wtf # top nginx -c $nginx_root/nginx.conf -s stop -coverage report --fail-under 75 --include 'certbot-nginx/*' --show-missing +coverage report --fail-under 72 --include 'certbot-nginx/*' --show-missing diff --git a/certbot/__init__.py b/certbot/__init__.py index c96f7b28e..bf68034c8 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.30.0.dev0' +__version__ = '0.31.0.dev0' diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index efee49143..3dfaaf26f 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -31,7 +31,7 @@ class AuthHandler(object): :class:`~acme.challenges.Challenge` types :type auth: :class:`certbot.interfaces.IAuthenticator` - :ivar acme.client.BackwardsCompatibleClientV2 acme: ACME client API. + :ivar acme.client.BackwardsCompatibleClientV2 acme_client: ACME client API. :ivar account: Client's Account :type account: :class:`certbot.account.Account` @@ -40,9 +40,9 @@ class AuthHandler(object): type strings with the most preferred challenge listed first """ - def __init__(self, auth, acme, account, pref_challs): + def __init__(self, auth, acme_client, account, pref_challs): self.auth = auth - self.acme = acme + self.acme = acme_client self.account = account self.pref_challs = pref_challs @@ -85,19 +85,26 @@ class AuthHandler(object): self.verify_authzr_complete(aauthzrs) # Only return valid authorizations - retVal = [aauthzr.authzr for aauthzr in aauthzrs - if aauthzr.authzr.body.status == messages.STATUS_VALID] + ret_val = [aauthzr.authzr for aauthzr in aauthzrs + if aauthzr.authzr.body.status == messages.STATUS_VALID] - if not retVal: + if not ret_val: raise errors.AuthorizationError( "Challenges failed for all domains") - return retVal + return ret_val def _choose_challenges(self, aauthzrs): - """Retrieve necessary challenges to satisfy server.""" - logger.info("Performing the following challenges:") - for aauthzr in aauthzrs: + """ + Retrieve necessary and pending challenges to satisfy server. + NB: Necessary and already validated challenges are not retrieved, + as they can be reused for a certificate issuance. + """ + pending_authzrs = [aauthzr for aauthzr in aauthzrs + if aauthzr.authzr.body.status != messages.STATUS_VALID] + if pending_authzrs: + logger.info("Performing the following challenges:") + for aauthzr in pending_authzrs: aauthzr_challenges = aauthzr.authzr.body.challenges if self.acme.acme_version == 1: combinations = aauthzr.authzr.body.combinations @@ -125,7 +132,7 @@ class AuthHandler(object): def _solve_challenges(self, aauthzrs): """Get Responses for challenges from authenticators.""" - resp = [] # type: Collection[acme.challenges.ChallengeResponse] + resp = [] # type: Collection[challenges.ChallengeResponse] all_achalls = self._get_all_achalls(aauthzrs) try: if all_achalls: @@ -531,7 +538,7 @@ def _report_failed_challs(failed_achalls): """ problems = collections.defaultdict(list)\ - # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] + # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] for achall in failed_achalls: if achall.error: problems[achall.error.typ].append(achall) diff --git a/certbot/cli.py b/certbot/cli.py index 8e599cfda..31f55711f 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1215,6 +1215,10 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): # pylint: dis " one will be run.") helpful.add("renew", "--renew-hook", action=_RenewHookAction, help=argparse.SUPPRESS) + helpful.add( + "renew", "--no-random-sleep-on-renew", action="store_false", + default=flag_default("random_sleep_on_renew"), dest="random_sleep_on_renew", + help=argparse.SUPPRESS) helpful.add( "renew", "--deploy-hook", action=_DeployHookAction, help='Command to be run in a shell once for each successfully' @@ -1307,7 +1311,8 @@ def _create_subparsers(helpful): helpful.add("revoke", "--delete-after-revoke", action="store_true", default=flag_default("delete_after_revoke"), - help="Delete certificates after revoking them.") + help="Delete certificates after revoking them, along with all previous and later " + "versions of those certificates.") helpful.add("revoke", "--no-delete-after-revoke", action="store_false", dest="delete_after_revoke", diff --git a/certbot/constants.py b/certbot/constants.py index eb4105f82..7d0e0c879 100644 --- a/certbot/constants.py +++ b/certbot/constants.py @@ -68,6 +68,7 @@ CLI_DEFAULTS = dict( directory_hooks=True, reuse_key=False, disable_renew_updates=False, + random_sleep_on_renew=True, eab_hmac_key=None, eab_kid=None, diff --git a/certbot/main.py b/certbot/main.py index b24c5c622..a0c0ab64d 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -4,9 +4,7 @@ from __future__ import print_function import functools import logging.handlers import os -import random import sys -import time import configobj import josepy as jose @@ -550,7 +548,8 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b attempt_deletion = config.delete_after_revoke if attempt_deletion is None: - msg = ("Would you like to delete the cert(s) you just revoked?") + msg = ("Would you like to delete the cert(s) you just revoked, along with all earlier and " + "later versions of the cert?") attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", force_interactive=True, default=True) @@ -1269,16 +1268,6 @@ def renew(config, unused_plugins): :rtype: None """ - if not sys.stdin.isatty(): - # Noninteractive renewals include a random delay in order to spread - # out the load on the certificate authority servers, even if many - # users all pick the same time for renewals. This delay precedes - # running any hooks, so that side effects of the hooks (such as - # shutting down a web service) aren't prolonged unnecessarily. - sleep_time = random.randint(1, 60*8) - logger.info("Non-interactive renewal: random delay of %s seconds", sleep_time) - time.sleep(sleep_time) - try: renewal.handle_renewal_request(config) finally: diff --git a/certbot/plugins/dns_common_lexicon.py b/certbot/plugins/dns_common_lexicon.py index f9610b816..5b50cc285 100644 --- a/certbot/plugins/dns_common_lexicon.py +++ b/certbot/plugins/dns_common_lexicon.py @@ -1,12 +1,22 @@ """Common code for DNS Authenticator Plugins built on Lexicon.""" - import logging from requests.exceptions import HTTPError, RequestException +from acme.magic_typing import Union, Dict, Any # pylint: disable=unused-import,no-name-in-module from certbot import errors from certbot.plugins import dns_common +# Lexicon is not declared as a dependency in Certbot itself, +# but in the Certbot plugins backed by Lexicon. +# So we catch import error here to allow this module to be +# always importable, even if it does not make sense to use it +# if Lexicon is not available, obviously. +try: + from lexicon.config import ConfigResolver +except ImportError: + ConfigResolver = None # type: ignore + logger = logging.getLogger(__name__) @@ -100,3 +110,28 @@ class LexiconClient(object): if not str(e).startswith('No domain found'): return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}' .format(domain_name, e)) + + +def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_options): + # type: (str, Dict, Dict) -> Union[ConfigResolver, Dict] + """ + Convenient function to build a Lexicon 2.x/3.x config object. + :param str lexicon_provider_name: the name of the lexicon provider to use + :param dict lexicon_options: options specific to lexicon + :param dict provider_options: options specific to provider + :return: configuration to apply to the provider + :rtype: ConfigurationResolver or dict + """ + config = {'provider_name': lexicon_provider_name} # type: Dict[str, Any] + config.update(lexicon_options) + if not ConfigResolver: + # Lexicon 2.x + config.update(provider_options) + else: + # Lexicon 3.x + provider_config = {} + provider_config.update(provider_options) + config[lexicon_provider_name] = provider_config + config = ConfigResolver().with_dict(config).with_env() + + return config diff --git a/certbot/renewal.py b/certbot/renewal.py index 5b2e00740..4c592b27f 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -5,6 +5,9 @@ import itertools import logging import os import traceback +import sys +import time +import random import six import zope.component @@ -372,7 +375,7 @@ def _renew_describe_results(config, renew_successes, renew_failures, disp.notification("\n".join(out), wrap=False) -def handle_renewal_request(config): +def handle_renewal_request(config): # pylint: disable=too-many-locals,too-many-branches,too-many-statements """Examine each lineage; renew if due and report results""" # This is trivially False if config.domains is empty @@ -396,6 +399,14 @@ def handle_renewal_request(config): renew_failures = [] renew_skipped = [] parse_failures = [] + + # Noninteractive renewals include a random delay in order to spread + # out the load on the certificate authority servers, even if many + # users all pick the same time for renewals. This delay precedes + # running any hooks, so that side effects of the hooks (such as + # shutting down a web service) aren't prolonged unnecessarily. + apply_random_sleep = not sys.stdin.isatty() and config.random_sleep_on_renew + for renewal_file in conf_files: disp = zope.component.getUtility(interfaces.IDisplay) disp.notification("Processing " + renewal_file, pause=False) @@ -424,6 +435,15 @@ def handle_renewal_request(config): from certbot import main plugins = plugins_disco.PluginsRegistry.find_all() if should_renew(lineage_config, renewal_candidate): + # Apply random sleep upon first renewal if needed + if apply_random_sleep: + sleep_time = random.randint(1, 60 * 8) + logger.info("Non-interactive renewal: random delay of %s seconds", + sleep_time) + time.sleep(sleep_time) + # We will sleep only once this day, folks. + apply_random_sleep = False + # domains have been restored into lineage_config by reconstitute # but they're unnecessary anyway because renew_cert here # will just grab them from the certificate diff --git a/certbot/storage.py b/certbot/storage.py index eb17e1d38..d17a0f29d 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -41,7 +41,9 @@ def renewal_conf_files(config): :rtype: `list` of `str` """ - return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) + result = glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) + result.sort() + return result def renewal_file_for_certname(config, certname): """Return /path/to/certname.conf in the renewal conf directory""" @@ -877,45 +879,6 @@ class RenewableCert(object): with open(target) as f: return crypto_util.get_names_from_cert(f.read()) - def autodeployment_is_enabled(self): - """Is automatic deployment enabled for this cert? - - If autodeploy is not specified, defaults to True. - - :returns: True if automatic deployment is enabled - :rtype: bool - - """ - return ("autodeploy" not in self.configuration or - self.configuration.as_bool("autodeploy")) - - def should_autodeploy(self, interactive=False): - """Should this lineage now automatically deploy a newer version? - - This is a policy question and does not only depend on whether - there is a newer version of the cert. (This considers whether - autodeployment is enabled, whether a relevant newer version - exists, and whether the time interval for autodeployment has - been reached.) - - :param bool interactive: set to True to examine the question - regardless of whether the renewal configuration allows - automated deployment (for interactive use). Default False. - - :returns: whether the lineage now ought to autodeploy an - existing newer cert version - :rtype: bool - - """ - if interactive or self.autodeployment_is_enabled(): - if self.has_pending_deployment(): - interval = self.configuration.get("deploy_before_expiry", - "5 days") - now = pytz.UTC.fromutc(datetime.datetime.utcnow()) - if self.target_expiry < add_time_interval(now, interval): - return True - return False - def ocsp_revoked(self, version=None): # pylint: disable=no-self-use,unused-argument """Is the specified cert version revoked according to OCSP? diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index e1319b614..fe0ece12e 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -57,7 +57,7 @@ class ChallengeFactoryTest(unittest.TestCase): errors.Error, self.handler._challenge_factory, authzr, [0]) -class HandleAuthorizationsTest(unittest.TestCase): +class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-public-methods """handle_authorizations test. This tests everything except for all functions under _poll_challenges. @@ -316,6 +316,24 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + def test_validated_challenge_not_rerun(self): + # With pending challenge, we expect the challenge to be tried, and fail. + authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "0", + [acme_util.HTTP01], + [messages.STATUS_PENDING], False) + mock_order = mock.MagicMock(authorizations=[authzr]) + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + + # With validated challenge; we expect the challenge not be tried again, and succeed. + authzr = acme_util.gen_authzr( + messages.STATUS_VALID, "0", + [acme_util.HTTP01], + [messages.STATUS_VALID], False) + mock_order = mock.MagicMock(authorizations=[authzr]) + self.handler.handle_authorizations(mock_order) + def _validate_all(self, aauthzrs, unused_1, unused_2): for i, aauthzr in enumerate(aauthzrs): azr = aauthzr.authzr diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index d75f4f595..e61ed2aca 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -388,8 +388,7 @@ class RenewableCertTests(BaseRenewableCertTest): @mock.patch("certbot.storage.cli") @mock.patch("certbot.storage.datetime") def test_time_interval_judgments(self, mock_datetime, mock_cli): - """Test should_autodeploy() and should_autorenew() on the basis - of expiry time windows.""" + """Test should_autorenew() on the basis of expiry time windows.""" test_cert = test_util.load_vector("cert_512.pem") self._write_out_ex_kinds() @@ -430,31 +429,8 @@ class RenewableCertTests(BaseRenewableCertTest): mock_datetime.datetime.utcnow.return_value = sometime self.test_rc.configuration["deploy_before_expiry"] = interval self.test_rc.configuration["renew_before_expiry"] = interval - self.assertEqual(self.test_rc.should_autodeploy(), result) self.assertEqual(self.test_rc.should_autorenew(), result) - def test_autodeployment_is_enabled(self): - self.assertTrue(self.test_rc.autodeployment_is_enabled()) - self.test_rc.configuration["autodeploy"] = "1" - self.assertTrue(self.test_rc.autodeployment_is_enabled()) - - self.test_rc.configuration["autodeploy"] = "0" - self.assertFalse(self.test_rc.autodeployment_is_enabled()) - - def test_should_autodeploy(self): - """Test should_autodeploy() on the basis of reasons other than - expiry time window.""" - # pylint: disable=too-many-statements - # Autodeployment turned off - self.test_rc.configuration["autodeploy"] = "0" - self.assertFalse(self.test_rc.should_autodeploy()) - self.test_rc.configuration["autodeploy"] = "1" - # No pending deployment - for ver in six.moves.range(1, 6): - for kind in ALL_FOUR: - self._write_out_kind(kind, ver) - self.assertFalse(self.test_rc.should_autodeploy()) - def test_autorenewal_is_enabled(self): self.test_rc.configuration["renewalparams"] = {} self.assertTrue(self.test_rc.autorenewal_is_enabled()) diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 37d67cfe0..bad6275e7 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -29,6 +29,7 @@ manage certificates: manage your account with Let's Encrypt: register Create a Let's Encrypt ACME account + update_account Update a Let's Encrypt ACME account --agree-tos Agree to the ACME server's Subscriber Agreement -m EMAIL Email address for important account notifications @@ -112,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.29.1 + "". (default: CertbotACMEClient/0.30.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the @@ -359,7 +360,7 @@ revoke: certificates. (default: None) register: - Options for account registration & modification + Options for account registration --register-unsafely-without-email Specifying this flag enables registering an account @@ -371,11 +372,6 @@ register: to the Subscriber Agreement will still affect you, and will be effective 14 days after posting an update to the web site. (default: False) - --update-registration - With the register verb, indicates that details - associated with an existing registration, such as the - e-mail address, should be updated, rather than - registering a new account. (default: False) -m EMAIL, --email EMAIL Email used for registration and recovery contact. Use comma to register multiple emails, ex: @@ -384,6 +380,9 @@ register: --no-eff-email Don't share your e-mail address with EFF (default: None) +update_account: + Options for account modification + unregister: Options for account deactivation. diff --git a/docs/using.rst b/docs/using.rst index 5e8675418..d77badd65 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -44,8 +44,7 @@ a combination_ of distinct authenticator and installer plugins. =========== ==== ==== =============================================================== ============================= Plugin Auth Inst Notes Challenge types (and port) =========== ==== ==== =============================================================== ============================= -apache_ Y Y | Automates obtaining and installing a certificate with Apache http-01_ (80) - | 2.4 on OSes with ``libaugeas0`` 1.0+. +apache_ Y Y | Automates obtaining and installing a certificate with Apache. http-01_ (80) nginx_ Y Y | Automates obtaining and installing a certificate with Nginx. http-01_ (80) webroot_ Y N | Obtains a certificate by writing to the webroot directory of http-01_ (80) | an already running webserver. @@ -83,8 +82,7 @@ the circumstances in which each plugin can be used, and how to use it. Apache ------ -The Apache plugin currently requires an OS with augeas version 1.0; currently `it -supports +The Apache plugin currently `supports `_ modern OSes based on Debian, Fedora, SUSE, Gentoo and Darwin. This automates both obtaining *and* installing certificates on an Apache @@ -136,9 +134,8 @@ the webserver. Nginx ----- -The Nginx plugin has been distributed with Certbot since version 0.9.0 and should -work for most configurations. We recommend backing up Nginx -configurations before using it (though you can also revert changes to +The Nginx plugin should work for most configurations. We recommend backing up +Nginx configurations before using it (though you can also revert changes to configurations with ``certbot --nginx rollback``). You can use it by providing the ``--nginx`` flag on the commandline. diff --git a/letsencrypt-auto b/letsencrypt-auto index 6003afcf4..be2c3679b 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.29.1" +LE_AUTO_VERSION="0.30.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1212,9 +1212,9 @@ requests-toolbelt==0.8.0 \ chardet==3.0.2 \ --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 -urllib3==1.21.1 \ - --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ - --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 certifi==2017.4.17 \ --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a @@ -1230,18 +1230,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.29.1 \ - --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ - --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 -acme==0.29.1 \ - --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ - --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c -certbot-apache==0.29.1 \ - --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ - --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 -certbot-nginx==0.29.1 \ - --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ - --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d +certbot==0.30.0 \ + --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ + --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 +acme==0.30.0 \ + --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ + --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 +certbot-apache==0.30.0 \ + --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ + --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf +certbot-nginx==0.30.0 \ + --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ + --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/Dockerfile.wheezy b/letsencrypt-auto-source/Dockerfile.jessie similarity index 98% rename from letsencrypt-auto-source/Dockerfile.wheezy rename to letsencrypt-auto-source/Dockerfile.jessie index f4f3fea15..9ee37b763 100644 --- a/letsencrypt-auto-source/Dockerfile.wheezy +++ b/letsencrypt-auto-source/Dockerfile.jessie @@ -1,7 +1,7 @@ # For running tests, build a docker image with a passwordless sudo and a trust # store we can manipulate. -FROM debian:wheezy +FROM debian:jessie # Add an unprivileged user: RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea diff --git a/letsencrypt-auto-source/Dockerfile.precise b/letsencrypt-auto-source/Dockerfile.xenial similarity index 98% rename from letsencrypt-auto-source/Dockerfile.precise rename to letsencrypt-auto-source/Dockerfile.xenial index 39a167c14..931f1c6d3 100644 --- a/letsencrypt-auto-source/Dockerfile.precise +++ b/letsencrypt-auto-source/Dockerfile.xenial @@ -1,7 +1,7 @@ # For running tests, build a docker image with a passwordless sudo and a trust # store we can manipulate. -FROM ubuntu:precise +FROM ubuntu:xenial # Add an unprivileged user: RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index b4e520fb2..4e2a700e5 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlwIbaAACgkQTRfJlc2X -dfLIGQf+JZr3oP89qyMREYAL3/Bx+vxsx+c01IuDaG1pBUeVwL5rdeU1kDZ1WkKb -61nCoMPbSLqLMor2IpobFj44lEFJS1WYrtfe8sgMLeSaQkWXlB3breLKI09p/IBm -X3lN7VIMPW8eLziSArGivKTpsW+cDau8Rbnn5FSMNiZojCp+bSPehOpZBmHb2OrS -2akgCBQYh2e+cadv5MmGPqta8iTDMDgMOrTzAstPKRfeHozyoOsRjeq1T9Yl65lk -XF+m0yl2r4w8VxemWzqaT53XHS+3tgCunDtB7pDXuM6zpNnWghxN1UJ8Ywle1nt4 -ecxxPwHfGAq9cAVCM7M0x4y+bxdnbw== -=8xDr +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAlwtH9cACgkQTRfJlc2X +dfIqUwf/RXLZAeFF/59PjTAzcV+eEISlvEmFcV0zL3vv23PsY3S5Iuuwcd6rTm5M +UWNtmUTmFVo0xmxAj6Eqfpnt0P+JPpPcnbLNIGKFekBWIshgH84RRFWPJjNh/hu1 +pyzkkcWaOB86egdVfjvuRJ0j7AGd0ih6ur2rlgfHVjTYR+0EdWszFDEFBlq8cpct +9d1gCgH7VWKSIQMhzGLMsmdMxNoDl4hiqVPU0FP5/mn2xGF7FgeKNW3+NiTouKuB +mZOeEl3f3uOze/suHPyfOu+49jk+TWWE05Xfqfowjf486nKPg6/uSA2izW/MwIKN +HuIuY3bBf+lx5yUVIraoZhH2MxODDQ== +=BZqz -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0ffe6c5f5..d86401904 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="0.30.0.dev0" +LE_AUTO_VERSION="0.31.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -333,63 +333,11 @@ BootstrapDebCommon() { fi augeas_pkg="libaugeas0 augeas-lenses" - AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` if [ "$ASSUME_YES" = 1 ]; then YES_FLAG="-y" fi - AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - add_backports=1 - else - read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response - case $response in - [yY][eE][sS]|[yY]|"") - add_backports=1;; - *) - add_backports=0;; - esac - fi - if [ "$add_backports" = 1 ]; then - sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - apt-get $QUIET_FLAG update - fi - fi - fi - if [ "$add_backports" != 0 ]; then - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - } - - - if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Certbot apache plugin..." - fi - # XXX add a case for ubuntu PPAs - fi - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ python \ python-dev \ @@ -1138,9 +1086,9 @@ parsedatetime==2.1 \ pbr==1.8.1 \ --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 -pyOpenSSL==16.2.0 \ - --hash=sha256:26ca380ddf272f7556e48064bbcd5bd71f83dfc144f3583501c7ddbd9434ee17 \ - --hash=sha256:7779a3bbb74e79db234af6a08775568c6769b5821faecf6e2f4143edb227516e +pyOpenSSL==18.0.0 \ + --hash=sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854 \ + --hash=sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580 pyparsing==2.1.8 \ --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \ --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \ @@ -1212,9 +1160,9 @@ requests-toolbelt==0.8.0 \ chardet==3.0.2 \ --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 -urllib3==1.21.1 \ - --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ - --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 certifi==2017.4.17 \ --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a @@ -1230,18 +1178,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==0.29.1 \ - --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ - --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 -acme==0.29.1 \ - --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ - --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c -certbot-apache==0.29.1 \ - --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ - --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 -certbot-nginx==0.29.1 \ - --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ - --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d +certbot==0.30.0 \ + --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ + --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 +acme==0.30.0 \ + --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ + --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 +certbot-apache==0.30.0 \ + --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ + --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf +certbot-nginx==0.30.0 \ + --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ + --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 7b3874da0..e4902c0ee 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh index eb22225e4..93bdc63b4 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh @@ -43,63 +43,11 @@ BootstrapDebCommon() { fi augeas_pkg="libaugeas0 augeas-lenses" - AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` if [ "$ASSUME_YES" = 1 ]; then YES_FLAG="-y" fi - AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - add_backports=1 - else - read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response - case $response in - [yY][eE][sS]|[yY]|"") - add_backports=1;; - *) - add_backports=0;; - esac - fi - if [ "$add_backports" = 1 ]; then - sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - apt-get $QUIET_FLAG update - fi - fi - fi - if [ "$add_backports" != 0 ]; then - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - } - - - if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Certbot apache plugin..." - fi - # XXX add a case for ubuntu PPAs - fi - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ python \ python-dev \ diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index cfe3f6fbe..08d8d553f 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==0.29.1 \ - --hash=sha256:2ba2c60fd1969e75d3e5048d3f7d95afd0949670b39a6a0037ba4a594e9f26a5 \ - --hash=sha256:6fc604d207c48b95dea3458bb33a11b17aa625628eb197927ffee8b458f62692 -acme==0.29.1 \ - --hash=sha256:4be3848f8813c455021f13519642d8ec2746b78d4d0bc2ae04c3dcb1d8862f60 \ - --hash=sha256:a2e203ade83cd1eaf19112004a63073830211cf7759d437f634babb08c49b47c -certbot-apache==0.29.1 \ - --hash=sha256:8d8b6b7c5f333cf5297153c6a1eacc09b4a5c73e8f93544800b3ad016d5e34d0 \ - --hash=sha256:c3af1c66c86cfeef7dac4fe9b16c7c755ebd12bc526408c27781bd34b9de8128 -certbot-nginx==0.29.1 \ - --hash=sha256:5ba3a7d93d3ce317fb8b3d0222c708fb79e96c7a9b1ba56e12e46892c2d12869 \ - --hash=sha256:0c1205ebb91eef4b7d15293c6778ffc962d09563b315120b2d226348d751e38d +certbot==0.30.0 \ + --hash=sha256:b3468e128e74d2295598f6d3fbf9d0edfb67fe5abaca3b985a9e858395bd027f \ + --hash=sha256:d631fe6c75700ce9b2fdae194ff8b53c7518545d87dd451a1704f7572dcd49e8 +acme==0.30.0 \ + --hash=sha256:eed9389f802ebf4988c9e43c28ad3d5c2734237371d78e97450a1d61189a15aa \ + --hash=sha256:984b6d00bec73dcfa616636a760e80ca14bd246fb908710a656547f542f09445 +certbot-apache==0.30.0 \ + --hash=sha256:d38c70fc6930db298ea992a3145362eebdce460d3d2651f86a8f2f43d838c6d0 \ + --hash=sha256:1d4bc207d53a3e5d37e5d9ebd05f26089aa21d1fbf384113ed9d1829b4d1e9bf +certbot-nginx==0.30.0 \ + --hash=sha256:6163c7d0080f59b4ebe510afcc6af2d2eebf15469275c3835866690db4d465d6 \ + --hash=sha256:e39a3f3d77cd4c653949cf066fb2211039fd2032665697c27b6e8501c7c2dd92 diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index 983d2bb95..1fac78836 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -114,9 +114,9 @@ parsedatetime==2.1 \ pbr==1.8.1 \ --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 -pyOpenSSL==16.2.0 \ - --hash=sha256:26ca380ddf272f7556e48064bbcd5bd71f83dfc144f3583501c7ddbd9434ee17 \ - --hash=sha256:7779a3bbb74e79db234af6a08775568c6769b5821faecf6e2f4143edb227516e +pyOpenSSL==18.0.0 \ + --hash=sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854 \ + --hash=sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580 pyparsing==2.1.8 \ --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \ --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \ @@ -188,9 +188,9 @@ requests-toolbelt==0.8.0 \ chardet==3.0.2 \ --hash=sha256:4f7832e7c583348a9eddd927ee8514b3bf717c061f57b21dbe7697211454d9bb \ --hash=sha256:6ebf56457934fdce01fb5ada5582762a84eed94cad43ed877964aebbdd8174c0 -urllib3==1.21.1 \ - --hash=sha256:8ed6d5c1ff9d6ba84677310060d6a3a78ca3072ce0684cb3c645023009c114b1 \ - --hash=sha256:b14486978518ca0901a76ba973d7821047409d7f726f22156b24e83fd71382a5 +urllib3==1.24.1 \ + --hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \ + --hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 certifi==2017.4.17 \ --hash=sha256:f4318671072f030a33c7ca6acaef720ddd50ff124d1388e50c1bda4cbd6d7010 \ --hash=sha256:f7527ebf7461582ce95f7a9e03dd141ce810d40590834f4ec20cddd54234c10a diff --git a/pull_request_template.md b/pull_request_template.md index c071d4135..60fd6da7e 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -1 +1,3 @@ -Be sure to edit the `master` section of `CHANGELOG.md` with a line describing this PR before it gets merged. +Be sure to edit the `master` section of `CHANGELOG.md`. This includes a +description of the change and ensuring the modified package(s) are listed as +having been changed. diff --git a/pytest.ini b/pytest.ini index 9a5807f34..8f009045c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,10 +3,14 @@ # directly. [pytest] addopts = --numprocesses auto --pyargs -# ResourceWarnings are ignored as errors, since they're raised at close -# decodestring: https://github.com/rthalley/dnspython/issues/338 -# ignore our own TLS-SNI-01 warning +# In general, all warnings are treated as errors. Here are the exceptions: +# 1- decodestring: https://github.com/rthalley/dnspython/issues/338 +# 2- ignore our own TLS-SNI-01 warning +# 3- ignore warn for importing abstract classes from collections instead of collections.abc, +# too much third party dependencies are still relying on this behavior, +# but it should be corrected to allow Certbot compatiblity with Python >= 3.8 filterwarnings = error ignore:decodestring:DeprecationWarning ignore:TLS-SNI-01:DeprecationWarning + ignore:.*collections\.abc:DeprecationWarning diff --git a/setup.py b/setup.py index 2a9b2c203..9e6af2d4f 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires = [ # in which we added 2.6 support (see #2243), so we relax the requirement. 'ConfigArgParse>=0.9.3', 'configobj', - 'cryptography>=1.2', # load_pem_x509_certificate + 'cryptography>=1.2.3', # load_pem_x509_certificate 'josepy', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index 31e0f6b30..f34deb74e 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -11,7 +11,6 @@ if [ ! -d ${BOULDERPATH} ]; then fi cd ${BOULDERPATH} -sed -i "s/FAKE_DNS: .*/FAKE_DNS: 10.77.77.1/" docker-compose.yml docker-compose up -d boulder @@ -28,3 +27,6 @@ if ! curl http://localhost:4000/directory 2>/dev/null; then echo "timed out waiting for boulder to start" exit 1 fi + +# Setup the DNS resolution used by boulder instance to docker host +curl -X POST -d '{"ip":"10.77.77.1"}' http://localhost:8055/set-default-ipv4 diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 9011d8ba3..c03b3def8 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -174,7 +174,7 @@ CheckRenewHook() { TotalAndDistinctLines() { total=$1 distinct=$2 - awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' + awk '{a[$1] = 1}; END {n = 0; for (i in a) { n++ }; exit(NR !='$total' || n !='$distinct')}' } # Cleanup coverage data @@ -221,20 +221,20 @@ common plugins --init --prepare | grep webroot # We start a server listening on the port for the # unrequested challenge to prevent regressions in #3601. -python ./tests/run_http_server.py $http_01_port & +python ./tests/run_http_server.py $tls_alpn_01_port & python_server_pid=$! - certname="le1.wtf" -common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ +common --domains le1.wtf --preferred-challenges http-01 auth \ --cert-name $certname \ --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ --deploy-hook 'echo deploy >> "$HOOK_TEST"' -kill $python_server_pid CheckDeployHook $certname -python ./tests/run_http_server.py $tls_sni_01_port & -python_server_pid=$! +# Previous test used to be a tls-sni-01 challenge that is not supported anymore. +# Now it is a http-01 challenge and this makes it a duplicate of the following test. +# But removing it would break many tests here, as they are strongly coupled. +# See https://github.com/certbot/certbot/pull/6679 certname="le2.wtf" common --domains le2.wtf --preferred-challenges http-01 run \ --cert-name $certname \ @@ -254,7 +254,7 @@ common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ CheckRenewHook $certname certname="dns.le.wtf" -common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ +common -a manual -d dns.le.wtf --preferred-challenges dns run \ --cert-name $certname \ --manual-auth-hook ./tests/manual-dns-auth.sh \ --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ @@ -396,7 +396,7 @@ CheckDirHooks 1 # with fail. common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ --allow-subset-of-names \ - --preferred-challenges dns,tls-sni \ + --preferred-challenges dns \ --manual-auth-hook ./tests/manual-dns-auth.sh \ --manual-cleanup-hook ./tests/manual-dns-cleanup.sh diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 1e444fa26..76b990ac4 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -3,12 +3,15 @@ root=${root:-$(mktemp -d -t leitXXXX)} echo "Root integration tests directory: $root" config_dir="$root/conf" -store_flags="--config-dir $config_dir --work-dir $root/work" -store_flags="$store_flags --logs-dir $root/logs" -tls_sni_01_port=5001 +tls_alpn_01_port=5001 http_01_port=5002 sources="acme/,$(ls -dm certbot*/ | tr -d ' \n')" -export root config_dir store_flags tls_sni_01_port http_01_port sources +export root config_dir tls_alpn_01_port http_01_port sources +certbot_path="$(command -v certbot)" +# Flags that are added here will be added to Certbot calls within +# certbot_test_no_force_renew. +other_flags="--config-dir $config_dir --work-dir $root/work" +other_flags="$other_flags --logs-dir $root/logs" certbot_test () { certbot_test_no_force_renew \ @@ -16,11 +19,35 @@ certbot_test () { "$@" } +# Succeeds if Certbot version is at least the given version number and fails +# otherwise. This is useful for making sure Certbot has certain features +# available. The patch version is currently ignored. +# +# Arguments: +# First argument is the minimum major version +# Second argument is the minimum minor version +version_at_least () { + # Certbot major and minor version (e.g. 0.30) + major_minor=$("$certbot_path" --version 2>&1 | cut -d' ' -f2 | cut -d. -f1,2) + major=$(echo "$major_minor" | cut -d. -f1) + minor=$(echo "$major_minor" | cut -d. -f2) + # Test that either the major version is greater or major version is equal + # and minor version is greater than or equal to. + [ \( "$major" -gt "$1" \) -o \( "$major" -eq "$1" -a "$minor" -ge "$2" \) ] +} + # Use local ACMEv2 endpoint if requested and SERVER isn't already set. if [ "${BOULDER_INTEGRATION:-v1}" = "v2" -a -z "${SERVER:+x}" ]; then SERVER="http://localhost:4001/directory" fi +# --no-random-sleep-on-renew was added in +# https://github.com/certbot/certbot/pull/6599 and first released in Certbot +# 0.30.0. +if version_at_least 0 30; then + other_flags="$other_flags --no-random-sleep-on-renew" +fi + certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*" @@ -30,13 +57,13 @@ certbot_test_no_force_renew () { --append \ --source $sources \ --omit $omit_patterns \ - $(command -v certbot) \ + "$certbot_path" \ --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ - --tls-sni-01-port $tls_sni_01_port \ + --tls-sni-01-port $tls_alpn_01_port \ --http-01-port $http_01_port \ --manual-public-ip-logging-ok \ - $store_flags \ + $other_flags \ --non-interactive \ --no-redirect \ --agree-tos \ diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 4036e6efa..d24de2458 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -54,6 +54,7 @@ if [ $? -ne 0 ] ; then fi if [ "$OS_TYPE" = "ubuntu" ] ; then + export SERVER="$BOULDER_URL" venv/bin/tox -e apacheconftest else echo Not running hackish apache tests on $OS_TYPE diff --git a/tools/_release.sh b/tools/_release.sh index 90c176b0e..ec2deda22 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -143,7 +143,7 @@ pip install -U pip # (or our dependencies) have conditional dependencies implemented with if # statements in setup.py and we have cached wheels lying around that would # cause those ifs to not be evaluated. -pip install \ +python ../tools/pip_install.py \ --no-cache-dir \ --extra-index-url http://localhost:$PORT \ $SUBPKGS @@ -166,7 +166,7 @@ fi mkdir kgs kgs="kgs/$version" pip freeze | tee $kgs -pip install pytest +python ../tools/pip_install.py pytest for module in $subpkgs_modules ; do echo testing $module # use an empty configuration file rather than the one in the repo root diff --git a/tools/_venv_common.py b/tools/_venv_common.py index ecd438f94..540842773 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -156,7 +156,7 @@ def main(venv_name, venv_args, args): new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), env=new_environ, shell=True) - subprocess_with_print("python -m pip install --upgrade 'setuptools>=30.3'", + subprocess_with_print('python -m pip install --upgrade "setuptools>=30.3"', env=new_environ, shell=True) subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), env=new_environ, shell=True) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 778012d31..88340cb00 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,5 +1,7 @@ -# Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. +# Specifies Python package versions for development. +# It includes in particular packages not specified in letsencrypt-auto's requirements file. +# Some dev package versions specified here may be overridden by higher level constraints +# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). alabaster==0.7.10 apipkg==1.4 asn1crypto==0.22.0 @@ -12,7 +14,7 @@ botocore==1.12.36 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.7.14 +dns-lexicon==3.0.8 dnspython==1.15.0 docutils==0.12 execnet==1.5.0 diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 79a7c2264..0142eeea4 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -17,16 +17,15 @@ import re SKIP_PROJECTS_ON_WINDOWS = [ 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + def call_with_print(command, cwd=None): print(command) subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + def main(args): - if os.environ.get('CERTBOT_NO_PIN') == '1': - command = [sys.executable, '-m', 'pip', '-e'] - else: - script_dir = os.path.dirname(os.path.abspath(__file__)) - command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] + script_dir = os.path.dirname(os.path.abspath(__file__)) + command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] new_args = [] for arg in args: diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index ad44a55d0..4205e6bcf 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -5,13 +5,14 @@ Requirements files specified later take precedence over earlier ones. Only simple SomeProject==1.2.3 format is currently supported. """ - from __future__ import print_function import sys + def read_file(file_path): """Reads in a Python requirements file. + Ignore empty lines, comments and editable requirements :param str file_path: path to requirements file @@ -19,16 +20,16 @@ def read_file(file_path): :rtype: dict """ - d = {} - with open(file_path) as f: - for line in f: + data = {} + with open(file_path) as file_h: + for line in file_h: line = line.strip() - if line and not line.startswith('#'): + if line and not line.startswith('#') and not line.startswith('-e'): project, version = line.split('==') if not version: raise ValueError("Unexpected syntax '{0}'".format(line)) - d[project] = version - return d + data[project] = version + return data def output_requirements(requirements): @@ -37,25 +38,24 @@ def output_requirements(requirements): :param dict requirements: mapping from a project to its pinned version """ - return '\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items())) + return '\n'.join('{0}=={1}'.format(key, value) + for key, value in sorted(requirements.items())) -def main(*files): +def main(*paths): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier files. - :param tuple files: paths to requirements files + :param tuple paths: paths to requirements files """ - d = {} - for f in files: - d.update(read_file(f)) - return output_requirements(d) + data = {} + for path in paths: + data.update(read_file(path)) + return output_requirements(data) if __name__ == '__main__': - merged_requirements = main(*sys.argv[1:]) - print(merged_requirements) + print(main(*sys.argv[1:])) # pylint: disable=star-args diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 80fce8b33..e48d6b13c 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -37,19 +37,24 @@ pytz==2012rc0 # Our setup.py constraints cloudflare==1.5.1 -cryptography==1.2.0 +cryptography==1.2.3 google-api-python-client==1.5 oauth2client==2.0 parsedatetime==1.3 pyparsing==1.5.5 python-digitalocean==1.11 -requests[security]==2.4.1 +requests[security]==2.6.0 # Ubuntu Xenial constraints ConfigArgParse==0.10.0 funcsigs==0.4 zope.hookable==4.0.4 +# Ubuntu Bionic constraints. +# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins +# using their local-oldest-requirements.txt +dns-lexicon==2.2.1 + # Plugin constraints # These aren't necessarily the oldest versions we need to support # Tracking at https://github.com/certbot/certbot/issues/6473 diff --git a/tools/pip_install.py b/tools/pip_install.py index 354dce32b..dd6302b48 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -32,10 +32,10 @@ def certbot_oldest_processing(tools_path, args, test_constraints): # remove any extras such as [dev] pkg_dir = re.sub(r'\[\w+\]', '', args[1]) requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') + shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) # packages like acme don't have any local oldest requirements if not os.path.isfile(requirements): - requirements = None - shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) + return None return requirements @@ -53,11 +53,19 @@ def certbot_normal_processing(tools_path, test_constraints): fd.write('{0}{1}'.format(search.group(1), os.linesep)) -def merge_requirements(tools_path, test_constraints, all_constraints): - merged_requirements = merge_module.main( - os.path.join(tools_path, 'dev_constraints.txt'), - test_constraints - ) +def merge_requirements(tools_path, requirements, test_constraints, all_constraints): + # Order of the files in the merge function matters. + # Indeed version retained for a given package will be the last version + # found when following all requirements in the given order. + # Here is the order by increasing priority: + # 1) The general development constraints (tools/dev_constraints.txt) + # 2) The general tests constraints (oldest_requirements.txt or + # certbot-auto's dependency-requirements.txt for the normal processing) + # 3) The local requirement file, typically local-oldest-requirement in oldest tests + files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] + if requirements: + files.append(requirements) + merged_requirements = merge_module.main(*files) with open(all_constraints, 'w') as fd: fd.write(merged_requirements) @@ -71,24 +79,37 @@ def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() + if os.environ.get('TRAVIS'): + # When this script is executed on Travis, the following print will make the log + # be folded until the end command is printed (see finally section). + print('travis_fold:start:install_certbot_deps') + try: test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') - requirements = None - if os.environ.get('CERTBOT_OLDEST') == '1': - requirements = certbot_oldest_processing(tools_path, args, test_constraints) + if os.environ.get('CERTBOT_NO_PIN') == '1': + # With unpinned dependencies, there is no constraint + call_with_print('"{0}" -m pip install {1}' + .format(sys.executable, ' '.join(args))) else: - certbot_normal_processing(tools_path, test_constraints) + # Otherwise, we merge requirements to build the constraints and pin dependencies + requirements = None + if os.environ.get('CERTBOT_OLDEST') == '1': + requirements = certbot_oldest_processing(tools_path, args, test_constraints) + else: + certbot_normal_processing(tools_path, test_constraints) - merge_requirements(tools_path, test_constraints, all_constraints) - if requirements: - call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' - .format(sys.executable, all_constraints, requirements)) + merge_requirements(tools_path, requirements, test_constraints, all_constraints) + if requirements: + call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' + .format(sys.executable, all_constraints, requirements)) - call_with_print('"{0}" -m pip install --constraint "{1}" {2}' - .format(sys.executable, all_constraints, ' '.join(args))) + call_with_print('"{0}" -m pip install --constraint "{1}" {2}' + .format(sys.executable, all_constraints, ' '.join(args))) finally: + if os.environ.get('TRAVIS'): + print('travis_fold:end:install_certbot_deps') shutil.rmtree(working_dir) diff --git a/tox.ini b/tox.ini index 61cee8eff..363f06bf2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py{34,35,36},py27-cover,lint +envlist = modification,py3,py27-cover,lint,mypy [base] # pip installs the requested packages in editable mode @@ -64,9 +64,8 @@ source_paths = tests/lock_test.py [testenv] -passenv = - TRAVIS - APPVEYOR +passenv = + CERTBOT_NO_PIN commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py @@ -155,6 +154,20 @@ commands = commands = {[base]pip_install} acme . certbot-apache certbot-compatibility-test {toxinidir}/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test --debian-modules +passenv = + SERVER + +[testenv:apacheconftest-with-pebble] +commands = + {toxinidir}/tests/pebble-fetch.sh + {[testenv:apacheconftest]commands} +passenv = + HOME + GOPATH + PEBBLEPATH + PEBBLE_STRICT +setenv = + SERVER=https://localhost:14000/dir [testenv:nginxroundtrip] commands = @@ -176,7 +189,6 @@ whitelist_externals = docker passenv = DOCKER_* - TRAVIS [testenv:nginx_compat] commands = @@ -187,19 +199,6 @@ whitelist_externals = docker passenv = DOCKER_* - TRAVIS - -[testenv:le_auto_precise] -# At the moment, this tests under Python 2.7 only, as only that version is -# readily available on the Precise Docker image. -commands = - docker build -f letsencrypt-auto-source/Dockerfile.precise -t lea letsencrypt-auto-source - docker run --rm -t -i lea -whitelist_externals = - docker -passenv = - DOCKER_* - TRAVIS [testenv:le_auto_trusty] # At the moment, this tests under Python 2.7 only, as only that version is @@ -212,14 +211,22 @@ whitelist_externals = docker passenv = DOCKER_* - TRAVIS TRAVIS_BRANCH -[testenv:le_auto_wheezy] +[testenv:le_auto_xenial] +# At the moment, this tests under Python 2.7 only. +commands = + docker build -f letsencrypt-auto-source/Dockerfile.xenial -t lea letsencrypt-auto-source + docker run --rm -t -i lea +whitelist_externals = + docker +passenv = DOCKER_* + +[testenv:le_auto_jessie] # At the moment, this tests under Python 2.7 only, as only that version is # readily available on the Wheezy Docker image. commands = - docker build -f letsencrypt-auto-source/Dockerfile.wheezy -t lea letsencrypt-auto-source + docker build -f letsencrypt-auto-source/Dockerfile.jessie -t lea letsencrypt-auto-source docker run --rm -t -i lea whitelist_externals = docker