diff --git a/.travis.yml b/.travis.yml index 2b8eafc13..16cb6f23f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,14 +4,13 @@ cache: directories: - $HOME/.cache/pip -before_install: - - '([ $TRAVIS_OS_NAME == linux ] && dpkg -s libaugeas0) || (brew update && brew install augeas python3 && brew upgrade python && brew link python)' - before_script: - 'if [ $TRAVIS_OS_NAME = osx ] ; then ulimit -n 1024 ; fi' + - export TOX_TESTENV_PASSENV=TRAVIS matrix: include: + # These environments are always executed - python: "2.7" env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=all TOXENV=py27_install sudo: required @@ -57,11 +56,119 @@ matrix: before_install: addons: - python: "2.7" - env: TOXENV=apacheconftest + env: TOXENV=apacheconftest-with-pebble sudo: required + services: docker - python: "2.7" env: TOXENV=nginxroundtrip + # These environments are executed on cron events only + - python: "3.7" + dist: xenial + env: TOXENV=py37 CERTBOT_NO_PIN=1 + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest + sudo: required + services: docker + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=certbot TOXENV=py27-certbot-oldest + sudo: required + services: docker + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v1 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest + sudo: required + services: docker + if: type = cron + - python: "2.7" + env: BOULDER_INTEGRATION=v2 INTEGRATION_TEST=nginx TOXENV=py27-nginx-oldest + sudo: required + services: docker + if: type = cron + - python: "3.4" + env: TOXENV=py34 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.4" + env: TOXENV=py34 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - python: "3.5" + env: TOXENV=py35 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.5" + env: TOXENV=py35 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - python: "3.6" + env: TOXENV=py36 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.6" + env: TOXENV=py36 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - python: "3.7" + dist: xenial + env: TOXENV=py37 BOULDER_INTEGRATION=v1 + sudo: required + services: docker + if: type = cron + - python: "3.7" + dist: xenial + env: TOXENV=py37 BOULDER_INTEGRATION=v2 + sudo: required + services: docker + if: type = cron + - sudo: required + env: TOXENV=le_auto_xenial + services: docker + if: type = cron + - sudo: required + env: TOXENV=le_auto_jessie + services: docker + if: type = cron + - sudo: required + env: TOXENV=le_auto_centos6 + services: docker + if: type = cron + - sudo: required + env: TOXENV=docker_dev + services: docker + addons: + apt: + packages: # don't install nginx and apache + - libaugeas0 + if: type = cron + - language: generic + env: TOXENV=py27 + os: osx + addons: + homebrew: + packages: + - augeas + - python2 + if: type = cron + - language: generic + env: TOXENV=py3 + os: osx + addons: + homebrew: + packages: + - augeas + - python3 + if: type = cron + + # Only build pushes to the master branch, PRs, and branches beginning with # `test-` or of the form `digit(s).digit(s).x`. This reduces the number of diff --git a/CHANGELOG.md b/CHANGELOG.md index 4de61860a..6085765c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,25 +6,39 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). ### Added -* +* Avoid to process again challenges that are already validated + when a certificate is issued. ### Changed -* +* Lexicon-based DNS plugins are now fully compatible with Lexicon 3.x (support + on 2.x branch is maintained). ### Fixed -* +* Fixed accessing josepy contents through acme.jose when the full acme.jose + path is used. +* Clarify behavior for deleting certs as part of revocation. Despite us having broken lockstep, we are continuing to release new versions of all Certbot components during releases for the time being, however, the only package with changes other than its version number was: -* +* acme +* certbot +* certbot-dns-cloudxns +* certbot-dns-dnsimple +* certbot-dns-dnsmadeeasy +* certbot-dns-gehirn +* certbot-dns-linode +* certbot-dns-luadns +* certbot-dns-nsone +* certbot-dns-ovh +* certbot-dns-sakuracloud More details about these changes can be found on our GitHub repo. -## 0.31.1 - 2019-01-24 +## 0.30.1 - 2019-01-24 ### Fixed @@ -52,7 +66,7 @@ More details about these changes can be found on our GitHub repo. * Copied account management functionality from the `register` subcommand to the `update_account` subcommand. -* Marked usage `register --update-registration` for deprecation and +* Marked usage `register --update-registration` for deprecation and removal in a future release. ### Fixed diff --git a/Dockerfile b/Dockerfile index d1296b30f..f3626dc8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ VOLUME /etc/letsencrypt /var/lib/letsencrypt WORKDIR /opt/certbot COPY CHANGELOG.md README.rst setup.py src/ +COPY letsencrypt-auto-source/pieces/dependency-requirements.txt . COPY acme src/acme COPY certbot src/certbot @@ -21,6 +22,7 @@ RUN apk add --no-cache --virtual .build-deps \ openssl-dev \ musl-dev \ libffi-dev \ + && pip install -r /opt/certbot/dependency-requirements.txt \ && pip install --no-cache-dir \ --editable /opt/certbot/src/acme \ --editable /opt/certbot/src \ diff --git a/README.rst b/README.rst index 62681c7eb..f55581268 100644 --- a/README.rst +++ b/README.rst @@ -99,8 +99,8 @@ ACME working area in github: https://github.com/ietf-wg-acme/acme |build-status| |coverage| |docs| |container| -.. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master - :target: https://travis-ci.org/certbot/certbot +.. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master + :target: https://travis-ci.com/certbot/certbot :alt: Travis CI status .. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg diff --git a/acme/acme/__init__.py b/acme/acme/__init__.py index d2cb1ee06..d91072a3b 100644 --- a/acme/acme/__init__.py +++ b/acme/acme/__init__.py @@ -1,25 +1,20 @@ """ACME protocol implementation. -This module is an implementation of the `ACME protocol`_. Latest -supported version: `draft-ietf-acme-01`_. - +This module is an implementation of the `ACME protocol`_. .. _`ACME protocol`: https://ietf-wg-acme.github.io/acme -.. _`draft-ietf-acme-01`: - https://github.com/ietf-wg-acme/acme/tree/draft-ietf-acme-acme-01 - """ import sys -import josepy - # This code exists to keep backwards compatibility with people using acme.jose # before it became the standalone josepy package. # # It is based on # https://github.com/requests/requests/blob/1278ecdf71a312dc2268f3bfc0aabfab3c006dcf/requests/packages.py +import josepy as jose + for mod in list(sys.modules): # This traversal is apparently necessary such that the identities are # preserved (acme.jose.* is josepy.*) diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 33ae3886b..b3d0f1921 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -707,6 +707,7 @@ class ClientTest(ClientTestBase): self.certr, self.rsn) + class ClientV2Test(ClientTestBase): """Tests for acme.client.ClientV2.""" @@ -950,7 +951,6 @@ class ClientNetworkTest(unittest.TestCase): self.assertEqual(jws.signature.combined.kid, u'acct-uri') self.assertEqual(jws.signature.combined.url, u'url') - def test_check_response_not_ok_jobj_no_error(self): self.response.ok = False self.response.json.return_value = {} @@ -1113,8 +1113,8 @@ class ClientNetworkTest(unittest.TestCase): # Requests Library Exceptions except requests.exceptions.ConnectionError as z: #pragma: no cover - self.assertTrue("('Connection aborted.', error(111, 'Connection refused'))" - == str(z) or "[WinError 10061]" in str(z)) + self.assertTrue("'Connection aborted.'" in str(z) or "[WinError 10061]" in str(z)) + class ClientNetworkWithMockedResponseTest(unittest.TestCase): """Tests for acme.client.ClientNetwork which mock out response.""" diff --git a/acme/acme/jose_test.py b/acme/acme/jose_test.py index ecd483662..340624a4f 100644 --- a/acme/acme/jose_test.py +++ b/acme/acme/jose_test.py @@ -12,11 +12,21 @@ class JoseTest(unittest.TestCase): else: acme_jose_path = 'acme.jose' josepy_path = 'josepy' - acme_jose = importlib.import_module(acme_jose_path) - josepy = importlib.import_module(josepy_path) + acme_jose_mod = importlib.import_module(acme_jose_path) + josepy_mod = importlib.import_module(josepy_path) - self.assertIs(acme_jose, josepy) - self.assertIs(getattr(acme_jose, attribute), getattr(josepy, attribute)) + self.assertIs(acme_jose_mod, josepy_mod) + self.assertIs(getattr(acme_jose_mod, attribute), getattr(josepy_mod, attribute)) + + # We use the imports below with eval, but pylint doesn't + # understand that. + # pylint: disable=eval-used,unused-variable + import acme + import josepy + acme_jose_mod = eval(acme_jose_path) + josepy_mod = eval(josepy_path) + self.assertIs(acme_jose_mod, josepy_mod) + self.assertIs(getattr(acme_jose_mod, attribute), getattr(josepy_mod, attribute)) def test_top_level(self): self._test_it('', 'RS512') diff --git a/acme/acme/messages.py b/acme/acme/messages.py index 4400a6c31..7c82c8507 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -1,7 +1,10 @@ """ACME protocol messages.""" -import collections import six import json +try: + from collections.abc import Hashable # pylint: disable=no-name-in-module +except ImportError: + from collections import Hashable import josepy as jose @@ -107,7 +110,7 @@ class Error(jose.JSONObjectWithFields, errors.Error): if part is not None).decode() -class _Constant(jose.JSONDeSerializable, collections.Hashable): # type: ignore +class _Constant(jose.JSONDeSerializable, Hashable): # type: ignore """ACME constant.""" __slots__ = ('name',) POSSIBLE_NAMES = NotImplemented diff --git a/acme/docs/index.rst b/acme/docs/index.rst index a200808da..a94c02e5c 100644 --- a/acme/docs/index.rst +++ b/acme/docs/index.rst @@ -16,13 +16,6 @@ Contents: .. automodule:: acme :members: - -Example client: - -.. include:: ../examples/example_client.py - :code: python - - Indices and tables ================== diff --git a/acme/examples/example_client.py b/acme/examples/example_client.py deleted file mode 100644 index abeedc082..000000000 --- a/acme/examples/example_client.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Example script showing how to use acme client API.""" -import logging -import os -import pkg_resources - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa -import josepy as jose -import OpenSSL - -from acme import client -from acme import messages - - -logging.basicConfig(level=logging.DEBUG) - - -DIRECTORY_URL = 'https://acme-staging.api.letsencrypt.org/directory' -BITS = 2048 # minimum for Boulder -DOMAIN = 'example1.com' # example.com is ignored by Boulder - -# generate_private_key requires cryptography>=0.5 -key = jose.JWKRSA(key=rsa.generate_private_key( - public_exponent=65537, - key_size=BITS, - backend=default_backend())) -acme = client.Client(DIRECTORY_URL, key) - -regr = acme.register() -logging.info('Auto-accepting TOS: %s', regr.terms_of_service) -acme.agree_to_tos(regr) -logging.debug(regr) - -authzr = acme.request_challenges( - identifier=messages.Identifier(typ=messages.IDENTIFIER_FQDN, value=DOMAIN)) -logging.debug(authzr) - -authzr, authzr_response = acme.poll(authzr) - -csr = OpenSSL.crypto.load_certificate_request( - OpenSSL.crypto.FILETYPE_ASN1, pkg_resources.resource_string( - 'acme', os.path.join('testdata', 'csr.der'))) -try: - acme.request_issuance(jose.util.ComparableX509(csr), (authzr,)) -except messages.Error as error: - print ("This script is doomed to fail as no authorization " - "challenges are ever solved. Error from server: {0}".format(error)) diff --git a/acme/setup.py b/acme/setup.py index 77ff7bae8..eac3974fa 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -9,15 +9,15 @@ version = '0.31.0.dev0' install_requires = [ # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) - 'cryptography>=0.8', + 'cryptography>=1.2.3', # formerly known as acme.jose: 'josepy>=1.0.0', # Connection.set_tlsext_host_name (>=0.13) 'mock', - 'PyOpenSSL>=0.13', + 'PyOpenSSL>=0.13.1', 'pyrfc3339', 'pytz', - 'requests[security]>=2.4.1', # security extras added in 2.4.1 + 'requests[security]>=2.6.0', # security extras added in 2.4.1 'requests-toolbelt>=0.3.0', 'setuptools', 'six>=1.9.0', # needed for python_2_unicode_compatible diff --git a/appveyor.yml b/appveyor.yml index ce2b5998c..2b6b82747 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,6 +24,7 @@ install: build: off test_script: + - set TOX_TESTENV_PASSENV=APPVEYOR # Test env is set by TOXENV env variable - tox diff --git a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test index dcbba9d3e..4838a6eee 100755 --- a/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test +++ b/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test @@ -3,6 +3,11 @@ # A hackish script to see if the client is behaving as expected # with each of the "passing" conf files. +if [ -z "$SERVER" ]; then + echo "Please set SERVER to the ACME server's directory URL." + exit 1 +fi + export EA=/etc/apache2/ TESTDIR="`dirname $0`" cd $TESTDIR/passing @@ -56,13 +61,16 @@ if [ "$1" = --debian-modules ] ; then done fi +CERTBOT_CMD="sudo $(command -v certbot) --server $SERVER -vvvv" +CERTBOT_CMD="$CERTBOT_CMD --debug --apache --register-unsafely-without-email" +CERTBOT_CMD="$CERTBOT_CMD --agree-tos certonly -t --no-verify-ssl" FAILS=0 trap CleanupExit INT for f in *.conf ; do echo -n testing "$f"... Setup - RESULT=`echo c | sudo $(command -v certbot) -vvvv --debug --staging --apache --register-unsafely-without-email --agree-tos certonly -t 2>&1` + RESULT=`echo c | $CERTBOT_CMD 2>&1` if echo $RESULT | grep -Eq \("Which names would you like"\|"mod_macro is not yet"\) ; then echo passed else diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py index 658db6072..5132137f8 100644 --- a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py @@ -69,13 +69,15 @@ class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key, secret_key, ttl): super(_CloudXNSLexiconClient, self).__init__() - self.provider = cloudxns.Provider({ - 'provider_name': 'cloudxns', + config = dns_common_lexicon.build_lexicon_config('cloudxns', { + 'ttl': ttl, + }, { 'auth_username': api_key, 'auth_token': secret_key, - 'ttl': ttl, }) + self.provider = cloudxns.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('400 Client Error:'): diff --git a/certbot-dns-cloudxns/local-oldest-requirements.txt b/certbot-dns-cloudxns/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-cloudxns/local-oldest-requirements.txt +++ b/certbot-dns-cloudxns/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 5c8709445..1a6f900d8 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py index 3eb56e37c..ad2a3fa30 100644 --- a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py @@ -65,12 +65,14 @@ class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, token, ttl): super(_DNSimpleLexiconClient, self).__init__() - self.provider = dnsimple.Provider({ - 'provider_name': 'dnssimple', - 'auth_token': token, + config = dns_common_lexicon.build_lexicon_config('dnssimple', { 'ttl': ttl, + }, { + 'auth_token': token, }) + self.provider = dnsimple.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('401 Client Error: Unauthorized for url:'): diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-dnsimple/local-oldest-requirements.txt +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 3bb43cf5d..1a2ce5d92 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py index 4236ce37a..4b63cb4b5 100644 --- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py +++ b/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy.py @@ -71,13 +71,15 @@ class _DNSMadeEasyLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key, secret_key, ttl): super(_DNSMadeEasyLexiconClient, self).__init__() - self.provider = dnsmadeeasy.Provider({ - 'provider_name': 'dnsmadeeasy', + config = dns_common_lexicon.build_lexicon_config('dnsmadeeasy', { + 'ttl': ttl, + }, { 'auth_username': api_key, 'auth_token': secret_key, - 'ttl': ttl, }) + self.provider = dnsmadeeasy.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and str(e).startswith('404 Client Error: Not Found for url:'): return diff --git a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt +++ b/certbot-dns-dnsmadeeasy/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index 599cec486..0a99f452d 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py index 9c35e72ab..edf530072 100644 --- a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py +++ b/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn.py @@ -72,13 +72,15 @@ class _GehirnLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_token, api_secret, ttl): super(_GehirnLexiconClient, self).__init__() - self.provider = gehirn.Provider({ - 'provider_name': 'gehirn', + config = dns_common_lexicon.build_lexicon_config('gehirn', { + 'ttl': ttl, + }, { 'auth_token': api_token, 'auth_secret': api_secret, - 'ttl': ttl, }) + self.provider = gehirn.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return # Expected errors when zone name guess is wrong diff --git a/certbot-dns-gehirn/local-oldest-requirements.txt b/certbot-dns-gehirn/local-oldest-requirements.txt new file mode 100644 index 000000000..65f5a758e --- /dev/null +++ b/certbot-dns-gehirn/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index 894d809ac..f4a75379c 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -6,8 +6,8 @@ version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', 'dns-lexicon>=2.1.22', 'mock', 'setuptools', diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode.py b/certbot-dns-linode/certbot_dns_linode/dns_linode.py index 01da2cf60..4e0500fa0 100644 --- a/certbot-dns-linode/certbot_dns_linode/dns_linode.py +++ b/certbot-dns-linode/certbot_dns_linode/dns_linode.py @@ -54,6 +54,7 @@ class Authenticator(dns_common.DNSAuthenticator): def _get_linode_client(self): return _LinodeLexiconClient(self.credentials.conf('key')) + class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): """ Encapsulates all communication with the Linode API. @@ -61,11 +62,13 @@ class _LinodeLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key): super(_LinodeLexiconClient, self).__init__() - self.provider = linode.Provider({ - 'provider_name': 'linode', - 'auth_token': api_key + + config = dns_common_lexicon.build_lexicon_config('linode', {}, { + 'auth_token': api_key, }) + self.provider = linode.Provider(config) + def _handle_general_error(self, e, domain_name): if not str(e).startswith('Domain not found'): return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}' diff --git a/certbot-dns-linode/local-oldest-requirements.txt b/certbot-dns-linode/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-linode/local-oldest-requirements.txt +++ b/certbot-dns-linode/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index 588b6a40a..31c2c20bc 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -5,8 +5,8 @@ version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', 'dns-lexicon>=2.2.1', 'mock', 'setuptools', diff --git a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py b/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py index bd6a16f69..7cdd4c8e1 100644 --- a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py +++ b/certbot-dns-luadns/certbot_dns_luadns/dns_luadns.py @@ -68,13 +68,15 @@ class _LuaDNSLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, email, token, ttl): super(_LuaDNSLexiconClient, self).__init__() - self.provider = luadns.Provider({ - 'provider_name': 'luadns', + config = dns_common_lexicon.build_lexicon_config('luadns', { + 'ttl': ttl, + }, { 'auth_username': email, 'auth_token': token, - 'ttl': ttl, }) + self.provider = luadns.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('401 Client Error: Unauthorized for url:'): diff --git a/certbot-dns-luadns/local-oldest-requirements.txt b/certbot-dns-luadns/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-luadns/local-oldest-requirements.txt +++ b/certbot-dns-luadns/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index b09a09762..31472e8cf 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py b/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py index 5f33efbba..3e23df11c 100644 --- a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py +++ b/certbot-dns-nsone/certbot_dns_nsone/dns_nsone.py @@ -65,12 +65,14 @@ class _NS1LexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_key, ttl): super(_NS1LexiconClient, self).__init__() - self.provider = nsone.Provider({ - 'provider_name': 'nsone', - 'auth_token': api_key, + config = dns_common_lexicon.build_lexicon_config('nsone', { 'ttl': ttl, + }, { + 'auth_token': api_key, }) + self.provider = nsone.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:') or \ str(e).startswith("400 Client Error: Bad Request for url:")): diff --git a/certbot-dns-nsone/local-oldest-requirements.txt b/certbot-dns-nsone/local-oldest-requirements.txt index 8368d266e..65f5a758e 100644 --- a/certbot-dns-nsone/local-oldest-requirements.txt +++ b/certbot-dns-nsone/local-oldest-requirements.txt @@ -1,2 +1,2 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index e48428191..41b99cc73 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py index 578ee8e89..84771b0a8 100644 --- a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py +++ b/certbot-dns-ovh/certbot_dns_ovh/dns_ovh.py @@ -77,15 +77,17 @@ class _OVHLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, endpoint, application_key, application_secret, consumer_key, ttl): super(_OVHLexiconClient, self).__init__() - self.provider = ovh.Provider({ - 'provider_name': 'ovh', + config = dns_common_lexicon.build_lexicon_config('ovh', { + 'ttl': ttl, + }, { 'auth_entrypoint': endpoint, 'auth_application_key': application_key, 'auth_application_secret': application_secret, 'auth_consumer_key': consumer_key, - 'ttl': ttl, }) + self.provider = ovh.Provider(config) + def _handle_http_error(self, e, domain_name): hint = None if str(e).startswith('400 Client Error:'): diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt index 8368d266e..01cbcb317 100644 --- a/certbot-dns-ovh/local-oldest-requirements.txt +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -1,2 +1,3 @@ -acme[dev]==0.21.1 -certbot[dev]==0.21.1 +-e acme[dev] +-e .[dev] +dns-lexicon==2.7.14 diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 8b6d73d33..5b3329568 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -7,9 +7,9 @@ version = '0.31.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', - 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', + 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider 'mock', 'setuptools', 'zope.interface', diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py index b892330f5..7fd6d3ef5 100644 --- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py +++ b/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud.py @@ -75,13 +75,15 @@ class _SakuraCloudLexiconClient(dns_common_lexicon.LexiconClient): def __init__(self, api_token, api_secret, ttl): super(_SakuraCloudLexiconClient, self).__init__() - self.provider = sakuracloud.Provider({ - 'provider_name': 'sakuracloud', + config = dns_common_lexicon.build_lexicon_config('sakuracloud', { + 'ttl': ttl, + }, { 'auth_token': api_token, 'auth_secret': api_secret, - 'ttl': ttl, }) + self.provider = sakuracloud.Provider(config) + def _handle_http_error(self, e, domain_name): if domain_name in str(e) and (str(e).startswith('404 Client Error: Not Found for url:')): return # Expected errors when zone name guess is wrong diff --git a/certbot-dns-sakuracloud/local-oldest-requirements.txt b/certbot-dns-sakuracloud/local-oldest-requirements.txt new file mode 100644 index 000000000..65f5a758e --- /dev/null +++ b/certbot-dns-sakuracloud/local-oldest-requirements.txt @@ -0,0 +1,2 @@ +-e acme[dev] +-e .[dev] diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 05843d2ed..4ebfc6e1d 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -6,8 +6,8 @@ version = '0.31.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'acme>=0.21.1', - 'certbot>=0.21.1', + 'acme>=0.31.0.dev0', + 'certbot>=0.31.0.dev0', 'dns-lexicon>=2.1.23', 'mock', 'setuptools', diff --git a/certbot-nginx/tests/boulder-integration.conf.sh b/certbot-nginx/tests/boulder-integration.conf.sh index 4374f9094..470eab28e 100755 --- a/certbot-nginx/tests/boulder-integration.conf.sh +++ b/certbot-nginx/tests/boulder-integration.conf.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Based on # https://www.exratione.com/2014/03/running-nginx-as-a-non-root-user/ # https://github.com/exratione/non-root-nginx/blob/9a77f62e5d5cb9c9026fd62eece76b9514011019/nginx.conf @@ -52,7 +53,7 @@ http { listen 5002 $default_server; # IPv6. listen [::]:5002 $default_server; - server_name nginx.wtf nginx2.wtf; + server_name nginx.wtf nginx-tls.wtf nginx2.wtf; root $root/webroot; diff --git a/certbot-nginx/tests/boulder-integration.sh b/certbot-nginx/tests/boulder-integration.sh index 2a24e645f..547502ccf 100755 --- a/certbot-nginx/tests/boulder-integration.sh +++ b/certbot-nginx/tests/boulder-integration.sh @@ -39,8 +39,6 @@ nginx -v reload_nginx certbot_test_nginx --domains nginx.wtf run test_deployment_and_rollback nginx.wtf -certbot_test_nginx --domains nginx.wtf run --preferred-challenges tls-sni -test_deployment_and_rollback nginx.wtf certbot_test_nginx --domains nginx2.wtf --preferred-challenges http test_deployment_and_rollback nginx2.wtf # Overlapping location block and server-block-level return 301 @@ -66,4 +64,4 @@ test_deployment_and_rollback nginx6.wtf # top nginx -c $nginx_root/nginx.conf -s stop -coverage report --fail-under 75 --include 'certbot-nginx/*' --show-missing +coverage report --fail-under 72 --include 'certbot-nginx/*' --show-missing diff --git a/certbot/auth_handler.py b/certbot/auth_handler.py index efee49143..3dfaaf26f 100644 --- a/certbot/auth_handler.py +++ b/certbot/auth_handler.py @@ -31,7 +31,7 @@ class AuthHandler(object): :class:`~acme.challenges.Challenge` types :type auth: :class:`certbot.interfaces.IAuthenticator` - :ivar acme.client.BackwardsCompatibleClientV2 acme: ACME client API. + :ivar acme.client.BackwardsCompatibleClientV2 acme_client: ACME client API. :ivar account: Client's Account :type account: :class:`certbot.account.Account` @@ -40,9 +40,9 @@ class AuthHandler(object): type strings with the most preferred challenge listed first """ - def __init__(self, auth, acme, account, pref_challs): + def __init__(self, auth, acme_client, account, pref_challs): self.auth = auth - self.acme = acme + self.acme = acme_client self.account = account self.pref_challs = pref_challs @@ -85,19 +85,26 @@ class AuthHandler(object): self.verify_authzr_complete(aauthzrs) # Only return valid authorizations - retVal = [aauthzr.authzr for aauthzr in aauthzrs - if aauthzr.authzr.body.status == messages.STATUS_VALID] + ret_val = [aauthzr.authzr for aauthzr in aauthzrs + if aauthzr.authzr.body.status == messages.STATUS_VALID] - if not retVal: + if not ret_val: raise errors.AuthorizationError( "Challenges failed for all domains") - return retVal + return ret_val def _choose_challenges(self, aauthzrs): - """Retrieve necessary challenges to satisfy server.""" - logger.info("Performing the following challenges:") - for aauthzr in aauthzrs: + """ + Retrieve necessary and pending challenges to satisfy server. + NB: Necessary and already validated challenges are not retrieved, + as they can be reused for a certificate issuance. + """ + pending_authzrs = [aauthzr for aauthzr in aauthzrs + if aauthzr.authzr.body.status != messages.STATUS_VALID] + if pending_authzrs: + logger.info("Performing the following challenges:") + for aauthzr in pending_authzrs: aauthzr_challenges = aauthzr.authzr.body.challenges if self.acme.acme_version == 1: combinations = aauthzr.authzr.body.combinations @@ -125,7 +132,7 @@ class AuthHandler(object): def _solve_challenges(self, aauthzrs): """Get Responses for challenges from authenticators.""" - resp = [] # type: Collection[acme.challenges.ChallengeResponse] + resp = [] # type: Collection[challenges.ChallengeResponse] all_achalls = self._get_all_achalls(aauthzrs) try: if all_achalls: @@ -531,7 +538,7 @@ def _report_failed_challs(failed_achalls): """ problems = collections.defaultdict(list)\ - # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] + # type: DefaultDict[str, List[achallenges.KeyAuthorizationAnnotatedChallenge]] for achall in failed_achalls: if achall.error: problems[achall.error.typ].append(achall) diff --git a/certbot/cli.py b/certbot/cli.py index ff1827cb1..31f55711f 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1311,7 +1311,8 @@ def _create_subparsers(helpful): helpful.add("revoke", "--delete-after-revoke", action="store_true", default=flag_default("delete_after_revoke"), - help="Delete certificates after revoking them.") + help="Delete certificates after revoking them, along with all previous and later " + "versions of those certificates.") helpful.add("revoke", "--no-delete-after-revoke", action="store_false", dest="delete_after_revoke", diff --git a/certbot/main.py b/certbot/main.py index ac639bc80..a0c0ab64d 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -548,7 +548,8 @@ def _delete_if_appropriate(config): # pylint: disable=too-many-locals,too-many-b attempt_deletion = config.delete_after_revoke if attempt_deletion is None: - msg = ("Would you like to delete the cert(s) you just revoked?") + msg = ("Would you like to delete the cert(s) you just revoked, along with all earlier and " + "later versions of the cert?") attempt_deletion = display.yesno(msg, yes_label="Yes (recommended)", no_label="No", force_interactive=True, default=True) diff --git a/certbot/plugins/dns_common_lexicon.py b/certbot/plugins/dns_common_lexicon.py index f9610b816..5b50cc285 100644 --- a/certbot/plugins/dns_common_lexicon.py +++ b/certbot/plugins/dns_common_lexicon.py @@ -1,12 +1,22 @@ """Common code for DNS Authenticator Plugins built on Lexicon.""" - import logging from requests.exceptions import HTTPError, RequestException +from acme.magic_typing import Union, Dict, Any # pylint: disable=unused-import,no-name-in-module from certbot import errors from certbot.plugins import dns_common +# Lexicon is not declared as a dependency in Certbot itself, +# but in the Certbot plugins backed by Lexicon. +# So we catch import error here to allow this module to be +# always importable, even if it does not make sense to use it +# if Lexicon is not available, obviously. +try: + from lexicon.config import ConfigResolver +except ImportError: + ConfigResolver = None # type: ignore + logger = logging.getLogger(__name__) @@ -100,3 +110,28 @@ class LexiconClient(object): if not str(e).startswith('No domain found'): return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}' .format(domain_name, e)) + + +def build_lexicon_config(lexicon_provider_name, lexicon_options, provider_options): + # type: (str, Dict, Dict) -> Union[ConfigResolver, Dict] + """ + Convenient function to build a Lexicon 2.x/3.x config object. + :param str lexicon_provider_name: the name of the lexicon provider to use + :param dict lexicon_options: options specific to lexicon + :param dict provider_options: options specific to provider + :return: configuration to apply to the provider + :rtype: ConfigurationResolver or dict + """ + config = {'provider_name': lexicon_provider_name} # type: Dict[str, Any] + config.update(lexicon_options) + if not ConfigResolver: + # Lexicon 2.x + config.update(provider_options) + else: + # Lexicon 3.x + provider_config = {} + provider_config.update(provider_options) + config[lexicon_provider_name] = provider_config + config = ConfigResolver().with_dict(config).with_env() + + return config diff --git a/certbot/storage.py b/certbot/storage.py index eb17e1d38..d17a0f29d 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -41,7 +41,9 @@ def renewal_conf_files(config): :rtype: `list` of `str` """ - return glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) + result = glob.glob(os.path.join(config.renewal_configs_dir, "*.conf")) + result.sort() + return result def renewal_file_for_certname(config, certname): """Return /path/to/certname.conf in the renewal conf directory""" @@ -877,45 +879,6 @@ class RenewableCert(object): with open(target) as f: return crypto_util.get_names_from_cert(f.read()) - def autodeployment_is_enabled(self): - """Is automatic deployment enabled for this cert? - - If autodeploy is not specified, defaults to True. - - :returns: True if automatic deployment is enabled - :rtype: bool - - """ - return ("autodeploy" not in self.configuration or - self.configuration.as_bool("autodeploy")) - - def should_autodeploy(self, interactive=False): - """Should this lineage now automatically deploy a newer version? - - This is a policy question and does not only depend on whether - there is a newer version of the cert. (This considers whether - autodeployment is enabled, whether a relevant newer version - exists, and whether the time interval for autodeployment has - been reached.) - - :param bool interactive: set to True to examine the question - regardless of whether the renewal configuration allows - automated deployment (for interactive use). Default False. - - :returns: whether the lineage now ought to autodeploy an - existing newer cert version - :rtype: bool - - """ - if interactive or self.autodeployment_is_enabled(): - if self.has_pending_deployment(): - interval = self.configuration.get("deploy_before_expiry", - "5 days") - now = pytz.UTC.fromutc(datetime.datetime.utcnow()) - if self.target_expiry < add_time_interval(now, interval): - return True - return False - def ocsp_revoked(self, version=None): # pylint: disable=no-self-use,unused-argument """Is the specified cert version revoked according to OCSP? diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index e1319b614..fe0ece12e 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -57,7 +57,7 @@ class ChallengeFactoryTest(unittest.TestCase): errors.Error, self.handler._challenge_factory, authzr, [0]) -class HandleAuthorizationsTest(unittest.TestCase): +class HandleAuthorizationsTest(unittest.TestCase): # pylint: disable=too-many-public-methods """handle_authorizations test. This tests everything except for all functions under _poll_challenges. @@ -316,6 +316,24 @@ class HandleAuthorizationsTest(unittest.TestCase): self.assertEqual( self.mock_auth.cleanup.call_args[0][0][0].typ, "tls-sni-01") + def test_validated_challenge_not_rerun(self): + # With pending challenge, we expect the challenge to be tried, and fail. + authzr = acme_util.gen_authzr( + messages.STATUS_PENDING, "0", + [acme_util.HTTP01], + [messages.STATUS_PENDING], False) + mock_order = mock.MagicMock(authorizations=[authzr]) + self.assertRaises( + errors.AuthorizationError, self.handler.handle_authorizations, mock_order) + + # With validated challenge; we expect the challenge not be tried again, and succeed. + authzr = acme_util.gen_authzr( + messages.STATUS_VALID, "0", + [acme_util.HTTP01], + [messages.STATUS_VALID], False) + mock_order = mock.MagicMock(authorizations=[authzr]) + self.handler.handle_authorizations(mock_order) + def _validate_all(self, aauthzrs, unused_1, unused_2): for i, aauthzr in enumerate(aauthzrs): azr = aauthzr.authzr diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index d75f4f595..e61ed2aca 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -388,8 +388,7 @@ class RenewableCertTests(BaseRenewableCertTest): @mock.patch("certbot.storage.cli") @mock.patch("certbot.storage.datetime") def test_time_interval_judgments(self, mock_datetime, mock_cli): - """Test should_autodeploy() and should_autorenew() on the basis - of expiry time windows.""" + """Test should_autorenew() on the basis of expiry time windows.""" test_cert = test_util.load_vector("cert_512.pem") self._write_out_ex_kinds() @@ -430,31 +429,8 @@ class RenewableCertTests(BaseRenewableCertTest): mock_datetime.datetime.utcnow.return_value = sometime self.test_rc.configuration["deploy_before_expiry"] = interval self.test_rc.configuration["renew_before_expiry"] = interval - self.assertEqual(self.test_rc.should_autodeploy(), result) self.assertEqual(self.test_rc.should_autorenew(), result) - def test_autodeployment_is_enabled(self): - self.assertTrue(self.test_rc.autodeployment_is_enabled()) - self.test_rc.configuration["autodeploy"] = "1" - self.assertTrue(self.test_rc.autodeployment_is_enabled()) - - self.test_rc.configuration["autodeploy"] = "0" - self.assertFalse(self.test_rc.autodeployment_is_enabled()) - - def test_should_autodeploy(self): - """Test should_autodeploy() on the basis of reasons other than - expiry time window.""" - # pylint: disable=too-many-statements - # Autodeployment turned off - self.test_rc.configuration["autodeploy"] = "0" - self.assertFalse(self.test_rc.should_autodeploy()) - self.test_rc.configuration["autodeploy"] = "1" - # No pending deployment - for ver in six.moves.range(1, 6): - for kind in ALL_FOUR: - self._write_out_kind(kind, ver) - self.assertFalse(self.test_rc.should_autodeploy()) - def test_autorenewal_is_enabled(self): self.test_rc.configuration["renewalparams"] = {} self.assertTrue(self.test_rc.autorenewal_is_enabled()) diff --git a/docs/using.rst b/docs/using.rst index 5e8675418..d77badd65 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -44,8 +44,7 @@ a combination_ of distinct authenticator and installer plugins. =========== ==== ==== =============================================================== ============================= Plugin Auth Inst Notes Challenge types (and port) =========== ==== ==== =============================================================== ============================= -apache_ Y Y | Automates obtaining and installing a certificate with Apache http-01_ (80) - | 2.4 on OSes with ``libaugeas0`` 1.0+. +apache_ Y Y | Automates obtaining and installing a certificate with Apache. http-01_ (80) nginx_ Y Y | Automates obtaining and installing a certificate with Nginx. http-01_ (80) webroot_ Y N | Obtains a certificate by writing to the webroot directory of http-01_ (80) | an already running webserver. @@ -83,8 +82,7 @@ the circumstances in which each plugin can be used, and how to use it. Apache ------ -The Apache plugin currently requires an OS with augeas version 1.0; currently `it -supports +The Apache plugin currently `supports `_ modern OSes based on Debian, Fedora, SUSE, Gentoo and Darwin. This automates both obtaining *and* installing certificates on an Apache @@ -136,9 +134,8 @@ the webserver. Nginx ----- -The Nginx plugin has been distributed with Certbot since version 0.9.0 and should -work for most configurations. We recommend backing up Nginx -configurations before using it (though you can also revert changes to +The Nginx plugin should work for most configurations. We recommend backing up +Nginx configurations before using it (though you can also revert changes to configurations with ``certbot --nginx rollback``). You can use it by providing the ``--nginx`` flag on the commandline. diff --git a/letsencrypt-auto-source/Dockerfile.wheezy b/letsencrypt-auto-source/Dockerfile.jessie similarity index 98% rename from letsencrypt-auto-source/Dockerfile.wheezy rename to letsencrypt-auto-source/Dockerfile.jessie index f4f3fea15..9ee37b763 100644 --- a/letsencrypt-auto-source/Dockerfile.wheezy +++ b/letsencrypt-auto-source/Dockerfile.jessie @@ -1,7 +1,7 @@ # For running tests, build a docker image with a passwordless sudo and a trust # store we can manipulate. -FROM debian:wheezy +FROM debian:jessie # Add an unprivileged user: RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea diff --git a/letsencrypt-auto-source/Dockerfile.precise b/letsencrypt-auto-source/Dockerfile.xenial similarity index 98% rename from letsencrypt-auto-source/Dockerfile.precise rename to letsencrypt-auto-source/Dockerfile.xenial index 39a167c14..931f1c6d3 100644 --- a/letsencrypt-auto-source/Dockerfile.precise +++ b/letsencrypt-auto-source/Dockerfile.xenial @@ -1,7 +1,7 @@ # For running tests, build a docker image with a passwordless sudo and a trust # store we can manipulate. -FROM ubuntu:precise +FROM ubuntu:xenial # Add an unprivileged user: RUN useradd --create-home --home-dir /home/lea --shell /bin/bash --groups sudo --uid 1000 lea diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 392fa738c..cb748ccd3 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -333,63 +333,11 @@ BootstrapDebCommon() { fi augeas_pkg="libaugeas0 augeas-lenses" - AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` if [ "$ASSUME_YES" = 1 ]; then YES_FLAG="-y" fi - AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - add_backports=1 - else - read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response - case $response in - [yY][eE][sS]|[yY]|"") - add_backports=1;; - *) - add_backports=0;; - esac - fi - if [ "$add_backports" = 1 ]; then - sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - apt-get $QUIET_FLAG update - fi - fi - fi - if [ "$add_backports" != 0 ]; then - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - } - - - if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Certbot apache plugin..." - fi - # XXX add a case for ubuntu PPAs - fi - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ python \ python-dev \ @@ -1140,9 +1088,9 @@ parsedatetime==2.1 \ pbr==1.8.1 \ --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 -pyOpenSSL==16.2.0 \ - --hash=sha256:26ca380ddf272f7556e48064bbcd5bd71f83dfc144f3583501c7ddbd9434ee17 \ - --hash=sha256:7779a3bbb74e79db234af6a08775568c6769b5821faecf6e2f4143edb227516e +pyOpenSSL==18.0.0 \ + --hash=sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854 \ + --hash=sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580 pyparsing==2.1.8 \ --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \ --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \ diff --git a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh index eb22225e4..93bdc63b4 100644 --- a/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh +++ b/letsencrypt-auto-source/pieces/bootstrappers/deb_common.sh @@ -43,63 +43,11 @@ BootstrapDebCommon() { fi augeas_pkg="libaugeas0 augeas-lenses" - AUGVERSION=`LC_ALL=C apt-cache show --no-all-versions libaugeas0 | grep ^Version: | cut -d" " -f2` if [ "$ASSUME_YES" = 1 ]; then YES_FLAG="-y" fi - AddBackportRepo() { - # ARGS: - BACKPORT_NAME="$1" - BACKPORT_SOURCELINE="$2" - say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." - if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then - # This can theoretically error if sources.list.d is empty, but in that case we don't care. - if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then - if [ "$ASSUME_YES" = 1 ]; then - /bin/echo -n "Installing augeas from $BACKPORT_NAME in 3 seconds..." - sleep 1s - /bin/echo -ne "\e[0K\rInstalling augeas from $BACKPORT_NAME in 2 seconds..." - sleep 1s - /bin/echo -e "\e[0K\rInstalling augeas from $BACKPORT_NAME in 1 second ..." - sleep 1s - add_backports=1 - else - read -p "Would you like to enable the $BACKPORT_NAME repository [Y/n]? " response - case $response in - [yY][eE][sS]|[yY]|"") - add_backports=1;; - *) - add_backports=0;; - esac - fi - if [ "$add_backports" = 1 ]; then - sh -c "echo $BACKPORT_SOURCELINE >> /etc/apt/sources.list.d/$BACKPORT_NAME.list" - apt-get $QUIET_FLAG update - fi - fi - fi - if [ "$add_backports" != 0 ]; then - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends -t "$BACKPORT_NAME" $augeas_pkg - augeas_pkg= - fi - } - - - if dpkg --compare-versions 1.0 gt "$AUGVERSION" ; then - if lsb_release -a | grep -q wheezy ; then - AddBackportRepo wheezy-backports "deb http://http.debian.net/debian wheezy-backports main" - elif lsb_release -a | grep -q precise ; then - # XXX add ARM case - AddBackportRepo precise-backports "deb http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse" - else - echo "No libaugeas0 version is available that's new enough to run the" - echo "Certbot apache plugin..." - fi - # XXX add a case for ubuntu PPAs - fi - apt-get install $QUIET_FLAG $YES_FLAG --no-install-recommends \ python \ python-dev \ diff --git a/letsencrypt-auto-source/pieces/dependency-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt index eb297bc6e..1fac78836 100644 --- a/letsencrypt-auto-source/pieces/dependency-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -114,9 +114,9 @@ parsedatetime==2.1 \ pbr==1.8.1 \ --hash=sha256:46c8db75ae75a056bd1cc07fa21734fe2e603d11a07833ecc1eeb74c35c72e0c \ --hash=sha256:e2127626a91e6c885db89668976db31020f0af2da728924b56480fc7ccf09649 -pyOpenSSL==16.2.0 \ - --hash=sha256:26ca380ddf272f7556e48064bbcd5bd71f83dfc144f3583501c7ddbd9434ee17 \ - --hash=sha256:7779a3bbb74e79db234af6a08775568c6769b5821faecf6e2f4143edb227516e +pyOpenSSL==18.0.0 \ + --hash=sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854 \ + --hash=sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580 pyparsing==2.1.8 \ --hash=sha256:2f0f5ceb14eccd5aef809d6382e87df22ca1da583c79f6db01675ce7d7f49c18 \ --hash=sha256:03a4869b9f3493807ee1f1cb405e6d576a1a2ca4d81a982677c0c1ad6177c56b \ diff --git a/pull_request_template.md b/pull_request_template.md index c071d4135..60fd6da7e 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -1 +1,3 @@ -Be sure to edit the `master` section of `CHANGELOG.md` with a line describing this PR before it gets merged. +Be sure to edit the `master` section of `CHANGELOG.md`. This includes a +description of the change and ensuring the modified package(s) are listed as +having been changed. diff --git a/pytest.ini b/pytest.ini index 9a5807f34..8f009045c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,10 +3,14 @@ # directly. [pytest] addopts = --numprocesses auto --pyargs -# ResourceWarnings are ignored as errors, since they're raised at close -# decodestring: https://github.com/rthalley/dnspython/issues/338 -# ignore our own TLS-SNI-01 warning +# In general, all warnings are treated as errors. Here are the exceptions: +# 1- decodestring: https://github.com/rthalley/dnspython/issues/338 +# 2- ignore our own TLS-SNI-01 warning +# 3- ignore warn for importing abstract classes from collections instead of collections.abc, +# too much third party dependencies are still relying on this behavior, +# but it should be corrected to allow Certbot compatiblity with Python >= 3.8 filterwarnings = error ignore:decodestring:DeprecationWarning ignore:TLS-SNI-01:DeprecationWarning + ignore:.*collections\.abc:DeprecationWarning diff --git a/setup.py b/setup.py index 2a9b2c203..9e6af2d4f 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires = [ # in which we added 2.6 support (see #2243), so we relax the requirement. 'ConfigArgParse>=0.9.3', 'configobj', - 'cryptography>=1.2', # load_pem_x509_certificate + 'cryptography>=1.2.3', # load_pem_x509_certificate 'josepy', 'mock', 'parsedatetime>=1.3', # Calendar.parseDT diff --git a/tests/certbot-boulder-integration.sh b/tests/certbot-boulder-integration.sh index 9011d8ba3..c03b3def8 100755 --- a/tests/certbot-boulder-integration.sh +++ b/tests/certbot-boulder-integration.sh @@ -174,7 +174,7 @@ CheckRenewHook() { TotalAndDistinctLines() { total=$1 distinct=$2 - awk '{a[$1] = 1}; END {exit(NR !='$total' || length(a) !='$distinct')}' + awk '{a[$1] = 1}; END {n = 0; for (i in a) { n++ }; exit(NR !='$total' || n !='$distinct')}' } # Cleanup coverage data @@ -221,20 +221,20 @@ common plugins --init --prepare | grep webroot # We start a server listening on the port for the # unrequested challenge to prevent regressions in #3601. -python ./tests/run_http_server.py $http_01_port & +python ./tests/run_http_server.py $tls_alpn_01_port & python_server_pid=$! - certname="le1.wtf" -common --domains le1.wtf --preferred-challenges tls-sni-01 auth \ +common --domains le1.wtf --preferred-challenges http-01 auth \ --cert-name $certname \ --pre-hook 'echo wtf.pre >> "$HOOK_TEST"' \ --post-hook 'echo wtf.post >> "$HOOK_TEST"'\ --deploy-hook 'echo deploy >> "$HOOK_TEST"' -kill $python_server_pid CheckDeployHook $certname -python ./tests/run_http_server.py $tls_sni_01_port & -python_server_pid=$! +# Previous test used to be a tls-sni-01 challenge that is not supported anymore. +# Now it is a http-01 challenge and this makes it a duplicate of the following test. +# But removing it would break many tests here, as they are strongly coupled. +# See https://github.com/certbot/certbot/pull/6679 certname="le2.wtf" common --domains le2.wtf --preferred-challenges http-01 run \ --cert-name $certname \ @@ -254,7 +254,7 @@ common certonly -a manual -d le.wtf --rsa-key-size 4096 --cert-name $certname \ CheckRenewHook $certname certname="dns.le.wtf" -common -a manual -d dns.le.wtf --preferred-challenges dns,tls-sni run \ +common -a manual -d dns.le.wtf --preferred-challenges dns run \ --cert-name $certname \ --manual-auth-hook ./tests/manual-dns-auth.sh \ --manual-cleanup-hook ./tests/manual-dns-cleanup.sh \ @@ -396,7 +396,7 @@ CheckDirHooks 1 # with fail. common -a manual -d dns1.le.wtf,fail.dns1.le.wtf \ --allow-subset-of-names \ - --preferred-challenges dns,tls-sni \ + --preferred-challenges dns \ --manual-auth-hook ./tests/manual-dns-auth.sh \ --manual-cleanup-hook ./tests/manual-dns-cleanup.sh diff --git a/tests/integration/_common.sh b/tests/integration/_common.sh index 1e444fa26..76b990ac4 100755 --- a/tests/integration/_common.sh +++ b/tests/integration/_common.sh @@ -3,12 +3,15 @@ root=${root:-$(mktemp -d -t leitXXXX)} echo "Root integration tests directory: $root" config_dir="$root/conf" -store_flags="--config-dir $config_dir --work-dir $root/work" -store_flags="$store_flags --logs-dir $root/logs" -tls_sni_01_port=5001 +tls_alpn_01_port=5001 http_01_port=5002 sources="acme/,$(ls -dm certbot*/ | tr -d ' \n')" -export root config_dir store_flags tls_sni_01_port http_01_port sources +export root config_dir tls_alpn_01_port http_01_port sources +certbot_path="$(command -v certbot)" +# Flags that are added here will be added to Certbot calls within +# certbot_test_no_force_renew. +other_flags="--config-dir $config_dir --work-dir $root/work" +other_flags="$other_flags --logs-dir $root/logs" certbot_test () { certbot_test_no_force_renew \ @@ -16,11 +19,35 @@ certbot_test () { "$@" } +# Succeeds if Certbot version is at least the given version number and fails +# otherwise. This is useful for making sure Certbot has certain features +# available. The patch version is currently ignored. +# +# Arguments: +# First argument is the minimum major version +# Second argument is the minimum minor version +version_at_least () { + # Certbot major and minor version (e.g. 0.30) + major_minor=$("$certbot_path" --version 2>&1 | cut -d' ' -f2 | cut -d. -f1,2) + major=$(echo "$major_minor" | cut -d. -f1) + minor=$(echo "$major_minor" | cut -d. -f2) + # Test that either the major version is greater or major version is equal + # and minor version is greater than or equal to. + [ \( "$major" -gt "$1" \) -o \( "$major" -eq "$1" -a "$minor" -ge "$2" \) ] +} + # Use local ACMEv2 endpoint if requested and SERVER isn't already set. if [ "${BOULDER_INTEGRATION:-v1}" = "v2" -a -z "${SERVER:+x}" ]; then SERVER="http://localhost:4001/directory" fi +# --no-random-sleep-on-renew was added in +# https://github.com/certbot/certbot/pull/6599 and first released in Certbot +# 0.30.0. +if version_at_least 0 30; then + other_flags="$other_flags --no-random-sleep-on-renew" +fi + certbot_test_no_force_renew () { omit_patterns="*/*.egg-info/*,*/dns_common*,*/setup.py,*/test_*,*/tests/*" omit_patterns="$omit_patterns,*_test.py,*_test_*,certbot-apache/*" @@ -30,13 +57,13 @@ certbot_test_no_force_renew () { --append \ --source $sources \ --omit $omit_patterns \ - $(command -v certbot) \ + "$certbot_path" \ --server "${SERVER:-http://localhost:4000/directory}" \ --no-verify-ssl \ - --tls-sni-01-port $tls_sni_01_port \ + --tls-sni-01-port $tls_alpn_01_port \ --http-01-port $http_01_port \ --manual-public-ip-logging-ok \ - $store_flags \ + $other_flags \ --non-interactive \ --no-redirect \ --agree-tos \ diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 4036e6efa..d24de2458 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -54,6 +54,7 @@ if [ $? -ne 0 ] ; then fi if [ "$OS_TYPE" = "ubuntu" ] ; then + export SERVER="$BOULDER_URL" venv/bin/tox -e apacheconftest else echo Not running hackish apache tests on $OS_TYPE diff --git a/tools/_venv_common.py b/tools/_venv_common.py index ecd438f94..540842773 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -156,7 +156,7 @@ def main(venv_name, venv_args, args): new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), env=new_environ, shell=True) - subprocess_with_print("python -m pip install --upgrade 'setuptools>=30.3'", + subprocess_with_print('python -m pip install --upgrade "setuptools>=30.3"', env=new_environ, shell=True) subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), env=new_environ, shell=True) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 778012d31..88340cb00 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,5 +1,7 @@ -# Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. +# Specifies Python package versions for development. +# It includes in particular packages not specified in letsencrypt-auto's requirements file. +# Some dev package versions specified here may be overridden by higher level constraints +# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). alabaster==0.7.10 apipkg==1.4 asn1crypto==0.22.0 @@ -12,7 +14,7 @@ botocore==1.12.36 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.7.14 +dns-lexicon==3.0.8 dnspython==1.15.0 docutils==0.12 execnet==1.5.0 diff --git a/tools/install_and_test.py b/tools/install_and_test.py index 79a7c2264..0142eeea4 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -17,16 +17,15 @@ import re SKIP_PROJECTS_ON_WINDOWS = [ 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + def call_with_print(command, cwd=None): print(command) subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + def main(args): - if os.environ.get('CERTBOT_NO_PIN') == '1': - command = [sys.executable, '-m', 'pip', '-e'] - else: - script_dir = os.path.dirname(os.path.abspath(__file__)) - command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] + script_dir = os.path.dirname(os.path.abspath(__file__)) + command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] new_args = [] for arg in args: diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index ad44a55d0..4205e6bcf 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -5,13 +5,14 @@ Requirements files specified later take precedence over earlier ones. Only simple SomeProject==1.2.3 format is currently supported. """ - from __future__ import print_function import sys + def read_file(file_path): """Reads in a Python requirements file. + Ignore empty lines, comments and editable requirements :param str file_path: path to requirements file @@ -19,16 +20,16 @@ def read_file(file_path): :rtype: dict """ - d = {} - with open(file_path) as f: - for line in f: + data = {} + with open(file_path) as file_h: + for line in file_h: line = line.strip() - if line and not line.startswith('#'): + if line and not line.startswith('#') and not line.startswith('-e'): project, version = line.split('==') if not version: raise ValueError("Unexpected syntax '{0}'".format(line)) - d[project] = version - return d + data[project] = version + return data def output_requirements(requirements): @@ -37,25 +38,24 @@ def output_requirements(requirements): :param dict requirements: mapping from a project to its pinned version """ - return '\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items())) + return '\n'.join('{0}=={1}'.format(key, value) + for key, value in sorted(requirements.items())) -def main(*files): +def main(*paths): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier files. - :param tuple files: paths to requirements files + :param tuple paths: paths to requirements files """ - d = {} - for f in files: - d.update(read_file(f)) - return output_requirements(d) + data = {} + for path in paths: + data.update(read_file(path)) + return output_requirements(data) if __name__ == '__main__': - merged_requirements = main(*sys.argv[1:]) - print(merged_requirements) + print(main(*sys.argv[1:])) # pylint: disable=star-args diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 80fce8b33..e48d6b13c 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -37,19 +37,24 @@ pytz==2012rc0 # Our setup.py constraints cloudflare==1.5.1 -cryptography==1.2.0 +cryptography==1.2.3 google-api-python-client==1.5 oauth2client==2.0 parsedatetime==1.3 pyparsing==1.5.5 python-digitalocean==1.11 -requests[security]==2.4.1 +requests[security]==2.6.0 # Ubuntu Xenial constraints ConfigArgParse==0.10.0 funcsigs==0.4 zope.hookable==4.0.4 +# Ubuntu Bionic constraints. +# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins +# using their local-oldest-requirements.txt +dns-lexicon==2.2.1 + # Plugin constraints # These aren't necessarily the oldest versions we need to support # Tracking at https://github.com/certbot/certbot/issues/6473 diff --git a/tools/pip_install.py b/tools/pip_install.py index 354dce32b..dd6302b48 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -32,10 +32,10 @@ def certbot_oldest_processing(tools_path, args, test_constraints): # remove any extras such as [dev] pkg_dir = re.sub(r'\[\w+\]', '', args[1]) requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') + shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) # packages like acme don't have any local oldest requirements if not os.path.isfile(requirements): - requirements = None - shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) + return None return requirements @@ -53,11 +53,19 @@ def certbot_normal_processing(tools_path, test_constraints): fd.write('{0}{1}'.format(search.group(1), os.linesep)) -def merge_requirements(tools_path, test_constraints, all_constraints): - merged_requirements = merge_module.main( - os.path.join(tools_path, 'dev_constraints.txt'), - test_constraints - ) +def merge_requirements(tools_path, requirements, test_constraints, all_constraints): + # Order of the files in the merge function matters. + # Indeed version retained for a given package will be the last version + # found when following all requirements in the given order. + # Here is the order by increasing priority: + # 1) The general development constraints (tools/dev_constraints.txt) + # 2) The general tests constraints (oldest_requirements.txt or + # certbot-auto's dependency-requirements.txt for the normal processing) + # 3) The local requirement file, typically local-oldest-requirement in oldest tests + files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] + if requirements: + files.append(requirements) + merged_requirements = merge_module.main(*files) with open(all_constraints, 'w') as fd: fd.write(merged_requirements) @@ -71,24 +79,37 @@ def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() + if os.environ.get('TRAVIS'): + # When this script is executed on Travis, the following print will make the log + # be folded until the end command is printed (see finally section). + print('travis_fold:start:install_certbot_deps') + try: test_constraints = os.path.join(working_dir, 'test_constraints.txt') all_constraints = os.path.join(working_dir, 'all_constraints.txt') - requirements = None - if os.environ.get('CERTBOT_OLDEST') == '1': - requirements = certbot_oldest_processing(tools_path, args, test_constraints) + if os.environ.get('CERTBOT_NO_PIN') == '1': + # With unpinned dependencies, there is no constraint + call_with_print('"{0}" -m pip install {1}' + .format(sys.executable, ' '.join(args))) else: - certbot_normal_processing(tools_path, test_constraints) + # Otherwise, we merge requirements to build the constraints and pin dependencies + requirements = None + if os.environ.get('CERTBOT_OLDEST') == '1': + requirements = certbot_oldest_processing(tools_path, args, test_constraints) + else: + certbot_normal_processing(tools_path, test_constraints) - merge_requirements(tools_path, test_constraints, all_constraints) - if requirements: - call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' - .format(sys.executable, all_constraints, requirements)) + merge_requirements(tools_path, requirements, test_constraints, all_constraints) + if requirements: + call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' + .format(sys.executable, all_constraints, requirements)) - call_with_print('"{0}" -m pip install --constraint "{1}" {2}' - .format(sys.executable, all_constraints, ' '.join(args))) + call_with_print('"{0}" -m pip install --constraint "{1}" {2}' + .format(sys.executable, all_constraints, ' '.join(args))) finally: + if os.environ.get('TRAVIS'): + print('travis_fold:end:install_certbot_deps') shutil.rmtree(working_dir) diff --git a/tox.ini b/tox.ini index 021c23949..363f06bf2 100644 --- a/tox.ini +++ b/tox.ini @@ -64,9 +64,8 @@ source_paths = tests/lock_test.py [testenv] -passenv = - TRAVIS - APPVEYOR +passenv = + CERTBOT_NO_PIN commands = {[base]install_and_test} {[base]all_packages} python tests/lock_test.py @@ -155,6 +154,20 @@ commands = commands = {[base]pip_install} acme . certbot-apache certbot-compatibility-test {toxinidir}/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test --debian-modules +passenv = + SERVER + +[testenv:apacheconftest-with-pebble] +commands = + {toxinidir}/tests/pebble-fetch.sh + {[testenv:apacheconftest]commands} +passenv = + HOME + GOPATH + PEBBLEPATH + PEBBLE_STRICT +setenv = + SERVER=https://localhost:14000/dir [testenv:nginxroundtrip] commands = @@ -176,7 +189,6 @@ whitelist_externals = docker passenv = DOCKER_* - TRAVIS [testenv:nginx_compat] commands = @@ -187,19 +199,6 @@ whitelist_externals = docker passenv = DOCKER_* - TRAVIS - -[testenv:le_auto_precise] -# At the moment, this tests under Python 2.7 only, as only that version is -# readily available on the Precise Docker image. -commands = - docker build -f letsencrypt-auto-source/Dockerfile.precise -t lea letsencrypt-auto-source - docker run --rm -t -i lea -whitelist_externals = - docker -passenv = - DOCKER_* - TRAVIS [testenv:le_auto_trusty] # At the moment, this tests under Python 2.7 only, as only that version is @@ -212,14 +211,22 @@ whitelist_externals = docker passenv = DOCKER_* - TRAVIS TRAVIS_BRANCH -[testenv:le_auto_wheezy] +[testenv:le_auto_xenial] +# At the moment, this tests under Python 2.7 only. +commands = + docker build -f letsencrypt-auto-source/Dockerfile.xenial -t lea letsencrypt-auto-source + docker run --rm -t -i lea +whitelist_externals = + docker +passenv = DOCKER_* + +[testenv:le_auto_jessie] # At the moment, this tests under Python 2.7 only, as only that version is # readily available on the Wheezy Docker image. commands = - docker build -f letsencrypt-auto-source/Dockerfile.wheezy -t lea letsencrypt-auto-source + docker build -f letsencrypt-auto-source/Dockerfile.jessie -t lea letsencrypt-auto-source docker run --rm -t -i lea whitelist_externals = docker