diff --git a/.azure-pipelines/advanced-test.yml b/.azure-pipelines/advanced-test.yml new file mode 100644 index 000000000..5be29ba79 --- /dev/null +++ b/.azure-pipelines/advanced-test.yml @@ -0,0 +1,13 @@ +# Advanced pipeline for running our full test suite on demand. +trigger: + # When changing these triggers, please ensure the documentation under + # "Running tests in CI" is still correct. + - azure-test-* + - test-* +pr: none + +jobs: + # Any addition here should be reflected in the advanced and release pipelines. + # It is advised to declare all jobs here as templates to improve maintainability. + - template: templates/tests-suite.yml + - template: templates/installer-tests.yml diff --git a/.azure-pipelines/advanced.yml b/.azure-pipelines/advanced.yml index dda7f9bfd..d950e6524 100644 --- a/.azure-pipelines/advanced.yml +++ b/.azure-pipelines/advanced.yml @@ -1,12 +1,7 @@ -# Advanced pipeline for isolated checks and release purpose +# Advanced pipeline for running our full test suite on protected branches. trigger: - # When changing these triggers, please ensure the documentation under - # "Running tests in CI" is still correct. - - azure-test-* - - test-* - '*.x' -pr: - - test-* +pr: none # This pipeline is also nightly run on master schedules: - cron: "0 4 * * *" @@ -17,7 +12,7 @@ schedules: always: true jobs: - # Any addition here should be reflected in the release pipeline. + # Any addition here should be reflected in the advanced-test and release pipelines. # It is advised to declare all jobs here as templates to improve maintainability. - template: templates/tests-suite.yml - template: templates/installer-tests.yml diff --git a/.azure-pipelines/release.yml b/.azure-pipelines/release.yml index aeb5ee327..e9acbc69a 100644 --- a/.azure-pipelines/release.yml +++ b/.azure-pipelines/release.yml @@ -6,7 +6,7 @@ trigger: pr: none jobs: - # Any addition here should be reflected in the advanced pipeline. + # Any addition here should be reflected in the advanced and advanced-test pipelines. # It is advised to declare all jobs here as templates to improve maintainability. - template: templates/tests-suite.yml - template: templates/installer-tests.yml diff --git a/.azure-pipelines/templates/installer-tests.yml b/.azure-pipelines/templates/installer-tests.yml index 6d5672339..ebadcb2dc 100644 --- a/.azure-pipelines/templates/installer-tests.yml +++ b/.azure-pipelines/templates/installer-tests.yml @@ -28,15 +28,20 @@ jobs: imageName: windows-2019 win2016: imageName: vs2017-win2016 - win2012r2: - imageName: vs2015-win2012r2 pool: vmImage: $(imageName) steps: - - powershell: Invoke-WebRequest https://www.python.org/ftp/python/3.8.1/python-3.8.1-amd64-webinstall.exe -OutFile C:\py3-setup.exe - displayName: Get Python - - script: C:\py3-setup.exe /quiet PrependPath=1 InstallAllUsers=1 Include_launcher=1 InstallLauncherAllUsers=1 Include_test=0 Include_doc=0 Include_dev=1 Include_debug=0 Include_tcltk=0 TargetDir=C:\py3 - displayName: Install Python + - powershell: | + $currentVersion = $PSVersionTable.PSVersion + if ($currentVersion.Major -ne 5) { + throw "Powershell version is not 5.x" + } + condition: eq(variables['imageName'], 'vs2017-win2016') + displayName: Check Powershell 5.x is used in vs2017-win2016 + - task: UsePythonVersion@0 + inputs: + versionSpec: 3.8 + addToPath: true - task: DownloadPipelineArtifact@2 inputs: artifact: windows-installer diff --git a/.azure-pipelines/templates/tests-suite.yml b/.azure-pipelines/templates/tests-suite.yml index 069ea94d6..d330b7954 100644 --- a/.azure-pipelines/templates/tests-suite.yml +++ b/.azure-pipelines/templates/tests-suite.yml @@ -25,8 +25,6 @@ jobs: PYTEST_ADDOPTS: --numprocesses 4 pool: vmImage: $(IMAGE_NAME) - variables: - - group: certbot-common steps: - bash: brew install augeas condition: startswith(variables['IMAGE_NAME'], 'macOS') @@ -39,14 +37,3 @@ jobs: displayName: Install dependencies - script: python -m tox displayName: Run tox - # We do not require codecov report upload to succeed. So to avoid to break the pipeline if - # something goes wrong, each command is suffixed with a command that hides any non zero exit - # codes and echoes an informative message instead. - - bash: | - curl -s https://codecov.io/bash -o codecov-bash || echo "Failed to download codecov-bash" - chmod +x codecov-bash || echo "Failed to apply execute permissions on codecov-bash" - ./codecov-bash -F windows || echo "Codecov did not collect coverage reports" - condition: in(variables['TOXENV'], 'py37-cover', 'integration-certbot') - env: - CODECOV_TOKEN: $(codecov_token) - displayName: Publish coverage diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 0a97fffe3..000000000 --- a/.codecov.yml +++ /dev/null @@ -1,18 +0,0 @@ -coverage: - status: - project: - default: off - linux: - flags: linux - # Fixed target instead of auto set by #7173, can - # be removed when flags in Codecov are added back. - target: 97.4 - threshold: 0.1 - base: auto - windows: - flags: windows - # Fixed target instead of auto set by #7173, can - # be removed when flags in Codecov are added back. - target: 97.4 - threshold: 0.1 - base: auto diff --git a/.gitignore b/.gitignore index 6dd422187..6505e716c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ tags tests/letstest/letest-*/ tests/letstest/*.pem tests/letstest/venv/ +tests/letstest/venv3/ .venv diff --git a/.travis.yml b/.travis.yml index e5354898d..d3eeb1e03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,8 +44,8 @@ matrix: <<: *not-on-master # This job is always executed, including on master - - python: "2.7" - env: TOXENV=py27-cover FYI="py27 tests + code coverage" + - python: "3.8" + env: TOXENV=py38-cover FYI="py38 tests + code coverage" - python: "3.7" env: TOXENV=lint @@ -60,12 +60,12 @@ matrix: dist: trusty env: TOXENV='py27-{acme,apache,apache-v2,certbot,dns,nginx}-oldest' <<: *not-on-master + - python: "2.7" + env: TOXENV=py27 + <<: *not-on-master - python: "3.5" env: TOXENV=py35 <<: *not-on-master - - python: "3.8" - env: TOXENV=py38 - <<: *not-on-master - sudo: required env: TOXENV=apache_compat services: docker @@ -90,24 +90,24 @@ matrix: before_install: addons: <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-apache2 - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-leauto-upgrades - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" git: depth: false # This is needed to have the history to checkout old versions of certbot-auto. <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-certonly-standalone - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" <<: *extended-test-suite - - python: "2.7" + - python: "3.7" env: - TOXENV=travis-test-farm-sdists - secure: "f+j/Lj9s1lcuKo5sEFrlRd1kIAMnIJI4z0MTI7QF8jl9Fkmbx7KECGzw31TNgzrOSzxSapHbcueFYvNCLKST+kE/8ogMZBbwqXfEDuKpyF6BY3uYoJn+wPVE5pIb8Hhe08xPte8TTDSMIyHI3EyTfcAKrIreauoArePvh/cRvSw=" @@ -247,15 +247,13 @@ addons: # version of virtualenv. The option "-I" is set so when CERTBOT_NO_PIN is also # set, pip updates dependencies it thinks are already satisfied to avoid some # problems with its lack of real dependency resolution. -install: 'tools/pip_install.py -I codecov tox virtualenv' +install: 'tools/pip_install.py -I tox virtualenv' # Most of the time TRAVIS_RETRY is an empty string, and has no effect on the # script command. It is set only to `travis_retry` during farm tests, in # order to trigger the Travis retry feature, and compensate the inherent # flakiness of these specific tests. script: '$TRAVIS_RETRY tox' -after_success: '[ "$TOXENV" == "py27-cover" ] && codecov -F linux' - notifications: email: false irc: diff --git a/AUTHORS.md b/AUTHORS.md index 80a24d3be..21a6e7773 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -21,6 +21,7 @@ Authors * [Andrzej Górski](https://github.com/andrzej3393) * [Anselm Levskaya](https://github.com/levskaya) * [Antoine Jacoutot](https://github.com/ajacoutot) +* [April King](https://github.com/april) * [asaph](https://github.com/asaph) * [Axel Beckert](https://github.com/xtaran) * [Bas](https://github.com/Mechazawa) @@ -103,6 +104,7 @@ Authors * [Henry Chen](https://github.com/henrychen95) * [Hugo van Kemenade](https://github.com/hugovk) * [Ingolf Becker](https://github.com/watercrossing) +* [Ivan Nejgebauer](https://github.com/inejge) * [Jaap Eldering](https://github.com/eldering) * [Jacob Hoffman-Andrews](https://github.com/jsha) * [Jacob Sachs](https://github.com/jsachs) @@ -266,5 +268,6 @@ Authors * [Yomna](https://github.com/ynasser) * [Yoni Jah](https://github.com/yonjah) * [YourDaddyIsHere](https://github.com/YourDaddyIsHere) +* [Yuseong Cho](https://github.com/g6123) * [Zach Shepherd](https://github.com/zjs) * [陈三](https://github.com/chenxsan) diff --git a/acme/acme/challenges.py b/acme/acme/challenges.py index 39c8d6269..b9c6b7eb2 100644 --- a/acme/acme/challenges.py +++ b/acme/acme/challenges.py @@ -1,15 +1,22 @@ """ACME Identifier Validation Challenges.""" import abc +import codecs import functools import hashlib import logging +import socket from cryptography.hazmat.primitives import hashes # type: ignore import josepy as jose import requests import six +from OpenSSL import SSL # type: ignore # https://github.com/python/typeshed/issues/2052 +from OpenSSL import crypto +from acme import crypto_util +from acme import errors from acme import fields +from acme.mixins import ResourceMixin, TypeMixin logger = logging.getLogger(__name__) @@ -28,7 +35,7 @@ class Challenge(jose.TypedJSONObjectWithFields): return UnrecognizedChallenge.from_json(jobj) -class ChallengeResponse(jose.TypedJSONObjectWithFields): +class ChallengeResponse(ResourceMixin, TypeMixin, jose.TypedJSONObjectWithFields): # _fields_to_partial_json """ACME challenge response.""" TYPES = {} # type: dict @@ -362,29 +369,163 @@ class HTTP01(KeyAuthorizationChallenge): @ChallengeResponse.register class TLSALPN01Response(KeyAuthorizationChallengeResponse): - """ACME TLS-ALPN-01 challenge response. - - This class only allows initiating a TLS-ALPN-01 challenge returned from the - CA. Full support for responding to TLS-ALPN-01 challenges by generating and - serving the expected response certificate is not currently provided. - """ + """ACME tls-alpn-01 challenge response.""" typ = "tls-alpn-01" + PORT = 443 + """Verification port as defined by the protocol. -@Challenge.register + You can override it (e.g. for testing) by passing ``port`` to + `simple_verify`. + + """ + + ID_PE_ACME_IDENTIFIER_V1 = b"1.3.6.1.5.5.7.1.30.1" + ACME_TLS_1_PROTOCOL = "acme-tls/1" + + @property + def h(self): + """Hash value stored in challenge certificate""" + return hashlib.sha256(self.key_authorization.encode('utf-8')).digest() + + def gen_cert(self, domain, key=None, bits=2048): + """Generate tls-alpn-01 certificate. + + :param unicode domain: Domain verified by the challenge. + :param OpenSSL.crypto.PKey key: Optional private key used in + certificate generation. If not provided (``None``), then + fresh key will be generated. + :param int bits: Number of bits for newly generated key. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + if key is None: + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, bits) + + + der_value = b"DER:" + codecs.encode(self.h, 'hex') + acme_extension = crypto.X509Extension(self.ID_PE_ACME_IDENTIFIER_V1, + critical=True, value=der_value) + + return crypto_util.gen_ss_cert(key, [domain], force_san=True, + extensions=[acme_extension]), key + + def probe_cert(self, domain, host=None, port=None): + """Probe tls-alpn-01 challenge certificate. + + :param unicode domain: domain being validated, required. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + """ + if host is None: + host = socket.gethostbyname(domain) + logger.debug('%s resolved to %s', domain, host) + if port is None: + port = self.PORT + + return crypto_util.probe_sni(host=host, port=port, name=domain, + alpn_protocols=[self.ACME_TLS_1_PROTOCOL]) + + def verify_cert(self, domain, cert): + """Verify tls-alpn-01 challenge certificate. + + :param unicode domain: Domain name being validated. + :param OpensSSL.crypto.X509 cert: Challenge certificate. + + :returns: Whether the certificate was successfully verified. + :rtype: bool + + """ + # pylint: disable=protected-access + names = crypto_util._pyopenssl_cert_or_req_all_names(cert) + logger.debug('Certificate %s. SANs: %s', cert.digest('sha256'), names) + if len(names) != 1 or names[0].lower() != domain.lower(): + return False + + for i in range(cert.get_extension_count()): + ext = cert.get_extension(i) + # FIXME: assume this is the ACME extension. Currently there is no + # way to get full OID of an unknown extension from pyopenssl. + if ext.get_short_name() == b'UNDEF': + data = ext.get_data() + return data == self.h + + return False + + # pylint: disable=too-many-arguments + def simple_verify(self, chall, domain, account_public_key, + cert=None, host=None, port=None): + """Simple verify. + + Verify ``validation`` using ``account_public_key``, optionally + probe tls-alpn-01 certificate and check using `verify_cert`. + + :param .challenges.TLSALPN01 chall: Corresponding challenge. + :param str domain: Domain name being validated. + :param JWK account_public_key: + :param OpenSSL.crypto.X509 cert: Optional certificate. If not + provided (``None``) certificate will be retrieved using + `probe_cert`. + :param string host: IP address used to probe the certificate. + :param int port: Port used to probe the certificate. + + + :returns: ``True`` if and only if client's control of the domain has been verified. + :rtype: bool + + """ + if not self.verify(chall, account_public_key): + logger.debug("Verification of key authorization in response failed") + return False + + if cert is None: + try: + cert = self.probe_cert(domain=domain, host=host, port=port) + except errors.Error as error: + logger.debug(str(error), exc_info=True) + return False + + return self.verify_cert(domain, cert) + + +@Challenge.register # pylint: disable=too-many-ancestors class TLSALPN01(KeyAuthorizationChallenge): - """ACME tls-alpn-01 challenge. - - This class simply allows parsing the TLS-ALPN-01 challenge returned from - the CA. Full TLS-ALPN-01 support is not currently provided. - - """ - typ = "tls-alpn-01" + """ACME tls-alpn-01 challenge.""" response_cls = TLSALPN01Response + typ = response_cls.typ def validation(self, account_key, **kwargs): - """Generate validation for the challenge.""" - raise NotImplementedError() + """Generate validation. + + :param JWK account_key: + :param unicode domain: Domain verified by the challenge. + :param OpenSSL.crypto.PKey cert_key: Optional private key used + in certificate generation. If not provided (``None``), then + fresh key will be generated. + + :rtype: `tuple` of `OpenSSL.crypto.X509` and `OpenSSL.crypto.PKey` + + """ + return self.response(account_key).gen_cert( + key=kwargs.get('cert_key'), + domain=kwargs.get('domain')) + + @staticmethod + def is_supported(): + """ + Check if TLS-ALPN-01 challenge is supported on this machine. + This implies that a recent version of OpenSSL is installed (>= 1.0.2), + or a recent cryptography version shipped with the OpenSSL library is installed. + + :returns: ``True`` if TLS-ALPN-01 is supported on this machine, ``False`` otherwise. + :rtype: bool + + """ + return (hasattr(SSL.Connection, "set_alpn_protos") + and hasattr(SSL.Context, "set_alpn_select_callback")) @Challenge.register diff --git a/acme/acme/client.py b/acme/acme/client.py index cecb727c7..3ce321ac9 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -25,6 +25,7 @@ from acme.magic_typing import Dict from acme.magic_typing import List from acme.magic_typing import Set from acme.magic_typing import Text +from acme.mixins import VersionedLEACMEMixin logger = logging.getLogger(__name__) @@ -987,6 +988,8 @@ class ClientNetwork(object): :rtype: `josepy.JWS` """ + if isinstance(obj, VersionedLEACMEMixin): + obj.le_acme_version = acme_version jobj = obj.json_dumps(indent=2).encode() if obj else b'' logger.debug('JWS payload:\n%s', jobj) kwargs = { @@ -1120,8 +1123,8 @@ class ClientNetwork(object): debug_content = response.content.decode("utf-8") logger.debug('Received response:\nHTTP %d\n%s\n\n%s', response.status_code, - "\n".join(["{0}: {1}".format(k, v) - for k, v in response.headers.items()]), + "\n".join("{0}: {1}".format(k, v) + for k, v in response.headers.items()), debug_content) return response diff --git a/acme/acme/crypto_util.py b/acme/acme/crypto_util.py index dc8fedad0..f8b7e2b30 100644 --- a/acme/acme/crypto_util.py +++ b/acme/acme/crypto_util.py @@ -27,19 +27,41 @@ logger = logging.getLogger(__name__) _DEFAULT_SSL_METHOD = SSL.SSLv23_METHOD # type: ignore -class SSLSocket(object): +class _DefaultCertSelection(object): + def __init__(self, certs): + self.certs = certs + + def __call__(self, connection): + server_name = connection.get_servername() + return self.certs.get(server_name, None) + + +class SSLSocket(object): # pylint: disable=too-few-public-methods """SSL wrapper for sockets. :ivar socket sock: Original wrapped socket. :ivar dict certs: Mapping from domain names (`bytes`) to `OpenSSL.crypto.X509`. :ivar method: See `OpenSSL.SSL.Context` for allowed values. + :ivar alpn_selection: Hook to select negotiated ALPN protocol for + connection. + :ivar cert_selection: Hook to select certificate for connection. If given, + `certs` parameter would be ignored, and therefore must be empty. """ - def __init__(self, sock, certs, method=_DEFAULT_SSL_METHOD): + def __init__(self, sock, certs=None, + method=_DEFAULT_SSL_METHOD, alpn_selection=None, + cert_selection=None): self.sock = sock - self.certs = certs + self.alpn_selection = alpn_selection self.method = method + if not cert_selection and not certs: + raise ValueError("Neither cert_selection or certs specified.") + if cert_selection and certs: + raise ValueError("Both cert_selection and certs specified.") + if cert_selection is None: + cert_selection = _DefaultCertSelection(certs) + self.cert_selection = cert_selection def __getattr__(self, name): return getattr(self.sock, name) @@ -56,18 +78,19 @@ class SSLSocket(object): :type connection: :class:`OpenSSL.Connection` """ - server_name = connection.get_servername() - try: - key, cert = self.certs[server_name] - except KeyError: - logger.debug("Server name (%s) not recognized, dropping SSL", - server_name) + pair = self.cert_selection(connection) + if pair is None: + logger.debug("Certificate selection for server name %s failed, dropping SSL", + connection.get_servername()) return + key, cert = pair new_context = SSL.Context(self.method) new_context.set_options(SSL.OP_NO_SSLv2) new_context.set_options(SSL.OP_NO_SSLv3) new_context.use_privatekey(key) new_context.use_certificate(cert) + if self.alpn_selection is not None: + new_context.set_alpn_select_callback(self.alpn_selection) connection.set_context(new_context) class FakeConnection(object): @@ -92,6 +115,8 @@ class SSLSocket(object): context.set_options(SSL.OP_NO_SSLv2) context.set_options(SSL.OP_NO_SSLv3) context.set_tlsext_servername_callback(self._pick_certificate_cb) + if self.alpn_selection is not None: + context.set_alpn_select_callback(self.alpn_selection) ssl_sock = self.FakeConnection(SSL.Connection(context, sock)) ssl_sock.set_accept_state() @@ -107,8 +132,9 @@ class SSLSocket(object): return ssl_sock, addr -def probe_sni(name, host, port=443, timeout=300, - method=_DEFAULT_SSL_METHOD, source_address=('', 0)): +def probe_sni(name, host, port=443, timeout=300, # pylint: disable=too-many-arguments + method=_DEFAULT_SSL_METHOD, source_address=('', 0), + alpn_protocols=None): """Probe SNI server for SSL certificate. :param bytes name: Byte string to send as the server name in the @@ -120,6 +146,8 @@ def probe_sni(name, host, port=443, timeout=300, :param tuple source_address: Enables multi-path probing (selection of source interface). See `socket.creation_connection` for more info. Available only in Python 2.7+. + :param alpn_protocols: Protocols to request using ALPN. + :type alpn_protocols: `list` of `bytes` :raises acme.errors.Error: In case of any problems. @@ -149,6 +177,8 @@ def probe_sni(name, host, port=443, timeout=300, client_ssl = SSL.Connection(context, client) client_ssl.set_connect_state() client_ssl.set_tlsext_host_name(name) # pyOpenSSL>=0.13 + if alpn_protocols is not None: + client_ssl.set_alpn_protos(alpn_protocols) try: client_ssl.do_handshake() client_ssl.shutdown() @@ -239,12 +269,14 @@ def _pyopenssl_cert_or_req_san(cert_or_req): def gen_ss_cert(key, domains, not_before=None, - validity=(7 * 24 * 60 * 60), force_san=True): + validity=(7 * 24 * 60 * 60), force_san=True, extensions=None): """Generate new self-signed certificate. :type domains: `list` of `unicode` :param OpenSSL.crypto.PKey key: :param bool force_san: + :param extensions: List of additional extensions to include in the cert. + :type extensions: `list` of `OpenSSL.crypto.X509Extension` If more than one domain is provided, all of the domains are put into ``subjectAltName`` X.509 extension and first domain is set as the @@ -257,10 +289,13 @@ def gen_ss_cert(key, domains, not_before=None, cert.set_serial_number(int(binascii.hexlify(os.urandom(16)), 16)) cert.set_version(2) - extensions = [ + if extensions is None: + extensions = [] + + extensions.append( crypto.X509Extension( b"basicConstraints", True, b"CA:TRUE, pathlen:0"), - ] + ) cert.get_subject().CN = domains[0] # TODO: what to put into cert.get_subject()? diff --git a/acme/acme/magic_typing.py b/acme/acme/magic_typing.py index d6b1ff056..7c5231c75 100644 --- a/acme/acme/magic_typing.py +++ b/acme/acme/magic_typing.py @@ -11,6 +11,5 @@ try: # mypy doesn't respect modifying sys.modules from typing import * # pylint: disable=wildcard-import, unused-wildcard-import from typing import Collection, IO # type: ignore - # pylint: enable=unused-import except ImportError: sys.modules[__name__] = TypingClass() diff --git a/acme/acme/messages.py b/acme/acme/messages.py index f8f4bfbe7..90059a6fb 100644 --- a/acme/acme/messages.py +++ b/acme/acme/messages.py @@ -9,6 +9,7 @@ from acme import errors from acme import fields from acme import jws from acme import util +from acme.mixins import ResourceMixin try: from collections.abc import Hashable @@ -356,13 +357,13 @@ class Registration(ResourceBody): @Directory.register -class NewRegistration(Registration): +class NewRegistration(ResourceMixin, Registration): """New registration.""" resource_type = 'new-reg' resource = fields.Resource(resource_type) -class UpdateRegistration(Registration): +class UpdateRegistration(ResourceMixin, Registration): """Update registration.""" resource_type = 'reg' resource = fields.Resource(resource_type) @@ -498,13 +499,13 @@ class Authorization(ResourceBody): @Directory.register -class NewAuthorization(Authorization): +class NewAuthorization(ResourceMixin, Authorization): """New authorization.""" resource_type = 'new-authz' resource = fields.Resource(resource_type) -class UpdateAuthorization(Authorization): +class UpdateAuthorization(ResourceMixin, Authorization): """Update authorization.""" resource_type = 'authz' resource = fields.Resource(resource_type) @@ -522,7 +523,7 @@ class AuthorizationResource(ResourceWithURI): @Directory.register -class CertificateRequest(jose.JSONObjectWithFields): +class CertificateRequest(ResourceMixin, jose.JSONObjectWithFields): """ACME new-cert request. :ivar josepy.util.ComparableX509 csr: @@ -548,7 +549,7 @@ class CertificateResource(ResourceWithURI): @Directory.register -class Revocation(jose.JSONObjectWithFields): +class Revocation(ResourceMixin, jose.JSONObjectWithFields): """Revocation message. :ivar .ComparableX509 certificate: `OpenSSL.crypto.X509` wrapped in diff --git a/acme/acme/mixins.py b/acme/acme/mixins.py new file mode 100644 index 000000000..1cd050ccc --- /dev/null +++ b/acme/acme/mixins.py @@ -0,0 +1,65 @@ +"""Useful mixins for Challenge and Resource objects""" + + +class VersionedLEACMEMixin(object): + """This mixin stores the version of Let's Encrypt's endpoint being used.""" + @property + def le_acme_version(self): + """Define the version of ACME protocol to use""" + return getattr(self, '_le_acme_version', 1) + + @le_acme_version.setter + def le_acme_version(self, version): + # We need to use object.__setattr__ to not depend on the specific implementation of + # __setattr__ in current class (eg. jose.TypedJSONObjectWithFields raises AttributeError + # for any attempt to set an attribute to make objects immutable). + object.__setattr__(self, '_le_acme_version', version) + + def __setattr__(self, key, value): + if key == 'le_acme_version': + # Required for @property to operate properly. See comment above. + object.__setattr__(self, key, value) + else: + super(VersionedLEACMEMixin, self).__setattr__(key, value) # pragma: no cover + + +class ResourceMixin(VersionedLEACMEMixin): + """ + This mixin generates a RFC8555 compliant JWS payload + by removing the `resource` field if needed (eg. ACME v2 protocol). + """ + def to_partial_json(self): + """See josepy.JSONDeserializable.to_partial_json()""" + return _safe_jobj_compliance(super(ResourceMixin, self), + 'to_partial_json', 'resource') + + def fields_to_partial_json(self): + """See josepy.JSONObjectWithFields.fields_to_partial_json()""" + return _safe_jobj_compliance(super(ResourceMixin, self), + 'fields_to_partial_json', 'resource') + + +class TypeMixin(VersionedLEACMEMixin): + """ + This mixin allows generation of a RFC8555 compliant JWS payload + by removing the `type` field if needed (eg. ACME v2 protocol). + """ + def to_partial_json(self): + """See josepy.JSONDeserializable.to_partial_json()""" + return _safe_jobj_compliance(super(TypeMixin, self), + 'to_partial_json', 'type') + + def fields_to_partial_json(self): + """See josepy.JSONObjectWithFields.fields_to_partial_json()""" + return _safe_jobj_compliance(super(TypeMixin, self), + 'fields_to_partial_json', 'type') + + +def _safe_jobj_compliance(instance, jobj_method, uncompliant_field): + if hasattr(instance, jobj_method): + jobj = getattr(instance, jobj_method)() + if instance.le_acme_version == 2: + jobj.pop(uncompliant_field, None) + return jobj + + raise AttributeError('Method {0}() is not implemented.'.format(jobj_method)) # pragma: no cover diff --git a/acme/acme/standalone.py b/acme/acme/standalone.py index 236f2c234..7a61ba868 100644 --- a/acme/acme/standalone.py +++ b/acme/acme/standalone.py @@ -33,7 +33,14 @@ class TLSServer(socketserver.TCPServer): def _wrap_sock(self): self.socket = crypto_util.SSLSocket( - self.socket, certs=self.certs, method=self.method) + self.socket, cert_selection=self._cert_selection, + alpn_selection=getattr(self, '_alpn_selection', None), + method=self.method) + + def _cert_selection(self, connection): # pragma: no cover + """Callback selecting certificate for connection.""" + server_name = connection.get_servername() + return self.certs.get(server_name, None) def server_bind(self): self._wrap_sock() @@ -120,6 +127,40 @@ class BaseDualNetworkedServers(object): self.threads = [] +class TLSALPN01Server(TLSServer, ACMEServerMixin): + """TLSALPN01 Server.""" + + ACME_TLS_1_PROTOCOL = b"acme-tls/1" + + def __init__(self, server_address, certs, challenge_certs, ipv6=False): + TLSServer.__init__( + self, server_address, _BaseRequestHandlerWithLogging, certs=certs, + ipv6=ipv6) + self.challenge_certs = challenge_certs + + def _cert_selection(self, connection): + # TODO: We would like to serve challenge cert only if asked for it via + # ALPN. To do this, we need to retrieve the list of protos from client + # hello, but this is currently impossible with openssl [0], and ALPN + # negotiation is done after cert selection. + # Therefore, currently we always return challenge cert, and terminate + # handshake in alpn_selection() if ALPN protos are not what we expect. + # [0] https://github.com/openssl/openssl/issues/4952 + server_name = connection.get_servername() + logger.debug("Serving challenge cert for server name %s", server_name) + return self.challenge_certs.get(server_name, None) + + def _alpn_selection(self, _connection, alpn_protos): + """Callback to select alpn protocol.""" + if len(alpn_protos) == 1 and alpn_protos[0] == self.ACME_TLS_1_PROTOCOL: + logger.debug("Agreed on %s ALPN", self.ACME_TLS_1_PROTOCOL) + return self.ACME_TLS_1_PROTOCOL + logger.debug("Cannot agree on ALPN proto. Got: %s", str(alpn_protos)) + # Explicitly close the connection now, by returning an empty string. + # See https://www.pyopenssl.org/en/stable/api/ssl.html#OpenSSL.SSL.Context.set_alpn_select_callback # pylint: disable=line-too-long + return b"" + + class HTTPServer(BaseHTTPServer.HTTPServer): """Generic HTTP Server.""" @@ -135,10 +176,10 @@ class HTTPServer(BaseHTTPServer.HTTPServer): class HTTP01Server(HTTPServer, ACMEServerMixin): """HTTP01 Server.""" - def __init__(self, server_address, resources, ipv6=False): + def __init__(self, server_address, resources, ipv6=False, timeout=30): HTTPServer.__init__( self, server_address, HTTP01RequestHandler.partial_init( - simple_http_resources=resources), ipv6=ipv6) + simple_http_resources=resources, timeout=timeout), ipv6=ipv6) class HTTP01DualNetworkedServers(BaseDualNetworkedServers): @@ -163,6 +204,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.simple_http_resources = kwargs.pop("simple_http_resources", set()) + self.timeout = kwargs.pop('timeout', 30) BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def log_message(self, format, *args): # pylint: disable=redefined-builtin @@ -212,7 +254,7 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.path) @classmethod - def partial_init(cls, simple_http_resources): + def partial_init(cls, simple_http_resources, timeout): """Partially initialize this handler. This is useful because `socketserver.BaseServer` takes @@ -221,4 +263,18 @@ class HTTP01RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """ return functools.partial( - cls, simple_http_resources=simple_http_resources) + cls, simple_http_resources=simple_http_resources, + timeout=timeout) + + +class _BaseRequestHandlerWithLogging(socketserver.BaseRequestHandler): + """BaseRequestHandler with logging.""" + + def log_message(self, format, *args): # pylint: disable=redefined-builtin + """Log arbitrary message.""" + logger.debug("%s - - %s", self.client_address[0], format % args) + + def handle(self): + """Handle request.""" + self.log_message("Incoming request") + socketserver.BaseRequestHandler.handle(self) diff --git a/acme/docs/conf.py b/acme/docs/conf.py index a9c69d538..ba1a3aa8b 100644 --- a/acme/docs/conf.py +++ b/acme/docs/conf.py @@ -13,7 +13,6 @@ # serve to show the default. import os -import shlex import sys here = os.path.abspath(os.path.dirname(__file__)) diff --git a/acme/setup.py b/acme/setup.py index 0e11779ba..356410efe 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -1,10 +1,12 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ @@ -15,7 +17,6 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'mock', # Connection.set_tlsext_host_name (>=0.13) 'PyOpenSSL>=0.13.1', 'pyrfc3339', @@ -26,6 +27,15 @@ install_requires = [ 'six>=1.9.0', # needed for python_2_unicode_compatible ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'pytest', 'pytest-xdist', diff --git a/acme/tests/challenges_test.py b/acme/tests/challenges_test.py index adebaffc5..70371051c 100644 --- a/acme/tests/challenges_test.py +++ b/acme/tests/challenges_test.py @@ -2,10 +2,16 @@ import unittest import josepy as jose -import mock +import OpenSSL +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import requests from six.moves.urllib import parse as urllib_parse +from acme import errors + import test_util CERT = test_util.load_comparable_cert('cert.pem') @@ -256,30 +262,87 @@ class HTTP01Test(unittest.TestCase): class TLSALPN01ResponseTest(unittest.TestCase): def setUp(self): - from acme.challenges import TLSALPN01Response - self.msg = TLSALPN01Response(key_authorization=u'foo') + from acme.challenges import TLSALPN01 + self.chall = TLSALPN01( + token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e')) + self.domain = u'example.com' + self.domain2 = u'example2.com' + + self.response = self.chall.response(KEY) self.jmsg = { 'resource': 'challenge', 'type': 'tls-alpn-01', - 'keyAuthorization': u'foo', + 'keyAuthorization': self.response.key_authorization, } - from acme.challenges import TLSALPN01 - self.chall = TLSALPN01(token=(b'x' * 16)) - self.response = self.chall.response(KEY) - def test_to_partial_json(self): self.assertEqual({k: v for k, v in self.jmsg.items() if k != 'keyAuthorization'}, - self.msg.to_partial_json()) + self.response.to_partial_json()) def test_from_json(self): from acme.challenges import TLSALPN01Response - self.assertEqual(self.msg, TLSALPN01Response.from_json(self.jmsg)) + self.assertEqual(self.response, TLSALPN01Response.from_json(self.jmsg)) def test_from_json_hashable(self): from acme.challenges import TLSALPN01Response hash(TLSALPN01Response.from_json(self.jmsg)) + def test_gen_verify_cert(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_gen_verify_cert_gen_key(self): + cert, key = self.response.gen_cert(self.domain) + self.assertTrue(isinstance(key, OpenSSL.crypto.PKey)) + self.assertTrue(self.response.verify_cert(self.domain, cert)) + + def test_verify_bad_cert(self): + self.assertFalse(self.response.verify_cert(self.domain, + test_util.load_cert('cert.pem'))) + + def test_verify_bad_domain(self): + key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem') + cert, key2 = self.response.gen_cert(self.domain, key1) + self.assertEqual(key1, key2) + self.assertFalse(self.response.verify_cert(self.domain2, cert)) + + def test_simple_verify_bad_key_authorization(self): + key2 = jose.JWKRSA.load(test_util.load_vector('rsa256_key.pem')) + self.response.simple_verify(self.chall, "local", key2.public_key()) + + @mock.patch('acme.challenges.TLSALPN01Response.verify_cert', autospec=True) + def test_simple_verify(self, mock_verify_cert): + mock_verify_cert.return_value = mock.sentinel.verification + self.assertEqual( + mock.sentinel.verification, self.response.simple_verify( + self.chall, self.domain, KEY.public_key(), + cert=mock.sentinel.cert)) + mock_verify_cert.assert_called_once_with( + self.response, self.domain, mock.sentinel.cert) + + @mock.patch('acme.challenges.socket.gethostbyname') + @mock.patch('acme.challenges.crypto_util.probe_sni') + def test_probe_cert(self, mock_probe_sni, mock_gethostbyname): + mock_gethostbyname.return_value = '127.0.0.1' + self.response.probe_cert('foo.com') + mock_gethostbyname.assert_called_once_with('foo.com') + mock_probe_sni.assert_called_once_with( + host='127.0.0.1', port=self.response.PORT, name='foo.com', + alpn_protocols=['acme-tls/1']) + + self.response.probe_cert('foo.com', host='8.8.8.8') + mock_probe_sni.assert_called_with( + host='8.8.8.8', port=mock.ANY, name='foo.com', + alpn_protocols=['acme-tls/1']) + + @mock.patch('acme.challenges.TLSALPN01Response.probe_cert') + def test_simple_verify_false_on_probe_error(self, mock_probe_cert): + mock_probe_cert.side_effect = errors.Error + self.assertFalse(self.response.simple_verify( + self.chall, self.domain, KEY.public_key())) + class TLSALPN01Test(unittest.TestCase): @@ -309,8 +372,13 @@ class TLSALPN01Test(unittest.TestCase): self.assertRaises( jose.DeserializationError, TLSALPN01.from_json, self.jmsg) - def test_validation(self): - self.assertRaises(NotImplementedError, self.msg.validation, KEY) + @mock.patch('acme.challenges.TLSALPN01Response.gen_cert') + def test_validation(self, mock_gen_cert): + mock_gen_cert.return_value = ('cert', 'key') + self.assertEqual(('cert', 'key'), self.msg.validation( + KEY, cert_key=mock.sentinel.cert_key, domain=mock.sentinel.domain)) + mock_gen_cert.assert_called_once_with(key=mock.sentinel.cert_key, + domain=mock.sentinel.domain) class DNSTest(unittest.TestCase): @@ -413,5 +481,18 @@ class DNSResponseTest(unittest.TestCase): self.msg.check_validation(self.chall, KEY.public_key())) +class JWSPayloadRFC8555Compliant(unittest.TestCase): + """Test for RFC8555 compliance of JWS generated from resources/challenges""" + def test_challenge_payload(self): + from acme.challenges import HTTP01Response + + challenge_body = HTTP01Response() + challenge_body.le_acme_version = 2 + + jobj = challenge_body.json_dumps(indent=2).encode() + # RFC8555 states that challenge responses must have an empty payload. + self.assertEqual(jobj, b'{}') + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/tests/client_test.py b/acme/tests/client_test.py index a4966140f..c90cad9b0 100644 --- a/acme/tests/client_test.py +++ b/acme/tests/client_test.py @@ -6,7 +6,10 @@ import json import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import requests from six.moves import http_client # pylint: disable=import-error @@ -15,7 +18,7 @@ from acme import challenges from acme import errors from acme import jws as acme_jws from acme import messages -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module +from acme.mixins import VersionedLEACMEMixin import messages_test import test_util @@ -886,7 +889,7 @@ class ClientV2Test(ClientTestBase): self.client.net.get.assert_not_called() -class MockJSONDeSerializable(jose.JSONDeSerializable): +class MockJSONDeSerializable(VersionedLEACMEMixin, jose.JSONDeSerializable): # pylint: disable=missing-docstring def __init__(self, value): self.value = value diff --git a/acme/tests/crypto_util_test.py b/acme/tests/crypto_util_test.py index 41640ed60..705a3c856 100644 --- a/acme/tests/crypto_util_test.py +++ b/acme/tests/crypto_util_test.py @@ -11,14 +11,12 @@ import six from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import errors -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module import test_util class SSLSocketAndProbeSNITest(unittest.TestCase): """Tests for acme.crypto_util.SSLSocket/probe_sni.""" - def setUp(self): self.cert = test_util.load_comparable_cert('rsa2048_cert.pem') key = test_util.load_pyopenssl_private_key('rsa2048_key.pem') @@ -32,7 +30,8 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): # six.moves.* | pylint: disable=attribute-defined-outside-init,no-init def server_bind(self): # pylint: disable=missing-docstring - self.socket = SSLSocket(socket.socket(), certs=certs) + self.socket = SSLSocket(socket.socket(), + certs) socketserver.TCPServer.server_bind(self) self.server = _TestServer(('', 0), socketserver.BaseRequestHandler) @@ -73,6 +72,18 @@ class SSLSocketAndProbeSNITest(unittest.TestCase): socket.setdefaulttimeout(original_timeout) +class SSLSocketTest(unittest.TestCase): + """Tests for acme.crypto_util.SSLSocket.""" + + def test_ssl_socket_invalid_arguments(self): + from acme.crypto_util import SSLSocket + with self.assertRaises(ValueError): + _ = SSLSocket(None, {'sni': ('key', 'cert')}, + cert_selection=lambda _: None) + with self.assertRaises(ValueError): + _ = SSLSocket(None) + + class PyOpenSSLCertOrReqAllNamesTest(unittest.TestCase): """Test for acme.crypto_util._pyopenssl_cert_or_req_all_names.""" diff --git a/acme/tests/errors_test.py b/acme/tests/errors_test.py index 1e5f3d479..fb90a3f0d 100644 --- a/acme/tests/errors_test.py +++ b/acme/tests/errors_test.py @@ -1,7 +1,10 @@ """Tests for acme.errors.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore class BadNonceTest(unittest.TestCase): @@ -35,7 +38,7 @@ class PollErrorTest(unittest.TestCase): def setUp(self): from acme.errors import PollError self.timeout = PollError( - exhausted=set([mock.sentinel.AR]), + exhausted={mock.sentinel.AR}, updated={}) self.invalid = PollError(exhausted=set(), updated={ mock.sentinel.AR: mock.sentinel.AR2}) diff --git a/acme/tests/magic_typing_test.py b/acme/tests/magic_typing_test.py index 23dfe3367..9e4fd29f5 100644 --- a/acme/tests/magic_typing_test.py +++ b/acme/tests/magic_typing_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore class MagicTypingTest(unittest.TestCase): @@ -18,7 +21,7 @@ class MagicTypingTest(unittest.TestCase): sys.modules['typing'] = typing_class_mock if 'acme.magic_typing' in sys.modules: del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text # pylint: disable=no-name-in-module + from acme.magic_typing import Text self.assertEqual(Text, text_mock) del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing @@ -31,7 +34,7 @@ class MagicTypingTest(unittest.TestCase): sys.modules['typing'] = None if 'acme.magic_typing' in sys.modules: del sys.modules['acme.magic_typing'] # pragma: no cover - from acme.magic_typing import Text # pylint: disable=no-name-in-module + from acme.magic_typing import Text self.assertTrue(Text is None) del sys.modules['acme.magic_typing'] sys.modules['typing'] = temp_typing diff --git a/acme/tests/messages_test.py b/acme/tests/messages_test.py index b9b70266b..890a5f413 100644 --- a/acme/tests/messages_test.py +++ b/acme/tests/messages_test.py @@ -2,10 +2,12 @@ import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from acme import challenges -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module import test_util CERT = test_util.load_comparable_cert('cert.der') @@ -453,6 +455,7 @@ class OrderResourceTest(unittest.TestCase): 'authorizations': None, }) + class NewOrderTest(unittest.TestCase): """Tests for acme.messages.NewOrder.""" @@ -467,5 +470,18 @@ class NewOrderTest(unittest.TestCase): }) +class JWSPayloadRFC8555Compliant(unittest.TestCase): + """Test for RFC8555 compliance of JWS generated from resources/challenges""" + def test_message_payload(self): + from acme.messages import NewAuthorization + + new_order = NewAuthorization() + new_order.le_acme_version = 2 + + jobj = new_order.json_dumps(indent=2).encode() + # RFC8555 states that JWS bodies must not have a resource field. + self.assertEqual(jobj, b'{}') + + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/acme/tests/standalone_test.py b/acme/tests/standalone_test.py index 83ced12b0..d03b56535 100644 --- a/acme/tests/standalone_test.py +++ b/acme/tests/standalone_test.py @@ -4,13 +4,18 @@ import threading import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import requests from six.moves import http_client # pylint: disable=import-error from six.moves import socketserver # type: ignore # pylint: disable=import-error from acme import challenges -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme import crypto_util +from acme import errors + import test_util @@ -83,6 +88,81 @@ class HTTP01ServerTest(unittest.TestCase): def test_http01_not_found(self): self.assertFalse(self._test_http01(add=False)) + def test_timely_shutdown(self): + from acme.standalone import HTTP01Server + server = HTTP01Server(('', 0), resources=set(), timeout=0.05) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.start() + + client = socket.socket() + client.connect(('localhost', server.socket.getsockname()[1])) + + stop_thread = threading.Thread(target=server.shutdown) + stop_thread.start() + server_thread.join(5.) + + is_hung = server_thread.is_alive() + try: + client.shutdown(socket.SHUT_RDWR) + except: # pragma: no cover, pylint: disable=bare-except + # may raise error because socket could already be closed + pass + + self.assertFalse(is_hung, msg='Server shutdown should not be hung') + + +@unittest.skipIf(not challenges.TLSALPN01.is_supported(), "pyOpenSSL too old") +class TLSALPN01ServerTest(unittest.TestCase): + """Test for acme.standalone.TLSALPN01Server.""" + + def setUp(self): + self.certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa2048_key.pem'), + test_util.load_cert('rsa2048_cert.pem'), + )} + # Use different certificate for challenge. + self.challenge_certs = {b'localhost': ( + test_util.load_pyopenssl_private_key('rsa1024_key.pem'), + test_util.load_cert('rsa1024_cert.pem'), + )} + from acme.standalone import TLSALPN01Server + self.server = TLSALPN01Server(("localhost", 0), certs=self.certs, + challenge_certs=self.challenge_certs) + # pylint: disable=no-member + self.thread = threading.Thread(target=self.server.serve_forever) + self.thread.start() + + def tearDown(self): + self.server.shutdown() # pylint: disable=no-member + self.thread.join() + + # TODO: This is not implemented yet, see comments in standalone.py + # def test_certs(self): + # host, port = self.server.socket.getsockname()[:2] + # cert = crypto_util.probe_sni( + # b'localhost', host=host, port=port, timeout=1) + # # Expect normal cert when connecting without ALPN. + # self.assertEqual(jose.ComparableX509(cert), + # jose.ComparableX509(self.certs[b'localhost'][1])) + + def test_challenge_certs(self): + host, port = self.server.socket.getsockname()[:2] + cert = crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"acme-tls/1"]) + # Expect challenge cert when connecting with ALPN. + self.assertEqual( + jose.ComparableX509(cert), + jose.ComparableX509(self.challenge_certs[b'localhost'][1]) + ) + + def test_bad_alpn(self): + host, port = self.server.socket.getsockname()[:2] + with self.assertRaises(errors.Error): + crypto_util.probe_sni( + b'localhost', host=host, port=port, timeout=1, + alpn_protocols=[b"bad-alpn"]) + class BaseDualNetworkedServersTest(unittest.TestCase): """Test for acme.standalone.BaseDualNetworkedServers.""" @@ -138,7 +218,6 @@ class BaseDualNetworkedServersTest(unittest.TestCase): class HTTP01DualNetworkedServersTest(unittest.TestCase): """Tests for acme.standalone.HTTP01DualNetworkedServers.""" - def setUp(self): self.account_key = jose.JWK.load( test_util.load_vector('rsa1024_key.pem')) diff --git a/acme/tests/testdata/README b/acme/tests/testdata/README index dfe3f5405..d65cc3018 100644 --- a/acme/tests/testdata/README +++ b/acme/tests/testdata/README @@ -10,6 +10,8 @@ and for the CSR: openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -outform DER > csr.der -and for the certificate: +and for the certificates: - openssl req -key rsa2047_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 -outform DER > cert.der + openssl req -key rsa2048_key.pem -new -subj '/CN=example.com' -x509 > rsa2048_cert.pem + openssl req -key rsa1024_key.pem -new -subj '/CN=example.com' -x509 > rsa1024_cert.pem diff --git a/acme/tests/testdata/rsa1024_cert.pem b/acme/tests/testdata/rsa1024_cert.pem new file mode 100644 index 000000000..1b7912181 --- /dev/null +++ b/acme/tests/testdata/rsa1024_cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB/TCCAWagAwIBAgIJAOyRIBs3QT8QMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV +BAMMC2V4YW1wbGUuY29tMB4XDTE4MDQyMzEwMzE0NFoXDTE4MDUyMzEwMzE0NFow +FjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJ +AoGBAJqJ87R8aVwByONxgQA9hwgvQd/QqI1r1UInXhEF2VnEtZGtUWLi100IpIqr +Mq4qusDwNZ3g8cUPtSkvJGs89djoajMDIJP7lQUEKUYnYrI0q755Tr/DgLWSk7iW +l5ezym0VzWUD0/xXUz8yRbNMTjTac80rS5SZk2ja2wWkYlRJAgMBAAGjUzBRMB0G +A1UdDgQWBBSsaX0IVZ4XXwdeffVAbG7gnxSYjTAfBgNVHSMEGDAWgBSsaX0IVZ4X +XwdeffVAbG7gnxSYjTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB +ADe7SVmvGH2nkwVfONk8TauRUDkePN1CJZKFb2zW1uO9ANJ2v5Arm/OQp0BG/xnI +Djw/aLTNVESF89oe15dkrUErtcaF413MC1Ld5lTCaJLHLGqDKY69e02YwRuxW7jY +qarpt7k7aR5FbcfO5r4V/FK/Gvp4Dmoky8uap7SJIW6x +-----END CERTIFICATE----- diff --git a/certbot-apache/MANIFEST.in b/certbot-apache/MANIFEST.in index 2316983bb..9e4913a03 100644 --- a/certbot-apache/MANIFEST.in +++ b/certbot-apache/MANIFEST.in @@ -1,7 +1,7 @@ include LICENSE.txt include README.rst recursive-include tests * -include certbot_apache/_internal/options-ssl-apache.conf recursive-include certbot_apache/_internal/augeas_lens *.aug +recursive-include certbot_apache/_internal/tls_configs *.conf global-exclude __pycache__ global-exclude *.py[cod] diff --git a/certbot-apache/certbot_apache/_internal/apache_util.py b/certbot-apache/certbot_apache/_internal/apache_util.py index 085ccddc8..862685027 100644 --- a/certbot-apache/certbot_apache/_internal/apache_util.py +++ b/certbot-apache/certbot_apache/_internal/apache_util.py @@ -5,6 +5,8 @@ import logging import re import subprocess +import pkg_resources + from certbot import errors from certbot import util @@ -128,7 +130,7 @@ def included_in_paths(filepath, paths): :rtype: bool """ - return any([fnmatch.fnmatch(filepath, path) for path in paths]) + return any(fnmatch.fnmatch(filepath, path) for path in paths) def parse_defines(apachectl): @@ -142,7 +144,7 @@ def parse_defines(apachectl): :rtype: dict """ - variables = dict() + variables = {} define_cmd = [apachectl, "-t", "-D", "DUMP_RUN_CFG"] matches = parse_from_subprocess(define_cmd, r"Define: ([^ \n]*)") @@ -241,3 +243,14 @@ def _get_runtime_cfg(command): "loaded because Apache is misconfigured.") return stdout + +def find_ssl_apache_conf(prefix): + """ + Find a TLS Apache config file in the dedicated storage. + :param str prefix: prefix of the TLS Apache config file to find + :return: the path the TLS Apache config file + :rtype: str + """ + return pkg_resources.resource_filename( + "certbot_apache", + os.path.join("_internal", "tls_configs", "{0}-options-ssl-apache.conf".format(prefix))) diff --git a/certbot-apache/certbot_apache/_internal/apacheparser.py b/certbot-apache/certbot_apache/_internal/apacheparser.py index 77f4517fe..c7b723ae6 100644 --- a/certbot-apache/certbot_apache/_internal/apacheparser.py +++ b/certbot-apache/certbot_apache/_internal/apacheparser.py @@ -73,7 +73,7 @@ class ApacheDirectiveNode(ApacheParserNode): self.metadata == other.metadata) return False - def set_parameters(self, _parameters): + def set_parameters(self, _parameters): # pragma: no cover """Sets the parameters for DirectiveNode""" return @@ -97,7 +97,8 @@ class ApacheBlockNode(ApacheDirectiveNode): self.metadata == other.metadata) return False - def add_child_block(self, name, parameters=None, position=None): # pylint: disable=unused-argument + # pylint: disable=unused-argument + def add_child_block(self, name, parameters=None, position=None): # pragma: no cover """Adds a new BlockNode to the sequence of children""" new_block = ApacheBlockNode(name=assertions.PASS, parameters=assertions.PASS, @@ -107,7 +108,8 @@ class ApacheBlockNode(ApacheDirectiveNode): self.children += (new_block,) return new_block - def add_child_directive(self, name, parameters=None, position=None): # pylint: disable=unused-argument + # pylint: disable=unused-argument + def add_child_directive(self, name, parameters=None, position=None): # pragma: no cover """Adds a new DirectiveNode to the sequence of children""" new_dir = ApacheDirectiveNode(name=assertions.PASS, parameters=assertions.PASS, @@ -144,7 +146,8 @@ class ApacheBlockNode(ApacheDirectiveNode): filepath=assertions.PASS, metadata=self.metadata)] - def find_comments(self, comment, exact=False): # pylint: disable=unused-argument + # pylint: disable=unused-argument + def find_comments(self, comment, exact=False): # pragma: no cover """Recursive search of DirectiveNodes from the sequence of children""" return [ApacheCommentNode(comment=assertions.PASS, ancestor=self, diff --git a/certbot-apache/certbot_apache/_internal/assertions.py b/certbot-apache/certbot_apache/_internal/assertions.py index e1b4cdcc8..2b2ce4f50 100644 --- a/certbot-apache/certbot_apache/_internal/assertions.py +++ b/certbot-apache/certbot_apache/_internal/assertions.py @@ -132,9 +132,9 @@ def assertEqualPathsList(first, second): # pragma: no cover Checks that the two lists of file paths match. This assertion allows for wildcard paths. """ - if any([isPass(path) for path in first]): + if any(isPass(path) for path in first): return - if any([isPass(path) for path in second]): + if any(isPass(path) for path in second): return for fpath in first: assert any([fnmatch.fnmatch(fpath, spath) for spath in second]) diff --git a/certbot-apache/certbot_apache/_internal/augeasparser.py b/certbot-apache/certbot_apache/_internal/augeasparser.py index e1d7c941d..3b2ce40d8 100644 --- a/certbot-apache/certbot_apache/_internal/augeasparser.py +++ b/certbot-apache/certbot_apache/_internal/augeasparser.py @@ -64,7 +64,7 @@ Translates over to: "/files/etc/apache2/apache2.conf/bLoCk[1]", ] """ -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module +from acme.magic_typing import Set from certbot import errors from certbot.compat import os @@ -339,7 +339,7 @@ class AugeasBlockNode(AugeasDirectiveNode): def find_blocks(self, name, exclude=True): """Recursive search of BlockNodes from the sequence of children""" - nodes = list() + nodes = [] paths = self._aug_find_blocks(name) if exclude: paths = self.parser.exclude_dirs(paths) @@ -351,7 +351,7 @@ class AugeasBlockNode(AugeasDirectiveNode): def find_directives(self, name, exclude=True): """Recursive search of DirectiveNodes from the sequence of children""" - nodes = list() + nodes = [] ownpath = self.metadata.get("augeaspath") directives = self.parser.find_dir(name, start=ownpath, exclude=exclude) @@ -374,7 +374,7 @@ class AugeasBlockNode(AugeasDirectiveNode): :param str comment: Comment content to search for. """ - nodes = list() + nodes = [] ownpath = self.metadata.get("augeaspath") comments = self.parser.find_comments(comment, start=ownpath) diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py index 8daa28173..dbfc15468 100644 --- a/certbot-apache/certbot_apache/_internal/configurator.py +++ b/certbot-apache/certbot_apache/_internal/configurator.py @@ -1,6 +1,7 @@ """Apache Configurator.""" # pylint: disable=too-many-lines from collections import defaultdict +from distutils.version import LooseVersion import copy import fnmatch import logging @@ -8,10 +9,14 @@ import re import socket import time -import pkg_resources import six import zope.component import zope.interface +try: + import apacheconfig + HAS_APACHECONFIG = True +except ImportError: # pragma: no cover + HAS_APACHECONFIG = False from acme import challenges from acme.magic_typing import DefaultDict @@ -110,14 +115,29 @@ class ApacheConfigurator(common.Installer): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def option(self, key): """Get a value from options""" return self.options.get(key) + def pick_apache_config(self, warn_on_no_mod_ssl=True): + """ + Pick the appropriate TLS Apache configuration file for current version of Apache and OS. + + :param bool warn_on_no_mod_ssl: True if we should warn if mod_ssl is not found. + + :return: the path to the TLS Apache configuration file to use + :rtype: str + """ + # Disabling TLS session tickets is supported by Apache 2.4.11+ and OpenSSL 1.0.2l+. + # So for old versions of Apache we pick a configuration without this option. + openssl_version = self.openssl_version(warn_on_no_mod_ssl) + if self.version < (2, 4, 11) or not openssl_version or\ + LooseVersion(openssl_version) < LooseVersion('1.0.2l'): + return apache_util.find_ssl_apache_conf("old") + return apache_util.find_ssl_apache_conf("current") + def _prepare_options(self): """ Set the values possibly changed by command line parameters to @@ -184,15 +204,16 @@ class ApacheConfigurator(common.Installer): """ version = kwargs.pop("version", None) use_parsernode = kwargs.pop("use_parsernode", False) + openssl_version = kwargs.pop("openssl_version", None) super(ApacheConfigurator, self).__init__(*args, **kwargs) # Add name_server association dict - self.assoc = dict() # type: Dict[str, obj.VirtualHost] + self.assoc = {} # type: Dict[str, obj.VirtualHost] # Outstanding challenges self._chall_out = set() # type: Set[KeyAuthorizationAnnotatedChallenge] # List of vhosts configured per wildcard domain on this run. # used by deploy_cert() and enhance() - self._wildcard_vhosts = dict() # type: Dict[str, List[obj.VirtualHost]] + self._wildcard_vhosts = {} # type: Dict[str, List[obj.VirtualHost]] # Maps enhancements to vhosts we've enabled the enhancement for self._enhanced_vhosts = defaultdict(set) # type: DefaultDict[str, Set[obj.VirtualHost]] # Temporary state for AutoHSTS enhancement @@ -209,6 +230,7 @@ class ApacheConfigurator(common.Installer): self.parser = None self.parser_root = None self.version = version + self._openssl_version = openssl_version self.vhosts = None self.options = copy.deepcopy(self.OS_DEFAULTS) self._enhance_func = {"redirect": self._enable_redirect, @@ -225,6 +247,52 @@ class ApacheConfigurator(common.Installer): """Full absolute path to digest of updated SSL configuration file.""" return os.path.join(self.config.config_dir, constants.UPDATED_MOD_SSL_CONF_DIGEST) + def _open_module_file(self, ssl_module_location): + """Extract the open lines of openssl_version for testing purposes""" + try: + with open(ssl_module_location, mode="rb") as f: + contents = f.read() + except IOError as error: + logger.debug(str(error), exc_info=True) + return None + return contents + + def openssl_version(self, warn_on_no_mod_ssl=True): + """Lazily retrieve openssl version + + :param bool warn_on_no_mod_ssl: `True` if we should warn if mod_ssl is not found. Set to + `False` when we know we'll try to enable mod_ssl later. This is currently debian/ubuntu, + when called from `prepare`. + + :return: the OpenSSL version as a string, or None. + :rtype: str or None + """ + if self._openssl_version: + return self._openssl_version + # Step 1. Check for LoadModule directive + try: + ssl_module_location = self.parser.modules['ssl_module'] + except KeyError: + if warn_on_no_mod_ssl: + logger.warning("Could not find ssl_module; not disabling session tickets.") + return None + if not ssl_module_location: + logger.warning("Could not find ssl_module; not disabling session tickets.") + return None + ssl_module_location = self.parser.standard_path_from_server_root(ssl_module_location) + # Step 2. Grep in the .so for openssl version + contents = self._open_module_file(ssl_module_location) + if not contents: + logger.warning("Unable to read ssl_module file; not disabling session tickets.") + return None + # looks like: OpenSSL 1.0.2s 28 May 2019 + matches = re.findall(br"OpenSSL ([0-9]\.[^ ]+) ", contents) + if not matches: + logger.warning("Could not find OpenSSL version; not disabling session tickets.") + return None + self._openssl_version = matches[0].decode('UTF-8') + return self._openssl_version + def prepare(self): """Prepare the authenticator/installer. @@ -271,8 +339,12 @@ class ApacheConfigurator(common.Installer): # Get all of the available vhosts self.vhosts = self.get_virtual_hosts() + # We may try to enable mod_ssl later. If so, we shouldn't warn if we can't find it now. + # This is currently only true for debian/ubuntu. + warn_on_no_mod_ssl = not self.option("handle_modules") self.install_ssl_options_conf(self.mod_ssl_conf, - self.updated_mod_ssl_conf_digest) + self.updated_mod_ssl_conf_digest, + warn_on_no_mod_ssl) # Prevent two Apache plugins from modifying a config at once try: @@ -363,11 +435,17 @@ class ApacheConfigurator(common.Installer): def get_parsernode_root(self, metadata): """Initializes the ParserNode parser root instance.""" - apache_vars = dict() - apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) - apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) - apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) - metadata["apache_vars"] = apache_vars + if HAS_APACHECONFIG: + apache_vars = {} + apache_vars["defines"] = apache_util.parse_defines(self.option("ctl")) + apache_vars["includes"] = apache_util.parse_includes(self.option("ctl")) + apache_vars["modules"] = apache_util.parse_modules(self.option("ctl")) + metadata["apache_vars"] = apache_vars + + with open(self.parser.loc["root"]) as f: + with apacheconfig.make_loader(writable=True, + **apacheconfig.flavors.NATIVE_APACHE) as loader: + metadata["ac_ast"] = loader.loads(f.read()) return dualparser.DualBlockNode( name=assertions.PASS, @@ -470,7 +548,7 @@ class ApacheConfigurator(common.Installer): # Go through the vhosts, making sure that we cover all the names # present, but preferring the SSL vhosts - filtered_vhosts = dict() + filtered_vhosts = {} for vhost in vhosts: for name in vhost.get_names(): if vhost.ssl: @@ -496,7 +574,7 @@ class ApacheConfigurator(common.Installer): # Make sure we create SSL vhosts for the ones that are HTTP only # if requested. - return_vhosts = list() + return_vhosts = [] for vhost in dialog_output: if not vhost.ssl: return_vhosts.append(self.make_vhost_ssl(vhost)) @@ -907,7 +985,7 @@ class ApacheConfigurator(common.Installer): """ v1_vhosts = self.get_virtual_hosts_v1() - if self.USE_PARSERNODE: + if self.USE_PARSERNODE and HAS_APACHECONFIG: v2_vhosts = self.get_virtual_hosts_v2() for v1_vh in v1_vhosts: @@ -1241,6 +1319,14 @@ class ApacheConfigurator(common.Installer): self.enable_mod("socache_shmcb", temp=temp) if "ssl_module" not in self.parser.modules: self.enable_mod("ssl", temp=temp) + # Make sure we're not throwing away any unwritten changes to the config + self.parser.ensure_augeas_state() + self.parser.aug.load() + self.parser.reset_modules() # Reset to load the new ssl_module path + # Call again because now we can gate on openssl version + self.install_ssl_options_conf(self.mod_ssl_conf, + self.updated_mod_ssl_conf_digest, + warn_on_no_mod_ssl=True) def make_vhost_ssl(self, nonssl_vhost): """Makes an ssl_vhost version of a nonssl_vhost. @@ -1493,7 +1579,7 @@ class ApacheConfigurator(common.Installer): result.append(comment) sift = True - result.append('\n'.join(['# ' + l for l in chunk])) + result.append('\n'.join('# ' + l for l in chunk)) else: result.append('\n'.join(chunk)) return result, sift @@ -1633,7 +1719,7 @@ class ApacheConfigurator(common.Installer): for addr in vhost.addrs: # In Apache 2.2, when a NameVirtualHost directive is not # set, "*" and "_default_" will conflict when sharing a port - addrs = set((addr,)) + addrs = {addr,} if addr.get_addr() in ("*", "_default_"): addrs.update(obj.Addr((a, addr.get_port(),)) for a in ("*", "_default_")) @@ -1824,7 +1910,7 @@ class ApacheConfigurator(common.Installer): try: self._autohsts = self.storage.fetch("autohsts") except KeyError: - self._autohsts = dict() + self._autohsts = {} def _autohsts_save_state(self): """ @@ -2385,7 +2471,7 @@ class ApacheConfigurator(common.Installer): if len(matches) != 1: raise errors.PluginError("Unable to find Apache version") - return tuple([int(i) for i in matches[0].split(".")]) + return tuple(int(i) for i in matches[0].split(".")) def more_info(self): """Human-readable string to help understand the module""" @@ -2454,14 +2540,19 @@ class ApacheConfigurator(common.Installer): self.restart() self.parser.reset_modules() - def install_ssl_options_conf(self, options_ssl, options_ssl_digest): - """Copy Certbot's SSL options file into the system's config dir if required.""" + def install_ssl_options_conf(self, options_ssl, options_ssl_digest, warn_on_no_mod_ssl=True): + """Copy Certbot's SSL options file into the system's config dir if required. + + :param bool warn_on_no_mod_ssl: True if we should warn if mod_ssl is not found. + """ # XXX if we ever try to enforce a local privilege boundary (eg, running # certbot for unprivileged users via setuid), this function will need # to be modified. - return common.install_version_controlled_file(options_ssl, options_ssl_digest, - self.option("MOD_SSL_CONF_SRC"), constants.ALL_SSL_OPTIONS_HASHES) + apache_config_path = self.pick_apache_config(warn_on_no_mod_ssl) + + return common.install_version_controlled_file( + options_ssl, options_ssl_digest, apache_config_path, constants.ALL_SSL_OPTIONS_HASHES) def enable_autohsts(self, _unused_lineage, domains): """ diff --git a/certbot-apache/certbot_apache/_internal/constants.py b/certbot-apache/certbot_apache/_internal/constants.py index a37bebac5..68d29406e 100644 --- a/certbot-apache/certbot_apache/_internal/constants.py +++ b/certbot-apache/certbot_apache/_internal/constants.py @@ -26,6 +26,7 @@ ALL_SSL_OPTIONS_HASHES = [ '06675349e457eae856120cdebb564efe546f0b87399f2264baeb41e442c724c7', '5cc003edd93fb9cd03d40c7686495f8f058f485f75b5e764b789245a386e6daf', '007cd497a56a3bb8b6a2c1aeb4997789e7e38992f74e44cc5d13a625a738ac73', + '34783b9e2210f5c4a23bced2dfd7ec289834716673354ed7c7abf69fe30192a3', ] """SHA256 hashes of the contents of previous versions of all versions of MOD_SSL_CONF_SRC""" diff --git a/certbot-apache/certbot_apache/_internal/display_ops.py b/certbot-apache/certbot_apache/_internal/display_ops.py index 1ae32bb47..dabf20606 100644 --- a/certbot-apache/certbot_apache/_internal/display_ops.py +++ b/certbot-apache/certbot_apache/_internal/display_ops.py @@ -21,7 +21,7 @@ def select_vhost_multiple(vhosts): :rtype: :class:`list`of type `~obj.Vhost` """ if not vhosts: - return list() + return [] tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] # Remove the extra newline from the last entry if tags_list: @@ -37,7 +37,7 @@ def select_vhost_multiple(vhosts): def _reversemap_vhosts(names, vhosts): """Helper function for select_vhost_multiple for mapping string representations back to actual vhost objects""" - return_vhosts = list() + return_vhosts = [] for selection in names: for vhost in vhosts: diff --git a/certbot-apache/certbot_apache/_internal/dualparser.py b/certbot-apache/certbot_apache/_internal/dualparser.py index aa66cf84c..eef8f2a0e 100644 --- a/certbot-apache/certbot_apache/_internal/dualparser.py +++ b/certbot-apache/certbot_apache/_internal/dualparser.py @@ -49,7 +49,7 @@ class DualNodeBase(object): pass_primary = assertions.isPassNodeList(primary_res) pass_secondary = assertions.isPassNodeList(secondary_res) - new_nodes = list() + new_nodes = [] if pass_primary and pass_secondary: # Both unimplemented @@ -221,7 +221,7 @@ class DualBlockNode(DualNodeBase): implementations to a list of tuples. """ - matched = list() + matched = [] for p in primary_list: match = None for s in secondary_list: diff --git a/certbot-apache/certbot_apache/_internal/entrypoint.py b/certbot-apache/certbot_apache/_internal/entrypoint.py index e31e1f4eb..79337b381 100644 --- a/certbot-apache/certbot_apache/_internal/entrypoint.py +++ b/certbot-apache/certbot_apache/_internal/entrypoint.py @@ -1,6 +1,4 @@ """ Entry point for Apache Plugin """ -# Pylint does not like disutils.version when running inside a venv. -# See: https://github.com/PyCQA/pylint/issues/73 from distutils.version import LooseVersion from certbot import util diff --git a/certbot-apache/certbot_apache/_internal/http_01.py b/certbot-apache/certbot_apache/_internal/http_01.py index 5ea0ce8ec..ce64d451e 100644 --- a/certbot-apache/certbot_apache/_internal/http_01.py +++ b/certbot-apache/certbot_apache/_internal/http_01.py @@ -1,5 +1,6 @@ """A class that performs HTTP-01 challenges for Apache""" import logging +import errno from acme.magic_typing import List from acme.magic_typing import Set @@ -168,7 +169,15 @@ class ApacheHttp01(common.ChallengePerformer): def _set_up_challenges(self): if not os.path.isdir(self.challenge_dir): - filesystem.makedirs(self.challenge_dir, 0o755) + old_umask = os.umask(0o022) + try: + filesystem.makedirs(self.challenge_dir, 0o755) + except OSError as exception: + if exception.errno not in (errno.EEXIST, errno.EISDIR): + raise errors.PluginError( + "Couldn't create root for http-01 challenge") + finally: + os.umask(old_umask) responses = [] for achall in self.achalls: diff --git a/certbot-apache/certbot_apache/_internal/interfaces.py b/certbot-apache/certbot_apache/_internal/interfaces.py index 1b67be5c8..647790c41 100644 --- a/certbot-apache/certbot_apache/_internal/interfaces.py +++ b/certbot-apache/certbot_apache/_internal/interfaces.py @@ -102,7 +102,6 @@ For this reason the internal representation of data should not ignore the case. import abc import six -from acme.magic_typing import Any, Dict, Optional, Tuple # pylint: disable=unused-import, no-name-in-module @six.add_metaclass(abc.ABCMeta) diff --git a/certbot-apache/certbot_apache/_internal/override_arch.py b/certbot-apache/certbot_apache/_internal/override_arch.py index 2765bd238..54202c087 100644 --- a/certbot-apache/certbot_apache/_internal/override_arch.py +++ b/certbot-apache/certbot_apache/_internal/override_arch.py @@ -1,9 +1,7 @@ """ Distribution specific override class for Arch Linux """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import configurator @@ -26,6 +24,4 @@ class ArchConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py index 2ab160c2f..0de882519 100644 --- a/certbot-apache/certbot_apache/_internal/override_centos.py +++ b/certbot-apache/certbot_apache/_internal/override_centos.py @@ -1,14 +1,12 @@ """ Distribution specific override class for CentOS family (RHEL, Fedora) """ import logging -import pkg_resources import zope.interface from acme.magic_typing import List from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import os from certbot.errors import MisconfigurationError from certbot_apache._internal import apache_util from certbot_apache._internal import configurator @@ -37,8 +35,6 @@ class CentOSConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def config_test(self): diff --git a/certbot-apache/certbot_apache/_internal/override_darwin.py b/certbot-apache/certbot_apache/_internal/override_darwin.py index 00faff623..f19823866 100644 --- a/certbot-apache/certbot_apache/_internal/override_darwin.py +++ b/certbot-apache/certbot_apache/_internal/override_darwin.py @@ -1,9 +1,7 @@ """ Distribution specific override class for macOS """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import configurator @@ -26,6 +24,4 @@ class DarwinConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/other", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) diff --git a/certbot-apache/certbot_apache/_internal/override_debian.py b/certbot-apache/certbot_apache/_internal/override_debian.py index 77ced6a3f..c977fb43e 100644 --- a/certbot-apache/certbot_apache/_internal/override_debian.py +++ b/certbot-apache/certbot_apache/_internal/override_debian.py @@ -1,7 +1,6 @@ """ Distribution specific override class for Debian family (Ubuntu/Debian) """ import logging -import pkg_resources import zope.interface from certbot import errors @@ -34,8 +33,6 @@ class DebianConfigurator(configurator.ApacheConfigurator): handle_modules=True, handle_sites=True, challenge_location="/etc/apache2", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def enable_site(self, vhost): diff --git a/certbot-apache/certbot_apache/_internal/override_fedora.py b/certbot-apache/certbot_apache/_internal/override_fedora.py index 8197b0dcd..2436c76cc 100644 --- a/certbot-apache/certbot_apache/_internal/override_fedora.py +++ b/certbot-apache/certbot_apache/_internal/override_fedora.py @@ -1,11 +1,9 @@ """ Distribution specific override class for Fedora 29+ """ -import pkg_resources import zope.interface from certbot import errors from certbot import interfaces from certbot import util -from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser @@ -31,9 +29,6 @@ class FedoraConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/httpd/conf.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - # TODO: eventually newest version of Fedora will need their own config - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def config_test(self): diff --git a/certbot-apache/certbot_apache/_internal/override_gentoo.py b/certbot-apache/certbot_apache/_internal/override_gentoo.py index c215771e6..6b7416c0d 100644 --- a/certbot-apache/certbot_apache/_internal/override_gentoo.py +++ b/certbot-apache/certbot_apache/_internal/override_gentoo.py @@ -1,9 +1,7 @@ """ Distribution specific override class for Gentoo Linux """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import apache_util from certbot_apache._internal import configurator from certbot_apache._internal import parser @@ -29,8 +27,6 @@ class GentooConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) def _prepare_options(self): diff --git a/certbot-apache/certbot_apache/_internal/override_suse.py b/certbot-apache/certbot_apache/_internal/override_suse.py index 0c9219e6d..895e0cb05 100644 --- a/certbot-apache/certbot_apache/_internal/override_suse.py +++ b/certbot-apache/certbot_apache/_internal/override_suse.py @@ -1,9 +1,7 @@ """ Distribution specific override class for OpenSUSE """ -import pkg_resources import zope.interface from certbot import interfaces -from certbot.compat import os from certbot_apache._internal import configurator @@ -26,6 +24,4 @@ class OpenSUSEConfigurator(configurator.ApacheConfigurator): handle_modules=False, handle_sites=False, challenge_location="/etc/apache2/vhosts.d", - MOD_SSL_CONF_SRC=pkg_resources.resource_filename( - "certbot_apache", os.path.join("_internal", "options-ssl-apache.conf")) ) diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py index 992672913..c9aebae54 100644 --- a/certbot-apache/certbot_apache/_internal/parser.py +++ b/certbot-apache/certbot_apache/_internal/parser.py @@ -9,7 +9,6 @@ import six from acme.magic_typing import Dict from acme.magic_typing import List -from acme.magic_typing import Set from certbot import errors from certbot.compat import os from certbot_apache._internal import apache_util @@ -31,7 +30,7 @@ class ApacheParser(object): """ arg_var_interpreter = re.compile(r"\$\{[^ \}]*}") - fnmatch_chars = set(["*", "?", "\\", "[", "]"]) + fnmatch_chars = {"*", "?", "\\", "[", "]"} def __init__(self, root, vhostroot=None, version=(2, 4), configurator=None): @@ -52,7 +51,7 @@ class ApacheParser(object): "version 1.2.0 or higher, please make sure you have you have " "those installed.") - self.modules = set() # type: Set[str] + self.modules = {} # type: Dict[str, str] self.parser_paths = {} # type: Dict[str, List[str]] self.variables = {} # type: Dict[str, str] @@ -249,14 +248,14 @@ class ApacheParser(object): def add_mod(self, mod_name): """Shortcut for updating parser modules.""" if mod_name + "_module" not in self.modules: - self.modules.add(mod_name + "_module") + self.modules[mod_name + "_module"] = None if "mod_" + mod_name + ".c" not in self.modules: - self.modules.add("mod_" + mod_name + ".c") + self.modules["mod_" + mod_name + ".c"] = None def reset_modules(self): """Reset the loaded modules list. This is called from cleanup to clear temporarily loaded modules.""" - self.modules = set() + self.modules = {} self.update_modules() self.parse_modules() @@ -267,7 +266,7 @@ class ApacheParser(object): the iteration issue. Else... parse and enable mods at same time. """ - mods = set() # type: Set[str] + mods = {} # type: Dict[str, str] matches = self.find_dir("LoadModule") iterator = iter(matches) # Make sure prev_size != cur_size for do: while: iteration @@ -281,8 +280,8 @@ class ApacheParser(object): mod_name = self.get_arg(match_name) mod_filename = self.get_arg(match_filename) if mod_name and mod_filename: - mods.add(mod_name) - mods.add(os.path.basename(mod_filename)[:-2] + "c") + mods[mod_name] = mod_filename + mods[os.path.basename(mod_filename)[:-2] + "c"] = mod_filename else: logger.debug("Could not read LoadModule directive from Augeas path: %s", match_name[6:]) @@ -621,7 +620,7 @@ class ApacheParser(object): def exclude_dirs(self, matches): """Exclude directives that are not loaded into the configuration.""" - filters = [("ifmodule", self.modules), ("ifdefine", self.variables)] + filters = [("ifmodule", self.modules.keys()), ("ifdefine", self.variables)] valid_matches = [] @@ -662,6 +661,25 @@ class ApacheParser(object): return True + def standard_path_from_server_root(self, arg): + """Ensure paths are consistent and absolute + + :param str arg: Argument of directive + + :returns: Standardized argument path + :rtype: str + """ + # Remove beginning and ending quotes + arg = arg.strip("'\"") + + # Standardize the include argument based on server root + if not arg.startswith("/"): + # Normpath will condense ../ + arg = os.path.normpath(os.path.join(self.root, arg)) + else: + arg = os.path.normpath(arg) + return arg + def _get_include_path(self, arg): """Converts an Apache Include directive into Augeas path. @@ -682,16 +700,7 @@ class ApacheParser(object): # if matchObj.group() != arg: # logger.error("Error: Invalid regexp characters in %s", arg) # return [] - - # Remove beginning and ending quotes - arg = arg.strip("'\"") - - # Standardize the include argument based on server root - if not arg.startswith("/"): - # Normpath will condense ../ - arg = os.path.normpath(os.path.join(self.root, arg)) - else: - arg = os.path.normpath(arg) + arg = self.standard_path_from_server_root(arg) # Attempts to add a transform to the file if one does not already exist if os.path.isdir(arg): @@ -732,7 +741,7 @@ class ApacheParser(object): """ if sys.version_info < (3, 6): # This strips off final /Z(?ms) - return fnmatch.translate(clean_fn_match)[:-7] + return fnmatch.translate(clean_fn_match)[:-7] # pragma: no cover # Since Python 3.6, it returns a different pattern like (?s:.*\.load)\Z return fnmatch.translate(clean_fn_match)[4:-3] # pragma: no cover @@ -936,8 +945,8 @@ def case_i(string): :param str string: string to make case i regex """ - return "".join(["[" + c.upper() + c.lower() + "]" - if c.isalpha() else c for c in re.escape(string)]) + return "".join("[" + c.upper() + c.lower() + "]" + if c.isalpha() else c for c in re.escape(string)) def get_aug_path(file_path): diff --git a/certbot-apache/certbot_apache/_internal/parsernode_util.py b/certbot-apache/certbot_apache/_internal/parsernode_util.py index d9646862a..0e39ec365 100644 --- a/certbot-apache/certbot_apache/_internal/parsernode_util.py +++ b/certbot-apache/certbot_apache/_internal/parsernode_util.py @@ -11,7 +11,7 @@ def validate_kwargs(kwargs, required_names): :param list required_names: List of required parameter names. """ - validated_kwargs = dict() + validated_kwargs = {} for name in required_names: try: validated_kwargs[name] = kwargs.pop(name) diff --git a/certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf b/certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf new file mode 100644 index 000000000..32a2c3335 --- /dev/null +++ b/certbot-apache/certbot_apache/_internal/tls_configs/current-options-ssl-apache.conf @@ -0,0 +1,19 @@ +# This file contains important security parameters. If you modify this file +# manually, Certbot will be unable to automatically provide future security +# updates. Instead, Certbot will print and log an error message with a path to +# the up-to-date file that you will need to refer to when manually updating +# this file. + +SSLEngine on + +# Intermediate configuration, tweak to your needs +SSLProtocol all -SSLv2 -SSLv3 -TLSv1 -TLSv1.1 +SSLCipherSuite ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 +SSLHonorCipherOrder off +SSLSessionTickets off + +SSLOptions +StrictRequire + +# Add vhost name to log entries: +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" vhost_combined +LogFormat "%v %h %l %u %t \"%r\" %>s %b" vhost_common diff --git a/certbot-apache/certbot_apache/_internal/options-ssl-apache.conf b/certbot-apache/certbot_apache/_internal/tls_configs/old-options-ssl-apache.conf similarity index 100% rename from certbot-apache/certbot_apache/_internal/options-ssl-apache.conf rename to certbot-apache/certbot_apache/_internal/tls_configs/old-options-ssl-apache.conf diff --git a/certbot-apache/local-oldest-requirements.txt b/certbot-apache/local-oldest-requirements.txt index cf61c15a5..e45a0f831 100644 --- a/certbot-apache/local-oldest-requirements.txt +++ b/certbot-apache/local-oldest-requirements.txt @@ -1,3 +1,3 @@ # Remember to update setup.py to match the package versions below. acme[dev]==0.29.0 -certbot[dev]==1.1.0 +certbot[dev]==1.1.0 \ No newline at end of file diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index b37ee3972..25fb920c2 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -1,25 +1,35 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', - 'mock', 'python-augeas', 'setuptools', 'zope.component', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ - 'apacheconfig>=0.3.1', + 'apacheconfig>=0.3.2', ] class PyTest(TestCommand): diff --git a/certbot-apache/tests/augeasnode_test.py b/certbot-apache/tests/augeasnode_test.py index 9d663a05f..abe72a5d0 100644 --- a/certbot-apache/tests/augeasnode_test.py +++ b/certbot-apache/tests/augeasnode_test.py @@ -1,14 +1,27 @@ """Tests for AugeasParserNode classes""" -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore +import os import util -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot_apache._internal import assertions +from certbot_apache._internal import augeasparser +def _get_augeasnode_mock(filepath): + """ Helper function for mocking out DualNode instance with an AugeasNode """ + def augeasnode_mock(metadata): + return augeasparser.AugeasBlockNode( + name=assertions.PASS, + ancestor=None, + filepath=filepath, + metadata=metadata) + return augeasnode_mock class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public-methods """Test AugeasParserNode using available test configurations""" @@ -16,8 +29,11 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- def setUp(self): # pylint: disable=arguments-differ super(AugeasParserNodeTest, self).setUp() - self.config = util.get_apache_configurator( - self.config_path, self.vhost_path, self.config_dir, self.work_dir, use_parsernode=True) + with mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.get_parsernode_root") as mock_parsernode: + mock_parsernode.side_effect = _get_augeasnode_mock( + os.path.join(self.config_path, "apache2.conf")) + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir, use_parsernode=True) self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") @@ -111,7 +127,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- name=servernames[0].name, parameters=["test", "setting", "these"], ancestor=assertions.PASS, - metadata=servernames[0].primary.metadata + metadata=servernames[0].metadata ) self.assertTrue(mock_set.called) self.assertEqual( @@ -146,26 +162,26 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.assertTrue(found) def test_add_child_comment(self): - newc = self.config.parser_root.primary.add_child_comment("The content") + newc = self.config.parser_root.add_child_comment("The content") comments = self.config.parser_root.find_comments("The content") self.assertEqual(len(comments), 1) self.assertEqual( newc.metadata["augeaspath"], - comments[0].primary.metadata["augeaspath"] + comments[0].metadata["augeaspath"] ) self.assertEqual(newc.comment, comments[0].comment) def test_delete_child(self): - listens = self.config.parser_root.primary.find_directives("Listen") + listens = self.config.parser_root.find_directives("Listen") self.assertEqual(len(listens), 1) - self.config.parser_root.primary.delete_child(listens[0]) + self.config.parser_root.delete_child(listens[0]) - listens = self.config.parser_root.primary.find_directives("Listen") + listens = self.config.parser_root.find_directives("Listen") self.assertEqual(len(listens), 0) def test_delete_child_not_found(self): listen = self.config.parser_root.find_directives("Listen")[0] - listen.primary.metadata["augeaspath"] = "/files/something/nonexistent" + listen.metadata["augeaspath"] = "/files/something/nonexistent" self.assertRaises( errors.PluginError, @@ -178,10 +194,10 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "NewBlock", ["first", "second"] ) - rpath, _, directive = nb.primary.metadata["augeaspath"].rpartition("/") + rpath, _, directive = nb.metadata["augeaspath"].rpartition("/") self.assertEqual( rpath, - self.config.parser_root.primary.metadata["augeaspath"] + self.config.parser_root.metadata["augeaspath"] ) self.assertTrue(directive.startswith("NewBlock")) @@ -190,8 +206,8 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "Beginning", position=0 ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Get first child first = parser.aug.match("{}/*[1]".format(root_path)) self.assertTrue(first[0].endswith("Beginning")) @@ -200,8 +216,8 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.config.parser_root.add_child_block( "VeryLast", ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Get last child last = parser.aug.match("{}/*[last()]".format(root_path)) self.assertTrue(last[0].endswith("VeryLast")) @@ -211,8 +227,8 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "VeryLastAlt", position=99999 ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Get last child last = parser.aug.match("{}/*[last()]".format(root_path)) self.assertTrue(last[0].endswith("VeryLastAlt")) @@ -222,15 +238,15 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- "Middle", position=5 ) - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # Augeas indices start at 1 :( middle = parser.aug.match("{}/*[6]".format(root_path)) self.assertTrue(middle[0].endswith("Middle")) def test_add_child_block_existing_name(self): - parser = self.config.parser_root.primary.parser - root_path = self.config.parser_root.primary.metadata["augeaspath"] + parser = self.config.parser_root.parser + root_path = self.config.parser_root.metadata["augeaspath"] # There already exists a single VirtualHost in the base config new_block = parser.aug.match("{}/VirtualHost[2]".format(root_path)) self.assertEqual(len(new_block), 0) @@ -239,7 +255,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- ) new_block = parser.aug.match("{}/VirtualHost[2]".format(root_path)) self.assertEqual(len(new_block), 1) - self.assertTrue(vh.primary.metadata["augeaspath"].endswith("VirtualHost[2]")) + self.assertTrue(vh.metadata["augeaspath"].endswith("VirtualHost[2]")) def test_node_init_error_bad_augeaspath(self): from certbot_apache._internal.augeasparser import AugeasBlockNode @@ -284,7 +300,7 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.assertEqual(len(dirs), 1) self.assertEqual(dirs[0].parameters, ("with", "parameters")) # The new directive was added to the very first line of the config - self.assertTrue(dirs[0].primary.metadata["augeaspath"].endswith("[1]")) + self.assertTrue(dirs[0].metadata["augeaspath"].endswith("[1]")) def test_add_child_directive_exception(self): self.assertRaises( @@ -314,6 +330,6 @@ class AugeasParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public- self.assertTrue(nonmacro_test) def test_find_ancestors_bad_path(self): - self.config.parser_root.primary.metadata["augeaspath"] = "" - ancs = self.config.parser_root.primary.find_ancestors("Anything") + self.config.parser_root.metadata["augeaspath"] = "" + ancs = self.config.parser_root.find_ancestors("Anything") self.assertEqual(len(ancs), 0) diff --git a/certbot-apache/tests/autohsts_test.py b/certbot-apache/tests/autohsts_test.py index c9901ecdb..d15600215 100644 --- a/certbot-apache/tests/autohsts_test.py +++ b/certbot-apache/tests/autohsts_test.py @@ -3,7 +3,10 @@ import re import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six # pylint: disable=unused-import # six is used in mock.patch() from certbot import errors @@ -20,10 +23,10 @@ class AutoHSTSTest(util.ApacheTest): self.config = util.get_apache_configurator( self.config_path, self.vhost_path, self.config_dir, self.work_dir) - self.config.parser.modules.add("headers_module") - self.config.parser.modules.add("mod_headers.c") - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["headers_module"] = None + self.config.parser.modules["mod_headers.c"] = None + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None self.vh_truth = util.get_vh_truth( self.temp_dir, "debian_apache_2_4/multiple_vhosts") @@ -42,8 +45,8 @@ class AutoHSTSTest(util.ApacheTest): @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.restart") @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod") def test_autohsts_enable_headers_mod(self, mock_enable, _restart): - self.config.parser.modules.discard("headers_module") - self.config.parser.modules.discard("mod_header.c") + self.config.parser.modules.pop("headers_module", None) + self.config.parser.modules.pop("mod_header.c", None) self.config.enable_autohsts(mock.MagicMock(), ["ocspvhost.com"]) self.assertTrue(mock_enable.called) diff --git a/certbot-apache/tests/centos6_test.py b/certbot-apache/tests/centos6_test.py index 15d086600..27b4f8e80 100644 --- a/certbot-apache/tests/centos6_test.py +++ b/certbot-apache/tests/centos6_test.py @@ -19,12 +19,12 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "test.example.com.conf"), os.path.join(aug_pre, "test.example.com.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "test.example.com"), obj.VirtualHost( os.path.join(prefix, "ssl.conf"), os.path.join(aug_pre, "ssl.conf/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), + {obj.Addr.fromstring("_default_:443")}, True, True, None) ] return vh_truth @@ -104,7 +104,7 @@ class CentOS6Tests(util.ApacheTest): pre_loadmods = self.config.parser.find_dir( "LoadModule", "ssl_module", exclude=False) # LoadModules are not within IfModule blocks - self.assertFalse(any(["ifmodule" in m.lower() for m in pre_loadmods])) + self.assertFalse(any("ifmodule" in m.lower() for m in pre_loadmods)) self.config.assoc["test.example.com"] = self.vh_truth[0] self.config.deploy_cert( "random.demo", "example/cert.pem", "example/key.pem", diff --git a/certbot-apache/tests/centos_test.py b/certbot-apache/tests/centos_test.py index 55fee3faa..9dc6fa5a7 100644 --- a/certbot-apache/tests/centos_test.py +++ b/certbot-apache/tests/centos_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.configurator for Centos overrides""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import filesystem @@ -21,12 +24,12 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "centos.example.com.conf"), os.path.join(aug_pre, "centos.example.com.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "centos.example.com"), obj.VirtualHost( os.path.join(prefix, "ssl.conf"), os.path.join(aug_pre, "ssl.conf/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), + {obj.Addr.fromstring("_default_:443")}, True, True, None) ] return vh_truth @@ -126,7 +129,7 @@ class MultipleVhostsTestCentOS(util.ApacheTest): return mod_val return "" mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() + self.config.parser.modules = {} self.config.parser.variables = {} with mock.patch("certbot.util.get_os_info") as mock_osi: diff --git a/certbot-apache/tests/configurator_reverter_test.py b/certbot-apache/tests/configurator_reverter_test.py index ad8e73347..8596195d8 100644 --- a/certbot-apache/tests/configurator_reverter_test.py +++ b/certbot-apache/tests/configurator_reverter_test.py @@ -2,7 +2,10 @@ import shutil import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors import util diff --git a/certbot-apache/tests/configurator_test.py b/certbot-apache/tests/configurator_test.py index cbb052155..8fd3cb750 100644 --- a/certbot-apache/tests/configurator_test.py +++ b/certbot-apache/tests/configurator_test.py @@ -6,7 +6,10 @@ import socket import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six # pylint: disable=unused-import # six is used in mock.patch() from acme import challenges @@ -99,7 +102,7 @@ class MultipleVhostsTest(util.ApacheTest): parserargs = ["server_root", "enmod", "dismod", "le_vhost_ext", "vhost_root", "logs_root", "challenge_location", "handle_modules", "handle_sites", "ctl"] - exp = dict() + exp = {} for k in ApacheConfigurator.OS_DEFAULTS: if k in parserargs: @@ -140,11 +143,9 @@ class MultipleVhostsTest(util.ApacheTest): mock_utility = mock_getutility() mock_utility.notification = mock.MagicMock(return_value=True) names = self.config.get_all_names() - self.assertEqual(names, set( - ["certbot.demo", "ocspvhost.com", "encryption-example.demo", + self.assertEqual(names, {"certbot.demo", "ocspvhost.com", "encryption-example.demo", "nonsym.link", "vhost.in.rootconf", "www.certbot.demo", - "duplicate.example.com"] - )) + "duplicate.example.com"}) @certbot_util.patch_get_utility() @mock.patch("certbot_apache._internal.configurator.socket.gethostbyaddr") @@ -154,9 +155,9 @@ class MultipleVhostsTest(util.ApacheTest): mock_utility.notification.return_value = True vhost = obj.VirtualHost( "fp", "ap", - set([obj.Addr(("8.8.8.8", "443")), + {obj.Addr(("8.8.8.8", "443")), obj.Addr(("zombo.com",)), - obj.Addr(("192.168.1.2"))]), + obj.Addr(("192.168.1.2"))}, True, False) self.config.vhosts.append(vhost) @@ -185,7 +186,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_bad_servername_alias(self): ssl_vh1 = obj.VirtualHost( - "fp1", "ap1", set([obj.Addr(("*", "443"))]), + "fp1", "ap1", {obj.Addr(("*", "443"))}, True, False) # pylint: disable=protected-access self.config._add_servernames(ssl_vh1) @@ -198,7 +199,7 @@ class MultipleVhostsTest(util.ApacheTest): # pylint: disable=protected-access self.config._add_servernames(self.vh_truth[2]) self.assertEqual( - self.vh_truth[2].get_names(), set(["*.le.co", "ip-172-30-0-17"])) + self.vh_truth[2].get_names(), {"*.le.co", "ip-172-30-0-17"}) def test_get_virtual_hosts(self): """Make sure all vhosts are being properly found.""" @@ -269,7 +270,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_choose_vhost_select_vhost_conflicting_non_ssl(self, mock_select): mock_select.return_value = self.vh_truth[3] conflicting_vhost = obj.VirtualHost( - "path", "aug_path", set([obj.Addr.fromstring("*:443")]), + "path", "aug_path", {obj.Addr.fromstring("*:443")}, True, True) self.config.vhosts.append(conflicting_vhost) @@ -278,14 +279,14 @@ class MultipleVhostsTest(util.ApacheTest): def test_find_best_http_vhost_default(self): vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr.fromstring("_default_:80")]), False, True) + "fp", "ap", {obj.Addr.fromstring("_default_:80")}, False, True) self.config.vhosts = [vh] self.assertEqual(self.config.find_best_http_vhost("foo.bar", False), vh) def test_find_best_http_vhost_port(self): port = "8080" vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr.fromstring("*:" + port)]), + "fp", "ap", {obj.Addr.fromstring("*:" + port)}, False, True, "encryption-example.demo") self.config.vhosts.append(vh) self.assertEqual(self.config.find_best_http_vhost("foo.bar", False, port), vh) @@ -313,8 +314,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_find_best_vhost_variety(self): # pylint: disable=protected-access ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), - obj.Addr(("zombo.com",))]), + "fp", "ap", {obj.Addr(("*", "443")), + obj.Addr(("zombo.com",))}, True, False) self.config.vhosts.append(ssl_vh) self.assertEqual(self.config._find_best_vhost("zombo.com"), ssl_vh) @@ -341,9 +342,9 @@ class MultipleVhostsTest(util.ApacheTest): def test_deploy_cert_enable_new_vhost(self): # Create ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.assertFalse(ssl_vhost.enabled) self.config.deploy_cert( @@ -377,9 +378,9 @@ class MultipleVhostsTest(util.ApacheTest): # pragma: no cover def test_deploy_cert(self): - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None # Patch _add_dummy_ssl_directives to make sure we write them correctly # pylint: disable=protected-access orig_add_dummy = self.config._add_dummy_ssl_directives @@ -459,9 +460,9 @@ class MultipleVhostsTest(util.ApacheTest): method is called with an invalid vhost parameter. Currently this tests that a PluginError is appropriately raised when important directives are missing in an SSL module.""" - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None def side_effect(*args): """Mocks case where an SSLCertificateFile directive can be found @@ -544,7 +545,8 @@ class MultipleVhostsTest(util.ApacheTest): call_found = True self.assertTrue(call_found) - def test_prepare_server_https(self): + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https(self, mock_reset): mock_enable = mock.Mock() self.config.enable_mod = mock_enable @@ -570,7 +572,8 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(mock_add_dir.call_count, 2) - def test_prepare_server_https_named_listen(self): + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https_named_listen(self, mock_reset): mock_find = mock.Mock() mock_find.return_value = ["test1", "test2", "test3"] mock_get = mock.Mock() @@ -608,7 +611,8 @@ class MultipleVhostsTest(util.ApacheTest): # self.config.prepare_server_https("8080", temp=True) # self.assertEqual(self.listens, 0) - def test_prepare_server_https_needed_listen(self): + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https_needed_listen(self, mock_reset): mock_find = mock.Mock() mock_find.return_value = ["test1", "test2"] mock_get = mock.Mock() @@ -624,8 +628,8 @@ class MultipleVhostsTest(util.ApacheTest): self.config.prepare_server_https("443") self.assertEqual(mock_add_dir.call_count, 1) - def test_prepare_server_https_mixed_listen(self): - + @mock.patch("certbot_apache._internal.parser.ApacheParser.reset_modules") + def test_prepare_server_https_mixed_listen(self, mock_reset): mock_find = mock.Mock() mock_find.return_value = ["test1", "test2"] mock_get = mock.Mock() @@ -682,7 +686,7 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(ssl_vhost.path, "/files" + ssl_vhost.filep + "/IfModule/Virtualhost") self.assertEqual(len(ssl_vhost.addrs), 1) - self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs) + self.assertEqual({obj.Addr.fromstring("*:443")}, ssl_vhost.addrs) self.assertEqual(ssl_vhost.name, "encryption-example.demo") self.assertTrue(ssl_vhost.ssl) self.assertFalse(ssl_vhost.enabled) @@ -904,10 +908,10 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache._internal.display_ops.select_vhost") @mock.patch("certbot.util.exe_exists") def test_enhance_unknown_vhost(self, mock_exe, mock_sel_vhost, mock_get): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None mock_exe.return_value = True ssl_vh1 = obj.VirtualHost( - "fp1", "ap1", set([obj.Addr(("*", "443"))]), + "fp1", "ap1", {obj.Addr(("*", "443"))}, True, False) ssl_vh1.name = "satoshi.com" self.config.vhosts.append(ssl_vh1) @@ -942,8 +946,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling(self, mock_exe): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True @@ -969,8 +973,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling_twice(self, mock_exe): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True @@ -997,8 +1001,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_ocsp_unsupported_apache_version(self, mock_exe): mock_exe.return_value = True self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None self.config.get_version = mock.Mock(return_value=(2, 2, 0)) self.config.choose_vhost("certbot.demo") @@ -1008,7 +1012,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_get_http_vhost_third_filter(self): ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443"))]), + "fp", "ap", {obj.Addr(("*", "443"))}, True, False) ssl_vh.name = "satoshi.com" self.config.vhosts.append(ssl_vh) @@ -1021,8 +1025,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_http_header_hsts(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None mock_exe.return_value = True # This will create an ssl vhost for certbot.demo @@ -1042,9 +1046,9 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(len(hsts_header), 4) def test_http_header_hsts_twice(self): - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None # skip the enable mod - self.config.parser.modules.add("headers_module") + self.config.parser.modules["headers_module"] = None # This will create an ssl vhost for encryption-example.demo self.config.choose_vhost("encryption-example.demo") @@ -1060,8 +1064,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_http_header_uir(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None mock_exe.return_value = True @@ -1084,9 +1088,9 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(len(uir_header), 4) def test_http_header_uir_twice(self): - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None # skip the enable mod - self.config.parser.modules.add("headers_module") + self.config.parser.modules["headers_module"] = None # This will create an ssl vhost for encryption-example.demo self.config.choose_vhost("encryption-example.demo") @@ -1101,7 +1105,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_well_formed_http(self, mock_exe, _): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2)) @@ -1127,7 +1131,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_rewrite_rule_exists(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.parser.add_dir( self.vh_truth[3].path, "RewriteRule", ["Unknown"]) @@ -1136,7 +1140,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_rewrite_engine_exists(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.parser.add_dir( self.vh_truth[3].path, "RewriteEngine", "on") @@ -1146,7 +1150,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_with_existing_rewrite(self, mock_exe, _): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2, 0)) @@ -1180,7 +1184,7 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot.util.run_script") @mock.patch("certbot.util.exe_exists") def test_redirect_with_old_https_redirection(self, mock_exe, _): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.parser.update_runtime_variables = mock.Mock() mock_exe.return_value = True self.config.get_version = mock.Mock(return_value=(2, 2, 0)) @@ -1209,10 +1213,10 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_with_conflict(self): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None ssl_vh = obj.VirtualHost( - "fp", "ap", set([obj.Addr(("*", "443")), - obj.Addr(("zombo.com",))]), + "fp", "ap", {obj.Addr(("*", "443")), + obj.Addr(("zombo.com",))}, True, False) # No names ^ this guy should conflict. @@ -1222,7 +1226,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_two_domains_one_vhost(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) # Creates ssl vhost for the domain @@ -1237,7 +1241,7 @@ class MultipleVhostsTest(util.ApacheTest): def test_redirect_from_previous_run(self): # Skip the enable mod - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) self.config.choose_vhost("red.blue.purple.com") self.config.enhance("red.blue.purple.com", "redirect") @@ -1250,22 +1254,22 @@ class MultipleVhostsTest(util.ApacheTest): self.config.enhance, "green.blue.purple.com", "redirect") def test_create_own_redirect(self): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 3, 9)) # For full testing... give names... self.vh_truth[1].name = "default.com" - self.vh_truth[1].aliases = set(["yes.default.com"]) + self.vh_truth[1].aliases = {"yes.default.com"} # pylint: disable=protected-access self.config._enable_redirect(self.vh_truth[1], "") self.assertEqual(len(self.config.vhosts), 13) def test_create_own_redirect_for_old_apache_version(self): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None self.config.get_version = mock.Mock(return_value=(2, 2)) # For full testing... give names... self.vh_truth[1].name = "default.com" - self.vh_truth[1].aliases = set(["yes.default.com"]) + self.vh_truth[1].aliases = {"yes.default.com"} # pylint: disable=protected-access self.config._enable_redirect(self.vh_truth[1], "") @@ -1326,9 +1330,9 @@ class MultipleVhostsTest(util.ApacheTest): def test_deploy_cert_not_parsed_path(self): # Make sure that we add include to root config for vhosts when # handle-sites is false - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("socache_shmcb_module") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["socache_shmcb_module"] = None tmp_path = filesystem.realpath(tempfile.mkdtemp("vhostroot")) filesystem.chmod(tmp_path, 0o755) mock_p = "certbot_apache._internal.configurator.ApacheConfigurator._get_ssl_vhost_path" @@ -1441,8 +1445,8 @@ class MultipleVhostsTest(util.ApacheTest): @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator._choose_vhosts_wildcard") def test_enhance_wildcard_after_install(self, mock_choose): # pylint: disable=protected-access - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None self.vh_truth[3].ssl = True self.config._wildcard_vhosts["*.certbot.demo"] = [self.vh_truth[3]] self.config.enhance("*.certbot.demo", "ensure-http-header", @@ -1453,8 +1457,8 @@ class MultipleVhostsTest(util.ApacheTest): def test_enhance_wildcard_no_install(self, mock_choose): self.vh_truth[3].ssl = True mock_choose.return_value = [self.vh_truth[3]] - self.config.parser.modules.add("mod_ssl.c") - self.config.parser.modules.add("headers_module") + self.config.parser.modules["mod_ssl.c"] = None + self.config.parser.modules["headers_module"] = None self.config.enhance("*.certbot.demo", "ensure-http-header", "Upgrade-Insecure-Requests") self.assertTrue(mock_choose.called) @@ -1606,7 +1610,7 @@ class MultiVhostsTest(util.ApacheTest): self.assertEqual(ssl_vhost.path, "/files" + ssl_vhost.filep + "/IfModule/VirtualHost") self.assertEqual(len(ssl_vhost.addrs), 1) - self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs) + self.assertEqual({obj.Addr.fromstring("*:443")}, ssl_vhost.addrs) self.assertEqual(ssl_vhost.name, "banana.vomit.com") self.assertTrue(ssl_vhost.ssl) self.assertFalse(ssl_vhost.enabled) @@ -1638,7 +1642,7 @@ class MultiVhostsTest(util.ApacheTest): @certbot_util.patch_get_utility() def test_make_vhost_ssl_with_existing_rewrite_rule(self, mock_get_utility): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[4]) @@ -1658,7 +1662,7 @@ class MultiVhostsTest(util.ApacheTest): @certbot_util.patch_get_utility() def test_make_vhost_ssl_with_existing_rewrite_conds(self, mock_get_utility): - self.config.parser.modules.add("rewrite_module") + self.config.parser.modules["rewrite_module"] = None ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[3]) @@ -1700,7 +1704,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.config.updated_mod_ssl_conf_digest) def _current_ssl_options_hash(self): - return crypto_util.sha256sum(self.config.option("MOD_SSL_CONF_SRC")) + return crypto_util.sha256sum(self.config.pick_apache_config()) def _assert_current_file(self): self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) @@ -1736,7 +1740,7 @@ class InstallSslOptionsConfTest(util.ApacheTest): self.assertFalse(mock_logger.warning.called) self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) self.assertEqual(crypto_util.sha256sum( - self.config.option("MOD_SSL_CONF_SRC")), + self.config.pick_apache_config()), self._current_ssl_options_hash()) self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf), self._current_ssl_options_hash()) @@ -1752,19 +1756,99 @@ class InstallSslOptionsConfTest(util.ApacheTest): "%s has been manually modified; updated file " "saved to %s. We recommend updating %s for security purposes.") self.assertEqual(crypto_util.sha256sum( - self.config.option("MOD_SSL_CONF_SRC")), + self.config.pick_apache_config()), self._current_ssl_options_hash()) # only print warning once with mock.patch("certbot.plugins.common.logger") as mock_logger: self._call() self.assertFalse(mock_logger.warning.called) - def test_current_file_hash_in_all_hashes(self): + def test_ssl_config_files_hash_in_all_hashes(self): + """ + It is really critical that all TLS Apache config files have their SHA256 hash registered in + constants.ALL_SSL_OPTIONS_HASHES. Otherwise Certbot will mistakenly assume that the config + file has been manually edited by the user, and will refuse to update it. + This test ensures that all necessary hashes are present. + """ from certbot_apache._internal.constants import ALL_SSL_OPTIONS_HASHES - self.assertTrue(self._current_ssl_options_hash() in ALL_SSL_OPTIONS_HASHES, - "Constants.ALL_SSL_OPTIONS_HASHES must be appended" - " with the sha256 hash of self.config.mod_ssl_conf when it is updated.") + import pkg_resources + tls_configs_dir = pkg_resources.resource_filename( + "certbot_apache", os.path.join("_internal", "tls_configs")) + all_files = [os.path.join(tls_configs_dir, name) for name in os.listdir(tls_configs_dir) + if name.endswith('options-ssl-apache.conf')] + self.assertTrue(all_files) + for one_file in all_files: + file_hash = crypto_util.sha256sum(one_file) + self.assertTrue(file_hash in ALL_SSL_OPTIONS_HASHES, + "Constants.ALL_SSL_OPTIONS_HASHES must be appended with the sha256 " + "hash of {0} when it is updated.".format(one_file)) + + def test_openssl_version(self): + self.config._openssl_version = None + some_string_contents = b""" + SSLOpenSSLConfCmd + OpenSSL configuration command + SSLv3 not supported by this version of OpenSSL + '%s': invalid OpenSSL configuration command + OpenSSL 1.0.2g 1 Mar 2016 + OpenSSL + AH02407: "SSLOpenSSLConfCmd %s %s" failed for %s + AH02556: "SSLOpenSSLConfCmd %s %s" applied to %s + OpenSSL 1.0.2g 1 Mar 2016 + """ + self.config.parser.modules['ssl_module'] = '/fake/path' + with mock.patch("certbot_apache._internal.configurator." + "ApacheConfigurator._open_module_file") as mock_omf: + mock_omf.return_value = some_string_contents + self.assertEqual(self.config.openssl_version(), "1.0.2g") + + def test_current_version(self): + self.config.version = (2, 4, 10) + self.config._openssl_version = '1.0.2m' + self.assertTrue('old' in self.config.pick_apache_config()) + + self.config.version = (2, 4, 11) + self.config._openssl_version = '1.0.2m' + self.assertTrue('current' in self.config.pick_apache_config()) + + self.config._openssl_version = '1.0.2a' + self.assertTrue('old' in self.config.pick_apache_config()) + + def test_openssl_version_warns(self): + self.config._openssl_version = '1.0.2a' + self.assertEqual(self.config.openssl_version(), '1.0.2a') + + self.config._openssl_version = None + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Could not find ssl_module" in mock_log.call_args[0][0]) + + self.config._openssl_version = None + self.config.parser.modules['ssl_module'] = None + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Could not find ssl_module" in mock_log.call_args[0][0]) + + self.config.parser.modules['ssl_module'] = "/fake/path" + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + # Check that correct logger.warning was printed + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Unable to read" in mock_log.call_args[0][0]) + + contents_missing_openssl = b"these contents won't match the regex" + with mock.patch("certbot_apache._internal.configurator." + "ApacheConfigurator._open_module_file") as mock_omf: + mock_omf.return_value = contents_missing_openssl + with mock.patch("certbot_apache._internal.configurator.logger.warning") as mock_log: + # Check that correct logger.warning was printed + self.assertEqual(self.config.openssl_version(), None) + self.assertTrue("Could not find OpenSSL" in mock_log.call_args[0][0]) + + def test_open_module_file(self): + mock_open = mock.mock_open(read_data="testing 12 3") + with mock.patch("six.moves.builtins.open", mock_open): + self.assertEqual(self.config._open_module_file("/nonsense/"), "testing 12 3") if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/tests/debian_test.py b/certbot-apache/tests/debian_test.py index 400e503fb..192e3cba5 100644 --- a/certbot-apache/tests/debian_test.py +++ b/certbot-apache/tests/debian_test.py @@ -2,7 +2,10 @@ import shutil import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os @@ -61,8 +64,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): def test_deploy_cert_enable_new_vhost(self): # Create ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None self.assertFalse(ssl_vhost.enabled) self.config.deploy_cert( "encryption-example.demo", "example/cert.pem", "example/key.pem", @@ -92,8 +95,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) self.config = self.mock_deploy_cert(self.config) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] @@ -128,8 +131,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 16)) self.config = self.mock_deploy_cert(self.config) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] @@ -143,8 +146,8 @@ class MultipleVhostsTestDebian(util.ApacheTest): self.config_path, self.vhost_path, self.config_dir, self.work_dir, version=(2, 4, 7)) self.config = self.mock_deploy_cert(self.config) - self.config.parser.modules.add("ssl_module") - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["ssl_module"] = None + self.config.parser.modules["mod_ssl.c"] = None # Get the default 443 vhost self.config.assoc["random.demo"] = self.vh_truth[1] @@ -157,7 +160,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ocsp_stapling_enable_mod(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None self.config.get_version = mock.Mock(return_value=(2, 4, 7)) mock_exe.return_value = True # This will create an ssl vhost for certbot.demo @@ -169,7 +172,7 @@ class MultipleVhostsTestDebian(util.ApacheTest): @mock.patch("certbot.util.exe_exists") def test_ensure_http_header_enable_mod(self, mock_exe, _): self.config.parser.update_runtime_variables = mock.Mock() - self.config.parser.modules.add("mod_ssl.c") + self.config.parser.modules["mod_ssl.c"] = None mock_exe.return_value = True # This will create an ssl vhost for certbot.demo diff --git a/certbot-apache/tests/display_ops_test.py b/certbot-apache/tests/display_ops_test.py index 50bdc03cf..4559668ac 100644 --- a/certbot-apache/tests/display_ops_test.py +++ b/certbot-apache/tests/display_ops_test.py @@ -1,7 +1,10 @@ """Test certbot_apache._internal.display_ops.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.display import util as display_util @@ -93,9 +96,9 @@ class SelectVhostTest(unittest.TestCase): self.vhosts.append( obj.VirtualHost( - "path", "aug_path", set([obj.Addr.fromstring("*:80")]), + "path", "aug_path", {obj.Addr.fromstring("*:80")}, False, False, - "wildcard.com", set(["*.wildcard.com"]))) + "wildcard.com", {"*.wildcard.com"})) self.assertEqual(self.vhosts[5], self._call(self.vhosts)) diff --git a/certbot-apache/tests/dualnode_test.py b/certbot-apache/tests/dualnode_test.py index 0871bac78..44cc69ff4 100644 --- a/certbot-apache/tests/dualnode_test.py +++ b/certbot-apache/tests/dualnode_test.py @@ -1,7 +1,10 @@ """Tests for DualParserNode implementation""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot_apache._internal import assertions from certbot_apache._internal import augeasparser diff --git a/certbot-apache/tests/entrypoint_test.py b/certbot-apache/tests/entrypoint_test.py index 04c393bdf..6f6f5bbb0 100644 --- a/certbot-apache/tests/entrypoint_test.py +++ b/certbot-apache/tests/entrypoint_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.entrypoint for override class resolution""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot_apache._internal import configurator from certbot_apache._internal import entrypoint diff --git a/certbot-apache/tests/fedora_test.py b/certbot-apache/tests/fedora_test.py index cb1614278..e0ee603c3 100644 --- a/certbot-apache/tests/fedora_test.py +++ b/certbot-apache/tests/fedora_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.configurator for Fedora 29+ overrides""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import filesystem @@ -120,7 +123,7 @@ class MultipleVhostsTestFedora(util.ApacheTest): return mod_val return "" mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() + self.config.parser.modules = {} self.config.parser.variables = {} with mock.patch("certbot.util.get_os_info") as mock_osi: diff --git a/certbot-apache/tests/gentoo_test.py b/certbot-apache/tests/gentoo_test.py index fb5d192d0..aa923c367 100644 --- a/certbot-apache/tests/gentoo_test.py +++ b/certbot-apache/tests/gentoo_test.py @@ -1,7 +1,10 @@ """Test for certbot_apache._internal.configurator for Gentoo overrides""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import filesystem @@ -21,19 +24,19 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "gentoo.example.com.conf"), os.path.join(aug_pre, "gentoo.example.com.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "gentoo.example.com"), obj.VirtualHost( os.path.join(prefix, "00_default_vhost.conf"), os.path.join(aug_pre, "00_default_vhost.conf/IfDefine/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "localhost"), obj.VirtualHost( os.path.join(prefix, "00_default_ssl_vhost.conf"), os.path.join(aug_pre, "00_default_ssl_vhost.conf" + "/IfDefine/IfDefine/IfModule/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), + {obj.Addr.fromstring("_default_:443")}, True, True, "localhost") ] return vh_truth @@ -117,7 +120,7 @@ class MultipleVhostsTestGentoo(util.ApacheTest): return mod_val return None # pragma: no cover mock_get.side_effect = mock_get_cfg - self.config.parser.modules = set() + self.config.parser.modules = {} with mock.patch("certbot.util.get_os_info") as mock_osi: # Make sure we have the have the Gentoo httpd constants diff --git a/certbot-apache/tests/http_01_test.py b/certbot-apache/tests/http_01_test.py index 643a6bdd5..696cd4a54 100644 --- a/certbot-apache/tests/http_01_test.py +++ b/certbot-apache/tests/http_01_test.py @@ -1,10 +1,13 @@ """Test for certbot_apache._internal.http_01.""" import unittest +import errno -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from acme import challenges -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot.compat import filesystem @@ -40,8 +43,8 @@ class ApacheHttp01Test(util.ApacheTest): modules = ["ssl", "rewrite", "authz_core", "authz_host"] for mod in modules: - self.config.parser.modules.add("mod_{0}.c".format(mod)) - self.config.parser.modules.add(mod + "_module") + self.config.parser.modules["mod_{0}.c".format(mod)] = None + self.config.parser.modules[mod + "_module"] = None from certbot_apache._internal.http_01 import ApacheHttp01 self.http = ApacheHttp01(self.config) @@ -52,24 +55,24 @@ class ApacheHttp01Test(util.ApacheTest): @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod") def test_enable_modules_apache_2_2(self, mock_enmod): self.config.version = (2, 2) - self.config.parser.modules.remove("authz_host_module") - self.config.parser.modules.remove("mod_authz_host.c") + del self.config.parser.modules["authz_host_module"] + del self.config.parser.modules["mod_authz_host.c"] enmod_calls = self.common_enable_modules_test(mock_enmod) self.assertEqual(enmod_calls[0][0][0], "authz_host") @mock.patch("certbot_apache._internal.configurator.ApacheConfigurator.enable_mod") def test_enable_modules_apache_2_4(self, mock_enmod): - self.config.parser.modules.remove("authz_core_module") - self.config.parser.modules.remove("mod_authz_core.c") + del self.config.parser.modules["authz_core_module"] + del self.config.parser.modules["mod_authz_host.c"] enmod_calls = self.common_enable_modules_test(mock_enmod) self.assertEqual(enmod_calls[0][0][0], "authz_core") def common_enable_modules_test(self, mock_enmod): """Tests enabling mod_rewrite and other modules.""" - self.config.parser.modules.remove("rewrite_module") - self.config.parser.modules.remove("mod_rewrite.c") + del self.config.parser.modules["rewrite_module"] + del self.config.parser.modules["mod_rewrite.c"] self.http.prepare_http01_modules() @@ -197,6 +200,12 @@ class ApacheHttp01Test(util.ApacheTest): self.assertTrue(os.path.exists(challenge_dir)) + @mock.patch("certbot_apache._internal.http_01.filesystem.makedirs") + def test_failed_makedirs(self, mock_makedirs): + mock_makedirs.side_effect = OSError(errno.EACCES, "msg") + self.http.add_chall(self.achalls[0]) + self.assertRaises(errors.PluginError, self.http.perform) + def _test_challenge_conf(self): with open(self.http.challenge_conf_pre) as f: pre_conf_contents = f.read() diff --git a/certbot-apache/tests/obj_test.py b/certbot-apache/tests/obj_test.py index 1761b9c94..eac6a64ef 100644 --- a/certbot-apache/tests/obj_test.py +++ b/certbot-apache/tests/obj_test.py @@ -14,13 +14,13 @@ class VirtualHostTest(unittest.TestCase): self.addr_default = Addr.fromstring("_default_:443") self.vhost1 = VirtualHost( - "filep", "vh_path", set([self.addr1]), False, False, "localhost") + "filep", "vh_path", {self.addr1}, False, False, "localhost") self.vhost1b = VirtualHost( - "filep", "vh_path", set([self.addr1]), False, False, "localhost") + "filep", "vh_path", {self.addr1}, False, False, "localhost") self.vhost2 = VirtualHost( - "fp", "vhp", set([self.addr2]), False, False, "localhost") + "fp", "vhp", {self.addr2}, False, False, "localhost") def test_repr(self): self.assertEqual(repr(self.addr2), @@ -42,7 +42,7 @@ class VirtualHostTest(unittest.TestCase): complex_vh = VirtualHost( "fp", "vhp", - set([Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")]), + {Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")}, False, False) self.assertTrue(complex_vh.conflicts([self.addr1])) self.assertTrue(complex_vh.conflicts([self.addr2])) @@ -57,14 +57,14 @@ class VirtualHostTest(unittest.TestCase): def test_same_server(self): from certbot_apache._internal.obj import VirtualHost no_name1 = VirtualHost( - "fp", "vhp", set([self.addr1]), False, False, None) + "fp", "vhp", {self.addr1}, False, False, None) no_name2 = VirtualHost( - "fp", "vhp", set([self.addr2]), False, False, None) + "fp", "vhp", {self.addr2}, False, False, None) no_name3 = VirtualHost( - "fp", "vhp", set([self.addr_default]), + "fp", "vhp", {self.addr_default}, False, False, None) no_name4 = VirtualHost( - "fp", "vhp", set([self.addr2, self.addr_default]), + "fp", "vhp", {self.addr2, self.addr_default}, False, False, None) self.assertTrue(self.vhost1.same_server(self.vhost2)) diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py index f5a0a3d11..7aedec31d 100644 --- a/certbot-apache/tests/parser_test.py +++ b/certbot-apache/tests/parser_test.py @@ -2,7 +2,10 @@ import shutil import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os @@ -114,7 +117,7 @@ class BasicParserTest(util.ParserTest): """ from certbot_apache._internal.parser import get_aug_path # This makes sure that find_dir will work - self.parser.modules.add("mod_ssl.c") + self.parser.modules["mod_ssl.c"] = "/fake/path" self.parser.add_dir_to_ifmodssl( get_aug_path(self.parser.loc["default"]), @@ -128,7 +131,7 @@ class BasicParserTest(util.ParserTest): def test_add_dir_to_ifmodssl_multiple(self): from certbot_apache._internal.parser import get_aug_path # This makes sure that find_dir will work - self.parser.modules.add("mod_ssl.c") + self.parser.modules["mod_ssl.c"] = "/fake/path" self.parser.add_dir_to_ifmodssl( get_aug_path(self.parser.loc["default"]), @@ -260,7 +263,7 @@ class BasicParserTest(util.ParserTest): expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443", "example_path": "Documents/path"} - self.parser.modules = set() + self.parser.modules = {} with mock.patch( "certbot_apache._internal.parser.ApacheParser.parse_file") as mock_parse: self.parser.update_runtime_variables() @@ -282,7 +285,7 @@ class BasicParserTest(util.ParserTest): os.path.dirname(self.parser.loc["root"])) mock_cfg.return_value = inc_val - self.parser.modules = set() + self.parser.modules = {} with mock.patch( "certbot_apache._internal.parser.ApacheParser.parse_file") as mock_parse: diff --git a/certbot-apache/tests/parsernode_configurator_test.py b/certbot-apache/tests/parsernode_configurator_test.py index 67d65995a..7fbec2540 100644 --- a/certbot-apache/tests/parsernode_configurator_test.py +++ b/certbot-apache/tests/parsernode_configurator_test.py @@ -1,11 +1,21 @@ """Tests for ApacheConfigurator for AugeasParserNode classes""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import util +try: + import apacheconfig + HAS_APACHECONFIG = True +except ImportError: # pragma: no cover + HAS_APACHECONFIG = False + +@unittest.skipIf(not HAS_APACHECONFIG, reason='Tests require apacheconfig dependency') class ConfiguratorParserNodeTest(util.ApacheTest): # pylint: disable=too-many-public-methods """Test AugeasParserNode using available test configurations""" diff --git a/certbot-apache/tests/util.py b/certbot-apache/tests/util.py index ccd0b274d..f2a6a0263 100644 --- a/certbot-apache/tests/util.py +++ b/certbot-apache/tests/util.py @@ -5,7 +5,10 @@ import unittest import augeas import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import zope.component from certbot.compat import os @@ -85,7 +88,8 @@ def get_apache_configurator( config_dir, work_dir, version=(2, 4, 7), os_info="generic", conf_vhost_path=None, - use_parsernode=False): + use_parsernode=False, + openssl_version="1.1.1a"): """Create an Apache Configurator with the specified options. :param conf: Function that returns binary paths. self.conf in Configurator @@ -118,7 +122,8 @@ def get_apache_configurator( except KeyError: config_class = configurator.ApacheConfigurator config = config_class(config=mock_le_config, name="apache", - version=version, use_parsernode=use_parsernode) + version=version, use_parsernode=use_parsernode, + openssl_version=openssl_version) if not conf_vhost_path: config_class.OS_DEFAULTS["vhost_root"] = vhost_path else: @@ -140,71 +145,71 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "encryption-example.conf"), os.path.join(aug_pre, "encryption-example.conf/Virtualhost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "encryption-example.demo"), obj.VirtualHost( os.path.join(prefix, "default-ssl.conf"), os.path.join(aug_pre, "default-ssl.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("_default_:443")]), True, True), + {obj.Addr.fromstring("_default_:443")}, True, True), obj.VirtualHost( os.path.join(prefix, "000-default.conf"), os.path.join(aug_pre, "000-default.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80"), - obj.Addr.fromstring("[::]:80")]), + {obj.Addr.fromstring("*:80"), + obj.Addr.fromstring("[::]:80")}, False, True, "ip-172-30-0-17"), obj.VirtualHost( os.path.join(prefix, "certbot.conf"), os.path.join(aug_pre, "certbot.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "certbot.demo", aliases=["www.certbot.demo"]), obj.VirtualHost( os.path.join(prefix, "mod_macro-example.conf"), os.path.join(aug_pre, "mod_macro-example.conf/Macro/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, modmacro=True), obj.VirtualHost( os.path.join(prefix, "default-ssl-port-only.conf"), os.path.join(aug_pre, ("default-ssl-port-only.conf/" "IfModule/VirtualHost")), - set([obj.Addr.fromstring("_default_:443")]), True, True), + {obj.Addr.fromstring("_default_:443")}, True, True), obj.VirtualHost( os.path.join(prefix, "wildcard.conf"), os.path.join(aug_pre, "wildcard.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "ip-172-30-0-17", aliases=["*.blue.purple.com"]), obj.VirtualHost( os.path.join(prefix, "ocsp-ssl.conf"), os.path.join(aug_pre, "ocsp-ssl.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("10.2.3.4:443")]), True, True, + {obj.Addr.fromstring("10.2.3.4:443")}, True, True, "ocspvhost.com"), obj.VirtualHost( os.path.join(prefix, "non-symlink.conf"), os.path.join(aug_pre, "non-symlink.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "nonsym.link"), obj.VirtualHost( os.path.join(prefix, "default-ssl-port-only.conf"), os.path.join(aug_pre, "default-ssl-port-only.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), True, True, ""), + {obj.Addr.fromstring("*:80")}, True, True, ""), obj.VirtualHost( os.path.join(temp_dir, config_name, "apache2/apache2.conf"), "/files" + os.path.join(temp_dir, config_name, "apache2/apache2.conf/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), False, True, + {obj.Addr.fromstring("*:80")}, False, True, "vhost.in.rootconf"), obj.VirtualHost( os.path.join(prefix, "duplicatehttp.conf"), os.path.join(aug_pre, "duplicatehttp.conf/VirtualHost"), - set([obj.Addr.fromstring("10.2.3.4:80")]), False, True, + {obj.Addr.fromstring("10.2.3.4:80")}, False, True, "duplicate.example.com"), obj.VirtualHost( os.path.join(prefix, "duplicatehttps.conf"), os.path.join(aug_pre, "duplicatehttps.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("10.2.3.4:443")]), True, True, + {obj.Addr.fromstring("10.2.3.4:443")}, True, True, "duplicate.example.com")] return vh_truth if config_name == "debian_apache_2_4/multi_vhosts": @@ -215,27 +220,27 @@ def get_vh_truth(temp_dir, config_name): obj.VirtualHost( os.path.join(prefix, "default.conf"), os.path.join(aug_pre, "default.conf/VirtualHost[1]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "ip-172-30-0-17"), obj.VirtualHost( os.path.join(prefix, "default.conf"), os.path.join(aug_pre, "default.conf/VirtualHost[2]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "banana.vomit.com"), obj.VirtualHost( os.path.join(prefix, "multi-vhost.conf"), os.path.join(aug_pre, "multi-vhost.conf/VirtualHost[1]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "1.multi.vhost.tld"), obj.VirtualHost( os.path.join(prefix, "multi-vhost.conf"), os.path.join(aug_pre, "multi-vhost.conf/IfModule/VirtualHost"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "2.multi.vhost.tld"), obj.VirtualHost( os.path.join(prefix, "multi-vhost.conf"), os.path.join(aug_pre, "multi-vhost.conf/VirtualHost[2]"), - set([obj.Addr.fromstring("*:80")]), + {obj.Addr.fromstring("*:80")}, False, True, "3.multi.vhost.tld")] return vh_truth return None # pragma: no cover diff --git a/certbot-auto b/certbot-auto index cea58e2cb..0ea3275c3 100755 --- a/certbot-auto +++ b/certbot-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.2.0" +LE_AUTO_VERSION="1.3.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1540,18 +1540,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.2.0 \ - --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \ - --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c -acme==1.2.0 \ - --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \ - --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab -certbot-apache==1.2.0 \ - --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \ - --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3 -certbot-nginx==1.2.0 \ - --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \ - --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db +certbot==1.3.0 \ + --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ + --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 +acme==1.3.0 \ + --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ + --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d +certbot-apache==1.3.0 \ + --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ + --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 +certbot-nginx==1.3.0 \ + --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ + --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py index 6f8670000..e295aefd7 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/context.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -1,5 +1,4 @@ """Module to handle the context of integration tests.""" -import logging import os import shutil import sys diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py index 94e76cf79..f0c5edd3f 100644 --- a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -595,6 +595,23 @@ def test_ocsp_status_live(context): assert output.count('REVOKED') == 1, 'Expected {0} to be REVOKED'.format(cert) +def test_ocsp_renew(context): + """Test that revoked certificates are renewed.""" + # Obtain a certificate + certname = context.get_domain('ocsp-renew') + context.certbot(['--domains', certname]) + + # Test that "certbot renew" does not renew the certificate + assert_cert_count_for_lineage(context.config_dir, certname, 1) + context.certbot(['renew'], force_renew=False) + assert_cert_count_for_lineage(context.config_dir, certname, 1) + + # Revoke the certificate and test that it does renew the certificate + context.certbot(['revoke', '--cert-name', certname, '--no-delete-after-revoke']) + context.certbot(['renew'], force_renew=False) + assert_cert_count_for_lineage(context.config_dir, certname, 2) + + def test_dry_run_deactivate_authzs(context): """Test that Certbot deactivates authorizations when performing a dry run""" diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py index 5483251e6..fb202005e 100755 --- a/certbot-ci/certbot_integration_tests/utils/acme_server.py +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -38,7 +38,7 @@ class ACMEServer(object): :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) :param list nodes: list of node names that will be setup by pytest xdist :param bool http_proxy: if False do not start the HTTP proxy - :param bool stdout: if True stream subprocesses stdout to standard stdout + :param bool stdout: if True stream all subprocesses stdout to standard stdout """ self._construct_acme_xdist(acme_server, nodes) @@ -131,7 +131,7 @@ class ACMEServer(object): environ['PEBBLE_AUTHZREUSE'] = '100' self._launch_process( - [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053'], + [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053', '-strict'], env=environ) self._launch_process( @@ -165,17 +165,24 @@ class ACMEServer(object): os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'), join(instance_path, 'test/rate-limit-policies.yml')) - # Launch the Boulder server - self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path) + try: + # Launch the Boulder server + self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path) - # Wait for the ACME CA server to be up. - print('=> Waiting for boulder instance to respond...') - misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=240) + # Wait for the ACME CA server to be up. + print('=> Waiting for boulder instance to respond...') + misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300) - # Configure challtestsrv to answer any A record request with ip of the docker host. - response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), - json={'ip': '10.77.77.1'}) - response.raise_for_status() + # Configure challtestsrv to answer any A record request with ip of the docker host. + response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT), + json={'ip': '10.77.77.1'}) + response.raise_for_status() + except BaseException: + # If we failed to set up boulder, print its logs. + print('=> Boulder setup failed. Boulder logs are:') + process = self._launch_process(['docker-compose', 'logs'], cwd=instance_path, force_stderr=True) + process.wait() + raise print('=> Finished boulder instance deployment.') @@ -188,11 +195,12 @@ class ACMEServer(object): self._launch_process(command) print('=> Finished configuring the HTTP proxy.') - def _launch_process(self, command, cwd=os.getcwd(), env=None): + def _launch_process(self, command, cwd=os.getcwd(), env=None, force_stderr=False): """Launch silently a subprocess OS command""" if not env: env = os.environ - process = subprocess.Popen(command, stdout=self._stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env) + stdout = sys.stderr if force_stderr else self._stdout + process = subprocess.Popen(command, stdout=stdout, stderr=subprocess.STDOUT, cwd=cwd, env=env) self._processes.append(process) return process diff --git a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py index 2b1557928..7fe03b990 100644 --- a/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py +++ b/certbot-ci/certbot_integration_tests/utils/pebble_artifacts.py @@ -7,7 +7,7 @@ import requests from certbot_integration_tests.utils.constants import MOCK_OCSP_SERVER_PORT -PEBBLE_VERSION = 'v2.2.1' +PEBBLE_VERSION = 'v2.3.0' ASSETS_PATH = pkg_resources.resource_filename('certbot_integration_tests', 'assets') diff --git a/certbot-compatibility-test/Dockerfile b/certbot-compatibility-test/Dockerfile index a9996f779..a6a0c93db 100644 --- a/certbot-compatibility-test/Dockerfile +++ b/certbot-compatibility-test/Dockerfile @@ -31,7 +31,7 @@ COPY certbot-nginx /opt/certbot/src/certbot-nginx/ COPY certbot-compatibility-test /opt/certbot/src/certbot-compatibility-test/ COPY tools /opt/certbot/src/tools -RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 /opt/certbot/venv && \ +RUN VIRTUALENV_NO_DOWNLOAD=1 virtualenv -p python2 /opt/certbot/venv && \ /opt/certbot/venv/bin/pip install -U setuptools && \ /opt/certbot/venv/bin/pip install -U pip ENV PATH /opt/certbot/venv/bin:$PATH diff --git a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py index a9b1ce87e..5d5542ffd 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py +++ b/certbot-compatibility-test/certbot_compatibility_test/configurators/apache/common.py @@ -3,7 +3,10 @@ import os import shutil import subprocess -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import zope.interface from certbot import errors as le_errors diff --git a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py index d719f583b..5140dc8ea 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/test_driver.py +++ b/certbot-compatibility-test/certbot_compatibility_test/test_driver.py @@ -96,7 +96,7 @@ def test_authenticator(plugin, config, temp_dir): def _create_achalls(plugin): """Returns a list of annotated challenges to test on plugin""" - achalls = list() + achalls = [] names = plugin.get_testable_domain_names() for domain in names: prefs = plugin.get_chall_pref(domain) diff --git a/certbot-compatibility-test/certbot_compatibility_test/validator_test.py b/certbot-compatibility-test/certbot_compatibility_test/validator_test.py index 235ce0e3c..0b1056561 100644 --- a/certbot-compatibility-test/certbot_compatibility_test/validator_test.py +++ b/certbot-compatibility-test/certbot_compatibility_test/validator_test.py @@ -1,7 +1,10 @@ """Tests for certbot_compatibility_test.validator.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import requests diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 1dbcefa75..ed8e3f861 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -1,19 +1,29 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup -version = '1.3.0.dev0' +version = '1.4.0.dev0' install_requires = [ 'certbot', 'certbot-apache', - 'mock', 'six', 'requests', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + if sys.version_info < (2, 7, 9): # For secure SSL connexion with Python 2.7 (InsecurePlatformWarning) install_requires.append('ndg-httpsclient') diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py index 9376bc1c4..9ac16bc67 100644 --- a/certbot-dns-cloudflare/setup.py +++ b/certbot-dns-cloudflare/setup.py @@ -1,10 +1,12 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'cloudflare>=1.5.1', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py index d38330191..4d2dcf4ca 100644 --- a/certbot-dns-cloudflare/tests/dns_cloudflare_test.py +++ b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py @@ -3,7 +3,10 @@ import unittest import CloudFlare -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py index 4e99ff5ff..f998027f9 100644 --- a/certbot-dns-cloudxns/setup.py +++ b/certbot-dns-cloudxns/setup.py @@ -1,10 +1,12 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py index a1e3cde89..43c69790f 100644 --- a/certbot-dns-cloudxns/tests/dns_cloudxns_test.py +++ b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py index 9c9d1717c..7aef67d75 100644 --- a/certbot-dns-digitalocean/setup.py +++ b/certbot-dns-digitalocean/setup.py @@ -1,23 +1,33 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', - 'mock', 'python-digitalocean>=1.11', 'setuptools', 'six', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py index 71301a47c..a752f52d0 100644 --- a/certbot-dns-digitalocean/tests/dns_digitalocean_test.py +++ b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py @@ -3,7 +3,10 @@ import unittest import digitalocean -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py index 9cde6214c..4a3f863f5 100644 --- a/certbot-dns-dnsimple/setup.py +++ b/certbot-dns-dnsimple/setup.py @@ -1,22 +1,32 @@ +from distutils.version import StrictVersion import os import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + # This package normally depends on dns-lexicon>=3.2.1 to address the # problem described in https://github.com/AnalogJ/lexicon/issues/387, # however, the fix there has been backported to older versions of diff --git a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py index ca5eb4f36..40eba4754 100644 --- a/certbot-dns-dnsimple/tests/dns_dnsimple_test.py +++ b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py index adaba6851..2dced23bf 100644 --- a/certbot-dns-dnsmadeeasy/setup.py +++ b/certbot-dns-dnsmadeeasy/setup.py @@ -1,10 +1,12 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py index b94cc7d05..4a69e977c 100644 --- a/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py +++ b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py index a849cef45..7c7acc503 100644 --- a/certbot-dns-gehirn/setup.py +++ b/certbot-dns-gehirn/setup.py @@ -1,21 +1,31 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.1.22', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-gehirn/tests/dns_gehirn_test.py b/certbot-dns-gehirn/tests/dns_gehirn_test.py index f5b95b6c3..0598a5eb5 100644 --- a/certbot-dns-gehirn/tests/dns_gehirn_test.py +++ b/certbot-dns-gehirn/tests/dns_gehirn_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py index 51d5b8a3f..1b51a781e 100644 --- a/certbot-dns-google/setup.py +++ b/certbot-dns-google/setup.py @@ -1,10 +1,12 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -12,7 +14,6 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'google-api-python-client>=1.5.5', - 'mock', 'oauth2client>=4.0', 'setuptools', 'zope.interface', @@ -20,6 +21,15 @@ install_requires = [ 'httplib2' ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-google/tests/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py index 647a75b05..5af027cef 100644 --- a/certbot-dns-google/tests/dns_google_test.py +++ b/certbot-dns-google/tests/dns_google_test.py @@ -6,7 +6,10 @@ from googleapiclient import discovery from googleapiclient.errors import Error from googleapiclient.http import HttpMock from httplib2 import ServerNotFoundError -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py index e7e91b929..860c40079 100644 --- a/certbot-dns-linode/setup.py +++ b/certbot-dns-linode/setup.py @@ -1,21 +1,31 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.3', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-linode/tests/dns_linode_test.py b/certbot-dns-linode/tests/dns_linode_test.py index 3cf615486..fb9b1aa93 100644 --- a/certbot-dns-linode/tests/dns_linode_test.py +++ b/certbot-dns-linode/tests/dns_linode_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py index ea64f79a2..83932e140 100644 --- a/certbot-dns-luadns/setup.py +++ b/certbot-dns-luadns/setup.py @@ -1,10 +1,12 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-luadns/tests/dns_luadns_test.py b/certbot-dns-luadns/tests/dns_luadns_test.py index 934d3e103..a1242582f 100644 --- a/certbot-dns-luadns/tests/dns_luadns_test.py +++ b/certbot-dns-luadns/tests/dns_luadns_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py index d6bedca1c..459c6d752 100644 --- a/certbot-dns-nsone/setup.py +++ b/certbot-dns-nsone/setup.py @@ -1,10 +1,12 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.2.1', # Support for >1 TXT record per name - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-nsone/tests/dns_nsone_test.py b/certbot-dns-nsone/tests/dns_nsone_test.py index dd6168f08..83371252f 100644 --- a/certbot-dns-nsone/tests/dns_nsone_test.py +++ b/certbot-dns-nsone/tests/dns_nsone_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py index 8f5b052a2..5823b237e 100644 --- a/certbot-dns-ovh/setup.py +++ b/certbot-dns-ovh/setup.py @@ -1,10 +1,12 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.7.14', # Correct proxy use on OVH provider - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-ovh/tests/dns_ovh_test.py b/certbot-dns-ovh/tests/dns_ovh_test.py index a420239ab..dd0f3058b 100644 --- a/certbot-dns-ovh/tests/dns_ovh_test.py +++ b/certbot-dns-ovh/tests/dns_ovh_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py index fa51c2108..94cda6f65 100644 --- a/certbot-dns-rfc2136/setup.py +++ b/certbot-dns-rfc2136/setup.py @@ -1,10 +1,12 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'dnspython', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py index c767dba23..4c14a8072 100644 --- a/certbot-dns-rfc2136/tests/dns_rfc2136_test.py +++ b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py @@ -5,7 +5,10 @@ import unittest import dns.flags import dns.rcode import dns.tsig -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py index f25e348ff..f140d3f8d 100644 --- a/certbot-dns-route53/setup.py +++ b/certbot-dns-route53/setup.py @@ -1,10 +1,12 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. @@ -12,11 +14,19 @@ install_requires = [ 'acme>=0.29.0', 'certbot>=1.1.0', 'boto3', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + class PyTest(TestCommand): user_options = [] diff --git a/certbot-dns-route53/tests/dns_route53_test.py b/certbot-dns-route53/tests/dns_route53_test.py index 85ec259b1..a77495313 100644 --- a/certbot-dns-route53/tests/dns_route53_test.py +++ b/certbot-dns-route53/tests/dns_route53_test.py @@ -4,7 +4,10 @@ import unittest from botocore.exceptions import ClientError from botocore.exceptions import NoCredentialsError -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot import errors from certbot.compat import os diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py index 8df2320ba..b57e28577 100644 --- a/certbot-dns-sakuracloud/setup.py +++ b/certbot-dns-sakuracloud/setup.py @@ -1,21 +1,31 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ 'acme>=0.31.0', 'certbot>=1.1.0', 'dns-lexicon>=2.1.23', - 'mock', 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + docs_extras = [ 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags 'sphinx_rtd_theme', diff --git a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py index 16890b5a9..af94336b3 100644 --- a/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py +++ b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from certbot.compat import os diff --git a/certbot-nginx/certbot_nginx/_internal/configurator.py b/certbot-nginx/certbot_nginx/_internal/configurator.py index 459950aa1..a903c12bf 100644 --- a/certbot-nginx/certbot_nginx/_internal/configurator.py +++ b/certbot-nginx/certbot_nginx/_internal/configurator.py @@ -1,5 +1,4 @@ """Nginx Configuration""" -# https://github.com/PyCQA/pylint/issues/73 from distutils.version import LooseVersion import logging import re @@ -749,7 +748,7 @@ class NginxConfigurator(common.Installer): # if there is no separate SSL block, break the block into two and # choose the SSL block. - if vhost.ssl and any([not addr.ssl for addr in vhost.addrs]): + if vhost.ssl and any(not addr.ssl for addr in vhost.addrs): _, vhost = self._split_block(vhost) header_directives = [ @@ -984,7 +983,7 @@ class NginxConfigurator(common.Installer): logger.warning("NGINX derivative %s is not officially supported by" " certbot", product_name) - nginx_version = tuple([int(i) for i in product_version.split(".")]) + nginx_version = tuple(int(i) for i in product_version.split(".")) # nginx < 0.8.48 uses machine hostname as default server_name instead of # the empty string diff --git a/certbot-nginx/certbot_nginx/_internal/display_ops.py b/certbot-nginx/certbot_nginx/_internal/display_ops.py index bbb47f98a..356cc506c 100644 --- a/certbot-nginx/certbot_nginx/_internal/display_ops.py +++ b/certbot-nginx/certbot_nginx/_internal/display_ops.py @@ -17,7 +17,7 @@ def select_vhost_multiple(vhosts): :rtype: :class:`list`of type `~obj.Vhost` """ if not vhosts: - return list() + return [] tags_list = [vhost.display_repr()+"\n" for vhost in vhosts] # Remove the extra newline from the last entry if tags_list: @@ -33,7 +33,7 @@ def select_vhost_multiple(vhosts): def _reversemap_vhosts(names, vhosts): """Helper function for select_vhost_multiple for mapping string representations back to actual vhost objects""" - return_vhosts = list() + return_vhosts = [] for selection in names: for vhost in vhosts: diff --git a/certbot-nginx/certbot_nginx/_internal/parser.py b/certbot-nginx/certbot_nginx/_internal/parser.py index f71d7c018..bb0bb7d6f 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser.py +++ b/certbot-nginx/certbot_nginx/_internal/parser.py @@ -2,6 +2,7 @@ import copy import functools import glob +import io import logging import re @@ -205,12 +206,16 @@ class NginxParser(object): if item in self.parsed and not override: continue try: - with open(item) as _file: + with io.open(item, "r", encoding="utf-8") as _file: parsed = nginxparser.load(_file) self.parsed[item] = parsed trees.append(parsed) except IOError: logger.warning("Could not open file: %s", item) + except UnicodeDecodeError: + logger.warning("Could not read file: %s due to invalid " + "character. Only UTF-8 encoding is " + "supported.", item) except pyparsing.ParseException as err: logger.debug("Could not parse file: %s due to %s", item, err) return trees @@ -399,9 +404,9 @@ class NginxParser(object): if directive and directive[0] == 'listen': # Exclude one-time use parameters which will cause an error if repeated. # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen - exclude = set(('default_server', 'default', 'setfib', 'fastopen', 'backlog', + exclude = {'default_server', 'default', 'setfib', 'fastopen', 'backlog', 'rcvbuf', 'sndbuf', 'accept_filter', 'deferred', 'bind', - 'ipv6only', 'reuseport', 'so_keepalive')) + 'ipv6only', 'reuseport', 'so_keepalive'} for param in exclude: # See: github.com/certbot/certbot/pull/6223#pullrequestreview-143019225 @@ -414,10 +419,13 @@ class NginxParser(object): def _parse_ssl_options(ssl_options): if ssl_options is not None: try: - with open(ssl_options) as _file: + with io.open(ssl_options, "r", encoding="utf-8") as _file: return nginxparser.load(_file) except IOError: logger.warning("Missing NGINX TLS options file: %s", ssl_options) + except UnicodeDecodeError: + logger.warning("Could not read file: %s due to invalid character. " + "Only UTF-8 encoding is supported.", ssl_options) except pyparsing.ParseBaseException as err: logger.debug("Could not parse file: %s due to %s", ssl_options, err) return [] @@ -570,7 +578,7 @@ def _update_or_add_directives(directives, insert_at_top, block): INCLUDE = 'include' -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE, 'rewrite', 'add_header']) +REPEATABLE_DIRECTIVES = {'server_name', 'listen', INCLUDE, 'rewrite', 'add_header'} COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] diff --git a/certbot-nginx/certbot_nginx/_internal/parser_obj.py b/certbot-nginx/certbot_nginx/_internal/parser_obj.py index 61b31b2d5..390e18e4d 100644 --- a/certbot-nginx/certbot_nginx/_internal/parser_obj.py +++ b/certbot-nginx/certbot_nginx/_internal/parser_obj.py @@ -206,7 +206,7 @@ class Sentence(Parsable): :returns: whether this lists is parseable by `Sentence`. """ return isinstance(lists, list) and len(lists) > 0 and \ - all([isinstance(elem, six.string_types) for elem in lists]) + all(isinstance(elem, six.string_types) for elem in lists) def parse(self, raw_list, add_spaces=False): """ Parses a list of string types into this object. @@ -214,7 +214,7 @@ class Sentence(Parsable): if add_spaces: raw_list = _space_list(raw_list) if not isinstance(raw_list, list) or \ - any([not isinstance(elem, six.string_types) for elem in raw_list]): + any(not isinstance(elem, six.string_types) for elem in raw_list): raise errors.MisconfigurationError("Sentence parsing expects a list of string types.") self._data = raw_list diff --git a/certbot-nginx/local-oldest-requirements.txt b/certbot-nginx/local-oldest-requirements.txt index cee142934..1782f15ba 100644 --- a/certbot-nginx/local-oldest-requirements.txt +++ b/certbot-nginx/local-oldest-requirements.txt @@ -1,3 +1,3 @@ # Remember to update setup.py to match the package versions below. -acme[dev]==1.0.0 -certbot[dev]==1.1.0 +-e acme[dev] +-e certbot[dev] diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index b180fe06a..0e6deceb3 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -1,23 +1,33 @@ +from distutils.version import StrictVersion import sys +from setuptools import __version__ as setuptools_version from setuptools import find_packages from setuptools import setup from setuptools.command.test import test as TestCommand -version = '1.3.0.dev0' +version = '1.4.0.dev0' # Remember to update local-oldest-requirements.txt when changing the minimum # acme/certbot version. install_requires = [ - 'acme>=1.0.0', - 'certbot>=1.1.0', - 'mock', + 'acme>=1.4.0.dev0', + 'certbot>=1.4.0.dev0', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support 'setuptools', 'zope.interface', ] +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + class PyTest(TestCommand): user_options = [] diff --git a/certbot-nginx/tests/configurator_test.py b/certbot-nginx/tests/configurator_test.py index 0d9d6d356..2c3264a5f 100644 --- a/certbot-nginx/tests/configurator_test.py +++ b/certbot-nginx/tests/configurator_test.py @@ -1,7 +1,10 @@ """Test for certbot_nginx._internal.configurator.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL from acme import challenges @@ -104,7 +107,7 @@ class NginxConfiguratorTest(util.NginxTest): filep = self.config.parser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(filep, None, None, None, - set(['.example.com', 'example.*']), + {'.example.com', 'example.*'}, None, [0]) self.config.parser.add_server_directives( mock_vhost, @@ -150,11 +153,11 @@ class NginxConfiguratorTest(util.NginxTest): self._test_choose_vhosts_common('ipv6.com', 'ipv6_conf') def _test_choose_vhosts_common(self, name, conf): - conf_names = {'localhost_conf': set(['localhost', r'~^(www\.)?(example|bar)\.']), - 'server_conf': set(['somename', 'another.alias', 'alias']), - 'example_conf': set(['.example.com', 'example.*']), - 'foo_conf': set(['*.www.foo.com', '*.www.example.com']), - 'ipv6_conf': set(['ipv6.com'])} + conf_names = {'localhost_conf': {'localhost', r'~^(www\.)?(example|bar)\.'}, + 'server_conf': {'somename', 'another.alias', 'alias'}, + 'example_conf': {'.example.com', 'example.*'}, + 'foo_conf': {'*.www.foo.com', '*.www.example.com'}, + 'ipv6_conf': {'ipv6.com'}} conf_path = {'localhost': "etc_nginx/nginx.conf", 'alias': "etc_nginx/nginx.conf", @@ -177,7 +180,7 @@ class NginxConfiguratorTest(util.NginxTest): self.assertTrue(vhost.ipv6_enabled()) # Make sure that we have SSL enabled also for IPv6 addr self.assertTrue( - any([True for x in vhost.addrs if x.ssl and x.ipv6])) + any(True for x in vhost.addrs if x.ssl and x.ipv6)) def test_choose_vhosts_bad(self): bad_results = ['www.foo.com', 'example', 't.www.bar.co', diff --git a/certbot-nginx/tests/http_01_test.py b/certbot-nginx/tests/http_01_test.py index 6418a8841..8f0673c1f 100644 --- a/certbot-nginx/tests/http_01_test.py +++ b/certbot-nginx/tests/http_01_test.py @@ -2,7 +2,10 @@ import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six from acme import challenges diff --git a/certbot-nginx/tests/obj_test.py b/certbot-nginx/tests/obj_test.py index db808229f..93b9493eb 100644 --- a/certbot-nginx/tests/obj_test.py +++ b/certbot-nginx/tests/obj_test.py @@ -98,10 +98,10 @@ class AddrTest(unittest.TestCase): def test_set_inclusion(self): from certbot_nginx._internal.obj import Addr - set_a = set([self.addr1, self.addr2]) + set_a = {self.addr1, self.addr2} addr1b = Addr.fromstring("192.168.1.1") addr2b = Addr.fromstring("192.168.1.1:* ssl") - set_b = set([addr1b, addr2b]) + set_b = {addr1b, addr2b} self.assertEqual(set_a, set_b) @@ -120,8 +120,8 @@ class VirtualHostTest(unittest.TestCase): ] self.vhost1 = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw1, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw1, []) raw2 = [ ['listen', '69.50.225.155:9000'], [['if', '($scheme', '!=', '"https") '], @@ -130,24 +130,24 @@ class VirtualHostTest(unittest.TestCase): ] self.vhost2 = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw2, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw2, []) raw3 = [ ['listen', '69.50.225.155:9000'], ['rewrite', '^(.*)$', '$scheme://www.domain.com$1', 'permanent'] ] self.vhost3 = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw3, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw3, []) raw4 = [ ['listen', '69.50.225.155:9000'], ['server_name', 'return.com'] ] self.vhost4 = VirtualHost( "filp", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw4, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw4, []) raw_has_hsts = [ ['listen', '69.50.225.155:9000'], ['server_name', 'return.com'], @@ -155,16 +155,16 @@ class VirtualHostTest(unittest.TestCase): ] self.vhost_has_hsts = VirtualHost( "filep", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), raw_has_hsts, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, raw_has_hsts, []) def test_eq(self): from certbot_nginx._internal.obj import Addr from certbot_nginx._internal.obj import VirtualHost vhost1b = VirtualHost( "filep", - set([Addr.fromstring("localhost blah")]), False, False, - set(['localhost']), [], []) + {Addr.fromstring("localhost blah")}, False, False, + {'localhost'}, [], []) self.assertEqual(vhost1b, self.vhost1) self.assertEqual(str(vhost1b), str(self.vhost1)) @@ -203,8 +203,8 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], []] vhost_haystack = VirtualHost( "filp", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), test_haystack, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, test_haystack, []) test_bad_haystack = [['listen', '80'], ['root', '/var/www/html'], ['index', 'index.html index.htm index.nginx-debian.html'], ['server_name', 'two.functorkitten.xyz'], ['listen', '443 ssl'], @@ -219,8 +219,8 @@ class VirtualHostTest(unittest.TestCase): ['#', ' managed by Certbot'], []] vhost_bad_haystack = VirtualHost( "filp", - set([Addr.fromstring("localhost")]), False, False, - set(['localhost']), test_bad_haystack, []) + {Addr.fromstring("localhost")}, False, False, + {'localhost'}, test_bad_haystack, []) self.assertTrue(vhost_haystack.contains_list(test_needle)) self.assertFalse(vhost_bad_haystack.contains_list(test_needle)) diff --git a/certbot-nginx/tests/parser_obj_test.py b/certbot-nginx/tests/parser_obj_test.py index 132f83771..8262c5f52 100644 --- a/certbot-nginx/tests/parser_obj_test.py +++ b/certbot-nginx/tests/parser_obj_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from certbot_nginx._internal.parser_obj import COMMENT_BLOCK from certbot_nginx._internal.parser_obj import parse_raw @@ -80,7 +83,7 @@ class ParsingHooksTest(unittest.TestCase): fake_parser1.should_parse = lambda x: False parsing_hooks.return_value = (fake_parser1,) self.assertRaises(errors.MisconfigurationError, parse_raw, []) - parsing_hooks.return_value = tuple() + parsing_hooks.return_value = () self.assertRaises(errors.MisconfigurationError, parse_raw, []) def test_parse_raw_passes_add_spaces(self): diff --git a/certbot-nginx/tests/parser_test.py b/certbot-nginx/tests/parser_test.py index f3a5665c5..21dd1043d 100644 --- a/certbot-nginx/tests/parser_test.py +++ b/certbot-nginx/tests/parser_test.py @@ -4,7 +4,6 @@ import re import shutil import unittest -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot.compat import os from certbot_nginx._internal import nginxparser @@ -127,7 +126,7 @@ class NginxParserTest(util.NginxTest): vhost = obj.VirtualHost(nparser.abs_path('sites-enabled/globalssl.com'), [obj.Addr('4.8.2.6', '57', True, False, False, False)], - True, True, set(['globalssl.com']), [], [0]) + True, True, {'globalssl.com'}, [], [0]) globalssl_com = [x for x in vhosts if 'globalssl.com' in x.filep][0] self.assertEqual(vhost, globalssl_com) @@ -140,8 +139,8 @@ class NginxParserTest(util.NginxTest): [obj.Addr('', '8080', False, False, False, False)], False, True, - set(['localhost', - r'~^(www\.)?(example|bar)\.']), + {'localhost', + r'~^(www\.)?(example|bar)\.'}, [], [10, 1, 9]) vhost2 = obj.VirtualHost(nparser.abs_path('nginx.conf'), [obj.Addr('somename', '8080', False, False, @@ -149,7 +148,7 @@ class NginxParserTest(util.NginxTest): obj.Addr('', '8000', False, False, False, False)], False, True, - set(['somename', 'another.alias', 'alias']), + {'somename', 'another.alias', 'alias'}, [], [10, 1, 12]) vhost3 = obj.VirtualHost(nparser.abs_path('sites-enabled/example.com'), [obj.Addr('69.50.225.155', '9000', @@ -157,19 +156,19 @@ class NginxParserTest(util.NginxTest): obj.Addr('127.0.0.1', '', False, False, False, False)], False, True, - set(['.example.com', 'example.*']), [], [0]) + {'.example.com', 'example.*'}, [], [0]) vhost4 = obj.VirtualHost(nparser.abs_path('sites-enabled/default'), [obj.Addr('myhost', '', False, True, False, False), obj.Addr('otherhost', '', False, True, False, False)], - False, True, set(['www.example.org']), + False, True, {'www.example.org'}, [], [0]) vhost5 = obj.VirtualHost(nparser.abs_path('foo.conf'), [obj.Addr('*', '80', True, True, False, False)], - True, True, set(['*.www.foo.com', - '*.www.example.com']), + True, True, {'*.www.foo.com', + '*.www.example.com'}, [], [2, 1, 0]) self.assertEqual(14, len(vhosts)) @@ -209,11 +208,11 @@ class NginxParserTest(util.NginxTest): nparser = parser.NginxParser(self.config_path) mock_vhost = obj.VirtualHost(nparser.abs_path('nginx.conf'), None, None, None, - set(['localhost', - r'~^(www\.)?(example|bar)\.']), + {'localhost', + r'~^(www\.)?(example|bar)\.'}, None, [10, 1, 9]) example_com = nparser.abs_path('sites-enabled/example.com') - names = set(['.example.com', 'example.*']) + names = {'.example.com', 'example.*'} mock_vhost.filep = example_com mock_vhost.names = names mock_vhost.path = [0] @@ -233,8 +232,8 @@ class NginxParserTest(util.NginxTest): nparser = parser.NginxParser(self.config_path) mock_vhost = obj.VirtualHost(nparser.abs_path('nginx.conf'), None, None, None, - set(['localhost', - r'~^(www\.)?(example|bar)\.']), + {'localhost', + r'~^(www\.)?(example|bar)\.'}, None, [10, 1, 9]) nparser.add_server_directives(mock_vhost, [['foo', 'bar'], ['\n ', 'ssl_certificate', ' ', @@ -244,7 +243,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual(1, len(re.findall(ssl_re, dump))) example_com = nparser.abs_path('sites-enabled/example.com') - names = set(['.example.com', 'example.*']) + names = {'.example.com', 'example.*'} mock_vhost.filep = example_com mock_vhost.names = names mock_vhost.path = [0] @@ -265,7 +264,7 @@ class NginxParserTest(util.NginxTest): ]]]) server_conf = nparser.abs_path('server.conf') - names = set(['alias', 'another.alias', 'somename']) + names = {'alias', 'another.alias', 'somename'} mock_vhost.filep = server_conf mock_vhost.names = names mock_vhost.path = [] @@ -280,7 +279,7 @@ class NginxParserTest(util.NginxTest): example_com = nparser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(example_com, None, None, None, - set(['.example.com', 'example.*']), + {'.example.com', 'example.*'}, None, [0]) nparser.add_server_directives(mock_vhost, [['\n ', '#', ' ', 'what a nice comment']]) @@ -302,7 +301,7 @@ class NginxParserTest(util.NginxTest): def test_replace_server_directives(self): nparser = parser.NginxParser(self.config_path) - target = set(['.example.com', 'example.*']) + target = {'.example.com', 'example.*'} filep = nparser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(filep, None, None, None, target, None, [0]) nparser.update_or_add_server_directives( @@ -315,7 +314,7 @@ class NginxParserTest(util.NginxTest): ['server_name', 'foobar.com'], ['#', COMMENT], ['server_name', 'example.*'], [] ]]]) - mock_vhost.names = set(['foobar.com', 'example.*']) + mock_vhost.names = {'foobar.com', 'example.*'} nparser.update_or_add_server_directives( mock_vhost, [['ssl_certificate', 'cert.pem']]) self.assertEqual( @@ -329,19 +328,19 @@ class NginxParserTest(util.NginxTest): def test_get_best_match(self): target_name = 'www.eff.org' - names = [set(['www.eff.org', 'irrelevant.long.name.eff.org', '*.org']), - set(['eff.org', 'ww2.eff.org', 'test.www.eff.org']), - set(['*.eff.org', '.www.eff.org']), - set(['.eff.org', '*.org']), - set(['www.eff.', 'www.eff.*', '*.www.eff.org']), - set(['example.com', r'~^(www\.)?(eff.+)', '*.eff.*']), - set(['*', r'~^(www\.)?(eff.+)']), - set(['www.*', r'~^(www\.)?(eff.+)', '.test.eff.org']), - set(['*.org', r'*.eff.org', 'www.eff.*']), - set(['*.www.eff.org', 'www.*']), - set(['*.org']), - set([]), - set(['example.com'])] + names = [{'www.eff.org', 'irrelevant.long.name.eff.org', '*.org'}, + {'eff.org', 'ww2.eff.org', 'test.www.eff.org'}, + {'*.eff.org', '.www.eff.org'}, + {'.eff.org', '*.org'}, + {'www.eff.', 'www.eff.*', '*.www.eff.org'}, + {'example.com', r'~^(www\.)?(eff.+)', '*.eff.*'}, + {'*', r'~^(www\.)?(eff.+)'}, + {'www.*', r'~^(www\.)?(eff.+)', '.test.eff.org'}, + {'*.org', r'*.eff.org', 'www.eff.*'}, + {'*.www.eff.org', 'www.*'}, + {'*.org'}, + set(), + {'example.com'}] winners = [('exact', 'www.eff.org'), (None, None), ('exact', '.www.eff.org'), @@ -482,7 +481,43 @@ class NginxParserTest(util.NginxTest): called = True self.assertTrue(called) + def test_valid_unicode_characters(self): + nparser = parser.NginxParser(self.config_path) + path = nparser.abs_path('valid_unicode_comments.conf') + parsed = nparser._parse_files(path) # pylint: disable=protected-access + self.assertEqual(['server'], parsed[0][2][0]) + self.assertEqual(['listen', '80'], parsed[0][2][1][3]) + def test_invalid_unicode_characters(self): + with self.assertLogs() as log: + nparser = parser.NginxParser(self.config_path) + path = nparser.abs_path('invalid_unicode_comments.conf') + parsed = nparser._parse_files(path) # pylint: disable=protected-access + + self.assertEqual([], parsed) + self.assertTrue(any( + ('invalid character' in output) and ('UTF-8' in output) + for output in log.output + )) + + def test_valid_unicode_characters_in_ssl_options(self): + nparser = parser.NginxParser(self.config_path) + path = nparser.abs_path('valid_unicode_comments.conf') + parsed = parser._parse_ssl_options(path) # pylint: disable=protected-access + self.assertEqual(['server'], parsed[2][0]) + self.assertEqual(['listen', '80'], parsed[2][1][3]) + + def test_invalid_unicode_characters_in_ssl_options(self): + with self.assertLogs() as log: + nparser = parser.NginxParser(self.config_path) + path = nparser.abs_path('invalid_unicode_comments.conf') + parsed = parser._parse_ssl_options(path) # pylint: disable=protected-access + + self.assertEqual([], parsed) + self.assertTrue(any( + ('invalid character' in output) and ('UTF-8' in output) + for output in log.output + )) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-nginx/tests/test_log_util.py b/certbot-nginx/tests/test_log_util.py new file mode 100644 index 000000000..7aebf2151 --- /dev/null +++ b/certbot-nginx/tests/test_log_util.py @@ -0,0 +1,125 @@ +"""Backport for `TestCase.assertLogs()`. + +Most of the idea and code are from CPython implementation. +https://github.com/python/cpython/blob/b76518d43fb82ed9e5d27025d18c90a23d525c90/Lib/unittest/case.py +""" +import logging +import collections + +__all__ = ['AssertLogsMixin'] + +LoggingWatcher = collections.namedtuple('LoggingWatcher', ['records', 'output']) + + +class CapturingHandler(logging.Handler): + """ + A logging handler capturing all (raw and formatted) logging output. + """ + + def __init__(self): + super(CapturingHandler, self).__init__() + self.watcher = LoggingWatcher([], []) + + def flush(self): + pass + + def emit(self, record): + self.watcher.records.append(record) + self.watcher.output.append(self.format(record)) + + + +class AssertLogsContext(object): + """ + A context manager used to implement `TestCase.assertLogs()`. + """ + + LOGGING_FORMAT = '%(levelname)s:%(name)s:%(message)s' + + def __init__(self, test_case, logger_name, level): + self.test_case = test_case + + self.logger_name = logger_name + self.logger_states = None + self.logger = None + + if level: + # pylint: disable=protected-access,no-member + try: + # In Python 3.x + name_to_level = logging._nameToLevel # type: ignore + except AttributeError: + # In Python 2.7 + name_to_level = logging._levelNames # type: ignore + + self.level = name_to_level.get(level, level) + else: + self.level = logging.INFO + + self.watcher = None + + def _save_logger_states(self): + self.logger_states = (self.logger.handlers[:], self.logger.level, self.logger.propagate) + + def _restore_logger_states(self): + self.logger.handlers, self.logger.level, self.logger.propagate = self.logger_states + + def __enter__(self): + if isinstance(self.logger_name, logging.Logger): + self.logger = self.logger_name + else: + self.logger = logging.getLogger(self.logger_name) + + formatter = logging.Formatter(self.LOGGING_FORMAT) + + handler = CapturingHandler() + handler.setFormatter(formatter) + + self._save_logger_states() + self.logger.handlers = [handler] + self.logger.setLevel(self.level) + self.logger.propagate = False + + self.watcher = handler.watcher + return handler.watcher + + def __exit__(self, exc_type, exc_value, tb): + self._restore_logger_states() + + if exc_type is not None: + # let unexpected exceptions pass through + return + + if not self.watcher.records: + self._raiseFailure( + "no logs of level {} or higher triggered on {}" + .format(logging.getLevelName(self.level), self.logger.name)) + + def _raiseFailure(self, message): + message = self.test_case._formatMessage(None, message) # pylint: disable=protected-access + raise self.test_case.failureException(message) + + +class AssertLogsMixin(object): + """ + A mixin that implements `TestCase.assertLogs()`. + """ + + def assertLogs(self, logger=None, level=None): + """Fail unless a log message of level *level* or higher is emitted + on *logger_name* or its children. If omitted, *level* defaults to + INFO and *logger* defaults to the root logger. + This method must be used as a context manager, and will yield + a recording object with two attributes: `output` and `records`. + At the end of the context manager, the `output` attribute will + be a list of the matching formatted log messages and the + `records` attribute will be a list of the corresponding LogRecord + objects. + Example:: + with self.assertLogs('foo', level='INFO') as cm: + logging.getLogger('foo').info('first message') + logging.getLogger('foo.bar').error('second message') + self.assertEqual(cm.output, ['INFO:foo:first message', + 'ERROR:foo.bar:second message']) + """ + return AssertLogsContext(self, logger, level) diff --git a/certbot-nginx/tests/test_util.py b/certbot-nginx/tests/test_util.py index 8dfd18637..4b26f7935 100644 --- a/certbot-nginx/tests/test_util.py +++ b/certbot-nginx/tests/test_util.py @@ -4,7 +4,10 @@ import shutil import tempfile import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import pkg_resources import zope.component @@ -14,9 +17,10 @@ from certbot.plugins import common from certbot.tests import util as test_util from certbot_nginx._internal import configurator from certbot_nginx._internal import nginxparser +import test_log_util -class NginxTest(test_util.ConfigTestCase): +class NginxTest(test_log_util.AssertLogsMixin, test_util.ConfigTestCase): def setUp(self): super(NginxTest, self).setUp() diff --git a/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf b/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf new file mode 100644 index 000000000..4d6384535 --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/invalid_unicode_comments.conf @@ -0,0 +1,7 @@ +# This configuration file is saved with EUC-KR (a.k.a. cp949) encoding, +# including some Korean letters. + +server { + # ȳϼ. 80 Ʈ û ٸ. + listen 80; +} diff --git a/certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf b/certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf new file mode 100644 index 000000000..89c978b2e --- /dev/null +++ b/certbot-nginx/tests/testdata/etc_nginx/valid_unicode_comments.conf @@ -0,0 +1,9 @@ +# This configuration file is saved with valid UTF-8 encoding, +# including some CJK alphabets. + +server { + # 안녕하세요. 80번 포트에서 요청을 기다린다. + # こんにちは。８０番ポートからリクエストを待つ。 + # 你好。等待端口80上的请求。 + listen 80; +} diff --git a/certbot/CHANGELOG.md b/certbot/CHANGELOG.md index 30479f25b..1b9379de7 100644 --- a/certbot/CHANGELOG.md +++ b/certbot/CHANGELOG.md @@ -2,25 +2,60 @@ Certbot adheres to [Semantic Versioning](https://semver.org/). -## 1.3.0 - master +## 1.4.0 - master ### Added -* Added certbot.ocsp Certbot's API. The certbot.ocsp module can be used to - determine the OCSP status of certificates. -* Don't verify the existing certificate in HTTP01Response.simple_verify, for - compatibility with the real-world ACME challenge checks. +* Turn off session tickets for apache plugin by default when appropriate. +* Added serial number of certificate to the output of `certbot certificates` +* Expose two new environment variables in the authenticator and cleanup scripts used by + the `manual` plugin: `CERTBOT_REMAINING_CHALLENGES` is equal to the number of challenges + remaining after the current challenge, `CERTBOT_ALL_DOMAINS` is a comma-separated list + of all domains challenged for the current certificate. +* Added TLS-ALPN-01 challenge support in the `acme` library. Support of this + challenge in the Certbot client is planned to be added in a future release. +* Added minimal proxy support for OCSP verification. ### Changed -* certbot._internal.cli is now a package split in submodules instead of a whole module. +* Stop asking interactively if the user would like to add a redirect. +* `mock` dependency is now conditional on Python 2 in all of our packages. + +### Fixed + +* When using an RFC 8555 compliant endpoint, the `acme` library no longer sends the + `resource` field in any requests or the `type` field when responding to challenges. +* Fix nginx plugin crash when non-ASCII configuration file is being read (instead, + the user will be warned that UTF-8 must be used). +* Fix hanging OCSP queries during revocation checking - added a 10 second timeout. +* Standalone servers now have a default socket timeout of 30 seconds, fixing + cases where an idle connection can cause the standalone plugin to hang. +* Parsing of the RFC 8555 application/pem-certificate-chain now tolerates CRLF line + endings. This should fix interoperability with Buypass' services. + +More details about these changes can be found on our GitHub repo. + +## 1.3.0 - 2020-03-03 + +### Added + +* Added certbot.ocsp Certbot's API. The certbot.ocsp module can be used to + determine the OCSP status of certificates. +* Don't verify the existing certificate in HTTP01Response.simple_verify, for + compatibility with the real-world ACME challenge checks. +* Added support for `$hostname` in nginx `server_name` directive + +### Changed + +* Certbot will now renew certificates early if they have been revoked according + to OCSP. * Fix acme module warnings when response Content-Type includes params (e.g. charset). -* Fixed issue where webroot plugin would incorrectly raise `Read-only file system` +* Fixed issue where webroot plugin would incorrectly raise `Read-only file system` error when creating challenge directories (issue #7165). ### Fixed -* +* Fix Apache plugin to use less restrictive umask for making the challenge directory when a restrictive umask was set when certbot was started. More details about these changes can be found on our GitHub repo. @@ -29,7 +64,6 @@ More details about these changes can be found on our GitHub repo. ### Added * Added support for Cloudflare's limited-scope API Tokens -* Added support for `$hostname` in nginx `server_name` directive ### Changed diff --git a/certbot/README.rst b/certbot/README.rst index d1b1e4fe2..5ed74f247 100644 --- a/certbot/README.rst +++ b/certbot/README.rst @@ -71,16 +71,12 @@ ACME spec: http://ietf-wg-acme.github.io/acme/ ACME working area in github: https://github.com/ietf-wg-acme/acme -|build-status| |coverage| |container| +|build-status| |container| .. |build-status| image:: https://travis-ci.com/certbot/certbot.svg?branch=master :target: https://travis-ci.com/certbot/certbot :alt: Travis CI status -.. |coverage| image:: https://codecov.io/gh/certbot/certbot/branch/master/graph/badge.svg - :target: https://codecov.io/gh/certbot/certbot - :alt: Coverage status - .. |container| image:: https://quay.io/repository/letsencrypt/letsencrypt/status :target: https://quay.io/repository/letsencrypt/letsencrypt :alt: Docker Repository on Quay.io diff --git a/certbot/certbot/__init__.py b/certbot/certbot/__init__.py index 84ade6b08..0ce7ff6b7 100644 --- a/certbot/certbot/__init__.py +++ b/certbot/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '1.3.0.dev0' +__version__ = '1.4.0.dev0' diff --git a/certbot/certbot/_internal/cert_manager.py b/certbot/certbot/_internal/cert_manager.py index e6cbd5c2c..2652b3d2c 100644 --- a/certbot/certbot/_internal/cert_manager.py +++ b/certbot/certbot/_internal/cert_manager.py @@ -276,12 +276,15 @@ def human_readable_cert_info(config, cert, skip_filter_checks=False): status = "VALID: {0} days".format(diff.days) valid_string = "{0} ({1})".format(cert.target_expiry, status) + serial = format(crypto_util.get_serial_from_cert(cert.cert_path), 'x') certinfo.append(" Certificate Name: {0}\n" - " Domains: {1}\n" - " Expiry Date: {2}\n" - " Certificate Path: {3}\n" - " Private Key Path: {4}".format( + " Serial Number: {1}\n" + " Domains: {2}\n" + " Expiry Date: {3}\n" + " Certificate Path: {4}\n" + " Private Key Path: {5}".format( cert.lineagename, + serial, " ".join(cert.names()), valid_string, cert.fullchain, diff --git a/certbot/certbot/_internal/cli/__init__.py b/certbot/certbot/_internal/cli/__init__.py index 96dfb163e..2f83edcc8 100644 --- a/certbot/certbot/_internal/cli/__init__.py +++ b/certbot/certbot/_internal/cli/__init__.py @@ -321,12 +321,14 @@ def prepare_and_parse_args(plugins, args, detect_defaults=False): "--redirect", action="store_true", dest="redirect", default=flag_default("redirect"), help="Automatically redirect all HTTP traffic to HTTPS for the newly " - "authenticated vhost. (default: Ask)") + "authenticated vhost. (default: redirect enabled for install and run, " + "disabled for enhance)") helpful.add( "security", "--no-redirect", action="store_false", dest="redirect", default=flag_default("redirect"), help="Do not automatically redirect all HTTP traffic to HTTPS for the newly " - "authenticated vhost. (default: Ask)") + "authenticated vhost. (default: redirect enabled for install and run, " + "disabled for enhance)") helpful.add( ["security", "enhance"], "--hsts", action="store_true", dest="hsts", default=flag_default("hsts"), diff --git a/certbot/certbot/_internal/cli/cli_constants.py b/certbot/certbot/_internal/cli/cli_constants.py index 748ae0d94..4bc84bfe7 100644 --- a/certbot/certbot/_internal/cli/cli_constants.py +++ b/certbot/certbot/_internal/cli/cli_constants.py @@ -91,17 +91,17 @@ ARGPARSE_PARAMS_TO_REMOVE = ("const", "nargs", "type",) # These sets are used when to help detect options set by the user. -EXIT_ACTIONS = set(("help", "version",)) +EXIT_ACTIONS = {"help", "version",} -ZERO_ARG_ACTIONS = set(("store_const", "store_true", - "store_false", "append_const", "count",)) +ZERO_ARG_ACTIONS = {"store_const", "store_true", + "store_false", "append_const", "count",} # Maps a config option to a set of config options that may have modified it. # This dictionary is used recursively, so if A modifies B and B modifies C, # it is determined that C was modified by the user if A was modified. -VAR_MODIFIERS = {"account": set(("server",)), - "renew_hook": set(("deploy_hook",)), - "server": set(("dry_run", "staging",)), - "webroot_map": set(("webroot_path",))} +VAR_MODIFIERS = {"account": {"server",}, + "renew_hook": {"deploy_hook",}, + "server": {"dry_run", "staging",}, + "webroot_map": {"webroot_path",}} diff --git a/certbot/certbot/_internal/cli/helpful.py b/certbot/certbot/_internal/cli/helpful.py index e63ab4b87..31d9396e5 100644 --- a/certbot/certbot/_internal/cli/helpful.py +++ b/certbot/certbot/_internal/cli/helpful.py @@ -11,9 +11,7 @@ import zope.interface from zope.interface import interfaces as zope_interfaces -# pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Any, Dict, Optional -# pylint: enable=unused-import, no-name-in-module +from acme.magic_typing import Any, Dict from certbot import crypto_util from certbot import errors diff --git a/certbot/certbot/_internal/client.py b/certbot/certbot/_internal/client.py index 526f4200f..a9bf946cc 100644 --- a/certbot/certbot/_internal/client.py +++ b/certbot/certbot/_internal/client.py @@ -28,7 +28,6 @@ from certbot._internal import constants from certbot._internal import eff from certbot._internal import error_handler from certbot._internal import storage -from certbot._internal.display import enhancements from certbot._internal.plugins import selection as plugin_selection from certbot.compat import os from certbot.display import ops as display_ops @@ -521,12 +520,13 @@ class Client(object): # sites may have been enabled / final cleanup self.installer.restart() - def enhance_config(self, domains, chain_path, ask_redirect=True): + def enhance_config(self, domains, chain_path, redirect_default=True): """Enhance the configuration. :param list domains: list of domains to configure :param chain_path: chain file path :type chain_path: `str` or `None` + :param redirect_default: boolean value that the "redirect" flag should default to :raises .errors.Error: if no installer is specified in the client. @@ -548,14 +548,8 @@ class Client(object): for config_name, enhancement_name, option in enhancement_info: config_value = getattr(self.config, config_name) if enhancement_name in supported: - if ask_redirect: - if config_name == "redirect" and config_value is None: - config_value = enhancements.ask(enhancement_name) - if not config_value: - logger.warning("Future versions of Certbot will automatically " - "configure the webserver so that all requests redirect to secure " - "HTTPS access. You can control this behavior and disable this " - "warning with the --redirect and --no-redirect flags.") + if config_name == "redirect" and config_value is None: + config_value = redirect_default if config_value: self.apply_enhancement(domains, enhancement_name, option) enhanced = True diff --git a/certbot/certbot/_internal/display/enhancements.py b/certbot/certbot/_internal/display/enhancements.py deleted file mode 100644 index ce6470708..000000000 --- a/certbot/certbot/_internal/display/enhancements.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Certbot Enhancement Display""" -import logging - -import zope.component - -from certbot import errors -from certbot import interfaces -from certbot.display import util as display_util - -logger = logging.getLogger(__name__) - -# Define a helper function to avoid verbose code -util = zope.component.getUtility - - -def ask(enhancement): - """Display the enhancement to the user. - - :param str enhancement: One of the - :const:`~certbot.plugins.enhancements.ENHANCEMENTS` enhancements - - :returns: True if feature is desired, False otherwise - :rtype: bool - - :raises .errors.Error: if the enhancement provided is not supported - - """ - try: - # Call the appropriate function based on the enhancement - return DISPATCH[enhancement]() - except KeyError: - logger.error("Unsupported enhancement given to ask(): %s", enhancement) - raise errors.Error("Unsupported Enhancement") - - -def redirect_by_default(): - """Determines whether the user would like to redirect to HTTPS. - - :returns: True if redirect is desired, False otherwise - :rtype: bool - - """ - choices = [ - ("No redirect", "Make no further changes to the webserver configuration."), - ("Redirect", "Make all requests redirect to secure HTTPS access. " - "Choose this for new sites, or if you're confident your site works on HTTPS. " - "You can undo this change by editing your web server's configuration."), - ] - - code, selection = util(interfaces.IDisplay).menu( - "Please choose whether or not to redirect HTTP traffic to HTTPS, removing HTTP access.", - choices, default=1, - cli_flag="--redirect / --no-redirect", force_interactive=True) - - if code != display_util.OK: - return False - - return selection == 1 - - -DISPATCH = { - "redirect": redirect_by_default -} diff --git a/certbot/certbot/_internal/main.py b/certbot/certbot/_internal/main.py index 8674cd151..264f9667e 100644 --- a/certbot/certbot/_internal/main.py +++ b/certbot/certbot/_internal/main.py @@ -394,7 +394,7 @@ def _find_domains_or_certname(config, installer, question=None): :param installer: Installer object :type installer: interfaces.IInstaller - :param `str` question: Overriding dialog question to ask the user if asked + :param `str` question: Overriding default question to ask the user if asked to choose from domain names. :returns: Two-part tuple of domains and certname @@ -890,7 +890,7 @@ def enhance(config, plugins): """ supported_enhancements = ["hsts", "redirect", "uir", "staple"] # Check that at least one enhancement was requested on command line - oldstyle_enh = any([getattr(config, enh) for enh in supported_enhancements]) + oldstyle_enh = any(getattr(config, enh) for enh in supported_enhancements) if not enhancements.are_requested(config) and not oldstyle_enh: msg = ("Please specify one or more enhancement types to configure. To list " "the available enhancement types, run:\n\n%s --help enhance\n") @@ -927,7 +927,7 @@ def enhance(config, plugins): config.chain_path = lineage.chain_path if oldstyle_enh: le_client = _init_le_client(config, authenticator=None, installer=installer) - le_client.enhance_config(domains, config.chain_path, ask_redirect=False) + le_client.enhance_config(domains, config.chain_path, redirect_default=False) if enhancements.are_requested(config): enhancements.enable(lineage, domains, installer, config) diff --git a/certbot/certbot/_internal/notify.py b/certbot/certbot/_internal/notify.py deleted file mode 100644 index dda0a85af..000000000 --- a/certbot/certbot/_internal/notify.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Send e-mail notification to system administrators.""" - -import email -import smtplib -import socket -import subprocess - - -def notify(subject, whom, what): - """Send email notification. - - Try to notify the addressee (``whom``) by e-mail, with Subject: - defined by ``subject`` and message body by ``what``. - - """ - msg = email.message_from_string(what) - msg.add_header("From", "Certbot renewal agent ") - msg.add_header("To", whom) - msg.add_header("Subject", subject) - msg = msg.as_string() - try: - lmtp = smtplib.LMTP() - lmtp.connect() - lmtp.sendmail("root", [whom], msg) - except (smtplib.SMTPHeloError, smtplib.SMTPRecipientsRefused, - smtplib.SMTPSenderRefused, smtplib.SMTPDataError, socket.error): - # We should try using /usr/sbin/sendmail in this case - try: - proc = subprocess.Popen(["/usr/sbin/sendmail", "-t"], - stdin=subprocess.PIPE) - proc.communicate(msg) - except OSError: - return False - return True diff --git a/certbot/certbot/_internal/plugins/disco.py b/certbot/certbot/_internal/plugins/disco.py index d98a4cb0c..f1d89f06a 100644 --- a/certbot/certbot/_internal/plugins/disco.py +++ b/certbot/certbot/_internal/plugins/disco.py @@ -192,9 +192,6 @@ class PluginsRegistry(Mapping): # This prevents deadlock caused by plugins acquiring a lock # and ensures at least one concurrent Certbot instance will run # successfully. - - # Pylint checks for super init, but also claims the super - # has no __init__member self._plugins = collections.OrderedDict(sorted(six.iteritems(plugins))) @classmethod diff --git a/certbot/certbot/_internal/plugins/manual.py b/certbot/certbot/_internal/plugins/manual.py index 3313256d7..430059445 100644 --- a/certbot/certbot/_internal/plugins/manual.py +++ b/certbot/certbot/_internal/plugins/manual.py @@ -36,7 +36,11 @@ class Authenticator(common.Plugin): 'is the validation string, and $CERTBOT_TOKEN is the filename of the ' 'resource requested when performing an HTTP-01 challenge. An additional ' 'cleanup script can also be provided and can use the additional variable ' - '$CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth script.') + '$CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth script.' + 'For both authenticator and cleanup script, on HTTP-01 and DNS-01 challenges,' + '$CERTBOT_REMAINING_CHALLENGES will be equal to the number of challenges that ' + 'remain after the current one, and $CERTBOT_ALL_DOMAINS contains a comma-separated ' + 'list of all domains that are challenged for the current certificate.') _DNS_INSTRUCTIONS = """\ Please deploy a DNS TXT record under the name {domain} with the following value: @@ -68,7 +72,7 @@ permitted by DNS standards.) super(Authenticator, self).__init__(*args, **kwargs) self.reverter = reverter.Reverter(self.config) self.reverter.recovery_routine() - self.env = dict() \ + self.env = {} \ # type: Dict[achallenges.KeyAuthorizationAnnotatedChallenge, Dict[str, str]] self.subsequent_dns_challenge = False self.subsequent_any_challenge = False @@ -110,14 +114,13 @@ permitted by DNS standards.) def perform(self, achalls): # pylint: disable=missing-function-docstring self._verify_ip_logging_ok() - if self.conf('auth-hook'): - perform_achall = self._perform_achall_with_script - else: - perform_achall = self._perform_achall_manually responses = [] for achall in achalls: - perform_achall(achall) + if self.conf('auth-hook'): + self._perform_achall_with_script(achall, achalls) + else: + self._perform_achall_manually(achall) responses.append(achall.response(achall.account_key)) return responses @@ -135,9 +138,11 @@ permitted by DNS standards.) else: raise errors.PluginError('Must agree to IP logging to proceed') - def _perform_achall_with_script(self, achall): + def _perform_achall_with_script(self, achall, achalls): env = dict(CERTBOT_DOMAIN=achall.domain, - CERTBOT_VALIDATION=achall.validation(achall.account_key)) + CERTBOT_VALIDATION=achall.validation(achall.account_key), + CERTBOT_ALL_DOMAINS=','.join(one_achall.domain for one_achall in achalls), + CERTBOT_REMAINING_CHALLENGES=str(len(achalls) - achalls.index(achall) - 1)) if isinstance(achall.chall, challenges.HTTP01): env['CERTBOT_TOKEN'] = achall.chall.encode('token') else: diff --git a/certbot/certbot/_internal/plugins/selection.py b/certbot/certbot/_internal/plugins/selection.py index 6d87e4b07..53cef3969 100644 --- a/certbot/certbot/_internal/plugins/selection.py +++ b/certbot/certbot/_internal/plugins/selection.py @@ -138,7 +138,7 @@ def choose_plugin(prepared, question): while True: disp = z_util(interfaces.IDisplay) - if "CERTBOT_AUTO" in os.environ and names == set(("apache", "nginx")): + if "CERTBOT_AUTO" in os.environ and names == {"apache", "nginx"}: # The possibility of being offered exactly apache and nginx here # is new interactivity brought by https://github.com/certbot/certbot/issues/4079, # so set apache as a default for those kinds of non-interactive use diff --git a/certbot/certbot/_internal/storage.py b/certbot/certbot/_internal/storage.py index 6a34355a8..2dac163e2 100644 --- a/certbot/certbot/_internal/storage.py +++ b/certbot/certbot/_internal/storage.py @@ -15,6 +15,7 @@ import certbot from certbot import crypto_util from certbot import errors from certbot import interfaces +from certbot import ocsp from certbot import util from certbot._internal import cli from certbot._internal import constants @@ -882,27 +883,33 @@ class RenewableCert(interfaces.RenewableCert): with open(target) as f: return crypto_util.get_names_from_cert(f.read()) - def ocsp_revoked(self, version=None): - # pylint: disable=unused-argument + def ocsp_revoked(self, version): """Is the specified cert version revoked according to OCSP? - Also returns True if the cert version is declared as intended - to be revoked according to Let's Encrypt OCSP extensions. - (If no version is specified, uses the current version.) - - This method is not yet implemented and currently always returns - False. + Also returns True if the cert version is declared as revoked + according to OCSP. If OCSP status could not be determined, False + is returned. :param int version: the desired version number - :returns: whether the certificate is or will be revoked + :returns: True if the certificate is revoked, otherwise, False :rtype: bool """ - # XXX: This query and its associated network service aren't - # implemented yet, so we currently return False (indicating that the - # certificate is not revoked). - return False + cert_path = self.version("cert", version) + chain_path = self.version("chain", version) + # While the RevocationChecker should return False if it failed to + # determine the OCSP status, let's ensure we don't crash Certbot by + # catching all exceptions here. + try: + return ocsp.RevocationChecker().ocsp_revoked_by_paths(cert_path, + chain_path) + except Exception as e: # pylint: disable=broad-except + logger.warning( + "An error occurred determining the OCSP status of %s.", + cert_path) + logger.debug(str(e)) + return False def autorenewal_is_enabled(self): """Is automatic renewal enabled for this cert? diff --git a/certbot/certbot/compat/filesystem.py b/certbot/certbot/compat/filesystem.py index 88c2916fa..b9b6e5cc6 100644 --- a/certbot/certbot/compat/filesystem.py +++ b/certbot/certbot/compat/filesystem.py @@ -6,8 +6,6 @@ import os # pylint: disable=os-module-forbidden import stat from acme.magic_typing import List -from acme.magic_typing import Tuple # pylint: disable=unused-import -from acme.magic_typing import Union # pylint: disable=unused-import try: import ntsecuritycon @@ -17,7 +15,6 @@ try: import win32file import pywintypes import winerror - # pylint: enable=import-error except ImportError: POSIX_MODE = True else: diff --git a/certbot/certbot/crypto_util.py b/certbot/certbot/crypto_util.py index 9136445bc..168531667 100644 --- a/certbot/certbot/crypto_util.py +++ b/certbot/certbot/crypto_util.py @@ -8,6 +8,7 @@ import hashlib import logging import warnings +import re # See https://github.com/pyca/cryptography/issues/4275 from cryptography import x509 # type: ignore from cryptography.exceptions import InvalidSignature @@ -478,6 +479,17 @@ def sha256sum(filename): sha256.update(file_d.read().encode('UTF-8')) return sha256.hexdigest() +# Finds one CERTIFICATE stricttextualmsg according to rfc7468#section-3. +# Does not validate the base64text - use crypto.load_certificate. +CERT_PEM_REGEX = re.compile( + b"""-----BEGIN CERTIFICATE-----\r? +.+?\r? +-----END CERTIFICATE-----\r? +""", + re.DOTALL # DOTALL (/s) because the base64text may include newlines +) + + def cert_and_chain_from_fullchain(fullchain_pem): """Split fullchain_pem into cert_pem and chain_pem @@ -486,8 +498,35 @@ def cert_and_chain_from_fullchain(fullchain_pem): :returns: tuple of string cert_pem and chain_pem :rtype: tuple + :raises errors.Error: If there are less than 2 certificates in the chain. + """ - cert = crypto.dump_certificate(crypto.FILETYPE_PEM, - crypto.load_certificate(crypto.FILETYPE_PEM, fullchain_pem)).decode() - chain = fullchain_pem[len(cert):].lstrip() - return (cert, chain) + # First pass: find the boundary of each certificate in the chain. + # TODO: This will silently skip over any "explanatory text" in between boundaries, + # which is prohibited by RFC8555. + certs = CERT_PEM_REGEX.findall(fullchain_pem.encode()) + if len(certs) < 2: + raise errors.Error("failed to parse fullchain into cert and chain: " + + "less than 2 certificates in chain") + + # Second pass: for each certificate found, parse it using OpenSSL and re-encode it, + # with the effect of normalizing any encoding variations (e.g. CRLF, whitespace). + certs_normalized = [crypto.dump_certificate(crypto.FILETYPE_PEM, + crypto.load_certificate(crypto.FILETYPE_PEM, cert)).decode() for cert in certs] + + # Since each normalized cert has a newline suffix, no extra newlines are required. + return (certs_normalized[0], "".join(certs_normalized[1:])) + +def get_serial_from_cert(cert_path): + """Retrieve the serial number of a certificate from certificate path + + :param str cert_path: path to a cert in PEM format + + :returns: serial number of the certificate + :rtype: int + """ + # pylint: disable=redefined-outer-name + with open(cert_path) as f: + x509 = crypto.load_certificate(crypto.FILETYPE_PEM, + f.read()) + return x509.get_serial_number() diff --git a/certbot/certbot/display/ops.py b/certbot/certbot/display/ops.py index eab9d251d..21d169a55 100644 --- a/certbot/certbot/display/ops.py +++ b/certbot/certbot/display/ops.py @@ -107,7 +107,7 @@ def choose_names(installer, question=None): :param installer: An installer object :type installer: :class:`certbot.interfaces.IInstaller` - :param `str` question: Overriding dialog question to ask the user if asked + :param `str` question: Overriding default question to ask the user if asked to choose from domain names. :returns: List of selected names @@ -195,7 +195,7 @@ def _choose_names_manually(prompt_prefix=""): cli_flag="--domains", force_interactive=True) if code == display_util.OK: - invalid_domains = dict() + invalid_domains = {} retry_message = "" try: domain_list = display_util.separate_list_input(input_) diff --git a/certbot/certbot/ocsp.py b/certbot/certbot/ocsp.py index 6a95f26fa..863c5f163 100644 --- a/certbot/certbot/ocsp.py +++ b/certbot/certbot/ocsp.py @@ -21,6 +21,7 @@ from acme.magic_typing import Tuple from certbot import crypto_util from certbot import errors from certbot import util +from certbot.compat.os import getenv from certbot.interfaces import RenewableCert # pylint: disable=unused-import try: @@ -68,8 +69,20 @@ class RevocationChecker(object): :rtype: bool """ - cert_path, chain_path = cert.cert_path, cert.chain_path + return self.ocsp_revoked_by_paths(cert.cert_path, cert.chain_path) + def ocsp_revoked_by_paths(self, cert_path, chain_path, timeout=10): + # type: (str, str, int) -> bool + """Performs the OCSP revocation check + + :param str cert_path: Certificate filepath + :param str chain_path: Certificate chain + :param int timeout: Timeout (in seconds) for the OCSP query + + :returns: True if revoked; False if valid or the check failed or cert is expired. + :rtype: bool + + """ if self.broken: return False @@ -85,21 +98,37 @@ class RevocationChecker(object): return False if self.use_openssl_binary: - return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url) - return _check_ocsp_cryptography(cert_path, chain_path, url) + return self._check_ocsp_openssl_bin(cert_path, chain_path, host, url, timeout) + return _check_ocsp_cryptography(cert_path, chain_path, url, timeout) - def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url): - # type: (str, str, str, str) -> bool + def _check_ocsp_openssl_bin(self, cert_path, chain_path, host, url, timeout): + # type: (str, str, str, str, int) -> bool + # Minimal implementation of proxy selection logic as seen in, e.g., cURL + # Some things that won't work, but may well be in use somewhere: + # - username and password for proxy authentication + # - proxies accepting TLS connections + # - proxy exclusion through NO_PROXY + env_http_proxy = getenv('http_proxy') + env_HTTP_PROXY = getenv('HTTP_PROXY') + proxy_host = None + if env_http_proxy is not None or env_HTTP_PROXY is not None: + proxy_host = env_http_proxy if env_http_proxy is not None else env_HTTP_PROXY + if proxy_host is None: + url_opts = ["-url", url] + else: + if proxy_host.startswith('http://'): + proxy_host = proxy_host[len('http://'):] + url_opts = ["-host", proxy_host, "-path", url] # jdkasten thanks "Bulletproof SSL and TLS - Ivan Ristic" for documenting this! cmd = ["openssl", "ocsp", "-no_nonce", "-issuer", chain_path, "-cert", cert_path, - "-url", url, "-CAfile", chain_path, "-verify_other", chain_path, "-trust_other", - "-header"] + self.host_args(host) + "-timeout", str(timeout), + "-header"] + self.host_args(host) + url_opts logger.debug("Querying OCSP for %s", cert_path) logger.debug(" ".join(cmd)) try: @@ -141,8 +170,8 @@ def _determine_ocsp_server(cert_path): return None, None -def _check_ocsp_cryptography(cert_path, chain_path, url): - # type: (str, str, str) -> bool +def _check_ocsp_cryptography(cert_path, chain_path, url, timeout): + # type: (str, str, str, int) -> bool # Retrieve OCSP response with open(chain_path, 'rb') as file_handler: issuer = x509.load_pem_x509_certificate(file_handler.read(), default_backend()) @@ -154,7 +183,8 @@ def _check_ocsp_cryptography(cert_path, chain_path, url): request_binary = request.public_bytes(serialization.Encoding.DER) try: response = requests.post(url, data=request_binary, - headers={'Content-Type': 'application/ocsp-request'}) + headers={'Content-Type': 'application/ocsp-request'}, + timeout=timeout) except requests.exceptions.RequestException: logger.info("OCSP check failed for %s (are we offline?)", cert_path, exc_info=True) return False diff --git a/certbot/certbot/plugins/dns_test_common.py b/certbot/certbot/plugins/dns_test_common.py index 9ef76c2c3..d5044d336 100644 --- a/certbot/certbot/plugins/dns_test_common.py +++ b/certbot/certbot/plugins/dns_test_common.py @@ -2,7 +2,10 @@ import configobj import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import six from acme import challenges diff --git a/certbot/certbot/plugins/dns_test_common_lexicon.py b/certbot/certbot/plugins/dns_test_common_lexicon.py index c77d6da9e..1bef06042 100644 --- a/certbot/certbot/plugins/dns_test_common_lexicon.py +++ b/certbot/certbot/plugins/dns_test_common_lexicon.py @@ -1,7 +1,10 @@ """Base test class for DNS authenticators built on Lexicon.""" import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore from requests.exceptions import HTTPError from requests.exceptions import RequestException diff --git a/certbot/certbot/plugins/storage.py b/certbot/certbot/plugins/storage.py index fe3a97dd5..9123087e7 100644 --- a/certbot/certbot/plugins/storage.py +++ b/certbot/certbot/plugins/storage.py @@ -42,7 +42,7 @@ class PluginStorage(object): :raises .errors.PluginStorageError: when unable to open or read the file """ - data = dict() # type: Dict[str, Any] + data = {} # type: Dict[str, Any] filedata = "" try: with open(self._storagepath, 'r') as fh: @@ -107,7 +107,7 @@ class PluginStorage(object): self._initialize_storage() if not self._classkey in self._data.keys(): - self._data[self._classkey] = dict() + self._data[self._classkey] = {} self._data[self._classkey][key] = value def fetch(self, key): diff --git a/certbot/certbot/tests/util.py b/certbot/certbot/tests/util.py index 8b28b1080..92f52a852 100644 --- a/certbot/certbot/tests/util.py +++ b/certbot/certbot/tests/util.py @@ -10,7 +10,10 @@ import unittest from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock # type: ignore import OpenSSL import pkg_resources import six diff --git a/certbot/certbot/util.py b/certbot/certbot/util.py index aff2952f7..e69b11543 100644 --- a/certbot/certbot/util.py +++ b/certbot/certbot/util.py @@ -25,11 +25,9 @@ from certbot._internal import lock from certbot.compat import filesystem from certbot.compat import os -if sys.platform.startswith('linux'): +_USE_DISTRO = sys.platform.startswith('linux') +if _USE_DISTRO: import distro - _USE_DISTRO = True -else: - _USE_DISTRO = False logger = logging.getLogger(__name__) diff --git a/certbot/docs/ciphers.rst b/certbot/docs/ciphers.rst index 04b24b526..325d6244c 100644 --- a/certbot/docs/ciphers.rst +++ b/certbot/docs/ciphers.rst @@ -241,7 +241,7 @@ Mozilla Mozilla's general server configuration guidance is available at https://wiki.mozilla.org/Security/Server_Side_TLS -Mozilla has also produced a configuration generator: https://mozilla.github.io/server-side-tls/ssl-config-generator/ +Mozilla has also produced a configuration generator: https://ssl-config.mozilla.org Dutch National Cyber Security Centre ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/certbot/docs/cli-help.txt b/certbot/docs/cli-help.txt index ff49609c4..3c2289030 100644 --- a/certbot/docs/cli-help.txt +++ b/certbot/docs/cli-help.txt @@ -113,7 +113,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/1.2.0 (certbot(-auto); + "". (default: CertbotACMEClient/1.3.0 (certbot(-auto); OS_NAME OS_VERSION) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/major.minor.patchlevel). The flags encoded in the user agent are: --duplicate, diff --git a/certbot/docs/contributing.rst b/certbot/docs/contributing.rst index 25d832761..06d0e1b8d 100644 --- a/certbot/docs/contributing.rst +++ b/certbot/docs/contributing.rst @@ -247,8 +247,8 @@ built-in Standalone authenticator, implement just one interface. There are also `~certbot.interfaces.IDisplay` plugins, which can change how prompts are displayed to a user. -.. _interfaces.py: https://github.com/certbot/certbot/blob/master/certbot/interfaces.py -.. _plugins/common.py: https://github.com/certbot/certbot/blob/master/certbot/plugins/common.py#L34 +.. _interfaces.py: https://github.com/certbot/certbot/blob/master/certbot/certbot/interfaces.py +.. _plugins/common.py: https://github.com/certbot/certbot/blob/master/certbot/certbot/plugins/common.py#L45 Authenticators diff --git a/certbot/docs/using.rst b/certbot/docs/using.rst index 27ae826bd..d3c2d1582 100644 --- a/certbot/docs/using.rst +++ b/certbot/docs/using.rst @@ -280,6 +280,7 @@ pritunl_ N Y Install certificates in pritunl distributed OpenVPN proxmox_ N Y Install certificates in Proxmox Virtualization servers dns-standalone_ Y N Obtain certificates via an integrated DNS server dns-ispconfig_ Y N DNS Authentication using ISPConfig as DNS server +dns-clouddns_ Y N DNS Authentication using CloudDNS API ================== ==== ==== =============================================================== .. _haproxy: https://github.com/greenhost/certbot-haproxy @@ -291,6 +292,7 @@ dns-ispconfig_ Y N DNS Authentication using ISPConfig as DNS server .. _external-auth: https://github.com/EnigmaBridge/certbot-external-auth .. _dns-standalone: https://github.com/siilike/certbot-dns-standalone .. _dns-ispconfig: https://github.com/m42e/certbot-dns-ispconfig +.. _dns-clouddns: https://github.com/vshosting/certbot-dns-clouddns If you're interested, you can also :ref:`write your own plugin `. @@ -485,43 +487,6 @@ If you want your hook to run only after a successful renewal, use ``certbot renew --deploy-hook /path/to/deploy-hook-script`` -For example, if you have a daemon that does not read its certificates as the -root user, a deploy hook like this can copy them to the correct location and -apply appropriate file permissions. - -/path/to/deploy-hook-script - -.. code-block:: none - - #!/bin/sh - - set -e - - for domain in $RENEWED_DOMAINS; do - case $domain in - example.com) - daemon_cert_root=/etc/some-daemon/certs - - # Make sure the certificate and private key files are - # never world readable, even just for an instant while - # we're copying them into daemon_cert_root. - umask 077 - - cp "$RENEWED_LINEAGE/fullchain.pem" "$daemon_cert_root/$domain.cert" - cp "$RENEWED_LINEAGE/privkey.pem" "$daemon_cert_root/$domain.key" - - # Apply the proper file ownership and permissions for - # the daemon to read its certificate and key. - chown some-daemon "$daemon_cert_root/$domain.cert" \ - "$daemon_cert_root/$domain.key" - chmod 400 "$daemon_cert_root/$domain.cert" \ - "$daemon_cert_root/$domain.key" - - service some-daemon restart >/dev/null - ;; - esac - done - You can also specify hooks by placing files in subdirectories of Certbot's configuration directory. Assuming your configuration directory is ``/etc/letsencrypt``, any executable files found in @@ -686,6 +651,17 @@ your (web) server configuration directly to those files (or create symlinks). During the renewal_, ``/etc/letsencrypt/live`` is updated with the latest necessary files. +For historical reasons, the containing directories are created with +permissions of ``0700`` meaning that certificates are accessible only +to servers that run as the root user. **If you will never downgrade +to an older version of Certbot**, then you can safely fix this using +``chmod 0755 /etc/letsencrypt/{live,archive}``. + +For servers that drop root privileges before attempting to read the +private key file, you will also need to use ``chgrp`` and ``chmod +0640`` to allow the server to read +``/etc/letsencrypt/live/$domain/privkey.pem``. + .. note:: ``/etc/letsencrypt/archive`` and ``/etc/letsencrypt/keys`` contain all previous keys and certificates, while ``/etc/letsencrypt/live`` symlinks to the latest versions. @@ -762,8 +738,10 @@ the ``cleanup.sh`` script. Additionally certbot will pass relevant environment variables to these scripts: - ``CERTBOT_DOMAIN``: The domain being authenticated -- ``CERTBOT_VALIDATION``: The validation string (HTTP-01 and DNS-01 only) +- ``CERTBOT_VALIDATION``: The validation string - ``CERTBOT_TOKEN``: Resource name part of the HTTP-01 challenge (HTTP-01 only) +- ``CERTBOT_REMAINING_CHALLENGES``: Number of challenges remaining after the current challenge +- ``CERTBOT_ALL_DOMAINS``: A comma-separated list of all domains challenged for the current certificate Additionally for cleanup: diff --git a/certbot/local-oldest-requirements.txt b/certbot/local-oldest-requirements.txt index f6d158890..0acc68652 100644 --- a/certbot/local-oldest-requirements.txt +++ b/certbot/local-oldest-requirements.txt @@ -1,2 +1,2 @@ # Remember to update setup.py to match the package versions below. -acme[dev]==0.40.0 +-e acme[dev] diff --git a/certbot/setup.py b/certbot/setup.py index d19327e5e..143e1a10a 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -36,7 +36,7 @@ version = meta['version'] # specified here to avoid masking the more specific request requirements in # acme. See https://github.com/pypa/pip/issues/988 for more info. install_requires = [ - 'acme>=0.40.0', + 'acme>=1.4.0.dev0', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. @@ -47,7 +47,6 @@ install_requires = [ # 1.1.0+ is required to avoid the warnings described at # https://github.com/certbot/josepy/issues/13. 'josepy>=1.1.0', - 'mock', 'parsedatetime>=1.3', # Calendar.parseDT 'pyrfc3339', 'pytz', @@ -62,7 +61,8 @@ install_requires = [ # So this dependency is not added for old Linux distributions with old setuptools, # in order to allow these systems to build certbot from sources. pywin32_req = 'pywin32>=227' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py -if StrictVersion(setuptools_version) >= StrictVersion('36.2'): +setuptools_known_environment_markers = (StrictVersion(setuptools_version) >= StrictVersion('36.2')) +if setuptools_known_environment_markers: install_requires.append(pywin32_req + " ; sys_platform == 'win32'") elif 'bdist_wheel' in sys.argv[1:]: raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' @@ -73,6 +73,14 @@ elif os.name == 'nt': # setuptools, pywin32 will not be specified as a dependency. install_requires.append(pywin32_req) +if setuptools_known_environment_markers: + install_requires.append('mock ; python_version < "3.3"') +elif 'bdist_wheel' in sys.argv[1:]: + raise RuntimeError('Error, you are trying to build certbot wheels using an old version ' + 'of setuptools. Version 36.2+ of setuptools is required.') +elif sys.version_info < (3,3): + install_requires.append('mock') + dev_extras = [ 'coverage', 'ipdb', diff --git a/certbot/tests/account_test.py b/certbot/tests/account_test.py index 4a6ed3e01..6c6e6c860 100644 --- a/certbot/tests/account_test.py +++ b/certbot/tests/account_test.py @@ -4,7 +4,10 @@ import json import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz from acme import messages diff --git a/certbot/tests/auth_handler_test.py b/certbot/tests/auth_handler_test.py index 7ab3a2baa..6cd207b4b 100644 --- a/certbot/tests/auth_handler_test.py +++ b/certbot/tests/auth_handler_test.py @@ -3,7 +3,10 @@ import functools import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from acme import challenges diff --git a/certbot/tests/cert_manager_test.py b/certbot/tests/cert_manager_test.py index eb8005b2b..d956fd04f 100644 --- a/certbot/tests/cert_manager_test.py +++ b/certbot/tests/cert_manager_test.py @@ -7,7 +7,10 @@ import tempfile import unittest import configobj -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot._internal import configuration @@ -200,9 +203,11 @@ class CertificatesTest(BaseCertManagerTest): self.assertTrue(mock_utility.called) shutil.rmtree(empty_tempdir) + @mock.patch('certbot.crypto_util.get_serial_from_cert') @mock.patch('certbot._internal.cert_manager.ocsp.RevocationChecker.ocsp_revoked') - def test_report_human_readable(self, mock_revoked): + def test_report_human_readable(self, mock_revoked, mock_serial): mock_revoked.return_value = None + mock_serial.return_value = 1234567890 from certbot._internal import cert_manager import datetime import pytz diff --git a/certbot/tests/cli_test.py b/certbot/tests/cli_test.py index 3a7fb57f8..592c40be7 100644 --- a/certbot/tests/cli_test.py +++ b/certbot/tests/cli_test.py @@ -4,7 +4,10 @@ import copy import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from six.moves import reload_module # pylint: disable=import-error @@ -30,15 +33,23 @@ class TestReadFile(TempDirTestCase): # However a relative path between two different drives is invalid. So we move to # self.tempdir to ensure that we stay on the same drive. os.chdir(self.tempdir) - rel_test_path = os.path.relpath(os.path.join(self.tempdir, 'foo')) + # The read-only filesystem introduced with macOS Catalina can break + # code using relative paths below. See + # https://bugs.python.org/issue38295 for another example of this. + # Eliminating any possible symlinks in self.tempdir before passing + # it to os.path.relpath solves the problem. This is done by calling + # filesystem.realpath which removes any symlinks in the path on + # POSIX systems. + real_path = filesystem.realpath(os.path.join(self.tempdir, 'foo')) + relative_path = os.path.relpath(real_path) self.assertRaises( - argparse.ArgumentTypeError, cli.read_file, rel_test_path) + argparse.ArgumentTypeError, cli.read_file, relative_path) test_contents = b'bar\n' - with open(rel_test_path, 'wb') as f: + with open(relative_path, 'wb') as f: f.write(test_contents) - path, contents = cli.read_file(rel_test_path) + path, contents = cli.read_file(relative_path) self.assertEqual(path, os.path.abspath(path)) self.assertEqual(contents, test_contents) finally: @@ -142,7 +153,6 @@ class ParseTest(unittest.TestCase): self.assertTrue("how a certificate is deployed" in out) self.assertTrue("--webroot-path" in out) self.assertTrue("--text" not in out) - self.assertTrue("--dialog" not in out) self.assertTrue("%s" not in out) self.assertTrue("{0}" not in out) self.assertTrue("--renew-hook" not in out) @@ -203,7 +213,6 @@ class ParseTest(unittest.TestCase): self.assertTrue("how a certificate is deployed" in out) self.assertTrue("--webroot-path" in out) self.assertTrue("--text" not in out) - self.assertTrue("--dialog" not in out) self.assertTrue("%s" not in out) self.assertTrue("{0}" not in out) diff --git a/certbot/tests/client_test.py b/certbot/tests/client_test.py index 7232ed84b..cbc058c7a 100644 --- a/certbot/tests/client_test.py +++ b/certbot/tests/client_test.py @@ -5,7 +5,10 @@ import tempfile import unittest from josepy import interfaces -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util @@ -577,8 +580,7 @@ class EnhanceConfigTest(ClientTestCommon): self.assertRaises( errors.Error, self.client.enhance_config, [self.domain], None) - @mock.patch("certbot._internal.client.enhancements") - def test_unsupported(self, mock_enhancements): + def test_unsupported(self): self.client.installer = mock.MagicMock() self.client.installer.supported_enhancements.return_value = [] @@ -588,7 +590,6 @@ class EnhanceConfigTest(ClientTestCommon): self.client.enhance_config([self.domain], None) self.assertEqual(mock_logger.warning.call_count, 1) self.client.installer.enhance.assert_not_called() - mock_enhancements.ask.assert_not_called() @mock.patch("certbot._internal.client.logger") def test_already_exists_header(self, mock_log): @@ -612,14 +613,11 @@ class EnhanceConfigTest(ClientTestCommon): self._test_with_already_existing() self.assertFalse(mock_log.warning.called) - @mock.patch("certbot._internal.client.enhancements.ask") @mock.patch("certbot._internal.client.logger") - def test_warn_redirect(self, mock_log, mock_ask): + def test_no_warn_redirect(self, mock_log): self.config.redirect = None - mock_ask.return_value = False - self._test_with_already_existing() - self.assertTrue(mock_log.warning.called) - self.assertTrue("disable" in mock_log.warning.call_args[0][0]) + self._test_with_all_supported() + self.assertFalse(mock_log.warning.called) def test_no_ask_hsts(self): self.config.hsts = True @@ -670,12 +668,6 @@ class EnhanceConfigTest(ClientTestCommon): self.client.installer = installer self._test_error_with_rollback() - @mock.patch("certbot._internal.client.enhancements.ask") - def test_ask(self, mock_ask): - self.config.redirect = None - mock_ask.return_value = True - self._test_with_all_supported() - def _test_error_with_rollback(self): self._test_error() self.assertTrue(self.client.installer.restart.called) diff --git a/certbot/tests/compat/filesystem_test.py b/certbot/tests/compat/filesystem_test.py index e721bbd48..1c2d2df0d 100644 --- a/certbot/tests/compat/filesystem_test.py +++ b/certbot/tests/compat/filesystem_test.py @@ -3,7 +3,10 @@ import contextlib import errno import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import util from certbot._internal import lock @@ -13,11 +16,9 @@ import certbot.tests.util as test_util from certbot.tests.util import TempDirTestCase try: - # pylint: disable=import-error import win32api import win32security import ntsecuritycon - # pylint: enable=import-error POSIX_MODE = False except ImportError: POSIX_MODE = True diff --git a/certbot/tests/configuration_test.py b/certbot/tests/configuration_test.py index d748b9bfb..1f8a0e803 100644 --- a/certbot/tests/configuration_test.py +++ b/certbot/tests/configuration_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.configuration.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot._internal import constants diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 1d642ae9e..481d83894 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -2,7 +2,10 @@ import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL import zope.component @@ -376,17 +379,33 @@ class Sha256sumTest(unittest.TestCase): class CertAndChainFromFullchainTest(unittest.TestCase): """Tests for certbot.crypto_util.cert_and_chain_from_fullchain""" + def _parse_and_reencode_pem(self, cert_pem): + from OpenSSL import crypto + return crypto.dump_certificate(crypto.FILETYPE_PEM, + crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)).decode() + def test_cert_and_chain_from_fullchain(self): cert_pem = CERT.decode() chain_pem = cert_pem + SS_CERT.decode() fullchain_pem = cert_pem + chain_pem spacey_fullchain_pem = cert_pem + u'\n' + chain_pem + crlf_fullchain_pem = fullchain_pem.replace(u'\n', u'\r\n') + + # In the ACME v1 code path, the fullchain is constructed by loading cert+chain DERs + # and using OpenSSL to dump them, so here we confirm that OpenSSL is producing certs + # that will be parseable by cert_and_chain_from_fullchain. + acmev1_fullchain_pem = self._parse_and_reencode_pem(cert_pem) + \ + self._parse_and_reencode_pem(cert_pem) + self._parse_and_reencode_pem(SS_CERT.decode()) + from certbot.crypto_util import cert_and_chain_from_fullchain - for fullchain in (fullchain_pem, spacey_fullchain_pem): + for fullchain in (fullchain_pem, spacey_fullchain_pem, crlf_fullchain_pem, + acmev1_fullchain_pem): cert_out, chain_out = cert_and_chain_from_fullchain(fullchain) self.assertEqual(cert_out, cert_pem) self.assertEqual(chain_out, chain_pem) + self.assertRaises(errors.Error, cert_and_chain_from_fullchain, cert_pem) + if __name__ == '__main__': unittest.main() # pragma: no cover diff --git a/certbot/tests/display/completer_test.py b/certbot/tests/display/completer_test.py index 5ddf69266..0852ab175 100644 --- a/certbot/tests/display/completer_test.py +++ b/certbot/tests/display/completer_test.py @@ -7,10 +7,12 @@ import string import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from six.moves import reload_module # pylint: disable=import-error -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot.compat import filesystem # pylint: disable=ungrouped-imports from certbot.compat import os # pylint: disable=ungrouped-imports import certbot.tests.util as test_util # pylint: disable=ungrouped-imports diff --git a/certbot/tests/display/enhancements_test.py b/certbot/tests/display/enhancements_test.py deleted file mode 100644 index edace29b1..000000000 --- a/certbot/tests/display/enhancements_test.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Module for enhancement UI.""" -import logging -import unittest - -import mock - -from certbot import errors -from certbot.display import util as display_util - - -class AskTest(unittest.TestCase): - """Test the ask method.""" - def setUp(self): - logging.disable(logging.CRITICAL) - - def tearDown(self): - logging.disable(logging.NOTSET) - - @classmethod - def _call(cls, enhancement): - from certbot._internal.display.enhancements import ask - return ask(enhancement) - - @mock.patch("certbot._internal.display.enhancements.util") - def test_redirect(self, mock_util): - mock_util().menu.return_value = (display_util.OK, 1) - self.assertTrue(self._call("redirect")) - - def test_key_error(self): - self.assertRaises(errors.Error, self._call, "unknown_enhancement") - - -class RedirectTest(unittest.TestCase): - """Test the redirect_by_default method.""" - @classmethod - def _call(cls): - from certbot._internal.display.enhancements import redirect_by_default - return redirect_by_default() - - @mock.patch("certbot._internal.display.enhancements.util") - def test_secure(self, mock_util): - mock_util().menu.return_value = (display_util.OK, 1) - self.assertTrue(self._call()) - - @mock.patch("certbot._internal.display.enhancements.util") - def test_cancel(self, mock_util): - mock_util().menu.return_value = (display_util.CANCEL, 1) - self.assertFalse(self._call()) - - @mock.patch("certbot._internal.display.enhancements.util") - def test_easy(self, mock_util): - mock_util().menu.return_value = (display_util.OK, 0) - self.assertFalse(self._call()) - - -if __name__ == "__main__": - unittest.main() # pragma: no cover diff --git a/certbot/tests/display/ops_test.py b/certbot/tests/display/ops_test.py index 5df7bfcf8..a683e1d3d 100644 --- a/certbot/tests/display/ops_test.py +++ b/certbot/tests/display/ops_test.py @@ -4,7 +4,10 @@ import sys import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component from acme import messages @@ -271,7 +274,7 @@ class ChooseNamesTest(unittest.TestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_valid_return(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = (display_util.OK, ["example.com"]) names = self._call(self.mock_install) @@ -280,7 +283,7 @@ class ChooseNamesTest(unittest.TestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_namees_override_question(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = (display_util.OK, ["example.com"]) names = self._call(self.mock_install, "Custom") self.assertEqual(names, ["example.com"]) @@ -289,14 +292,14 @@ class ChooseNamesTest(unittest.TestCase): @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_nothing_selected(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = (display_util.OK, []) self.assertEqual(self._call(self.mock_install), []) @test_util.patch_get_utility("certbot.display.ops.z_util") def test_filter_names_cancel(self, mock_util): - self.mock_install.get_all_names.return_value = set(["example.com"]) + self.mock_install.get_all_names.return_value = {"example.com"} mock_util().checklist.return_value = ( display_util.CANCEL, ["example.com"]) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index 615f33406..3e492e9ab 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -4,7 +4,10 @@ import socket import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from certbot import errors @@ -173,14 +176,14 @@ class FileOutputDisplayTest(unittest.TestCase): code, tag_list = self.displayer.checklist( "msg", TAGS, force_interactive=True) self.assertEqual( - (code, set(tag_list)), (display_util.OK, set(["tag1", "tag2"]))) + (code, set(tag_list)), (display_util.OK, {"tag1", "tag2"})) @mock.patch("certbot.display.util.input_with_timeout") def test_checklist_empty(self, mock_input): mock_input.return_value = "" code, tag_list = self.displayer.checklist("msg", TAGS, force_interactive=True) self.assertEqual( - (code, set(tag_list)), (display_util.OK, set(["tag1", "tag2", "tag3"]))) + (code, set(tag_list)), (display_util.OK, {"tag1", "tag2", "tag3"})) @mock.patch("certbot.display.util.input_with_timeout") def test_checklist_miss_valid(self, mock_input): @@ -212,9 +215,9 @@ class FileOutputDisplayTest(unittest.TestCase): ["2", "3"], ] exp = [ - set(["tag1"]), - set(["tag1", "tag2"]), - set(["tag2", "tag3"]), + {"tag1"}, + {"tag1", "tag2"}, + {"tag2", "tag3"}, ] for i, list_ in enumerate(indices): set_tags = set( diff --git a/certbot/tests/eff_test.py b/certbot/tests/eff_test.py index cdd7908a3..c4a25da69 100644 --- a/certbot/tests/eff_test.py +++ b/certbot/tests/eff_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.eff.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import requests from certbot._internal import constants diff --git a/certbot/tests/error_handler_test.py b/certbot/tests/error_handler_test.py index 45fec7f39..e5a95c3a8 100644 --- a/certbot/tests/error_handler_test.py +++ b/certbot/tests/error_handler_test.py @@ -4,11 +4,11 @@ import signal import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock -from acme.magic_typing import Callable # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Union # pylint: disable=unused-import, no-name-in-module from certbot.compat import os @@ -45,7 +45,7 @@ class ErrorHandlerTest(unittest.TestCase): from certbot._internal import error_handler self.init_func = mock.MagicMock() - self.init_args = set((42,)) + self.init_args = {42,} self.init_kwargs = {'foo': 'bar'} self.handler = error_handler.ErrorHandler(self.init_func, *self.init_args, diff --git a/certbot/tests/errors_test.py b/certbot/tests/errors_test.py index d6c829322..792868df0 100644 --- a/certbot/tests/errors_test.py +++ b/certbot/tests/errors_test.py @@ -1,7 +1,10 @@ """Tests for certbot.errors.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import messages from certbot import achallenges diff --git a/certbot/tests/hook_test.py b/certbot/tests/hook_test.py index 3a8759251..b8fd02461 100644 --- a/certbot/tests/hook_test.py +++ b/certbot/tests/hook_test.py @@ -1,9 +1,11 @@ """Tests for certbot._internal.hooks.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import util from certbot.compat import filesystem @@ -25,7 +27,7 @@ class ValidateHooksTest(unittest.TestCase): self._call(config) types = [call[0][1] for call in mock_validate_hook.call_args_list] - self.assertEqual(set(("pre", "post", "deploy",)), set(types[:-1])) + self.assertEqual({"pre", "post", "deploy",}, set(types[:-1])) # This ensures error messages are about deploy hooks when appropriate self.assertEqual("renew", types[-1]) diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py index 5a48009fd..2f887d33e 100644 --- a/certbot/tests/lock_test.py +++ b/certbot/tests/lock_test.py @@ -3,7 +3,10 @@ import functools import multiprocessing import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot.compat import os diff --git a/certbot/tests/log_test.py b/certbot/tests/log_test.py index 3b9adbbf2..5cd287c2e 100644 --- a/certbot/tests/log_test.py +++ b/certbot/tests/log_test.py @@ -5,11 +5,13 @@ import sys import time import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import messages -from acme.magic_typing import Optional # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import util from certbot._internal import constants diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 7ebe5e66a..8113b2bc4 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -13,12 +13,14 @@ import traceback import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz import six from six.moves import reload_module # pylint: disable=import-error -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import crypto_util from certbot import errors from certbot import interfaces # pylint: disable=unused-import @@ -1584,9 +1586,9 @@ class EnhanceTest(test_util.ConfigTestCase): not_req_enh = ["uir"] self.assertTrue(mock_client.enhance_config.called) self.assertTrue( - all([getattr(mock_client.config, e) for e in req_enh])) + all(getattr(mock_client.config, e) for e in req_enh)) self.assertFalse( - any([getattr(mock_client.config, e) for e in not_req_enh])) + any(getattr(mock_client.config, e) for e in not_req_enh)) self.assertTrue( "example.com" in mock_client.enhance_config.call_args[0][0]) diff --git a/certbot/tests/notify_test.py b/certbot/tests/notify_test.py deleted file mode 100644 index d6f7d2239..000000000 --- a/certbot/tests/notify_test.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Tests for certbot._internal.notify.""" -import socket -import unittest - -import mock - - -class NotifyTests(unittest.TestCase): - """Tests for the notifier.""" - - @mock.patch("certbot._internal.notify.smtplib.LMTP") - def test_smtp_success(self, mock_lmtp): - from certbot._internal.notify import notify - lmtp_obj = mock.MagicMock() - mock_lmtp.return_value = lmtp_obj - self.assertTrue(notify("Goose", "auntrhody@example.com", - "The old grey goose is dead.")) - self.assertEqual(lmtp_obj.connect.call_count, 1) - self.assertEqual(lmtp_obj.sendmail.call_count, 1) - - @mock.patch("certbot._internal.notify.smtplib.LMTP") - @mock.patch("certbot._internal.notify.subprocess.Popen") - def test_smtp_failure(self, mock_popen, mock_lmtp): - from certbot._internal.notify import notify - lmtp_obj = mock.MagicMock() - mock_lmtp.return_value = lmtp_obj - lmtp_obj.sendmail.side_effect = socket.error(17) - proc = mock.MagicMock() - mock_popen.return_value = proc - self.assertTrue(notify("Goose", "auntrhody@example.com", - "The old grey goose is dead.")) - self.assertEqual(lmtp_obj.sendmail.call_count, 1) - self.assertEqual(proc.communicate.call_count, 1) - - @mock.patch("certbot._internal.notify.smtplib.LMTP") - @mock.patch("certbot._internal.notify.subprocess.Popen") - def test_everything_fails(self, mock_popen, mock_lmtp): - from certbot._internal.notify import notify - lmtp_obj = mock.MagicMock() - mock_lmtp.return_value = lmtp_obj - lmtp_obj.sendmail.side_effect = socket.error(17) - proc = mock.MagicMock() - mock_popen.return_value = proc - proc.communicate.side_effect = OSError("What we have here is a " - "failure to communicate.") - self.assertFalse(notify("Goose", "auntrhody@example.com", - "The old grey goose is dead.")) - self.assertEqual(lmtp_obj.sendmail.call_count, 1) - self.assertEqual(proc.communicate.call_count, 1) - -if __name__ == "__main__": - unittest.main() # pragma: no cover diff --git a/certbot/tests/ocsp_test.py b/certbot/tests/ocsp_test.py index 6b05a8d3c..af54844cf 100644 --- a/certbot/tests/ocsp_test.py +++ b/certbot/tests/ocsp_test.py @@ -10,7 +10,10 @@ from cryptography.exceptions import InvalidSignature from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes # type: ignore -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz from certbot import errors @@ -162,11 +165,11 @@ class OSCPTestCryptography(unittest.TestCase): @mock.patch('certbot.ocsp._determine_ocsp_server') @mock.patch('certbot.ocsp._check_ocsp_cryptography') - def test_ensure_cryptography_toggled(self, mock_revoke, mock_determine): + def test_ensure_cryptography_toggled(self, mock_check, mock_determine): mock_determine.return_value = ('http://example.com', 'example.com') self.checker.ocsp_revoked(self.cert_obj) - mock_revoke.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com') + mock_check.assert_called_once_with(self.cert_path, self.chain_path, 'http://example.com', 10) def test_revoke(self): with _ocsp_mock(ocsp_lib.OCSPCertStatus.REVOKED, ocsp_lib.OCSPResponseStatus.SUCCESSFUL): diff --git a/certbot/tests/plugins/common_test.py b/certbot/tests/plugins/common_test.py index 915a3ae6c..344e6312f 100644 --- a/certbot/tests/plugins/common_test.py +++ b/certbot/tests/plugins/common_test.py @@ -4,7 +4,10 @@ import shutil import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges from certbot import achallenges @@ -94,7 +97,7 @@ class InstallerTest(test_util.ConfigTestCase): self.reverter = self.installer.reverter def test_add_to_real_checkpoint(self): - files = set(("foo.bar", "baz.qux",)) + files = {"foo.bar", "baz.qux",} save_notes = "foo bar baz qux" self._test_wrapped_method("add_to_checkpoint", files, save_notes) @@ -105,7 +108,7 @@ class InstallerTest(test_util.ConfigTestCase): self._test_add_to_checkpoint_common(True) def _test_add_to_checkpoint_common(self, temporary): - files = set(("foo.bar", "baz.qux",)) + files = {"foo.bar", "baz.qux",} save_notes = "foo bar baz qux" installer_func = functools.partial(self.installer.add_to_checkpoint, @@ -177,7 +180,7 @@ class InstallerTest(test_util.ConfigTestCase): class AddrTest(unittest.TestCase): - """Tests for certbot._internal.client.plugins.common.Addr.""" + """Tests for certbot.plugins.common.Addr.""" def setUp(self): from certbot.plugins.common import Addr @@ -242,17 +245,17 @@ class AddrTest(unittest.TestCase): def test_set_inclusion(self): from certbot.plugins.common import Addr - set_a = set([self.addr1, self.addr2]) + set_a = {self.addr1, self.addr2} addr1b = Addr.fromstring("192.168.1.1") addr2b = Addr.fromstring("192.168.1.1:*") - set_b = set([addr1b, addr2b]) + set_b = {addr1b, addr2b} self.assertEqual(set_a, set_b) - set_c = set([self.addr4, self.addr5]) + set_c = {self.addr4, self.addr5} addr4b = Addr.fromstring("[fe00::1]") addr5b = Addr.fromstring("[fe00::1]:*") - set_d = set([addr4b, addr5b]) + set_d = {addr4b, addr5b} self.assertEqual(set_c, set_d) diff --git a/certbot/tests/plugins/disco_test.py b/certbot/tests/plugins/disco_test.py index 6d3c7d97e..5a0a392b0 100644 --- a/certbot/tests/plugins/disco_test.py +++ b/certbot/tests/plugins/disco_test.py @@ -3,12 +3,14 @@ import functools import string import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pkg_resources import six import zope.interface -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot._internal.plugins import standalone diff --git a/certbot/tests/plugins/dns_common_lexicon_test.py b/certbot/tests/plugins/dns_common_lexicon_test.py index 986362ca9..a67430f3e 100644 --- a/certbot/tests/plugins/dns_common_lexicon_test.py +++ b/certbot/tests/plugins/dns_common_lexicon_test.py @@ -2,7 +2,10 @@ import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.plugins import dns_common_lexicon from certbot.plugins import dns_test_common_lexicon diff --git a/certbot/tests/plugins/dns_common_test.py b/certbot/tests/plugins/dns_common_test.py index eba3c89d6..993f3b461 100644 --- a/certbot/tests/plugins/dns_common_test.py +++ b/certbot/tests/plugins/dns_common_test.py @@ -4,7 +4,10 @@ import collections import logging import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot import util diff --git a/certbot/tests/plugins/enhancements_test.py b/certbot/tests/plugins/enhancements_test.py index 05fbc5028..a20a6864f 100644 --- a/certbot/tests/plugins/enhancements_test.py +++ b/certbot/tests/plugins/enhancements_test.py @@ -1,7 +1,10 @@ """Tests for new style enhancements""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot._internal.plugins import null from certbot.plugins import enhancements diff --git a/certbot/tests/plugins/manual_test.py b/certbot/tests/plugins/manual_test.py index bd11a9538..933c759d6 100644 --- a/certbot/tests/plugins/manual_test.py +++ b/certbot/tests/plugins/manual_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import challenges @@ -72,16 +75,23 @@ class AuthenticatorTest(test_util.TempDirTestCase): self.config.manual_public_ip_logging_ok = True self.config.manual_auth_hook = ( '{0} -c "from __future__ import print_function;' - 'from certbot.compat import os; print(os.environ.get(\'CERTBOT_DOMAIN\'));' + 'from certbot.compat import os;' + 'print(os.environ.get(\'CERTBOT_DOMAIN\'));' 'print(os.environ.get(\'CERTBOT_TOKEN\', \'notoken\'));' - 'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));"' + 'print(os.environ.get(\'CERTBOT_VALIDATION\', \'novalidation\'));' + 'print(os.environ.get(\'CERTBOT_ALL_DOMAINS\'));' + 'print(os.environ.get(\'CERTBOT_REMAINING_CHALLENGES\'));"' .format(sys.executable)) - dns_expected = '{0}\n{1}\n{2}'.format( + dns_expected = '{0}\n{1}\n{2}\n{3}\n{4}'.format( self.dns_achall.domain, 'notoken', - self.dns_achall.validation(self.dns_achall.account_key)) - http_expected = '{0}\n{1}\n{2}'.format( + self.dns_achall.validation(self.dns_achall.account_key), + ','.join(achall.domain for achall in self.achalls), + len(self.achalls) - self.achalls.index(self.dns_achall) - 1) + http_expected = '{0}\n{1}\n{2}\n{3}\n{4}'.format( self.http_achall.domain, self.http_achall.chall.encode('token'), - self.http_achall.validation(self.http_achall.account_key)) + self.http_achall.validation(self.http_achall.account_key), + ','.join(achall.domain for achall in self.achalls), + len(self.achalls) - self.achalls.index(self.http_achall) - 1) self.assertEqual( self.auth.perform(self.achalls), diff --git a/certbot/tests/plugins/null_test.py b/certbot/tests/plugins/null_test.py index db0213813..47708e340 100644 --- a/certbot/tests/plugins/null_test.py +++ b/certbot/tests/plugins/null_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.plugins.null.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six diff --git a/certbot/tests/plugins/selection_test.py b/certbot/tests/plugins/selection_test.py index ac846af7b..e5e6db031 100644 --- a/certbot/tests/plugins/selection_test.py +++ b/certbot/tests/plugins/selection_test.py @@ -2,10 +2,12 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import zope.component -from acme.magic_typing import List # pylint: disable=unused-import, no-name-in-module from certbot import errors from certbot import interfaces from certbot._internal.plugins.disco import PluginsRegistry diff --git a/certbot/tests/plugins/standalone_test.py b/certbot/tests/plugins/standalone_test.py index 5d9ff5244..751b9d943 100644 --- a/certbot/tests/plugins/standalone_test.py +++ b/certbot/tests/plugins/standalone_test.py @@ -5,15 +5,15 @@ from socket import errno as socket_errors # type: ignore import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import OpenSSL.crypto # pylint: disable=unused-import import six from acme import challenges from acme import standalone as acme_standalone # pylint: disable=unused-import -from acme.magic_typing import Dict # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Set # pylint: disable=unused-import, no-name-in-module -from acme.magic_typing import Tuple # pylint: disable=unused-import, no-name-in-module from certbot import achallenges from certbot import errors from certbot.tests import acme_util @@ -164,7 +164,7 @@ class AuthenticatorTest(unittest.TestCase): self.auth.cleanup(["chall1"]) self.assertEqual(self.auth.served, { - "server1": set(), "server2": set(["chall2", "chall3"])}) + "server1": set(), "server2": {"chall2", "chall3"}}) self.auth.servers.stop.assert_called_once_with(1) self.auth.servers.running.return_value = { @@ -172,12 +172,12 @@ class AuthenticatorTest(unittest.TestCase): } self.auth.cleanup(["chall2"]) self.assertEqual(self.auth.served, { - "server1": set(), "server2": set(["chall3"])}) + "server1": set(), "server2": {"chall3"}}) self.assertEqual(1, self.auth.servers.stop.call_count) self.auth.cleanup(["chall3"]) self.assertEqual(self.auth.served, { - "server1": set(), "server2": set([])}) + "server1": set(), "server2": set()}) self.auth.servers.stop.assert_called_with(2) diff --git a/certbot/tests/plugins/storage_test.py b/certbot/tests/plugins/storage_test.py index e9ca2007f..4b0d1da83 100644 --- a/certbot/tests/plugins/storage_test.py +++ b/certbot/tests/plugins/storage_test.py @@ -2,7 +2,10 @@ import json import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import errors from certbot.compat import filesystem diff --git a/certbot/tests/plugins/util_test.py b/certbot/tests/plugins/util_test.py index c41e55222..9387b4ae7 100644 --- a/certbot/tests/plugins/util_test.py +++ b/certbot/tests/plugins/util_test.py @@ -1,7 +1,10 @@ """Tests for certbot.plugins.util.""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot.compat import os diff --git a/certbot/tests/plugins/webroot_test.py b/certbot/tests/plugins/webroot_test.py index fade12bb1..e57e09eae 100644 --- a/certbot/tests/plugins/webroot_test.py +++ b/certbot/tests/plugins/webroot_test.py @@ -10,7 +10,10 @@ import tempfile import unittest import josepy as jose -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from acme import challenges diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index e92211ea2..1fc54b42e 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -1,7 +1,10 @@ """Tests for certbot._internal.renewal""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from acme import challenges from certbot import errors diff --git a/certbot/tests/renewupdater_test.py b/certbot/tests/renewupdater_test.py index c6f8f3713..b5ecddb5a 100644 --- a/certbot/tests/renewupdater_test.py +++ b/certbot/tests/renewupdater_test.py @@ -1,7 +1,10 @@ """Tests for renewal updater interfaces""" import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock from certbot import interfaces from certbot._internal import main diff --git a/certbot/tests/reporter_test.py b/certbot/tests/reporter_test.py index 3d7c80172..7d03f1821 100644 --- a/certbot/tests/reporter_test.py +++ b/certbot/tests/reporter_test.py @@ -2,7 +2,10 @@ import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six diff --git a/certbot/tests/reverter_test.py b/certbot/tests/reverter_test.py index fc9133c5f..d67aa431a 100644 --- a/certbot/tests/reverter_test.py +++ b/certbot/tests/reverter_test.py @@ -5,7 +5,10 @@ import shutil import tempfile import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from certbot import errors @@ -87,7 +90,7 @@ class ReverterCheckpointLocalTest(test_util.ConfigTestCase): # Check to make sure new files are also checked... self.assertRaises(errors.ReverterError, self.reverter.add_to_checkpoint, - set([config3]), "invalid save") + {config3}, "invalid save") def test_multiple_saves_and_temp_revert(self): self.reverter.add_to_temp_checkpoint(self.sets[0], "save1") @@ -414,9 +417,9 @@ def setup_test_files(): with open(config2, "w") as file_fd: file_fd.write("directive-dir2") - sets = [set([config1]), - set([config2]), - set([config1, config2])] + sets = [{config1}, + {config2}, + {config1, config2}] return config1, config2, dir1, dir2, sets diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index 6208974ec..5aa37824d 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -6,7 +6,10 @@ import stat import unittest import configobj -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import pytz import six @@ -672,10 +675,35 @@ class RenewableCertTests(BaseRenewableCertTest): errors.CertStorageError, self.test_rc._update_link_to, "elephant", 17) - def test_ocsp_revoked(self): - # XXX: This is currently hardcoded to False due to a lack of an - # OCSP server to test against. - self.assertFalse(self.test_rc.ocsp_revoked()) + @mock.patch("certbot.ocsp.RevocationChecker.ocsp_revoked_by_paths") + def test_ocsp_revoked(self, mock_checker): + # Write out test files + for kind in ALL_FOUR: + self._write_out_kind(kind, 1) + version = self.test_rc.latest_common_version() + expected_cert_path = self.test_rc.version("cert", version) + expected_chain_path = self.test_rc.version("chain", version) + + # Test with cert revoked + mock_checker.return_value = True + self.assertTrue(self.test_rc.ocsp_revoked(version)) + self.assertEqual(mock_checker.call_args[0][0], expected_cert_path) + self.assertEqual(mock_checker.call_args[0][1], expected_chain_path) + + # Test with cert not revoked + mock_checker.return_value = False + self.assertFalse(self.test_rc.ocsp_revoked(version)) + self.assertEqual(mock_checker.call_args[0][0], expected_cert_path) + self.assertEqual(mock_checker.call_args[0][1], expected_chain_path) + + # Test with error + mock_checker.side_effect = ValueError + with mock.patch("certbot._internal.storage.logger.warning") as logger: + self.assertFalse(self.test_rc.ocsp_revoked(version)) + self.assertEqual(mock_checker.call_args[0][0], expected_cert_path) + self.assertEqual(mock_checker.call_args[0][1], expected_chain_path) + log_msg = logger.call_args[0][0] + self.assertIn("An error occurred determining the OCSP status", log_msg) def test_add_time_interval(self): from certbot._internal import storage diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 3ff09a83f..6dd0f964c 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -4,7 +4,10 @@ import errno import sys import unittest -import mock +try: + import mock +except ImportError: # pragma: no cover + from unittest import mock import six from six.moves import reload_module # pylint: disable=import-error diff --git a/letsencrypt-auto b/letsencrypt-auto index cea58e2cb..0ea3275c3 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.2.0" +LE_AUTO_VERSION="1.3.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1540,18 +1540,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.2.0 \ - --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \ - --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c -acme==1.2.0 \ - --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \ - --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab -certbot-apache==1.2.0 \ - --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \ - --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3 -certbot-nginx==1.2.0 \ - --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \ - --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db +certbot==1.3.0 \ + --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ + --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 +acme==1.3.0 \ + --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ + --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d +certbot-apache==1.3.0 \ + --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ + --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 +certbot-nginx==1.3.0 \ + --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ + --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 488d0bf2e..84473dc30 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- -iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl456ZoACgkQTRfJlc2X -dfJx8wf/addMw4kUlwu6poHqLvsifZzHAESgvq+qybgFvl5yTh2U+99PGBgxRYx+ -bENIWBi6+XB+CiVuLzIXWw/VkXh+za99orRkkVK9PI33Xr7jBMZo5Oa3JviYjl3X -PcfjioRQCD+a9Tf9RO25LXQmxn87Ql9x3nxJuk//YeSpuImFmYjIBPE4n/LPEf7z -8WHU4oxxa/bgqGCPgv6O7ZBw7ipd3g+VHcDZcNQMP4tWYb6m7x/nN61yirid7q3M -uqQ1lbitN48ISyru6xPyE6WGTvfl1SIQd21FNRETpcoesx+MTv3ApWT4dqXjZvaX -FeM55IS65e7ci6yLV9qdAbqGKzhX0Q== -=uLcV +iQEzBAABCAAdFiEEos+1H6J1pyhiNOeyTRfJlc2XdfIFAl5ewVUACgkQTRfJlc2X +dfJnZAf+KmxYl1YoP/FlTG5Npb64qaDdxm59SeEVJez6fZh15xq71tRPYR+4xszE +XTeyGt7uAxjYqeiBJU5xBvGC1Veprhj5AbflVOTP+5yiBr9iNWC35zmgaE63UlZ/ +V94sfL0pkax7wLngil7a0OuzUjikzK3gXOqrY8LoUdr4mAA9AhSjajWHmyY3tpDR +84GKrVhybIt0sjy/172VuPPbXZKno/clztkKMZHXNrDeL5jgJ15Va4Ts5FK0j9VT +HQvuazbGkYVCuvlp8Np5ESDje69LCJfPZxl34htoa8WNJoVIOsQWZpoXp5B5huSP +vGrh4LabZ5UDsl+k11ikHBRUpO7E5w== +=IgRH -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index e2813853b..ca0bda2d5 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -31,7 +31,7 @@ if [ -z "$VENV_PATH" ]; then fi VENV_BIN="$VENV_PATH/bin" BOOTSTRAP_VERSION_PATH="$VENV_PATH/certbot-auto-bootstrap-version.txt" -LE_AUTO_VERSION="1.3.0.dev0" +LE_AUTO_VERSION="1.4.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -1540,18 +1540,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -certbot==1.2.0 \ - --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \ - --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c -acme==1.2.0 \ - --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \ - --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab -certbot-apache==1.2.0 \ - --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \ - --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3 -certbot-nginx==1.2.0 \ - --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \ - --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db +certbot==1.3.0 \ + --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ + --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 +acme==1.3.0 \ + --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ + --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d +certbot-apache==1.3.0 \ + --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ + --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 +certbot-nginx==1.3.0 \ + --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ + --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index fefc81b37..8c4f52d6e 100644 Binary files a/letsencrypt-auto-source/letsencrypt-auto.sig and b/letsencrypt-auto-source/letsencrypt-auto.sig differ diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index eb9027edb..a8e275a45 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -certbot==1.2.0 \ - --hash=sha256:e25c17125c00b3398c8e9b9d54ef473c0e8f5aff53389f313a51b06cf472d335 \ - --hash=sha256:95dcbae085f8e4eb18442fe7b12994b08964a9a6e8e352e556cdb4a8a625373c -acme==1.2.0 \ - --hash=sha256:284d22fde75687a8ea72d737cac6bcbdc91f3c796221aa25378b8732ba6f6875 \ - --hash=sha256:0630c740d49bda945e97bd35fc8d6f02d082c8cb9e18f8fec0dbb3d395ac26ab -certbot-apache==1.2.0 \ - --hash=sha256:3f7493918353d3bd6067d446a2cf263e03831c4c10ec685b83d644b47767090d \ - --hash=sha256:b46e9def272103a68108e48bf7e410ea46801529b1ea6954f6506b14dd9df9b3 -certbot-nginx==1.2.0 \ - --hash=sha256:efd32a2b32f2439279da446b6bf67684f591f289323c5f494ebfd86a566a28fd \ - --hash=sha256:6fd7cf4f2545ad66e57000343227df9ccccaf04420e835e05cb3250fac1fa6db +certbot==1.3.0 \ + --hash=sha256:979793b36151be26c159f1946d065a0cbbcaed3e9ac452c19a142b0d2d2b42e3 \ + --hash=sha256:bc2091cbbc2f432872ed69309046e79771d9c81cd441bde3e6a6553ecd04b1d8 +acme==1.3.0 \ + --hash=sha256:b888757c750e393407a3cdf0eb5c2d06036951e10c41db4c83537617568561b6 \ + --hash=sha256:c0de9e1fbcb4a28509825a4d19ab5455910862b23fa338acebc7bbe7c0abd20d +certbot-apache==1.3.0 \ + --hash=sha256:1050cd262bcc598957c45a6fa1febdf5e41e87176c0aebad3a1ab7268b0d82d9 \ + --hash=sha256:4a6bb818a7a70803127590a54bb25c1e79810761c9d4c92cf9f16a56b518bd52 +certbot-nginx==1.3.0 \ + --hash=sha256:46106b96429d1aaf3765635056352d2372941027a3bc26bbf964e4329202adc7 \ + --hash=sha256:9aa0869c1250b7ea0a1eb1df6bdb5d0d6190d6ca0400da1033a8decc0df6f65b diff --git a/letsencrypt-auto-source/rebuild_dependencies.py b/letsencrypt-auto-source/rebuild_dependencies.py old mode 100755 new mode 100644 index 6d1ec15ff..3093b6bb0 --- a/letsencrypt-auto-source/rebuild_dependencies.py +++ b/letsencrypt-auto-source/rebuild_dependencies.py @@ -103,7 +103,7 @@ def _requirements_from_one_distribution(distribution, verbose): os.chmod(script, 0o755) _write_to(authoritative_constraints, '\n'.join( - ['{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items()])) + '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) command = ['docker', 'run', '--rm', '--cidfile', cid_file, '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), diff --git a/tests/letstest/README.md b/tests/letstest/README.md index f8a15208e..5bd326e2a 100644 --- a/tests/letstest/README.md +++ b/tests/letstest/README.md @@ -15,12 +15,12 @@ Simple AWS testfarm scripts for certbot client testing are needed, they need to be requested via online webform. ## Installation and configuration -These tests require Python 2.7, awscli, boto3, PyYAML, and fabric<2.0. If you -have Python 2.7 and virtualenv installed, you can use requirements.txt to -create a virtual environment with a known set of dependencies by running: +These tests require Python 3, awscli, boto3, PyYAML, and fabric 2.0+. If you +have Python 3 installed, you can use requirements.txt to create a virtual +environment with a known set of dependencies by running: ``` -virtualenv --python $(command -v python2.7 || command -v python2 || command -v python) venv -. ./venv/bin/activate +python3 -m venv venv3 +. ./venv3/bin/activate pip install --requirement requirements.txt ``` diff --git a/tests/letstest/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml index 1450a8578..76415b27e 100644 --- a/tests/letstest/apache2_targets.yaml +++ b/tests/letstest/apache2_targets.yaml @@ -1,11 +1,6 @@ targets: #----------------------------------------------------------------------------- #Ubuntu - - ami: ami-08ab45c4343f5f5c6 - name: ubuntu19.04 - type: ubuntu - virt: hvm - user: ubuntu - ami: ami-095192256fe1477ad name: ubuntu18.04LTS type: ubuntu diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py index 9ea9fe76b..09821e7dd 100644 --- a/tests/letstest/multitester.py +++ b/tests/letstest/multitester.py @@ -40,23 +40,16 @@ import socket import sys import time import traceback -import urllib2 import boto3 from botocore.exceptions import ClientError +from six.moves.urllib import error as urllib_error +from six.moves.urllib import request as urllib_request import yaml -import fabric -from fabric.api import cd -from fabric.api import env -from fabric.api import execute -from fabric.api import lcd -from fabric.api import local -from fabric.api import run -from fabric.api import sudo -from fabric.context_managers import shell_env -from fabric.operations import get -from fabric.operations import put +from fabric import Config +from fabric import Connection + # Command line parser #------------------------------------------------------------------------------- @@ -203,11 +196,11 @@ def block_until_http_ready(urlstring, wait_time=10, timeout=240): try: sys.stdout.write('.') sys.stdout.flush() - req = urllib2.Request(urlstring) - response = urllib2.urlopen(req) + req = urllib_request.Request(urlstring) + response = urllib_request.urlopen(req) #if response.code == 200: server_ready = True - except urllib2.URLError: + except urllib_error.URLError: pass time.sleep(wait_time) t_elapsed += wait_time @@ -244,76 +237,85 @@ def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20 # Fabric Routines #------------------------------------------------------------------------------- -def local_git_clone(repo_url): +def local_git_clone(local_cxn, repo_url): "clones master of repo_url" - with lcd(LOGDIR): - local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s letsencrypt'% repo_url) - local('tar czf le.tar.gz letsencrypt') + local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR) + local_cxn.local('cd %s && git clone %s letsencrypt'% (LOGDIR, repo_url)) + local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt'% LOGDIR) -def local_git_branch(repo_url, branch_name): +def local_git_branch(local_cxn, repo_url, branch_name): "clones branch of repo_url" - with lcd(LOGDIR): - local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s letsencrypt --branch %s --single-branch'%(repo_url, branch_name)) - local('tar czf le.tar.gz letsencrypt') + local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR) + local_cxn.local('cd %s && git clone %s letsencrypt --branch %s --single-branch'% + (LOGDIR, repo_url, branch_name)) + local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt' % LOGDIR) -def local_git_PR(repo_url, PRnumstr, merge_master=True): +def local_git_PR(local_cxn, repo_url, PRnumstr, merge_master=True): "clones specified pull request from repo_url and optionally merges into master" - with lcd(LOGDIR): - local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi') - local('git clone %s letsencrypt'% repo_url) - local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr) - local('cd letsencrypt && git checkout lePRtest') - if merge_master: - local('cd letsencrypt && git remote update origin') - local('cd letsencrypt && git merge origin/master -m "testmerge"') - local('tar czf le.tar.gz letsencrypt') + local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR) + local_cxn.local('cd %s && git clone %s letsencrypt' % (LOGDIR, repo_url)) + local_cxn.local('cd %s && cd letsencrypt && ' + 'git fetch origin pull/%s/head:lePRtest' % (LOGDIR, PRnumstr)) + local_cxn.local('cd %s && cd letsencrypt && git checkout lePRtest' % LOGDIR) + if merge_master: + local_cxn.local('cd %s && cd letsencrypt && git remote update origin' % LOGDIR) + local_cxn.local('cd %s && cd letsencrypt && ' + 'git merge origin/master -m "testmerge"' % LOGDIR) + local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt' % LOGDIR) -def local_repo_to_remote(): +def local_repo_to_remote(cxn): "copies local tarball of repo to remote" - with lcd(LOGDIR): - put(local_path='le.tar.gz', remote_path='') - run('tar xzf le.tar.gz') + filename = 'le.tar.gz' + local_path = os.path.join(LOGDIR, filename) + cxn.put(local=local_path, remote='') + cxn.run('tar xzf %s' % filename) -def local_repo_clean(): +def local_repo_clean(local_cxn): "delete tarball" - with lcd(LOGDIR): - local('rm le.tar.gz') + filename = 'le.tar.gz' + local_path = os.path.join(LOGDIR, filename) + local_cxn.local('rm %s' % local_path) -def deploy_script(scriptpath, *args): +def deploy_script(cxn, scriptpath, *args): "copies to remote and executes local script" - #with lcd('scripts'): - put(local_path=scriptpath, remote_path='', mirror_local_mode=True) + cxn.put(local=scriptpath, remote='', preserve_mode=True) scriptfile = os.path.split(scriptpath)[1] args_str = ' '.join(args) - run('./'+scriptfile+' '+args_str) + cxn.run('./'+scriptfile+' '+args_str) -def run_boulder(): - with cd('$GOPATH/src/github.com/letsencrypt/boulder'): - run('sudo docker-compose up -d') +def run_boulder(cxn): + boulder_path = '$GOPATH/src/github.com/letsencrypt/boulder' + cxn.run('cd %s && sudo docker-compose up -d' % boulder_path) -def config_and_launch_boulder(instance): - execute(deploy_script, 'scripts/boulder_config.sh') - execute(run_boulder) +def config_and_launch_boulder(cxn, instance): + # yes, we're hardcoding the gopath. it's a predetermined AMI. + with cxn.prefix('export GOPATH=/home/ubuntu/gopath'): + deploy_script(cxn, 'scripts/boulder_config.sh') + run_boulder(cxn) -def install_and_launch_certbot(instance, boulder_url, target): - execute(local_repo_to_remote) - with shell_env(BOULDER_URL=boulder_url, - PUBLIC_IP=instance.public_ip_address, - PRIVATE_IP=instance.private_ip_address, - PUBLIC_HOSTNAME=instance.public_dns_name, - PIP_EXTRA_INDEX_URL=cl_args.alt_pip, - OS_TYPE=target['type']): - execute(deploy_script, cl_args.test_script) +def install_and_launch_certbot(cxn, instance, boulder_url, target): + local_repo_to_remote(cxn) + # This needs to be like this, I promise. 1) The env argument to run doesn't work. + # See https://github.com/fabric/fabric/issues/1744. 2) prefix() sticks an && between + # the commands, so it needs to be exports rather than no &&s in between for the script subshell. + with cxn.prefix('export BOULDER_URL=%s && export PUBLIC_IP=%s && export PRIVATE_IP=%s && ' + 'export PUBLIC_HOSTNAME=%s && export PIP_EXTRA_INDEX_URL=%s && ' + 'export OS_TYPE=%s' % + (boulder_url, + instance.public_ip_address, + instance.private_ip_address, + instance.public_dns_name, + cl_args.alt_pip, + target['type'])): + deploy_script(cxn, cl_args.test_script) -def grab_certbot_log(): +def grab_certbot_log(cxn): "grabs letsencrypt.log via cat into logged stdout" - sudo('if [ -f /var/log/letsencrypt/letsencrypt.log ]; then \ - cat /var/log/letsencrypt/letsencrypt.log; else echo "[novarlog]"; fi') + cxn.sudo('/bin/bash -l -i -c \'if [ -f "/var/log/letsencrypt/letsencrypt.log" ]; then ' + + 'cat "/var/log/letsencrypt/letsencrypt.log"; else echo "[novarlog]"; fi\'') # fallback file if /var/log is unwriteable...? correct? - sudo('if [ -f ./certbot.log ]; then \ - cat ./certbot.log; else echo "[nolocallog]"; fi') + cxn.sudo('/bin/bash -l -i -c \'if [ -f ./certbot.log ]; then ' + + 'cat ./certbot.log; else echo "[nolocallog]"; fi\'') def create_client_instance(ec2_client, target, security_group_id, subnet_id): @@ -341,7 +343,7 @@ def create_client_instance(ec2_client, target, security_group_id, subnet_id): userdata=userdata) -def test_client_process(inqueue, outqueue, boulder_url): +def test_client_process(fab_config, inqueue, outqueue, boulder_url): cur_proc = mp.current_process() for inreq in iter(inqueue.get, SENTINEL): ii, instance_id, target = inreq @@ -358,30 +360,31 @@ def test_client_process(inqueue, outqueue, boulder_url): print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name'])) instance = block_until_instance_ready(instance) print("server %s at %s"%(instance, instance.public_ip_address)) - env.host_string = "%s@%s"%(target['user'], instance.public_ip_address) - print(env.host_string) + host_string = "%s@%s"%(target['user'], instance.public_ip_address) + print(host_string) - try: - install_and_launch_certbot(instance, boulder_url, target) - outqueue.put((ii, target, Status.PASS)) - print("%s - %s SUCCESS"%(target['ami'], target['name'])) - except: - outqueue.put((ii, target, Status.FAIL)) - print("%s - %s FAIL"%(target['ami'], target['name'])) - traceback.print_exc(file=sys.stdout) - pass + with Connection(host_string, config=fab_config) as cxn: + try: + install_and_launch_certbot(cxn, instance, boulder_url, target) + outqueue.put((ii, target, Status.PASS)) + print("%s - %s SUCCESS"%(target['ami'], target['name'])) + except: + outqueue.put((ii, target, Status.FAIL)) + print("%s - %s FAIL"%(target['ami'], target['name'])) + traceback.print_exc(file=sys.stdout) + pass - # append server certbot.log to each per-machine output log - print("\n\ncertbot.log\n" + "-"*80 + "\n") - try: - execute(grab_certbot_log) - except: - print("log fail\n") - traceback.print_exc(file=sys.stdout) - pass + # append server certbot.log to each per-machine output log + print("\n\ncertbot.log\n" + "-"*80 + "\n") + try: + grab_certbot_log(cxn) + except: + print("log fail\n") + traceback.print_exc(file=sys.stdout) + pass -def cleanup(cl_args, instances, targetlist): +def cleanup(cl_args, instances, targetlist, boulder_server): print('Logs in ', LOGDIR) # If lengths of instances and targetlist aren't equal, instances failed to # start before running tests so leaving instances running for debugging @@ -402,19 +405,25 @@ def cleanup(cl_args, instances, targetlist): def main(): # Fabric library controlled through global env parameters - env.key_filename = KEYFILE - env.shell = '/bin/bash -l -i -c' - env.connection_attempts = 5 - env.timeout = 10 - # replace default SystemExit thrown by fabric during trouble - class FabricException(Exception): - pass - env['abort_exception'] = FabricException + fab_config = Config(overrides={ + "connect_kwargs": { + "key_filename": [KEYFILE], # https://github.com/fabric/fabric/issues/2007 + }, + "run": { + "echo": True, + "pty": True, + }, + "timeouts": { + "connect": 10, + }, + }) + # no network connection, so don't worry about closing this one. + local_cxn = Connection('localhost', config=fab_config) # Set up local copy of git repo #------------------------------------------------------------------------------- print("Making local dir for test repo and logs: %s"%LOGDIR) - local('mkdir %s'%LOGDIR) + local_cxn.local('mkdir %s'%LOGDIR) # figure out what git object to test and locally create it in LOGDIR print("Making local git repo") @@ -422,14 +431,14 @@ def main(): if cl_args.pull_request != '~': print('Testing PR %s '%cl_args.pull_request, "MERGING into master" if cl_args.merge_master else "") - execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master) + local_git_PR(local_cxn, cl_args.repo, cl_args.pull_request, cl_args.merge_master) elif cl_args.branch != '~': print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo)) - execute(local_git_branch, cl_args.repo, cl_args.branch) + local_git_branch(local_cxn, cl_args.repo, cl_args.branch) else: print('Testing master of %s'%cl_args.repo) - execute(local_git_clone, cl_args.repo) - except FabricException: + local_git_clone(local_cxn, cl_args.repo) + except BaseException: print("FAIL: trouble with git repo") traceback.print_exc() exit() @@ -437,7 +446,7 @@ def main(): # Set up EC2 instances #------------------------------------------------------------------------------- - configdata = yaml.load(open(cl_args.config_file, 'r')) + configdata = yaml.safe_load(open(cl_args.config_file, 'r')) targetlist = configdata['targets'] print('Testing against these images: [%d total]'%len(targetlist)) for target in targetlist: @@ -511,15 +520,16 @@ def main(): print(" server %s"%boulder_server) - # env.host_string defines the ssh user and host for connection - env.host_string = "ubuntu@%s"%boulder_server.public_ip_address - print("Boulder Server at (SSH):", env.host_string) + # host_string defines the ssh user and host for connection + host_string = "ubuntu@%s"%boulder_server.public_ip_address + print("Boulder Server at (SSH):", host_string) if not boulder_preexists: print("Configuring and Launching Boulder") - config_and_launch_boulder(boulder_server) - # blocking often unnecessary, but cheap EC2 VMs can get very slow - block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, - wait_time=10, timeout=500) + with Connection(host_string, config=fab_config) as boulder_cxn: + config_and_launch_boulder(boulder_cxn, boulder_server) + # blocking often unnecessary, but cheap EC2 VMs can get very slow + block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address, + wait_time=10, timeout=500) boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address) @@ -545,7 +555,7 @@ def main(): # initiate process execution for i in range(num_processes): - p = mp.Process(target=test_client_process, args=(inqueue, outqueue, boulder_url)) + p = mp.Process(target=test_client_process, args=(fab_config, inqueue, outqueue, boulder_url)) jobs.append(p) p.daemon = True # kills subprocesses if parent is killed p.start() @@ -569,7 +579,7 @@ def main(): outqueue.put(SENTINEL) # clean up - execute(local_repo_clean) + local_repo_clean(local_cxn) # print and save summary results results_file = open(LOGDIR+'/results', 'w') @@ -594,10 +604,7 @@ def main(): sys.exit(1) finally: - cleanup(cl_args, instances, targetlist) - - # kill any connections - fabric.network.disconnect_all() + cleanup(cl_args, instances, targetlist, boulder_server) if __name__ == '__main__': diff --git a/tests/letstest/requirements.txt b/tests/letstest/requirements.txt index 64e1f6a0c..840e3e5d5 100644 --- a/tests/letstest/requirements.txt +++ b/tests/letstest/requirements.txt @@ -1,25 +1,19 @@ -asn1crypto==0.24.0 -awscli==1.16.157 -bcrypt==3.1.6 -boto3==1.9.146 -botocore==1.12.147 -cffi==1.12.3 -colorama==0.3.9 -cryptography==2.4.2 -docutils==0.14 -enum34==1.1.6 -Fabric==1.14.1 -futures==3.2.0 -idna==2.8 -ipaddress==1.0.22 -jmespath==0.9.4 -paramiko==2.4.2 -pyasn1==0.4.5 +bcrypt==3.1.7 +boto3==1.12.7 +botocore==1.15.7 +cffi==1.14.0 +cryptography==2.8 +docutils==0.15.2 +enum34==1.1.9 +Fabric==2.5.0 +ipaddress==1.0.23 +Invoke==1.4.1 +jmespath==0.9.5 +paramiko==2.7.1 pycparser==2.19 PyNaCl==1.3.0 -python-dateutil==2.8.0 -PyYAML==3.10 -rsa==3.4.2 -s3transfer==0.2.0 -six==1.12.0 -urllib3==1.24.3 +python-dateutil==2.8.1 +PyYAML==5.3 +s3transfer==0.3.3 +six==1.14.0 +urllib3==1.25.8 diff --git a/tests/letstest/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh index 9af39e8bb..9c3b95a31 100755 --- a/tests/letstest/scripts/test_apache2.sh +++ b/tests/letstest/scripts/test_apache2.sh @@ -58,6 +58,22 @@ if [ $? -ne 0 ] ; then FAIL=1 fi +# Check that ssl_module detection is working on various systems +if [ "$OS_TYPE" = "ubuntu" ] ; then + MOD_SSL_LOCATION="/usr/lib/apache2/modules/mod_ssl.so" + APACHE_NAME=apache2ctl +elif [ "$OS_TYPE" = "centos" ]; then + MOD_SSL_LOCATION="/etc/httpd/modules/mod_ssl.so" + APACHE_NAME=httpd +fi +OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9) +APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8) +"$PYTHON_NAME" tests/letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION" +if [ $? -ne 0 ] ; then + FAIL=1 +fi + + if [ "$OS_TYPE" = "ubuntu" ] ; then export SERVER="$BOULDER_URL" "$VENV_PATH/bin/tox" -e apacheconftest diff --git a/tests/letstest/scripts/test_openssl_version.py b/tests/letstest/scripts/test_openssl_version.py new file mode 100644 index 000000000..c55441c5d --- /dev/null +++ b/tests/letstest/scripts/test_openssl_version.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# Test script for OpenSSL version checking +from distutils.version import LooseVersion +import sys + + +def main(openssl_version, apache_version): + if not openssl_version.strip(): + raise Exception("No OpenSSL version found.") + if not apache_version.strip(): + raise Exception("No Apache version found.") + conf_file_location = "/etc/letsencrypt/options-ssl-apache.conf" + with open(conf_file_location) as f: + contents = f.read() + if LooseVersion(apache_version.strip()) < LooseVersion('2.4.11') or \ + LooseVersion(openssl_version.strip()) < LooseVersion('1.0.2l'): + # should be old version + # assert SSLSessionTickets not in conf file + if "SSLSessionTickets" in contents: + raise Exception("Apache or OpenSSL version is too old, " + "but SSLSessionTickets is set.") + else: + # should be current version + # assert SSLSessionTickets in conf file + if "SSLSessionTickets" not in contents: + raise Exception("Apache and OpenSSL versions are sufficiently new, " + "but SSLSessionTickets is not set.") + +if __name__ == '__main__': + main(*sys.argv[1:]) diff --git a/tests/letstest/targets.yaml b/tests/letstest/targets.yaml index 188be8e24..06055a9a5 100644 --- a/tests/letstest/targets.yaml +++ b/tests/letstest/targets.yaml @@ -1,11 +1,6 @@ targets: #----------------------------------------------------------------------------- #Ubuntu - - ami: ami-08ab45c4343f5f5c6 - name: ubuntu19.04 - type: ubuntu - virt: hvm - user: ubuntu - ami: ami-095192256fe1477ad name: ubuntu18.04LTS type: ubuntu diff --git a/tools/_release.sh b/tools/_release.sh index 1819adad2..97d5f5eb8 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -59,7 +59,7 @@ mv "dist.$version" "dist.$version.$(date +%s).bak" || true git tag --delete "$tag" || true tmpvenv=$(mktemp -d) -VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages -p python2 $tmpvenv +VIRTUALENV_NO_DOWNLOAD=1 virtualenv -p python2 $tmpvenv . $tmpvenv/bin/activate # update setuptools/pip just like in other places in the repo pip install -U setuptools @@ -160,7 +160,7 @@ cd "dist.$version" python -m SimpleHTTPServer $PORT & # cd .. is NOT done on purpose: we make sure that all subpackages are # installed from local PyPI rather than current directory (repo root) -VIRTUALENV_NO_DOWNLOAD=1 virtualenv --no-site-packages ../venv +VIRTUALENV_NO_DOWNLOAD=1 virtualenv ../venv . ../venv/bin/activate pip install -U setuptools pip install -U pip diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 75d0d5d33..5196cf9c4 100644 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -124,7 +124,9 @@ def _check_version(version_str, major_version): return False -def subprocess_with_print(cmd, env=os.environ, shell=False): +def subprocess_with_print(cmd, env=None, shell=False): + if env is None: + env = os.environ print('+ {0}'.format(subprocess.list2cmdline(cmd)) if isinstance(cmd, list) else cmd) subprocess.check_call(cmd, env=env, shell=shell) diff --git a/tools/deactivate.py b/tools/deactivate.py index 10c9ecd35..214c0595c 100644 --- a/tools/deactivate.py +++ b/tools/deactivate.py @@ -17,7 +17,6 @@ import sys from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa import josepy as jose from acme import client as acme_client diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 7d2013c7a..6e692841b 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -3,7 +3,7 @@ # Some dev package versions specified here may be overridden by higher level constraints # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). alabaster==0.7.10 -apacheconfig==0.3.1 +apacheconfig==0.3.2 apipkg==1.4 appnope==0.1.0 asn1crypto==0.22.0 @@ -18,7 +18,6 @@ boto3==1.11.7 botocore==1.14.7 cached-property==1.5.1 cloudflare==2.3.1 -codecov==2.0.15 configparser==3.7.4 contextlib2==0.6.0.post1 coverage==4.5.4 diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 85d058796..402f3fef1 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -39,7 +39,7 @@ pytz==2012rc0 google-api-python-client==1.5.5 # Our setup.py constraints -apacheconfig==0.3.1 +apacheconfig==0.3.2 cloudflare==1.5.1 cryptography==1.2.3 parsedatetime==1.3 diff --git a/tox.cover.py b/tox.cover.py index 3e69a14d6..4848b2740 100755 --- a/tox.cover.py +++ b/tox.cover.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function + import argparse import os import subprocess @@ -48,18 +50,23 @@ def cover(package): subprocess.check_call([sys.executable, '-m', 'pytest', '--cov', pkg_dir, '--cov-append', '--cov-report=', pkg_dir]) - subprocess.check_call([ - sys.executable, '-m', 'coverage', 'report', '--fail-under', str(threshold), '--include', - '{0}/*'.format(pkg_dir), '--show-missing']) + try: + subprocess.check_call([ + sys.executable, '-m', 'coverage', 'report', '--fail-under', + str(threshold), '--include', '{0}/*'.format(pkg_dir), + '--show-missing']) + except subprocess.CalledProcessError as err: + print(err) + print('Test coverage on', pkg_dir, + 'did not meet threshold of {0}%.'.format(threshold)) + sys.exit(1) def main(): description = """ This script is used by tox.ini (and thus by Travis CI and Azure Pipelines) in order to generate separate stats for each package. It should be removed once -those packages are moved to a separate repo. - -Option -e makes sure we fail fast and don't submit to codecov.""" +those packages are moved to a separate repo.""" parser = argparse.ArgumentParser(description=description) parser.add_argument('--packages', nargs='+') diff --git a/tox.ini b/tox.ini index 3903cdf45..7f5b7bd5a 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist = true -envlist = modification,py3,py27-cover,lint,mypy +envlist = modification,py3-cover,lint,mypy [base] # pip installs the requested packages in editable mode @@ -63,8 +63,11 @@ source_paths = passenv = CERTBOT_NO_PIN commands = - {[base]install_and_test} {[base]all_packages} - python tests/lock_test.py + !cover: {[base]install_and_test} {[base]all_packages} + !cover: python tests/lock_test.py + cover: {[base]install_packages} + cover: {[base]pip_install} certbot-apache[dev] + cover: python tox.cover.py # We always recreate the virtual environment to avoid problems like # https://github.com/certbot/certbot/issues/7745. recreate = true @@ -116,18 +119,6 @@ commands = setenv = {[testenv:py27-oldest]setenv} -[testenv:py27-cover] -basepython = python2.7 -commands = - {[base]install_packages} - python tox.cover.py - -[testenv:py37-cover] -basepython = python3.7 -commands = - {[base]install_packages} - python tox.cover.py - [testenv:lint] basepython = python3 # separating into multiple invocations disables cross package diff --git a/windows-installer/construct.py b/windows-installer/construct.py index f0724f5f4..4b05c926a 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -153,7 +153,7 @@ extra_preamble=pywin32_paths.py '''.format(certbot_version=certbot_version, installer_suffix='win_amd64' if PYTHON_BITNESS == 64 else 'win32', python_bitness=PYTHON_BITNESS, - python_version='.'.join([str(item) for item in PYTHON_VERSION]))) + python_version='.'.join(str(item) for item in PYTHON_VERSION))) return installer_cfg_path @@ -184,7 +184,7 @@ if __name__ == '__main__': if sys.version_info[:2] != PYTHON_VERSION[:2]: raise RuntimeError('This script must be run with Python {0}' - .format('.'.join([str(item) for item in PYTHON_VERSION[0:2]]))) + .format('.'.join(str(item) for item in PYTHON_VERSION[0:2]))) if struct.calcsize('P') * 8 != PYTHON_BITNESS: raise RuntimeError('This script must be run with a {0} bit version of Python.'